From 5018ff06b30e97d22f6ca83a9a6b221e56fd901d Mon Sep 17 00:00:00 2001 From: streamer45 Date: Sun, 25 Jan 2026 12:11:23 +0100 Subject: [PATCH 1/5] feat: pocket-tts plugin --- crates/plugin-native/src/wrapper.rs | 83 +- justfile | 40 +- plugins/native/pocket-tts/Cargo.lock | 3821 ++++ plugins/native/pocket-tts/Cargo.toml | 48 + plugins/native/pocket-tts/README.md | 101 + .../native/pocket-tts/config/b6369a24.yaml | 59 + plugins/native/pocket-tts/download-models.py | 83 + plugins/native/pocket-tts/src/config.rs | 99 + plugins/native/pocket-tts/src/lib.rs | 15 + plugins/native/pocket-tts/src/model.rs | 168 + .../native/pocket-tts/src/pocket_tts_node.rs | 500 + .../pocket-tts/src/sentence_splitter.rs | 74 + plugins/native/pocket-tts/src/voice.rs | 254 + .../vendor/pocket-tts/.cargo/config.toml | 6 + .../pocket-tts/vendor/pocket-tts/Cargo.toml | 73 + .../vendor/pocket-tts/assets/tokenizer.json | 16129 ++++++++++++++++ .../pocket-tts/benches/attention_bench.rs | 70 + .../pocket-tts/benches/full_benchmark.rs | 96 + .../pocket-tts/benches/streaming_bench.rs | 75 + .../vendor/pocket-tts/config/b6369a24.yaml | 57 + .../vendor/pocket-tts/examples/bench_sdpa.rs | 91 + .../pocket-tts/examples/check_config.rs | 12 + .../pocket-tts/examples/inspect_hound.rs | 8 + .../pocket-tts/examples/quantize_demo.rs | 73 + .../pocket-tts/examples/scaling_bench.rs | 32 + .../vendor/pocket-tts/examples/verify_sdpa.rs | 84 + .../pocket-tts/examples/wasm/index.html | 766 + .../pocket-tts/vendor/pocket-tts/src/audio.rs | 301 + .../vendor/pocket-tts/src/conditioners/mod.rs | 1 + .../pocket-tts/src/conditioners/text.rs | 161 + .../vendor/pocket-tts/src/config.rs | 168 + .../pocket-tts/vendor/pocket-tts/src/lib.rs | 18 + .../pocket-tts/src/modules/attention.rs | 233 + .../vendor/pocket-tts/src/modules/conv.rs | 346 + .../vendor/pocket-tts/src/modules/mlp.rs | 418 + .../vendor/pocket-tts/src/modules/mod.rs | 5 + .../vendor/pocket-tts/src/modules/rope.rs | 79 + .../vendor/pocket-tts/src/modules/sdpa.rs | 312 + .../pocket-tts/vendor/pocket-tts/src/pause.rs | 249 + .../vendor/pocket-tts/src/quantize.rs | 219 + .../vendor/pocket-tts/src/tts_model.rs | 1237 ++ .../vendor/pocket-tts/src/voice_state.rs | 93 + .../pocket-tts/vendor/pocket-tts/src/wasm.rs | 235 + .../vendor/pocket-tts/src/weights.rs | 108 + .../pocket-tts/tests/integration_tests.rs | 320 + .../vendor/pocket-tts/tests/memory_usage.rs | 40 + .../vendor/pocket-tts/tests/parity_tests.rs | 612 + .../pocket-tts/tests/streaming_tests.rs | 156 + .../vendor/pocket-tts/wasm-pack.toml | 22 + .../oneshot/pocket-tts-voice-clone.yml | 61 + samples/pipelines/oneshot/pocket-tts.yml | 34 + ui/src/components/converter/FileUpload.tsx | 5 +- ui/src/views/ConvertView.tsx | 342 +- 53 files changed, 28618 insertions(+), 74 deletions(-) create mode 100644 plugins/native/pocket-tts/Cargo.lock create mode 100644 plugins/native/pocket-tts/Cargo.toml create mode 100644 plugins/native/pocket-tts/README.md create mode 100644 plugins/native/pocket-tts/config/b6369a24.yaml create mode 100644 plugins/native/pocket-tts/download-models.py create mode 100644 plugins/native/pocket-tts/src/config.rs create mode 100644 plugins/native/pocket-tts/src/lib.rs create mode 100644 plugins/native/pocket-tts/src/model.rs create mode 100644 plugins/native/pocket-tts/src/pocket_tts_node.rs create mode 100644 plugins/native/pocket-tts/src/sentence_splitter.rs create mode 100644 plugins/native/pocket-tts/src/voice.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/.cargo/config.toml create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/Cargo.toml create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/assets/tokenizer.json create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/config/b6369a24.yaml create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/wasm-pack.toml create mode 100644 samples/pipelines/oneshot/pocket-tts-voice-clone.yml create mode 100644 samples/pipelines/oneshot/pocket-tts.yml diff --git a/crates/plugin-native/src/wrapper.rs b/crates/plugin-native/src/wrapper.rs index d99887ae..93e0d6e4 100644 --- a/crates/plugin-native/src/wrapper.rs +++ b/crates/plugin-native/src/wrapper.rs @@ -218,15 +218,62 @@ impl ProcessorNode for NativeNodeWrapper { warn!(error = %e, node = %node_name, "Failed to send initializing state"); } - tracing::debug!(node = %node_name, "Getting input channel"); + tracing::debug!(node = %node_name, "Getting input channels"); - // Get input channel - let mut input_rx = context.take_input("in").map_err(|e| { - tracing::error!(node = %node_name, error = %e, "Failed to get input channel"); - StreamKitError::Runtime(format!("Failed to get input channel: {e}")) - })?; + let mut inputs = std::mem::take(&mut context.inputs); + if inputs.is_empty() { + return Err(StreamKitError::Runtime( + "Engine did not provide any input pin receivers".to_string(), + )); + } - tracing::debug!(node = %node_name, "Got input channel, entering main loop"); + let mut input_pin_names = Vec::with_capacity(inputs.len()); + let mut input_pin_cstrs = Vec::with_capacity(inputs.len()); + let mut input_tasks = Vec::with_capacity(inputs.len()); + let (merged_tx, mut merged_rx) = + tokio::sync::mpsc::channel::<(usize, Packet)>(context.batch_size.max(1)); + let cancellation_token = context.cancellation_token.clone(); + + for (pin_name, mut rx) in inputs.drain() { + let pin_cstr = CString::new(pin_name.as_str()).map_err(|e| { + StreamKitError::Runtime(format!("Invalid pin name '{pin_name}': {e}")) + })?; + let pin_index = input_pin_names.len(); + input_pin_names.push(pin_name); + input_pin_cstrs.push(Arc::new(pin_cstr)); + + let tx = merged_tx.clone(); + let token = cancellation_token.clone(); + let handle = tokio::spawn(async move { + loop { + let packet = if let Some(token) = &token { + tokio::select! { + () = token.cancelled() => None, + packet = rx.recv() => packet, + } + } else { + rx.recv().await + }; + + let Some(packet) = packet else { + break; + }; + + if tx.send((pin_index, packet)).await.is_err() { + break; + } + } + }); + input_tasks.push(handle); + } + + drop(merged_tx); + + tracing::debug!( + node = %node_name, + inputs = ?input_pin_names, + "Got input channels, entering main loop" + ); // Emit running state if let Err(e) = @@ -311,8 +358,8 @@ impl ProcessorNode for NativeNodeWrapper { } } - maybe_packet = input_rx.recv() => { - let Some(packet) = maybe_packet else { + maybe_packet = merged_rx.recv() => { + let Some((pin_index, packet)) = maybe_packet else { // Input closed - flush any buffered data before shutting down tracing::debug!(node = %node_name, "Native plugin input closed, flushing buffers"); @@ -394,6 +441,9 @@ impl ProcessorNode for NativeNodeWrapper { let node_id = node_name.clone(); // spawn_blocking can only fail with JoinError if the task panics. // If that happens, it's a serious bug that should crash. + let pin_cstr = Arc::clone(&input_pin_cstrs[pin_index]); + // spawn_blocking can only fail with JoinError if the task panics. + // If that happens, it's a serious bug that should crash. #[allow(clippy::expect_used)] let (outputs, error) = tokio::task::spawn_blocking(move || { let Some(handle) = state.begin_call() else { @@ -405,10 +455,6 @@ impl ProcessorNode for NativeNodeWrapper { // Convert packet to C representation let packet_repr = conversions::packet_to_c(&packet); - // Prepare input pin name - hardcoded ASCII string "in" can never contain null bytes - #[allow(clippy::expect_used)] - let pin_cstr = CString::new("in").expect("Hardcoded ASCII string is always valid C string"); - // Create callback context let mut callback_ctx = CallbackContext { output_packets: Vec::new(), @@ -479,12 +525,19 @@ impl ProcessorNode for NativeNodeWrapper { warn!(error = %e, node = %node_name, "Failed to send failed state"); } - return Err(StreamKitError::Runtime(error_msg)); - } + for handle in &input_tasks { + handle.abort(); + } + return Err(StreamKitError::Runtime(error_msg)); + } } } } + for handle in &input_tasks { + handle.abort(); + } + // Input closed, emit stopped state info!(node = %node_name, "Input closed, shutting down"); if let Err(e) = context diff --git a/justfile b/justfile index 3203d93f..c7204875 100644 --- a/justfile +++ b/justfile @@ -259,6 +259,7 @@ lint-plugins: @cd plugins/native/sensevoice && cargo fmt -- --check && cargo clippy -- -D warnings @cd plugins/native/vad && cargo fmt -- --check && cargo clippy -- -D warnings @cd plugins/native/matcha && cargo fmt -- --check && cargo clippy -- -D warnings + @cd plugins/native/pocket-tts && cargo fmt -- --check && cargo clippy -- -D warnings @cd plugins/native/nllb && cargo fmt -- --check && CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$$(pwd)/target/cmake-install" cargo clippy -- -D warnings @echo "✓ All native plugins passed linting" @@ -271,6 +272,7 @@ fix-plugins: @cd plugins/native/sensevoice && cargo fmt && cargo clippy --fix --allow-dirty --allow-staged -- -D warnings @cd plugins/native/vad && cargo fmt && cargo clippy --fix --allow-dirty --allow-staged -- -D warnings @cd plugins/native/matcha && cargo fmt && cargo clippy --fix --allow-dirty --allow-staged -- -D warnings + @cd plugins/native/pocket-tts && cargo fmt && cargo clippy --fix --allow-dirty --allow-staged -- -D warnings @cd plugins/native/nllb && cargo fmt && CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$$(pwd)/target/cmake-install" cargo clippy --fix --allow-dirty --allow-staged -- -D warnings @echo "✓ All native plugins fixed" @@ -513,6 +515,16 @@ download-piper-models: @echo "Downloading Piper TTS models..." @cd plugins/native/piper && ./download-models.sh +# Download Pocket TTS models and voices +download-pocket-tts-models: + @echo "Downloading Pocket TTS models and voices..." + @echo "⚠️ This requires Python with huggingface-hub installed." + @echo "⚠️ Install with: pip3 install --user huggingface-hub" + @echo "⚠️ Model weights are gated; set HF_TOKEN to authenticate." + @echo "" + @HF_HOME=models/hf python3 plugins/native/pocket-tts/download-models.py + @echo "✓ Pocket TTS models copied to models/pocket-tts (HF cache in models/hf)" + # Setup Piper TTS (install dependencies + download models) setup-piper: install-sherpa-onnx download-piper-models @echo "✓ Piper TTS setup complete!" @@ -558,6 +570,19 @@ build-plugin-native-matcha: @echo "Building native Matcha TTS plugin..." @cargo build --release +# Build native Pocket TTS plugin +[working-directory: 'plugins/native/pocket-tts'] +build-plugin-native-pocket-tts: + @echo "Building native Pocket TTS plugin..." + @cargo build --release + +# Upload Pocket TTS plugin to running server +[working-directory: 'plugins/native/pocket-tts'] +upload-pocket-tts-plugin: build-plugin-native-pocket-tts + @echo "Uploading Pocket TTS plugin to server..." + @curl -X POST -F plugin=@target/release/libpocket_tts.so \ + http://127.0.0.1:4545/api/v1/plugins + # Upload Matcha plugin to running server [working-directory: 'plugins/native/matcha'] upload-matcha-plugin: build-plugin-native-matcha @@ -663,6 +688,9 @@ download-models: download-whisper-models download-silero-vad download-kokoro-mod @echo "Optional: To download NLLB translation models (CC-BY-NC-4.0 license - non-commercial only):" @echo " just download-nllb-models" @echo "" + @echo "Optional: To download Pocket TTS models (gated; requires HF_TOKEN):" + @echo " just download-pocket-tts-models" + @echo "" @du -sh models/ # Setup VAD (install dependencies + download models) @@ -719,7 +747,7 @@ build-plugin-native name: @just build-plugin-native-{{name}} # Build all native plugin examples -build-plugins-native: build-plugin-native-gain build-plugin-native-whisper build-plugin-native-kokoro build-plugin-native-piper build-plugin-native-matcha build-plugin-native-sensevoice build-plugin-native-nllb build-plugin-native-vad build-plugin-native-helsinki +build-plugins-native: build-plugin-native-gain build-plugin-native-whisper build-plugin-native-kokoro build-plugin-native-piper build-plugin-native-matcha build-plugin-native-pocket-tts build-plugin-native-sensevoice build-plugin-native-nllb build-plugin-native-vad build-plugin-native-helsinki ## Combined @@ -765,6 +793,16 @@ copy-plugins-native: fi done done + for f in \ + plugins/native/pocket-tts/target/release/libpocket_tts.so \ + plugins/native/pocket-tts/target/release/libpocket_tts.so.* \ + plugins/native/pocket-tts/target/release/libpocket_tts.dylib \ + plugins/native/pocket-tts/target/release/pocket_tts.dll + do + if [[ -f "$f" ]]; then + cp -f "$f" .plugins/native/ + fi + done echo "✓ Native plugins copied to .plugins/native/" # --- License Headers (REUSE) --- diff --git a/plugins/native/pocket-tts/Cargo.lock b/plugins/native/pocket-tts/Cargo.lock new file mode 100644 index 00000000..81ddbf9d --- /dev/null +++ b/plugins/native/pocket-tts/Cargo.lock @@ -0,0 +1,3821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "candle-core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091" +dependencies = [ + "byteorder", + "float8", + "gemm", + "half", + "libm", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.9.2", + "rand_distr 0.5.1", + "rayon", + "safetensors 0.7.0", + "thiserror 2.0.18", + "yoke", + "zip", +] + +[[package]] +name = "candle-nn" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045fa9e7aef8567d209a27d56b692f60b96f4d0569f4c3011f8ca6715c65e03" +dependencies = [ + "candle-core", + "half", + "libc", + "num-traits", + "rayon", + "safetensors 0.7.0", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "candle-transformers" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b538ec4aa807c416a2ddd3621044888f188827862e2a6fcacba4738e89795d01" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex 0.17.0", + "num-traits", + "rand 0.9.2", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys 0.4.1", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys 0.5.0", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.5.2", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "flate2" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "float8" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f463a8a37ede13dac13316d1a1eeafa992300906a0c4c7fa1177f366d10bcbf" +dependencies = [ + "half", + "num-traits", + "rand 0.9.2", + "rand_distr 0.5.1", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gemm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "rand 0.9.2", + "rand_distr 0.5.1", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", + "serde", + "serde_core", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hf-hub" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" +dependencies = [ + "dirs", + "futures", + "http", + "indicatif", + "libc", + "log", + "native-tls", + "num_cpus", + "rand 0.9.2", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "ureq", + "windows-sys 0.60.2", +] + +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "intel-mkl-src" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee70586cd5b3e772a8739a1bd43eaa90d4f4bf0fb2a4edc202e979937ee7f5e" +dependencies = [ + "anyhow", + "intel-mkl-tool", + "ocipkg", +] + +[[package]] +name = "intel-mkl-tool" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887a16b4537d82227af54d3372971cfa5e0cde53322e60f57584056c16ada1b4" +dependencies = [ + "anyhow", + "log", + "walkdir", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lenient_semver" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de8de3f4f3754c280ce1c8c42ed8dd26a9c8385c2e5ad4ec5a77e774cea9c1ec" +dependencies = [ + "lenient_semver_parser", + "semver", +] + +[[package]] +name = "lenient_semver_parser" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f650c1d024ddc26b4bb79c3076b30030f2cf2b18292af698c81f7337a64d7d6" +dependencies = [ + "lenient_semver_version_builder", + "semver", +] + +[[package]] +name = "lenient_semver_version_builder" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9049f8ff49f75b946f95557148e70230499c8a642bf2d6528246afc7d0282d17" +dependencies = [ + "semver", +] + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", + "stable_deref_trait", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "oci-spec" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f5a3fe998d50101ae009351fec56d88a69f4ed182e11000e711068c2f5abf72" +dependencies = [ + "derive_builder", + "getset", + "once_cell", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 1.0.69", +] + +[[package]] +name = "ocipkg" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb3293021f06540803301af45e7ab81693d50e89a7398a3420bdab139e7ba5e" +dependencies = [ + "base16ct", + "base64 0.22.1", + "chrono", + "directories", + "flate2", + "lazy_static", + "log", + "oci-spec", + "regex", + "serde", + "serde_json", + "sha2", + "tar", + "thiserror 1.0.69", + "toml", + "ureq", + "url", + "uuid", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "pocket-tts" +version = "0.3.1" +dependencies = [ + "anyhow", + "byteorder", + "candle-core", + "candle-nn", + "candle-transformers", + "console_error_panic_hook", + "getrandom 0.3.4", + "hf-hub", + "hound", + "intel-mkl-src", + "js-sys", + "lenient_semver", + "memmap2", + "rand 0.8.5", + "rand_distr 0.4.3", + "rayon", + "regex", + "rubato", + "safetensors 0.5.3", + "sentencepiece", + "serde", + "serde_json", + "serde_yaml", + "thiserror 2.0.18", + "tokenizers", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "pocket-tts-plugin-native" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "candle-core", + "pocket-tts", + "serde", + "serde_json", + "streamkit-plugin-sdk-native", + "tracing", +] + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pulp" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e205bb30d5b916c55e584c22201771bcf2bad9aabd5d4127f38387140c38632" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e24eee682d89fb193496edf918a7f407d30175b2e785fe057e4392dfd182e0" + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rubato" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6dd52e80cfc21894deadf554a5673002938ae4625f7a283e536f9cf7c17b0d5" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "realfft", +] + +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "safetensors" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0cdb7198d738a111f6df8fef42cb175412c311d0c4ac9126ff4e550ad1a0e8" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "safetensors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5" +dependencies = [ + "hashbrown", + "serde", + "serde_json", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "sentencepiece" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3baa1506c7718f6b70bcac5475e563176dd51bb669dae38181abbf3070517453" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "prost", + "prost-derive", + "sentencepiece-sys", + "thiserror 2.0.18", +] + +[[package]] +name = "sentencepiece-sys" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f9b54dc005df8ec1c3f9e2347cea6b6657ec5460977a982e7a4e09ef49411" +dependencies = [ + "cc", + "cmake", + "pkg-config", +] + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streamkit-core" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "schemars", + "serde", + "serde_json", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "ts-rs", +] + +[[package]] +name = "streamkit-plugin-sdk-native" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "serde", + "serde_json", + "streamkit-core", + "tracing", +] + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "hf-hub", + "indicatif", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.2", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ts-rs" +version = "11.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4994acea2522cd2b3b85c1d9529a55991e3ad5e25cdcd3de9d505972c4379424" +dependencies = [ + "serde_json", + "thiserror 2.0.18", + "ts-rs-macros", +] + +[[package]] +name = "ts-rs-macros" +version = "11.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6ff59666c9cbaec3533964505d39154dc4e0a56151fdea30a09ed0301f62e2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "termcolor", +] + +[[package]] +name = "typed-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e43ffa54726cdc9ea78392023ffe9fe9cf9ac779e1c6fcb0d23f9862e3879d20" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.5", +] + +[[package]] +name = "webpki-roots" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" diff --git a/plugins/native/pocket-tts/Cargo.toml b/plugins/native/pocket-tts/Cargo.toml new file mode 100644 index 00000000..f975f595 --- /dev/null +++ b/plugins/native/pocket-tts/Cargo.toml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# +# SPDX-License-Identifier: MPL-2.0 + +[package] +name = "pocket-tts-plugin-native" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" +description = "Pocket TTS text-to-speech plugin for StreamKit using Candle" + +[lib] +name = "pocket_tts" +crate-type = ["cdylib"] + +[dependencies] +streamkit-plugin-sdk-native = { path = "../../../sdks/plugin-sdk/native" } + +# Rust/Candle port of Pocket TTS (vendored) +pocket-tts = { path = "vendor/pocket-tts" } + +candle-core = "0.9.1" +base64 = "0.22" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tracing = "0.1" + +[features] +default = [] +quantized = ["pocket-tts/quantized"] + +[lints.clippy] +# Categories +pedantic = { level = "warn", priority = -1 } +nursery = { level = "warn", priority = -1 } +# Safety +unwrap_used = "warn" +expect_used = "warn" +# Complexity +cognitive_complexity = "warn" +# Math +cast_possible_truncation = "warn" +cast_precision_loss = "warn" +cast_sign_loss = "warn" +# Allow-list (Noise reduction) +module_name_repetitions = "allow" +must_use_candidate = "allow" +doc_markdown = "allow" diff --git a/plugins/native/pocket-tts/README.md b/plugins/native/pocket-tts/README.md new file mode 100644 index 00000000..cb646c18 --- /dev/null +++ b/plugins/native/pocket-tts/README.md @@ -0,0 +1,101 @@ +# Pocket TTS Native Plugin + +A native StreamKit plugin for Kyutai Pocket TTS using the Rust/Candle port. +This plugin runs fully on CPU and streams 24kHz mono audio. + +## Build + +```bash +just build-plugin-native-pocket-tts +``` + +Plugin binary: +`plugins/native/pocket-tts/target/release/libpocket_tts.so` + +## Download models (offline-friendly) + +This prefetches weights, tokenizer, and stock voice embeddings into the HF cache +(`models/hf`) and mirrors voice embeddings into `models/pocket-tts/embeddings`. +The main weights repo is gated, so set `HF_TOKEN` first. + +```bash +HF_TOKEN=your_token_here just download-pocket-tts-models +``` + +To use the cached files offline, keep `HF_HOME` consistent when running the server +and set `HF_HUB_OFFLINE=1`: + +```bash +export HF_HOME=models/hf +export HF_HUB_OFFLINE=1 +``` + +If you want fully local paths (no HF cache at runtime), pass `weights_path` and +`tokenizer_path`, and optionally set `voice_embeddings_dir`: + +```yaml +params: + weights_path: "models/pocket-tts/tts_b6369a24.safetensors" + tokenizer_path: "models/pocket-tts/tokenizer.model" + voice_embeddings_dir: "models/pocket-tts/embeddings" + voice: alba +``` + +## Loading the plugin + +```bash +curl -X POST -F plugin=@plugins/native/pocket-tts/target/release/libpocket_tts.so \ + http://127.0.0.1:4545/api/v1/plugins +``` + +The plugin appears as: `plugin::native::pocket-tts`. + +## Usage + +```yaml +nodes: + tts: + kind: plugin::native::pocket-tts + params: + voice: alba + temperature: 0.7 + lsd_decode_steps: 1 + eos_threshold: -4.0 +``` + +## Parameters + +- `variant` (string): Model variant config (default: `b6369a24`). +- `config_path` (string | null): Optional config YAML path for custom variants/offline use. +- `weights_path` (string | null): Local weights path for offline loading. +- `tokenizer_path` (string | null): Local tokenizer path for offline loading. +- `voice_embeddings_dir` (string | null): Directory containing predefined voice embeddings. +- `voice` (string): Voice name, local `.wav`/`.safetensors`, `hf://` URL, or base64 audio (default: `alba`). +- `temperature` (number): Sampling temperature (default: `0.7`). +- `lsd_decode_steps` (int): LSD decode steps (default: `1`). +- `eos_threshold` (number): End-of-sequence threshold (default: `-4.0`). +- `noise_clamp` (number | null): Optional noise clamp (default: `null`). +- `min_sentence_length` (int): Minimum characters before TTS triggers (default: `10`). +- `quantized` (bool): Enable int8 weights (requires plugin built with feature `quantized`). + +## Voices + +Predefined voices: +`alba`, `marius`, `javert`, `jean`, `fantine`, `cosette`, `eponine`, `azelma` + +You can also pass: +- Local `.wav` (voice cloning) +- Local `.safetensors` (precomputed embeddings) +- `hf://` URLs +- Base64-encoded WAV (with or without `data:audio/...;base64,` prefix) + +## Notes + +- Weights and tokenizer are downloaded via Hugging Face on first use unless you + provide `weights_path` and `tokenizer_path`. +- Model licenses and voice usage terms are governed by Kyutai; review upstream usage restrictions. + +## License + +- Plugin code: MPL-2.0 (StreamKit Contributors) +- Pocket TTS models/voices: see upstream licenses and terms. diff --git a/plugins/native/pocket-tts/config/b6369a24.yaml b/plugins/native/pocket-tts/config/b6369a24.yaml new file mode 100644 index 00000000..bb3bca6c --- /dev/null +++ b/plugins/native/pocket-tts/config/b6369a24.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# +# SPDX-License-Identifier: MPL-2.0 + +# sig: b6369a24 + +weights_path: hf://kyutai/pocket-tts/tts_b6369a24.safetensors@427e3d61b276ed69fdd03de0d185fa8a8d97fc5b +weights_path_without_voice_cloning: hf://kyutai/pocket-tts-without-voice-cloning/tts_b6369a24.safetensors@d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3 + +flow_lm: + dtype: float32 + flow: + depth: 6 + dim: 512 + transformer: + d_model: 1024 + hidden_scale: 4 + max_period: 10000 + num_heads: 16 + num_layers: 6 + lookup_table: + dim: 1024 + n_bins: 4000 + tokenizer: sentencepiece + tokenizer_path: hf://kyutai/pocket-tts-without-voice-cloning/tokenizer.model@d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3 + +mimi: + dtype: float32 + sample_rate: 24000 + channels: 1 + frame_rate: 12.5 + seanet: + dimension: 512 + channels: 1 + n_filters: 64 + n_residual_layers: 1 + ratios: + - 6 + - 5 + - 4 + kernel_size: 7 + residual_kernel_size: 3 + last_kernel_size: 3 + dilation_base: 2 + pad_mode: constant + compress: 2 + transformer: + d_model: 512 + num_heads: 8 + num_layers: 2 + layer_scale: 0.01 + context: 250 + dim_feedforward: 2048 + input_dimension: 512 + output_dimensions: + - 512 + quantizer: + dimension: 32 + output_dimension: 512 diff --git a/plugins/native/pocket-tts/download-models.py b/plugins/native/pocket-tts/download-models.py new file mode 100644 index 00000000..fa7caa87 --- /dev/null +++ b/plugins/native/pocket-tts/download-models.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# +# SPDX-License-Identifier: MPL-2.0 + +import shutil +import sys +from pathlib import Path + +try: + from huggingface_hub import hf_hub_download +except ImportError: # pragma: no cover - runtime dependency check + print("Missing dependency: huggingface-hub", file=sys.stderr) + print("Install with: pip3 install --user huggingface-hub", file=sys.stderr) + sys.exit(1) + +WEIGHTS_REV = "427e3d61b276ed69fdd03de0d185fa8a8d97fc5b" +TOKENIZER_REV = "d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3" + +PREDEFINED_VOICES = [ + "alba", + "marius", + "javert", + "jean", + "fantine", + "cosette", + "eponine", + "azelma", +] + + +def download(repo_id: str, filename: str, revision: str | None = None) -> Path: + path = hf_hub_download( + repo_id=repo_id, + filename=filename, + revision=revision, + ) + path = Path(path) + print(f"Downloaded {repo_id}/{filename} -> {path}") + return path + + +def copy_to_output(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + print(f"Copied {src} -> {dest}") + + +def main() -> None: + print("Downloading Pocket TTS model weights and tokenizer...") + print("Note: kyutai/pocket-tts is gated; set HF_TOKEN to authenticate.") + + output_dir = Path("models/pocket-tts") + embeddings_dir = output_dir / "embeddings" + + weights_path = download( + "kyutai/pocket-tts", + "tts_b6369a24.safetensors", + WEIGHTS_REV, + ) + copy_to_output(weights_path, output_dir / "tts_b6369a24.safetensors") + + tokenizer_path = download( + "kyutai/pocket-tts-without-voice-cloning", + "tokenizer.model", + TOKENIZER_REV, + ) + copy_to_output(tokenizer_path, output_dir / "tokenizer.model") + + print("Downloading Pocket TTS voice embeddings...") + for voice in PREDEFINED_VOICES: + voice_path = download( + "kyutai/pocket-tts-without-voice-cloning", + f"embeddings/{voice}.safetensors", + ) + copy_to_output(voice_path, embeddings_dir / f"{voice}.safetensors") + + print("Pocket TTS downloads complete.") + print(f"Local model directory: {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/plugins/native/pocket-tts/src/config.rs b/plugins/native/pocket-tts/src/config.rs new file mode 100644 index 00000000..5230da04 --- /dev/null +++ b/plugins/native/pocket-tts/src/config.rs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PocketTtsConfig { + /// Model variant (defaults to b6369a24). + #[serde(default = "default_variant")] + pub variant: String, + + /// Optional config YAML path for offline loading or custom variants. + #[serde(default)] + pub config_path: Option, + + /// Optional local weights path for offline loading. + #[serde(default)] + pub weights_path: Option, + + /// Optional local tokenizer path for offline loading. + #[serde(default)] + pub tokenizer_path: Option, + + /// Optional directory containing predefined voice embeddings. + #[serde(default)] + pub voice_embeddings_dir: Option, + + /// Voice specification (predefined name, hf:// URL, local file, or base64 audio). + #[serde(default = "default_voice")] + pub voice: String, + + /// Sampling temperature (higher = more variation). + #[serde(default = "default_temperature")] + pub temperature: f32, + + /// LSD decode steps (higher = better quality, slower). + #[serde(default = "default_lsd_decode_steps")] + pub lsd_decode_steps: usize, + + /// EOS threshold (more negative = longer audio). + #[serde(default = "default_eos_threshold")] + pub eos_threshold: f32, + + /// Optional noise clamp for sampling. + #[serde(default)] + pub noise_clamp: Option, + + /// Minimum characters before triggering TTS. + #[serde(default = "default_min_sentence_length")] + pub min_sentence_length: usize, + + /// Enable simulated int8 quantization (requires plugin built with feature "quantized"). + #[serde(default)] + pub quantized: bool, +} + +const fn default_temperature() -> f32 { + 0.7 +} + +const fn default_lsd_decode_steps() -> usize { + 1 +} + +const fn default_eos_threshold() -> f32 { + -4.0 +} + +const fn default_min_sentence_length() -> usize { + 10 +} + +fn default_voice() -> String { + "alba".to_string() +} + +fn default_variant() -> String { + "b6369a24".to_string() +} + +impl Default for PocketTtsConfig { + fn default() -> Self { + Self { + variant: "b6369a24".to_string(), + config_path: None, + weights_path: None, + tokenizer_path: None, + voice_embeddings_dir: None, + voice: default_voice(), + temperature: default_temperature(), + lsd_decode_steps: default_lsd_decode_steps(), + eos_threshold: default_eos_threshold(), + noise_clamp: None, + min_sentence_length: default_min_sentence_length(), + quantized: false, + } + } +} diff --git a/plugins/native/pocket-tts/src/lib.rs b/plugins/native/pocket-tts/src/lib.rs new file mode 100644 index 00000000..ffd3be80 --- /dev/null +++ b/plugins/native/pocket-tts/src/lib.rs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +mod config; +mod model; +mod sentence_splitter; +mod voice; + +mod pocket_tts_node; + +use pocket_tts_node::PocketTtsNode; +use streamkit_plugin_sdk_native::{native_plugin_entry, NativeProcessorNode}; + +native_plugin_entry!(PocketTtsNode); diff --git a/plugins/native/pocket-tts/src/model.rs b/plugins/native/pocket-tts/src/model.rs new file mode 100644 index 00000000..a4fde4d8 --- /dev/null +++ b/plugins/native/pocket-tts/src/model.rs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, LazyLock, Mutex}; + +use pocket_tts::TTSModel; +use streamkit_plugin_sdk_native::prelude::Logger; +use streamkit_plugin_sdk_native::{plugin_info, plugin_warn}; + +use crate::config::PocketTtsConfig; + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct ModelCacheKey { + pub variant: String, + pub quantized: bool, + pub config_path: Option, + pub weights_path: Option, + pub tokenizer_path: Option, +} + +impl ModelCacheKey { + pub fn from_config(config: &PocketTtsConfig) -> Self { + Self { + variant: config.variant.clone(), + quantized: config.quantized, + config_path: config.config_path.as_deref().map(normalize_path), + weights_path: config.weights_path.as_deref().map(normalize_path), + tokenizer_path: config.tokenizer_path.as_deref().map(normalize_path), + } + } +} + +static MODEL_CACHE: LazyLock>>> = LazyLock::new(|| { + tracing::info!("[Pocket TTS Plugin] Initializing model cache"); + Mutex::new(HashMap::new()) +}); + +pub fn get_or_load_model( + key: &ModelCacheKey, + config: &PocketTtsConfig, + logger: &Logger, +) -> Result, String> { + { + let cache = MODEL_CACHE.lock().map_err(|e| format!("Failed to lock model cache: {e}"))?; + if let Some(model) = cache.get(key) { + plugin_info!( + logger, + "Pocket TTS model cache hit (variant={}, quantized={})", + key.variant, + key.quantized + ); + return Ok(Arc::clone(model)); + } + } + + plugin_info!( + logger, + "Loading Pocket TTS model (variant={}, quantized={}, offline={})", + key.variant, + key.quantized, + config.weights_path.is_some() + || config.tokenizer_path.is_some() + || config.config_path.is_some() + ); + + let model = if config.weights_path.is_some() + || config.tokenizer_path.is_some() + || config.config_path.is_some() + { + load_model_from_files(config, logger)? + } else if key.quantized { + #[cfg(feature = "quantized")] + { + TTSModel::load_quantized(&key.variant).map_err(|e| e.to_string())? + } + #[cfg(not(feature = "quantized"))] + { + return Err( + "Quantized model requested but plugin was built without feature \"quantized\"" + .to_string(), + ); + } + } else { + TTSModel::load(&key.variant).map_err(|e| e.to_string())? + }; + + let model = Arc::new(model); + + let mut cache = MODEL_CACHE.lock().map_err(|e| format!("Failed to lock model cache: {e}"))?; + if let Some(existing) = cache.get(key) { + return Ok(Arc::clone(existing)); + } + cache.insert(key.clone(), Arc::clone(&model)); + drop(cache); + + plugin_info!( + logger, + "Pocket TTS model loaded and cached (variant={}, quantized={})", + key.variant, + key.quantized + ); + + Ok(model) +} + +const DEFAULT_VARIANT: &str = "b6369a24"; +const DEFAULT_CONFIG_YAML: &str = include_str!("../config/b6369a24.yaml"); + +fn normalize_path(raw: &str) -> String { + let path = PathBuf::from(raw); + path.canonicalize().unwrap_or(path).to_string_lossy().to_string() +} + +fn load_model_from_files(config: &PocketTtsConfig, logger: &Logger) -> Result { + if config.quantized { + return Err("Quantized mode is not supported with local weights/tokenizer".to_string()); + } + + let weights_path = config + .weights_path + .as_ref() + .ok_or_else(|| "weights_path is required for offline loading".to_string())?; + let tokenizer_path = config + .tokenizer_path + .as_ref() + .ok_or_else(|| "tokenizer_path is required for offline loading".to_string())?; + + let config_yaml = if let Some(path) = &config.config_path { + std::fs::read(path).map_err(|e| format!("Failed to read config_path {path}: {e}"))? + } else if config.variant == DEFAULT_VARIANT { + DEFAULT_CONFIG_YAML.as_bytes().to_vec() + } else { + return Err(format!( + "config_path is required for variant '{}' when using local weights", + config.variant + )); + }; + + let weights_bytes = std::fs::read(weights_path) + .map_err(|e| format!("Failed to read weights_path {weights_path}: {e}"))?; + let tokenizer_bytes = std::fs::read(tokenizer_path) + .map_err(|e| format!("Failed to read tokenizer_path {tokenizer_path}: {e}"))?; + + plugin_info!( + logger, + "Loading Pocket TTS model from local files (weights={}, tokenizer={})", + weights_path, + tokenizer_path + ); + + TTSModel::load_from_bytes(&config_yaml, &weights_bytes, &tokenizer_bytes) + .map_err(|e| e.to_string()) +} + +pub fn configure_model(model: &mut TTSModel, config: &PocketTtsConfig, logger: &Logger) { + model.temp = config.temperature; + model.lsd_decode_steps = config.lsd_decode_steps; + model.eos_threshold = config.eos_threshold; + model.noise_clamp = config.noise_clamp; + model.flow_lm.noise_clamp = config.noise_clamp; + + if config.temperature <= 0.0 { + plugin_warn!(logger, "Pocket TTS temperature <= 0.0 may result in invalid sampling"); + } +} diff --git a/plugins/native/pocket-tts/src/pocket_tts_node.rs b/plugins/native/pocket-tts/src/pocket_tts_node.rs new file mode 100644 index 00000000..f2cabdbc --- /dev/null +++ b/plugins/native/pocket-tts/src/pocket_tts_node.rs @@ -0,0 +1,500 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +use std::borrow::Cow; +use std::sync::Arc; + +use pocket_tts::ModelState; +use streamkit_plugin_sdk_native::prelude::*; +use streamkit_plugin_sdk_native::streamkit_core::types::{AudioFormat, SampleFormat}; +use streamkit_plugin_sdk_native::{plugin_debug, plugin_error, plugin_info, plugin_warn}; + +use crate::config::PocketTtsConfig; +use crate::model::{configure_model, get_or_load_model, ModelCacheKey}; +use crate::sentence_splitter::SentenceSplitter; +use crate::voice::{ + get_or_load_voice_state, normalize_voice_spec, voice_state_from_base64, + voice_state_from_wav_bytes, VoiceCacheKey, +}; + +pub struct PocketTtsNode { + model: pocket_tts::TTSModel, + model_key: ModelCacheKey, + voice_state: Arc, + voice_buffer: Vec, + voice_expected_len: Option, + voice_input_seen: bool, + voice_ready: bool, + config: PocketTtsConfig, + text_buffer: String, + sentence_splitter: SentenceSplitter, + logger: Logger, +} + +impl NativeProcessorNode for PocketTtsNode { + fn metadata() -> NodeMetadata { + NodeMetadata::builder("pocket-tts") + .description( + "Lightweight CPU TTS using Kyutai Pocket TTS (Candle). \ + English-only voices with streaming output. \ + Outputs 24kHz mono audio.", + ) + .input("in", &[PacketType::Text, PacketType::Binary]) + .input("in_0", &[PacketType::Text, PacketType::Binary]) + .input("in_1", &[PacketType::Binary, PacketType::Text]) + .output( + "out", + PacketType::RawAudio(AudioFormat { + sample_rate: 24000, + channels: 1, + sample_format: SampleFormat::F32, + }), + ) + .param_schema(serde_json::json!({ + "type": "object", + "properties": { + "variant": { + "type": "string", + "description": "Model variant (config in pocket-tts crate)", + "default": "b6369a24" + }, + "config_path": { + "type": ["string", "null"], + "description": "Optional config YAML path for custom variants/offline use", + "default": null + }, + "weights_path": { + "type": ["string", "null"], + "description": "Local weights path for offline loading", + "default": null + }, + "tokenizer_path": { + "type": ["string", "null"], + "description": "Local tokenizer path for offline loading", + "default": null + }, + "voice_embeddings_dir": { + "type": ["string", "null"], + "description": "Directory with predefined voice embeddings (alba, marius, ...)", + "default": null + }, + "voice": { + "type": "string", + "description": "Voice name, local .wav/.safetensors, hf:// URL, or base64 audio", + "default": "alba" + }, + "temperature": { + "type": "number", + "description": "Sampling temperature (higher = more variation)", + "default": 0.7, + "minimum": 0.1, + "maximum": 2.0 + }, + "lsd_decode_steps": { + "type": "integer", + "description": "LSD decode steps (higher = better quality, slower)", + "default": 1, + "minimum": 1, + "maximum": 8 + }, + "eos_threshold": { + "type": "number", + "description": "End-of-sequence threshold (more negative = longer output)", + "default": -4.0, + "minimum": -10.0, + "maximum": 0.0 + }, + "noise_clamp": { + "type": ["number", "null"], + "description": "Optional noise clamp (null disables)", + "default": null + }, + "min_sentence_length": { + "type": "integer", + "description": "Minimum chars before triggering TTS", + "default": 10, + "minimum": 1 + }, + "quantized": { + "type": "boolean", + "description": "Enable int8 quantized weights (requires plugin built with feature 'quantized')", + "default": false + } + } + })) + .category("audio") + .category("tts") + .category("ml") + .build() + } + + fn new(params: Option, logger: Logger) -> Result { + plugin_info!(logger, "Pocket TTS plugin new() called with params: {:?}", params); + + let config: PocketTtsConfig = if let Some(p) = params { + serde_json::from_value(p).map_err(|e| { + let msg = format!("Config parse error: {e}"); + plugin_error!(logger, "{msg}"); + msg + })? + } else { + PocketTtsConfig::default() + }; + + plugin_info!( + logger, + "Pocket TTS config: variant={}, quantized={}, offline_weights={}, offline_tokenizer={}, config_path={}, voice_dir={}", + config.variant, + config.quantized, + config.weights_path.is_some(), + config.tokenizer_path.is_some(), + config.config_path.as_deref().unwrap_or("none"), + config.voice_embeddings_dir.as_deref().unwrap_or("none") + ); + + let model_key = ModelCacheKey::from_config(&config); + let base_model = get_or_load_model(&model_key, &config, &logger).map_err(|e| { + plugin_error!(logger, "Pocket TTS model load failed: {e}"); + e + })?; + let mut model = (*base_model).clone(); + configure_model(&mut model, &config, &logger); + + let voice_dir = config.voice_embeddings_dir.as_deref(); + let voice_spec = normalize_voice_spec(&config.voice, voice_dir); + let voice_key = VoiceCacheKey { model_key: model_key.clone(), voice_spec }; + let voice_state = + get_or_load_voice_state(&model, &voice_key, voice_dir, &logger).map_err(|e| { + plugin_error!(logger, "Pocket TTS voice load failed: {e}"); + e + })?; + + Ok(Self { + model, + model_key, + voice_state, + voice_buffer: Vec::new(), + voice_expected_len: None, + voice_input_seen: false, + voice_ready: false, + config: config.clone(), + text_buffer: String::new(), + sentence_splitter: SentenceSplitter::new(config.min_sentence_length), + logger, + }) + } + + fn process(&mut self, pin: &str, packet: Packet, output: &OutputSender) -> Result<(), String> { + match pin { + "in" | "in_0" => self.handle_text(&packet, output), + "in_1" | "voice" => self.handle_voice(packet, output), + other => Err(format!("Unsupported input pin '{other}'")), + } + } + + fn update_params(&mut self, params: Option) -> Result<(), String> { + if let Some(p) = params { + let new_config: PocketTtsConfig = serde_json::from_value(p).map_err(|e| { + let msg = format!("Config parse error: {e}"); + plugin_error!(self.logger, "{msg}"); + msg + })?; + + let new_model_key = ModelCacheKey::from_config(&new_config); + let model_changed = new_model_key != self.model_key; + if model_changed { + plugin_info!(self.logger, "Model parameters changed, reloading model"); + let base_model = get_or_load_model(&new_model_key, &new_config, &self.logger) + .map_err(|e| { + plugin_error!(self.logger, "Pocket TTS model load failed: {e}"); + e + })?; + let mut model = (*base_model).clone(); + configure_model(&mut model, &new_config, &self.logger); + self.model = model; + self.model_key = new_model_key.clone(); + } else { + configure_model(&mut self.model, &new_config, &self.logger); + } + + let new_voice_dir = new_config.voice_embeddings_dir.as_deref(); + let current_voice_dir = self.config.voice_embeddings_dir.as_deref(); + let new_voice_spec = normalize_voice_spec(&new_config.voice, new_voice_dir); + let current_voice_spec = normalize_voice_spec(&self.config.voice, current_voice_dir); + let voice_dir_changed = + new_config.voice_embeddings_dir != self.config.voice_embeddings_dir; + + if new_voice_spec != current_voice_spec || model_changed || voice_dir_changed { + let voice_key = + VoiceCacheKey { model_key: new_model_key, voice_spec: new_voice_spec }; + self.voice_state = + get_or_load_voice_state(&self.model, &voice_key, new_voice_dir, &self.logger) + .map_err(|e| { + plugin_error!(self.logger, "Pocket TTS voice load failed: {e}"); + e + })?; + self.voice_buffer.clear(); + self.voice_expected_len = None; + self.voice_input_seen = false; + self.voice_ready = false; + } + + if new_config.min_sentence_length != self.config.min_sentence_length { + self.sentence_splitter = SentenceSplitter::new(new_config.min_sentence_length); + } + + self.config = new_config; + } + Ok(()) + } + + fn flush(&mut self, output: &OutputSender) -> Result<(), String> { + if self.voice_input_seen && !self.voice_ready { + if self.voice_buffer.is_empty() { + return Err("Voice prompt not received for voice cloning".to_string()); + } + self.load_voice_from_buffer()?; + } + + if !self.text_buffer.is_empty() { + self.flush_text_buffer(output)?; + + if !self.text_buffer.is_empty() { + let text = self.text_buffer.clone(); + plugin_info!(self.logger, "Flushing remaining text buffer"); + self.generate_and_send(&text, output)?; + self.text_buffer.clear(); + } + } + Ok(()) + } + + fn cleanup(&mut self) { + if !self.text_buffer.is_empty() { + plugin_warn!(self.logger, "Text buffer not empty at cleanup"); + } + } +} + +impl PocketTtsNode { + fn handle_text(&mut self, packet: &Packet, output: &OutputSender) -> Result<(), String> { + let text: Cow<'_, str> = match packet { + Packet::Text(text) => Cow::Borrowed(text.as_ref()), + Packet::Binary { data, .. } => Cow::Owned( + String::from_utf8(data.to_vec()) + .map_err(|e| format!("Failed to decode binary text as UTF-8: {e}"))?, + ), + _ => return Err("Only accepts Text or Binary packets on text input".to_string()), + }; + + let mut sanitized = Self::sanitize_text(text.as_ref()); + if sanitized.is_empty() { + return Ok(()); + } + + if !sanitized.ends_with('.') && !sanitized.ends_with('!') && !sanitized.ends_with('?') { + sanitized.push('.'); + } + + self.text_buffer.push_str(&sanitized); + + if self.voice_input_seen && !self.voice_ready { + plugin_debug!( + self.logger, + "Voice prompt pending; buffering text (buffer_len={})", + self.text_buffer.len() + ); + return Ok(()); + } + + self.flush_text_buffer(output) + } + + fn handle_voice(&mut self, packet: Packet, output: &OutputSender) -> Result<(), String> { + match packet { + Packet::Binary { data, .. } => { + if data.is_empty() { + return Ok(()); + } + + self.voice_input_seen = true; + if self.voice_ready { + self.voice_buffer.clear(); + self.voice_expected_len = None; + self.voice_ready = false; + } + + self.voice_buffer.extend_from_slice(&data); + self.update_voice_expected_len(); + + if let Some(expected) = self.voice_expected_len { + if self.voice_buffer.len() >= expected { + self.load_voice_from_buffer()?; + if !self.text_buffer.is_empty() { + self.flush_text_buffer(output)?; + } + } + } + Ok(()) + }, + Packet::Text(text) => { + let trimmed = text.trim(); + if trimmed.is_empty() { + return Ok(()); + } + + let voice_state = voice_state_from_base64(&self.model, trimmed).map_err(|e| { + plugin_error!(self.logger, "Voice base64 decode failed: {e}"); + e + })?; + + self.voice_input_seen = true; + self.voice_state = Arc::new(voice_state); + self.voice_ready = true; + self.voice_buffer.clear(); + self.voice_expected_len = None; + + plugin_info!(self.logger, "Loaded voice prompt from text input"); + + if !self.text_buffer.is_empty() { + self.flush_text_buffer(output)?; + } + Ok(()) + }, + _ => Err("Voice input must be Binary (wav) or Text (base64)".to_string()), + } + } + + fn update_voice_expected_len(&mut self) { + if self.voice_expected_len.is_some() { + return; + } + self.voice_expected_len = Self::read_wav_expected_len(&self.voice_buffer); + } + + fn read_wav_expected_len(bytes: &[u8]) -> Option { + if bytes.len() < 12 { + return None; + } + + if &bytes[0..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { + return None; + } + + let riff_size = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) as usize; + if riff_size == 0 { + return None; + } + + let expected = riff_size.saturating_add(8); + if expected < 12 { + return None; + } + + Some(expected) + } + + fn load_voice_from_buffer(&mut self) -> Result<(), String> { + if self.voice_buffer.is_empty() { + return Err("Voice prompt buffer is empty".to_string()); + } + + let bytes = if let Some(expected) = self.voice_expected_len { + if self.voice_buffer.len() < expected { + return Err(format!( + "Voice prompt incomplete: {}/{} bytes", + self.voice_buffer.len(), + expected + )); + } + if self.voice_buffer.len() > expected { + plugin_warn!( + self.logger, + "Voice prompt larger than expected ({} > {} bytes)", + self.voice_buffer.len(), + expected + ); + } + &self.voice_buffer[..expected] + } else { + &self.voice_buffer[..] + }; + + let voice_state = voice_state_from_wav_bytes(&self.model, bytes).map_err(|e| { + plugin_error!(self.logger, "Voice prompt decode failed: {e}"); + e + })?; + + self.voice_state = Arc::new(voice_state); + self.voice_ready = true; + self.voice_buffer.clear(); + self.voice_expected_len = None; + + plugin_info!(self.logger, "Loaded voice prompt from multipart input"); + Ok(()) + } + + fn flush_text_buffer(&mut self, output: &OutputSender) -> Result<(), String> { + while let Some(sentence) = self.sentence_splitter.extract_sentence(&mut self.text_buffer) { + self.generate_and_send(&sentence, output)?; + } + Ok(()) + } + + fn generate_and_send(&self, text: &str, output: &OutputSender) -> Result<(), String> { + let voice_state = self.voice_state.as_ref(); + for chunk in self.model.generate_stream_long(text, voice_state) { + let chunk = chunk.map_err(|e| format!("TTS generation failed: {e}"))?; + let (samples, channels) = Self::tensor_to_interleaved_samples(&chunk)?; + if samples.is_empty() { + continue; + } + + let sample_rate = u32::try_from(self.model.sample_rate).map_err(|_| { + format!("Model sample rate {} does not fit in u32", self.model.sample_rate) + })?; + let frame = AudioFrame::new(sample_rate, channels, samples); + output + .send("out", &Packet::Audio(frame)) + .map_err(|e| format!("Failed to send audio: {e}"))?; + } + Ok(()) + } + + fn tensor_to_interleaved_samples( + chunk: &candle_core::Tensor, + ) -> Result<(Vec, u16), String> { + let chunk = chunk.squeeze(0).map_err(|e| format!("Failed to squeeze audio tensor: {e}"))?; + let data = + chunk.to_vec2::().map_err(|e| format!("Failed to read audio tensor data: {e}"))?; + + if data.is_empty() { + return Ok((Vec::new(), 1)); + } + + let channels = data.len(); + let samples_per_channel = data[0].len(); + for channel in &data { + if channel.len() != samples_per_channel { + return Err("Inconsistent channel lengths in audio tensor".to_string()); + } + } + + let mut interleaved = Vec::with_capacity(channels * samples_per_channel); + for i in 0..samples_per_channel { + for channel in &data { + interleaved.push(channel[i].clamp(-1.0, 1.0)); + } + } + + let channels_u16 = u16::try_from(channels) + .map_err(|_| format!("Channel count {channels} does not fit in u16"))?; + + Ok((interleaved, channels_u16)) + } + + fn sanitize_text(text: &str) -> String { + text.replace(['\n', '\r', '\t'], " ").split_whitespace().collect::>().join(" ") + } +} diff --git a/plugins/native/pocket-tts/src/sentence_splitter.rs b/plugins/native/pocket-tts/src/sentence_splitter.rs new file mode 100644 index 00000000..c0fdd6b2 --- /dev/null +++ b/plugins/native/pocket-tts/src/sentence_splitter.rs @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +pub struct SentenceSplitter { + min_length: usize, +} + +impl SentenceSplitter { + pub const fn new(min_length: usize) -> Self { + Self { min_length } + } + + /// Extract complete sentence from buffer if available. + pub fn extract_sentence(&self, buffer: &mut String) -> Option { + if buffer.len() < self.min_length { + return None; + } + + let boundaries = [ + ". ", ".\n", "! ", "!\n", "? ", "?\n", // English + ]; + + for boundary in &boundaries { + if let Some(pos) = buffer.find(boundary) { + let end_pos = pos + boundary.len(); + let sentence: String = buffer.drain(..end_pos).collect(); + return Some(sentence.trim().to_string()); + } + } + + if buffer.ends_with('.') || buffer.ends_with('!') || buffer.ends_with('?') { + let sentence = std::mem::take(buffer); + return Some(sentence); + } + + None + } + + #[allow(dead_code)] + pub fn flush(buffer: &mut String) -> Option { + if buffer.is_empty() { + None + } else { + Some(std::mem::take(buffer)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sentence_extraction() { + let splitter = SentenceSplitter::new(5); + let mut buffer = "Hello world. How are you?".to_string(); + + assert_eq!(splitter.extract_sentence(&mut buffer), Some("Hello world.".to_string())); + assert_eq!(buffer, "How are you?"); + + assert_eq!(splitter.extract_sentence(&mut buffer), Some("How are you?".to_string())); + assert_eq!(buffer, ""); + } + + #[test] + fn test_min_length() { + let splitter = SentenceSplitter::new(20); + let mut buffer = "Hi.".to_string(); + + assert_eq!(splitter.extract_sentence(&mut buffer), None); + assert_eq!(buffer, "Hi."); + } +} diff --git a/plugins/native/pocket-tts/src/voice.rs b/plugins/native/pocket-tts/src/voice.rs new file mode 100644 index 00000000..723fe085 --- /dev/null +++ b/plugins/native/pocket-tts/src/voice.rs @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, LazyLock, Mutex}; + +use base64::{engine::general_purpose, Engine as _}; +use pocket_tts::weights::download_if_necessary; +use pocket_tts::{audio, ModelState, TTSModel}; +use streamkit_plugin_sdk_native::plugin_info; +use streamkit_plugin_sdk_native::prelude::Logger; + +use crate::model::ModelCacheKey; + +pub const PREDEFINED_VOICES: &[&str] = + &["alba", "marius", "javert", "jean", "fantine", "cosette", "eponine", "azelma"]; + +const STOCK_VOICE_REPO: &str = "kyutai/pocket-tts-without-voice-cloning"; +const DEFAULT_VOICE: &str = "alba"; + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct VoiceCacheKey { + pub model_key: ModelCacheKey, + pub voice_spec: String, +} + +static VOICE_CACHE: LazyLock>>> = + LazyLock::new(|| { + tracing::info!("[Pocket TTS Plugin] Initializing voice cache"); + Mutex::new(HashMap::new()) + }); + +pub fn normalize_voice_spec(voice: &str, voice_dir: Option<&str>) -> String { + let trimmed = voice.trim(); + if trimmed.is_empty() { + return DEFAULT_VOICE.to_string(); + } + + if PREDEFINED_VOICES.contains(&trimmed) { + if let Some(path) = resolve_predefined_voice_path(voice_dir, trimmed) { + return path.to_string_lossy().to_string(); + } + } + + if is_base64_audio(trimmed) { + return trimmed.to_string(); + } + + if trimmed.starts_with("hf://") { + return trimmed.to_string(); + } + + let path = PathBuf::from(trimmed); + if path.exists() { + return path.canonicalize().unwrap_or(path).to_string_lossy().to_string(); + } + + trimmed.to_string() +} + +pub fn get_or_load_voice_state( + model: &TTSModel, + key: &VoiceCacheKey, + voice_dir: Option<&str>, + logger: &Logger, +) -> Result, String> { + { + let cache = VOICE_CACHE.lock().map_err(|e| format!("Failed to lock voice cache: {e}"))?; + if let Some(state) = cache.get(key) { + plugin_info!(logger, "Pocket TTS voice cache hit (voice={})", key.voice_spec); + return Ok(Arc::clone(state)); + } + } + + plugin_info!(logger, "Loading voice state: {}", key.voice_spec); + + let state = resolve_voice_spec(model, &key.voice_spec, voice_dir)?; + let state = Arc::new(state); + + let mut cache = VOICE_CACHE.lock().map_err(|e| format!("Failed to lock voice cache: {e}"))?; + if let Some(existing) = cache.get(key) { + return Ok(Arc::clone(existing)); + } + cache.insert(key.clone(), Arc::clone(&state)); + drop(cache); + + Ok(state) +} + +fn resolve_voice_spec( + model: &TTSModel, + spec: &str, + voice_dir: Option<&str>, +) -> Result { + let spec = spec.trim(); + + if spec.is_empty() { + return resolve_predefined_voice(model, DEFAULT_VOICE, voice_dir); + } + + if PREDEFINED_VOICES.contains(&spec) { + return resolve_predefined_voice(model, spec, voice_dir); + } + + if spec.starts_with("hf://") { + return resolve_hf_voice(model, spec); + } + + let path = PathBuf::from(spec); + if path.exists() { + return resolve_file_voice(model, &path); + } + + if is_base64_audio(spec) { + return resolve_base64_voice(model, spec); + } + + Err(format!( + "Voice '{spec}' not found. Expected predefined name, local .wav/.safetensors, hf:// URL, or base64 audio", + )) +} + +fn resolve_predefined_voice( + model: &TTSModel, + name: &str, + voice_dir: Option<&str>, +) -> Result { + if let Some(path) = resolve_predefined_voice_path(voice_dir, name) { + return model + .get_voice_state_from_prompt_file(&path) + .map_err(|e| format!("Failed to load local voice embeddings {}: {e}", path.display())); + } + + if let Some(dir) = voice_dir { + return Err(format!("Voice embeddings for '{name}' not found in {dir}")); + } + + let hf_path = format!("hf://{STOCK_VOICE_REPO}/embeddings/{name}.safetensors"); + let local_path = download_if_necessary(&hf_path) + .map_err(|e| format!("Failed to download stock voice '{name}': {e}"))?; + + model + .get_voice_state_from_prompt_file(&local_path) + .map_err(|e| format!("Failed to load voice embeddings {}: {e}", local_path.display())) +} + +fn resolve_predefined_voice_path(voice_dir: Option<&str>, name: &str) -> Option { + let dir = voice_dir?; + let base = PathBuf::from(dir); + let direct = base.join(format!("{name}.safetensors")); + if direct.exists() { + return Some(direct.canonicalize().unwrap_or(direct)); + } + let nested = base.join("embeddings").join(format!("{name}.safetensors")); + if nested.exists() { + return Some(nested.canonicalize().unwrap_or(nested)); + } + None +} + +fn resolve_hf_voice(model: &TTSModel, url: &str) -> Result { + let local_path = download_if_necessary(url) + .map_err(|e| format!("Failed to download voice from '{url}': {e}"))?; + + resolve_file_voice(model, &local_path) +} + +fn resolve_file_voice(model: &TTSModel, path: &PathBuf) -> Result { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase(); + + match ext.as_str() { + "safetensors" => model + .get_voice_state_from_prompt_file(path) + .map_err(|e| format!("Failed to load embeddings from {}: {e}", path.display())), + "wav" | "wave" => model + .get_voice_state(path) + .map_err(|e| format!("Failed to process voice audio {}: {e}", path.display())), + _ => { + Err(format!("Unsupported voice file extension '{ext}'. Expected .wav or .safetensors")) + }, + } +} + +fn resolve_base64_voice(model: &TTSModel, spec: &str) -> Result { + voice_state_from_base64(model, spec) +} + +fn is_base64_audio(spec: &str) -> bool { + if spec.starts_with("data:audio/") && spec.contains("base64,") { + return true; + } + + if spec.len() > 100 { + let clean = spec.trim(); + return clean + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '/' || c == '='); + } + + false +} + +pub fn voice_state_from_base64(model: &TTSModel, spec: &str) -> Result { + let b64_str = + if spec.starts_with("data:") { spec.split(',').nth(1).unwrap_or(spec) } else { spec }; + + let bytes = general_purpose::STANDARD + .decode(b64_str) + .map_err(|e| format!("Failed to decode base64 audio: {e}"))?; + + voice_state_from_wav_bytes(model, &bytes) +} + +pub fn voice_state_from_wav_bytes(model: &TTSModel, bytes: &[u8]) -> Result { + let (audio, sample_rate) = + audio::read_wav_from_bytes(bytes).map_err(|e| format!("WAV decode failed: {e}"))?; + + let model_sample_rate = u32::try_from(model.sample_rate) + .map_err(|_| format!("Model sample rate {} does not fit in u32", model.sample_rate))?; + let audio = if sample_rate == model_sample_rate { + audio + } else { + audio::resample(&audio, sample_rate, model_sample_rate) + .map_err(|e| format!("Failed to resample voice audio: {e}"))? + }; + + let audio = audio.unsqueeze(0).map_err(|e| format!("Failed to add batch dimension: {e}"))?; + + model + .get_voice_state_from_tensor(&audio) + .map_err(|e| format!("Failed to encode voice audio: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_predefined_voices() { + assert!(PREDEFINED_VOICES.contains(&"alba")); + assert!(!PREDEFINED_VOICES.contains(&"unknown")); + } + + #[test] + fn test_is_base64_audio() { + assert!(is_base64_audio( + "data:audio/wav;base64,UklGRi4AAABXQVZFZm10IBAAAAABAAIAQB8AAEAfAAABAAgAZGF0YQoAAAAA" + )); + assert!(!is_base64_audio("alba")); + assert!(!is_base64_audio("/path/to/file.wav")); + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/.cargo/config.toml b/plugins/native/pocket-tts/vendor/pocket-tts/.cargo/config.toml new file mode 100644 index 00000000..77471aca --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/.cargo/config.toml @@ -0,0 +1,6 @@ +# WASM build configuration +# Required for getrandom 0.3+ to work on wasm32-unknown-unknown +# See: https://docs.rs/getrandom/latest/#webassembly-support + +[target.wasm32-unknown-unknown] +rustflags = ['--cfg', 'getrandom_backend="wasm_js"'] diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/Cargo.toml b/plugins/native/pocket-tts/vendor/pocket-tts/Cargo.toml new file mode 100644 index 00000000..92bad058 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/Cargo.toml @@ -0,0 +1,73 @@ +[package] +name = "pocket-tts" +version = "0.3.1" +edition = "2024" +authors = ["Pocket TTS Contributors"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/babybirdprd/pocket-tts" +description = "High-performance CPU-based Text-to-Speech library using Candle" +keywords = ["tts", "candle", "speech-synthesis", "audio"] +categories = ["multimedia::audio", "science"] + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = [] +mkl = ["candle-core/mkl", "candle-nn/mkl"] +quantized = [] +# WASM is handled by target, not feature flag - candle-core auto-detects wasm32 target +wasm = [] +metal = ["candle-core/metal", "candle-nn/metal"] + +[dependencies] +candle-core = "0.9.1" +candle-nn = "0.9.1" +candle-transformers = "0.9.1" +safetensors = "0.5.2" +anyhow = "1.0" +thiserror = "2.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" +byteorder = "1.5" +tokenizers = { version = "0.21.0", default-features = false, features = ["fancy-regex"] } +lenient_semver = "0.4.2" +rubato = "0.14.1" +regex = "1" +hound = "3.5" +rand = "0.8" +rand_distr = "0.4" +rayon = "1.11.0" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +hf-hub = "0.4.3" +memmap2 = "0.9" +sentencepiece = "0.13" +tokenizers = { version = "0.21.0", default-features = false, features = ["onig", "progressbar", "http"] } +intel-mkl-src = { version = "0.8.1", features = ["mkl-static-lp64-iomp"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +js-sys = "0.3" +web-sys = { version = "0.3", features = ["console"] } +getrandom = { version = "0.3", features = ["wasm_js"] } +console_error_panic_hook = "0.1" + +[[bench]] +name = "streaming_bench" +harness = false + +[dev-dependencies] +criterion = "0.8.1" +candle-core = "0.9.1" +anyhow = "1.0" + +[[bench]] +name = "full_benchmark" +harness = false + +[[bench]] +name = "attention_bench" +harness = false diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/assets/tokenizer.json b/plugins/native/pocket-tts/vendor/pocket-tts/assets/tokenizer.json new file mode 100644 index 00000000..638a1594 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/assets/tokenizer.json @@ -0,0 +1,16129 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "Metaspace", + "replacement": "▁", + "prepend_scheme": "never", + "split": false + }, + "post_processor": { + "type": "TemplateProcessing", + "single": [ + { + "SpecialToken": { + "id": "", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + } + ], + "pair": [ + { + "SpecialToken": { + "id": "", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "SpecialToken": { + "id": "", + "type_id": 1 + } + }, + { + "Sequence": { + "id": "B", + "type_id": 1 + } + } + ], + "special_tokens": { + "": { + "id": "", + "ids": [ + 1 + ], + "tokens": [ + "" + ] + } + } + }, + "decoder": { + "type": "Sequence", + "decoders": [ + { + "type": "Replace", + "pattern": { + "String": "▁" + }, + "content": " " + }, + { + "type": "ByteFallback" + }, + { + "type": "Fuse" + } + ] + }, + "model": { + "type": "Unigram", + "unk_id": 0, + "vocab": [ + [ + "", + 0.0 + ], + [ + "", + 0.0 + ], + [ + "", + 0.0 + ], + [ + "", + 0.0 + ], + [ + "<0x00>", + 0.0 + ], + [ + "<0x01>", + 0.0 + ], + [ + "<0x02>", + 0.0 + ], + [ + "<0x03>", + 0.0 + ], + [ + "<0x04>", + 0.0 + ], + [ + "<0x05>", + 0.0 + ], + [ + "<0x06>", + 0.0 + ], + [ + "<0x07>", + 0.0 + ], + [ + "<0x08>", + 0.0 + ], + [ + "<0x09>", + 0.0 + ], + [ + "<0x0A>", + 0.0 + ], + [ + "<0x0B>", + 0.0 + ], + [ + "<0x0C>", + 0.0 + ], + [ + "<0x0D>", + 0.0 + ], + [ + "<0x0E>", + 0.0 + ], + [ + "<0x0F>", + 0.0 + ], + [ + "<0x10>", + 0.0 + ], + [ + "<0x11>", + 0.0 + ], + [ + "<0x12>", + 0.0 + ], + [ + "<0x13>", + 0.0 + ], + [ + "<0x14>", + 0.0 + ], + [ + "<0x15>", + 0.0 + ], + [ + "<0x16>", + 0.0 + ], + [ + "<0x17>", + 0.0 + ], + [ + "<0x18>", + 0.0 + ], + [ + "<0x19>", + 0.0 + ], + [ + "<0x1A>", + 0.0 + ], + [ + "<0x1B>", + 0.0 + ], + [ + "<0x1C>", + 0.0 + ], + [ + "<0x1D>", + 0.0 + ], + [ + "<0x1E>", + 0.0 + ], + [ + "<0x1F>", + 0.0 + ], + [ + "<0x20>", + 0.0 + ], + [ + "<0x21>", + 0.0 + ], + [ + "<0x22>", + 0.0 + ], + [ + "<0x23>", + 0.0 + ], + [ + "<0x24>", + 0.0 + ], + [ + "<0x25>", + 0.0 + ], + [ + "<0x26>", + 0.0 + ], + [ + "<0x27>", + 0.0 + ], + [ + "<0x28>", + 0.0 + ], + [ + "<0x29>", + 0.0 + ], + [ + "<0x2A>", + 0.0 + ], + [ + "<0x2B>", + 0.0 + ], + [ + "<0x2C>", + 0.0 + ], + [ + "<0x2D>", + 0.0 + ], + [ + "<0x2E>", + 0.0 + ], + [ + "<0x2F>", + 0.0 + ], + [ + "<0x30>", + 0.0 + ], + [ + "<0x31>", + 0.0 + ], + [ + "<0x32>", + 0.0 + ], + [ + "<0x33>", + 0.0 + ], + [ + "<0x34>", + 0.0 + ], + [ + "<0x35>", + 0.0 + ], + [ + "<0x36>", + 0.0 + ], + [ + "<0x37>", + 0.0 + ], + [ + "<0x38>", + 0.0 + ], + [ + "<0x39>", + 0.0 + ], + [ + "<0x3A>", + 0.0 + ], + [ + "<0x3B>", + 0.0 + ], + [ + "<0x3C>", + 0.0 + ], + [ + "<0x3D>", + 0.0 + ], + [ + "<0x3E>", + 0.0 + ], + [ + "<0x3F>", + 0.0 + ], + [ + "<0x40>", + 0.0 + ], + [ + "<0x41>", + 0.0 + ], + [ + "<0x42>", + 0.0 + ], + [ + "<0x43>", + 0.0 + ], + [ + "<0x44>", + 0.0 + ], + [ + "<0x45>", + 0.0 + ], + [ + "<0x46>", + 0.0 + ], + [ + "<0x47>", + 0.0 + ], + [ + "<0x48>", + 0.0 + ], + [ + "<0x49>", + 0.0 + ], + [ + "<0x4A>", + 0.0 + ], + [ + "<0x4B>", + 0.0 + ], + [ + "<0x4C>", + 0.0 + ], + [ + "<0x4D>", + 0.0 + ], + [ + "<0x4E>", + 0.0 + ], + [ + "<0x4F>", + 0.0 + ], + [ + "<0x50>", + 0.0 + ], + [ + "<0x51>", + 0.0 + ], + [ + "<0x52>", + 0.0 + ], + [ + "<0x53>", + 0.0 + ], + [ + "<0x54>", + 0.0 + ], + [ + "<0x55>", + 0.0 + ], + [ + "<0x56>", + 0.0 + ], + [ + "<0x57>", + 0.0 + ], + [ + "<0x58>", + 0.0 + ], + [ + "<0x59>", + 0.0 + ], + [ + "<0x5A>", + 0.0 + ], + [ + "<0x5B>", + 0.0 + ], + [ + "<0x5C>", + 0.0 + ], + [ + "<0x5D>", + 0.0 + ], + [ + "<0x5E>", + 0.0 + ], + [ + "<0x5F>", + 0.0 + ], + [ + "<0x60>", + 0.0 + ], + [ + "<0x61>", + 0.0 + ], + [ + "<0x62>", + 0.0 + ], + [ + "<0x63>", + 0.0 + ], + [ + "<0x64>", + 0.0 + ], + [ + "<0x65>", + 0.0 + ], + [ + "<0x66>", + 0.0 + ], + [ + "<0x67>", + 0.0 + ], + [ + "<0x68>", + 0.0 + ], + [ + "<0x69>", + 0.0 + ], + [ + "<0x6A>", + 0.0 + ], + [ + "<0x6B>", + 0.0 + ], + [ + "<0x6C>", + 0.0 + ], + [ + "<0x6D>", + 0.0 + ], + [ + "<0x6E>", + 0.0 + ], + [ + "<0x6F>", + 0.0 + ], + [ + "<0x70>", + 0.0 + ], + [ + "<0x71>", + 0.0 + ], + [ + "<0x72>", + 0.0 + ], + [ + "<0x73>", + 0.0 + ], + [ + "<0x74>", + 0.0 + ], + [ + "<0x75>", + 0.0 + ], + [ + "<0x76>", + 0.0 + ], + [ + "<0x77>", + 0.0 + ], + [ + "<0x78>", + 0.0 + ], + [ + "<0x79>", + 0.0 + ], + [ + "<0x7A>", + 0.0 + ], + [ + "<0x7B>", + 0.0 + ], + [ + "<0x7C>", + 0.0 + ], + [ + "<0x7D>", + 0.0 + ], + [ + "<0x7E>", + 0.0 + ], + [ + "<0x7F>", + 0.0 + ], + [ + "<0x80>", + 0.0 + ], + [ + "<0x81>", + 0.0 + ], + [ + "<0x82>", + 0.0 + ], + [ + "<0x83>", + 0.0 + ], + [ + "<0x84>", + 0.0 + ], + [ + "<0x85>", + 0.0 + ], + [ + "<0x86>", + 0.0 + ], + [ + "<0x87>", + 0.0 + ], + [ + "<0x88>", + 0.0 + ], + [ + "<0x89>", + 0.0 + ], + [ + "<0x8A>", + 0.0 + ], + [ + "<0x8B>", + 0.0 + ], + [ + "<0x8C>", + 0.0 + ], + [ + "<0x8D>", + 0.0 + ], + [ + "<0x8E>", + 0.0 + ], + [ + "<0x8F>", + 0.0 + ], + [ + "<0x90>", + 0.0 + ], + [ + "<0x91>", + 0.0 + ], + [ + "<0x92>", + 0.0 + ], + [ + "<0x93>", + 0.0 + ], + [ + "<0x94>", + 0.0 + ], + [ + "<0x95>", + 0.0 + ], + [ + "<0x96>", + 0.0 + ], + [ + "<0x97>", + 0.0 + ], + [ + "<0x98>", + 0.0 + ], + [ + "<0x99>", + 0.0 + ], + [ + "<0x9A>", + 0.0 + ], + [ + "<0x9B>", + 0.0 + ], + [ + "<0x9C>", + 0.0 + ], + [ + "<0x9D>", + 0.0 + ], + [ + "<0x9E>", + 0.0 + ], + [ + "<0x9F>", + 0.0 + ], + [ + "<0xA0>", + 0.0 + ], + [ + "<0xA1>", + 0.0 + ], + [ + "<0xA2>", + 0.0 + ], + [ + "<0xA3>", + 0.0 + ], + [ + "<0xA4>", + 0.0 + ], + [ + "<0xA5>", + 0.0 + ], + [ + "<0xA6>", + 0.0 + ], + [ + "<0xA7>", + 0.0 + ], + [ + "<0xA8>", + 0.0 + ], + [ + "<0xA9>", + 0.0 + ], + [ + "<0xAA>", + 0.0 + ], + [ + "<0xAB>", + 0.0 + ], + [ + "<0xAC>", + 0.0 + ], + [ + "<0xAD>", + 0.0 + ], + [ + "<0xAE>", + 0.0 + ], + [ + "<0xAF>", + 0.0 + ], + [ + "<0xB0>", + 0.0 + ], + [ + "<0xB1>", + 0.0 + ], + [ + "<0xB2>", + 0.0 + ], + [ + "<0xB3>", + 0.0 + ], + [ + "<0xB4>", + 0.0 + ], + [ + "<0xB5>", + 0.0 + ], + [ + "<0xB6>", + 0.0 + ], + [ + "<0xB7>", + 0.0 + ], + [ + "<0xB8>", + 0.0 + ], + [ + "<0xB9>", + 0.0 + ], + [ + "<0xBA>", + 0.0 + ], + [ + "<0xBB>", + 0.0 + ], + [ + "<0xBC>", + 0.0 + ], + [ + "<0xBD>", + 0.0 + ], + [ + "<0xBE>", + 0.0 + ], + [ + "<0xBF>", + 0.0 + ], + [ + "<0xC0>", + 0.0 + ], + [ + "<0xC1>", + 0.0 + ], + [ + "<0xC2>", + 0.0 + ], + [ + "<0xC3>", + 0.0 + ], + [ + "<0xC4>", + 0.0 + ], + [ + "<0xC5>", + 0.0 + ], + [ + "<0xC6>", + 0.0 + ], + [ + "<0xC7>", + 0.0 + ], + [ + "<0xC8>", + 0.0 + ], + [ + "<0xC9>", + 0.0 + ], + [ + "<0xCA>", + 0.0 + ], + [ + "<0xCB>", + 0.0 + ], + [ + "<0xCC>", + 0.0 + ], + [ + "<0xCD>", + 0.0 + ], + [ + "<0xCE>", + 0.0 + ], + [ + "<0xCF>", + 0.0 + ], + [ + "<0xD0>", + 0.0 + ], + [ + "<0xD1>", + 0.0 + ], + [ + "<0xD2>", + 0.0 + ], + [ + "<0xD3>", + 0.0 + ], + [ + "<0xD4>", + 0.0 + ], + [ + "<0xD5>", + 0.0 + ], + [ + "<0xD6>", + 0.0 + ], + [ + "<0xD7>", + 0.0 + ], + [ + "<0xD8>", + 0.0 + ], + [ + "<0xD9>", + 0.0 + ], + [ + "<0xDA>", + 0.0 + ], + [ + "<0xDB>", + 0.0 + ], + [ + "<0xDC>", + 0.0 + ], + [ + "<0xDD>", + 0.0 + ], + [ + "<0xDE>", + 0.0 + ], + [ + "<0xDF>", + 0.0 + ], + [ + "<0xE0>", + 0.0 + ], + [ + "<0xE1>", + 0.0 + ], + [ + "<0xE2>", + 0.0 + ], + [ + "<0xE3>", + 0.0 + ], + [ + "<0xE4>", + 0.0 + ], + [ + "<0xE5>", + 0.0 + ], + [ + "<0xE6>", + 0.0 + ], + [ + "<0xE7>", + 0.0 + ], + [ + "<0xE8>", + 0.0 + ], + [ + "<0xE9>", + 0.0 + ], + [ + "<0xEA>", + 0.0 + ], + [ + "<0xEB>", + 0.0 + ], + [ + "<0xEC>", + 0.0 + ], + [ + "<0xED>", + 0.0 + ], + [ + "<0xEE>", + 0.0 + ], + [ + "<0xEF>", + 0.0 + ], + [ + "<0xF0>", + 0.0 + ], + [ + "<0xF1>", + 0.0 + ], + [ + "<0xF2>", + 0.0 + ], + [ + "<0xF3>", + 0.0 + ], + [ + "<0xF4>", + 0.0 + ], + [ + "<0xF5>", + 0.0 + ], + [ + "<0xF6>", + 0.0 + ], + [ + "<0xF7>", + 0.0 + ], + [ + "<0xF8>", + 0.0 + ], + [ + "<0xF9>", + 0.0 + ], + [ + "<0xFA>", + 0.0 + ], + [ + "<0xFB>", + 0.0 + ], + [ + "<0xFC>", + 0.0 + ], + [ + "<0xFD>", + 0.0 + ], + [ + "<0xFE>", + 0.0 + ], + [ + "<0xFF>", + 0.0 + ], + [ + "▁", + -2.884568691253662 + ], + [ + "s", + -3.2484734058380127 + ], + [ + ",", + -3.293511152267456 + ], + [ + ".", + -3.3522086143493652 + ], + [ + "'", + -3.691579580307007 + ], + [ + "▁the", + -3.741624355316162 + ], + [ + "▁to", + -4.045279026031494 + ], + [ + "▁a", + -4.066256999969482 + ], + [ + "▁I", + -4.0755815505981445 + ], + [ + "▁and", + -4.241729259490967 + ], + [ + "▁you", + -4.310611248016357 + ], + [ + "▁that", + -4.342050075531006 + ], + [ + "▁of", + -4.390951633453369 + ], + [ + "ing", + -4.502481937408447 + ], + [ + "t", + -4.5397047996521 + ], + [ + "▁it", + -4.542901515960693 + ], + [ + "▁in", + -4.638480186462402 + ], + [ + "▁is", + -4.943281650543213 + ], + [ + "ed", + -4.943328857421875 + ], + [ + "▁be", + -5.043965816497803 + ], + [ + "re", + -5.14577054977417 + ], + [ + "▁we", + -5.182724952697754 + ], + [ + "▁like", + -5.221669673919678 + ], + [ + "m", + -5.342883110046387 + ], + [ + "▁for", + -5.343321800231934 + ], + [ + "▁this", + -5.363556861877441 + ], + [ + "▁was", + -5.375865936279297 + ], + [ + "▁And", + -5.445766925811768 + ], + [ + "▁on", + -5.466545581817627 + ], + [ + "▁know", + -5.47384786605835 + ], + [ + "▁have", + -5.571041107177734 + ], + [ + "▁with", + -5.58411169052124 + ], + [ + "?", + -5.620762348175049 + ], + [ + "▁they", + -5.640599727630615 + ], + [ + "▁so", + -5.654904842376709 + ], + [ + "er", + -5.673520088195801 + ], + [ + "▁just", + -5.677209377288818 + ], + [ + "▁what", + -5.72856330871582 + ], + [ + "▁So", + -5.789290904998779 + ], + [ + "▁do", + -5.848136901855469 + ], + [ + "▁not", + -5.849341869354248 + ], + [ + "▁some", + -5.853160858154297 + ], + [ + "▁he", + -5.866663932800293 + ], + [ + "▁can", + -5.883192539215088 + ], + [ + "▁are", + -5.899616241455078 + ], + [ + "▁but", + -5.9202561378479 + ], + [ + "n", + -5.930228233337402 + ], + [ + "d", + -5.950862407684326 + ], + [ + "▁me", + -5.984400272369385 + ], + [ + "▁at", + -5.9927825927734375 + ], + [ + "▁there", + -5.994783401489258 + ], + [ + "▁or", + -5.998849391937256 + ], + [ + "▁your", + -6.006519317626953 + ], + [ + "▁my", + -6.017693996429443 + ], + [ + "ve", + -6.01771354675293 + ], + [ + "▁about", + -6.023505687713623 + ], + [ + "0", + -6.0314812660217285 + ], + [ + "▁all", + -6.051407814025879 + ], + [ + "▁get", + -6.057258605957031 + ], + [ + "▁think", + -6.073665142059326 + ], + [ + "cause", + -6.094570159912109 + ], + [ + "▁out", + -6.11068058013916 + ], + [ + "▁one", + -6.11893367767334 + ], + [ + "ly", + -6.123717784881592 + ], + [ + "▁p", + -6.127171993255615 + ], + [ + "▁going", + -6.136852264404297 + ], + [ + "al", + -6.18625545501709 + ], + [ + "y", + -6.193512916564941 + ], + [ + "▁if", + -6.2064433097839355 + ], + [ + "▁as", + -6.226637363433838 + ], + [ + "▁up", + -6.236567974090576 + ], + [ + "▁c", + -6.243194103240967 + ], + [ + "▁don", + -6.256308078765869 + ], + [ + "▁It", + -6.295454502105713 + ], + [ + "r", + -6.328939437866211 + ], + [ + "ll", + -6.33448600769043 + ], + [ + "e", + -6.3421173095703125 + ], + [ + "-", + -6.345717430114746 + ], + [ + "▁re", + -6.3473334312438965 + ], + [ + "▁thing", + -6.348674774169922 + ], + [ + "▁from", + -6.352749824523926 + ], + [ + "1", + -6.3541717529296875 + ], + [ + "really", + -6.389099597930908 + ], + [ + "▁You", + -6.401214599609375 + ], + [ + "thing", + -6.417024612426758 + ], + [ + "▁right", + -6.424452304840088 + ], + [ + "people", + -6.425020217895508 + ], + [ + "▁want", + -6.425827980041504 + ], + [ + "▁when", + -6.43815803527832 + ], + [ + "ally", + -6.453556537628174 + ], + [ + "▁go", + -6.476170063018799 + ], + [ + "▁then", + -6.486055850982666 + ], + [ + "▁But", + -6.499558448791504 + ], + [ + "▁an", + -6.500698566436768 + ], + [ + "▁no", + -6.534000873565674 + ], + [ + "▁had", + -6.5463056564331055 + ], + [ + "in", + -6.5568647384643555 + ], + [ + "▁Yeah", + -6.568700790405273 + ], + [ + "▁would", + -6.5701375007629395 + ], + [ + "▁say", + -6.590750694274902 + ], + [ + "▁them", + -6.595056056976318 + ], + [ + "▁time", + -6.602702617645264 + ], + [ + "a", + -6.629042148590088 + ], + [ + "tion", + -6.633601665496826 + ], + [ + "▁The", + -6.638084411621094 + ], + [ + "2", + -6.6401753425598145 + ], + [ + "▁more", + -6.643313884735107 + ], + [ + "▁how", + -6.651402950286865 + ], + [ + "▁see", + -6.664009094238281 + ], + [ + "▁got", + -6.672194957733154 + ], + [ + "▁our", + -6.6888508796691895 + ], + [ + "es", + -6.693766117095947 + ], + [ + "▁now", + -6.706151962280273 + ], + [ + "▁who", + -6.730064392089844 + ], + [ + "▁We", + -6.735322952270508 + ], + [ + "▁here", + -6.757986545562744 + ], + [ + "▁his", + -6.763290882110596 + ], + [ + "▁will", + -6.763925075531006 + ], + [ + "▁by", + -6.77561092376709 + ], + [ + "▁were", + -6.781393527984619 + ], + [ + "▁every", + -6.806923866271973 + ], + [ + "▁kind", + -6.819945812225342 + ], + [ + "▁back", + -6.8263397216796875 + ], + [ + "▁A", + -6.831277370452881 + ], + [ + "▁look", + -6.834610939025879 + ], + [ + "▁been", + -6.865054607391357 + ], + [ + "▁their", + -6.865737438201904 + ], + [ + "▁very", + -6.8663811683654785 + ], + [ + "▁which", + -6.8790283203125 + ], + [ + "▁That", + -6.8807878494262695 + ], + [ + "▁into", + -6.887390613555908 + ], + [ + "▁where", + -6.889959335327148 + ], + [ + "▁has", + -6.893083095550537 + ], + [ + "o", + -6.901517868041992 + ], + [ + "▁way", + -6.914261341094971 + ], + [ + "▁good", + -6.918059349060059 + ], + [ + "▁she", + -6.91853141784668 + ], + [ + "▁other", + -6.925738334655762 + ], + [ + "▁lot", + -6.9355573654174805 + ], + [ + "▁mean", + -6.942234992980957 + ], + [ + "▁work", + -6.9438157081604 + ], + [ + "p", + -6.944849967956543 + ], + [ + "▁e", + -6.95320987701416 + ], + [ + "▁these", + -6.957985877990723 + ], + [ + "▁de", + -6.967844486236572 + ], + [ + "i", + -6.985270977020264 + ], + [ + "▁i", + -6.990671634674072 + ], + [ + "▁yeah", + -6.996245384216309 + ], + [ + "▁over", + -6.997771263122559 + ], + [ + "▁us", + -7.0103960037231445 + ], + [ + "le", + -7.013932704925537 + ], + [ + "▁him", + -7.016696929931641 + ], + [ + "▁talk", + -7.034051895141602 + ], + [ + "ers", + -7.039214134216309 + ], + [ + "▁He", + -7.044933319091797 + ], + [ + "little", + -7.05172872543335 + ], + [ + "▁well", + -7.058437347412109 + ], + [ + "▁pr", + -7.066596031188965 + ], + [ + "▁even", + -7.093531608581543 + ], + [ + "ion", + -7.095380783081055 + ], + [ + "▁could", + -7.097424030303955 + ], + [ + "▁make", + -7.106647968292236 + ], + [ + "▁need", + -7.112405300140381 + ], + [ + "▁those", + -7.135005950927734 + ], + [ + "▁actu", + -7.140950679779053 + ], + [ + "▁said", + -7.144470691680908 + ], + [ + "▁start", + -7.147336483001709 + ], + [ + "▁t", + -7.153529167175293 + ], + [ + "l", + -7.156661510467529 + ], + [ + "▁her", + -7.170865535736084 + ], + [ + "▁con", + -7.171987533569336 + ], + [ + "ation", + -7.1775288581848145 + ], + [ + "▁did", + -7.179727077484131 + ], + [ + "▁also", + -7.182616233825684 + ], + [ + "▁two", + -7.185066223144531 + ], + [ + "▁down", + -7.191295623779297 + ], + [ + "▁much", + -7.212818145751953 + ], + [ + "5", + -7.213355541229248 + ], + [ + "▁They", + -7.230929851531982 + ], + [ + "en", + -7.234388828277588 + ], + [ + "c", + -7.24249267578125 + ], + [ + "▁feel", + -7.249005317687988 + ], + [ + "ting", + -7.255661487579346 + ], + [ + "▁first", + -7.2672271728515625 + ], + [ + "▁C", + -7.270016193389893 + ], + [ + "though", + -7.2734270095825195 + ], + [ + "▁S", + -7.279673099517822 + ], + [ + "▁any", + -7.285599231719971 + ], + [ + "or", + -7.297698020935059 + ], + [ + "on", + -7.302798748016357 + ], + [ + "3", + -7.310871124267578 + ], + [ + "ment", + -7.319054126739502 + ], + [ + "▁let", + -7.325043201446533 + ], + [ + "g", + -7.329464435577393 + ], + [ + "▁take", + -7.329853057861328 + ], + [ + "one", + -7.338484764099121 + ], + [ + "an", + -7.363990306854248 + ], + [ + "il", + -7.365907192230225 + ], + [ + "▁part", + -7.373332977294922 + ], + [ + "rent", + -7.378327369689941 + ], + [ + "hrough", + -7.378621578216553 + ], + [ + "▁play", + -7.383774280548096 + ], + [ + "▁diffe", + -7.385295867919922 + ], + [ + "▁love", + -7.386297702789307 + ], + [ + "▁com", + -7.391439437866211 + ], + [ + "th", + -7.3933281898498535 + ], + [ + "▁show", + -7.403681755065918 + ], + [ + "▁come", + -7.410576343536377 + ], + [ + "▁than", + -7.415188312530518 + ], + [ + "▁bit", + -7.420681953430176 + ], + [ + "▁put", + -7.427697658538818 + ], + [ + "ra", + -7.429073333740234 + ], + [ + "ness", + -7.4318695068359375 + ], + [ + "▁doing", + -7.437935829162598 + ], + [ + "▁This", + -7.442959308624268 + ], + [ + "▁off", + -7.451347351074219 + ], + [ + "ity", + -7.457302570343018 + ], + [ + "▁What", + -7.463436603546143 + ], + [ + "▁life", + -7.4657464027404785 + ], + [ + "w", + -7.471033096313477 + ], + [ + "▁Oh", + -7.478610992431641 + ], + [ + "▁great", + -7.481228351593018 + ], + [ + "ever", + -7.486627578735352 + ], + [ + "u", + -7.487160682678223 + ], + [ + "happen", + -7.495415210723877 + ], + [ + "ic", + -7.495570182800293 + ], + [ + "ar", + -7.49574613571167 + ], + [ + "4", + -7.497114658355713 + ], + [ + "▁day", + -7.501342296600342 + ], + [ + "ro", + -7.528869152069092 + ], + [ + "▁st", + -7.530606269836426 + ], + [ + "▁only", + -7.533508777618408 + ], + [ + "ate", + -7.534730434417725 + ], + [ + "▁didn", + -7.55030632019043 + ], + [ + "▁still", + -7.555561542510986 + ], + [ + "able", + -7.555562973022461 + ], + [ + "▁F", + -7.555634498596191 + ], + [ + "▁P", + -7.562912940979004 + ], + [ + "▁un", + -7.567101955413818 + ], + [ + "▁under", + -7.569869041442871 + ], + [ + "▁No", + -7.576674461364746 + ], + [ + "9", + -7.579963684082031 + ], + [ + "te", + -7.58042049407959 + ], + [ + "▁being", + -7.581863880157471 + ], + [ + "ch", + -7.591626167297363 + ], + [ + "▁Well", + -7.5994791984558105 + ], + [ + "h", + -7.601132392883301 + ], + [ + "ry", + -7.60433292388916 + ], + [ + "▁inter", + -7.609300136566162 + ], + [ + "should", + -7.609492301940918 + ], + [ + "around", + -7.611624717712402 + ], + [ + "▁man", + -7.616955757141113 + ], + [ + "b", + -7.618578910827637 + ], + [ + "ur", + -7.6237335205078125 + ], + [ + "▁new", + -7.624754905700684 + ], + [ + "▁years", + -7.627285957336426 + ], + [ + "el", + -7.629648685455322 + ], + [ + "▁again", + -7.630054950714111 + ], + [ + "k", + -7.630280494689941 + ], + [ + "▁point", + -7.633115291595459 + ], + [ + "▁E", + -7.6337409019470215 + ], + [ + "▁f", + -7.637404441833496 + ], + [ + "▁too", + -7.642826557159424 + ], + [ + "▁give", + -7.6462554931640625 + ], + [ + "▁God", + -7.647794723510742 + ], + [ + "▁help", + -7.6489458084106445 + ], + [ + "▁why", + -7.651645660400391 + ], + [ + "▁tell", + -7.654996871948242 + ], + [ + "▁big", + -7.660550117492676 + ], + [ + "▁There", + -7.66077184677124 + ], + [ + "ent", + -7.662092208862305 + ], + [ + "before", + -7.66905403137207 + ], + [ + "ne", + -7.6692681312561035 + ], + [ + "person", + -7.68609094619751 + ], + [ + "always", + -7.697056770324707 + ], + [ + "▁gonna", + -7.708845138549805 + ], + [ + "ke", + -7.714349746704102 + ], + [ + "body", + -7.715152740478516 + ], + [ + "▁If", + -7.717781066894531 + ], + [ + "▁Be", + -7.722033500671387 + ], + [ + "▁three", + -7.726672172546387 + ], + [ + "la", + -7.727085113525391 + ], + [ + "▁find", + -7.7324018478393555 + ], + [ + "6", + -7.739603042602539 + ], + [ + "▁B", + -7.7404375076293945 + ], + [ + "▁many", + -7.741635322570801 + ], + [ + "▁year", + -7.7457594871521 + ], + [ + "8", + -7.748380184173584 + ], + [ + "▁same", + -7.74919319152832 + ], + [ + "▁most", + -7.749341011047363 + ], + [ + "ta", + -7.76569128036499 + ], + [ + "▁oh", + -7.765693664550781 + ], + [ + "est", + -7.769061088562012 + ], + [ + "▁Now", + -7.769343376159668 + ], + [ + "▁after", + -7.7787652015686035 + ], + [ + "▁last", + -7.78427791595459 + ], + [ + "▁maybe", + -7.787981033325195 + ], + [ + "▁b", + -7.79763126373291 + ], + [ + "▁never", + -7.800626277923584 + ], + [ + "na", + -7.806248188018799 + ], + [ + "▁might", + -7.814441680908203 + ], + [ + "op", + -7.823919773101807 + ], + [ + "it", + -7.825128555297852 + ], + [ + "se", + -7.825497150421143 + ], + [ + "▁end", + -7.827725887298584 + ], + [ + "▁In", + -7.8292555809021 + ], + [ + "li", + -7.834815502166748 + ], + [ + "▁use", + -7.842201232910156 + ], + [ + "x", + -7.843822956085205 + ], + [ + "▁Like", + -7.845688343048096 + ], + [ + "▁long", + -7.8490400314331055 + ], + [ + "▁co", + -7.849045753479004 + ], + [ + "▁stuff", + -7.85164213180542 + ], + [ + "▁dis", + -7.856433868408203 + ], + [ + "ol", + -7.860621929168701 + ], + [ + "▁sure", + -7.8607025146484375 + ], + [ + "ce", + -7.86693000793457 + ], + [ + "obably", + -7.869808673858643 + ], + [ + "▁world", + -7.871005535125732 + ], + [ + "7", + -7.8722758293151855 + ], + [ + "nother", + -7.874476909637451 + ], + [ + "▁does", + -7.886646270751953 + ], + [ + "ri", + -7.894267559051514 + ], + [ + "▁call", + -7.897050857543945 + ], + [ + "▁real", + -7.904669761657715 + ], + [ + "member", + -7.9059062004089355 + ], + [ + "us", + -7.914706707000732 + ], + [ + "▁sort", + -7.920995712280273 + ], + [ + "▁place", + -7.921472072601318 + ], + [ + "lo", + -7.925868988037109 + ], + [ + "age", + -7.932684421539307 + ], + [ + "▁own", + -7.936575889587402 + ], + [ + "▁imp", + -7.9380202293396 + ], + [ + "▁made", + -7.9458794593811035 + ], + [ + "id", + -7.946288585662842 + ], + [ + "is", + -7.946658134460449 + ], + [ + "▁cha", + -7.948122501373291 + ], + [ + "et", + -7.9484333992004395 + ], + [ + "▁next", + -7.948547840118408 + ], + [ + "ence", + -7.951374530792236 + ], + [ + "▁keep", + -7.95743989944458 + ], + [ + "f", + -7.95897102355957 + ], + [ + "▁T", + -7.959143161773682 + ], + [ + "ck", + -7.959611892700195 + ], + [ + "listen", + -7.962512493133545 + ], + [ + "▁g", + -7.97217321395874 + ], + [ + "▁ques", + -7.983057498931885 + ], + [ + "▁doesn", + -7.986578941345215 + ], + [ + "de", + -7.993881702423096 + ], + [ + "tive", + -7.995202541351318 + ], + [ + "▁Okay", + -7.995222091674805 + ], + [ + "▁okay", + -8.001017570495605 + ], + [ + "to", + -8.010042190551758 + ], + [ + "ma", + -8.011226654052734 + ], + [ + "▁y", + -8.014019966125488 + ], + [ + "▁done", + -8.016657829284668 + ], + [ + "▁try", + -8.017557144165039 + ], + [ + "▁game", + -8.023670196533203 + ], + [ + "▁G", + -8.030261039733887 + ], + [ + "better", + -8.031174659729004 + ], + [ + "▁guys", + -8.04036808013916 + ], + [ + "▁di", + -8.04699993133545 + ], + [ + "pretty", + -8.05696964263916 + ], + [ + "▁went", + -8.064704895019531 + ], + [ + "ful", + -8.066405296325684 + ], + [ + "ir", + -8.071381568908691 + ], + [ + "▁watch", + -8.072752952575684 + ], + [ + "number", + -8.073490142822266 + ], + [ + "odcast", + -8.074795722961426 + ], + [ + "out", + -8.078509330749512 + ], + [ + "▁today", + -8.080411911010742 + ], + [ + "▁week", + -8.081343650817871 + ], + [ + "tions", + -8.083463668823242 + ], + [ + "▁ma", + -8.090441703796387 + ], + [ + "ti", + -8.09251880645752 + ], + [ + "▁exper", + -8.099031448364258 + ], + [ + "friend", + -8.099273681640625 + ], + [ + "A", + -8.10132122039795 + ], + [ + "▁She", + -8.101981163024902 + ], + [ + "▁whole", + -8.10409927368164 + ], + [ + "S", + -8.111672401428223 + ], + [ + "ter", + -8.114810943603516 + ], + [ + "stand", + -8.123491287231445 + ], + [ + "▁away", + -8.123980522155762 + ], + [ + "▁hard", + -8.126935005187988 + ], + [ + "course", + -8.133652687072754 + ], + [ + "▁high", + -8.150348663330078 + ], + [ + "ru", + -8.150856971740723 + ], + [ + "▁home", + -8.152706146240234 + ], + [ + "▁All", + -8.155635833740234 + ], + [ + "▁busi", + -8.156524658203125 + ], + [ + "as", + -8.157022476196289 + ], + [ + "▁learn", + -8.159526824951172 + ], + [ + "v", + -8.160807609558105 + ], + [ + "▁able", + -8.165253639221191 + ], + [ + "ia", + -8.166909217834473 + ], + [ + "ary", + -8.167830467224121 + ], + [ + "▁M", + -8.171135902404785 + ], + [ + "▁best", + -8.172327041625977 + ], + [ + "▁team", + -8.175614356994629 + ], + [ + "ble", + -8.176185607910156 + ], + [ + "▁men", + -8.177753448486328 + ], + [ + "▁came", + -8.179851531982422 + ], + [ + "ad", + -8.181218147277832 + ], + [ + "mo", + -8.182390213012695 + ], + [ + "change", + -8.187325477600098 + ], + [ + "ty", + -8.18980884552002 + ], + [ + "ist", + -8.191882133483887 + ], + [ + "▁How", + -8.192187309265137 + ], + [ + "ical", + -8.193853378295898 + ], + [ + "▁r", + -8.194097518920898 + ], + [ + "▁may", + -8.199860572814941 + ], + [ + "having", + -8.203511238098145 + ], + [ + "at", + -8.2042818069458 + ], + [ + "▁money", + -8.207857131958008 + ], + [ + "▁pro", + -8.207982063293457 + ], + [ + "▁o", + -8.208535194396973 + ], + [ + "side", + -8.222607612609863 + ], + [ + "▁uh", + -8.22454833984375 + ], + [ + "▁name", + -8.231927871704102 + ], + [ + "me", + -8.232441902160645 + ], + [ + "▁pre", + -8.238127708435059 + ], + [ + "!", + -8.239486694335938 + ], + [ + "second", + -8.239890098571777 + ], + [ + "nd", + -8.243569374084473 + ], + [ + "ine", + -8.244396209716797 + ], + [ + "▁am", + -8.245768547058105 + ], + [ + "▁De", + -8.247952461242676 + ], + [ + "▁side", + -8.249322891235352 + ], + [ + "▁Do", + -8.250293731689453 + ], + [ + "z", + -8.252396583557129 + ], + [ + "▁guy", + -8.256911277770996 + ], + [ + "ver", + -8.256974220275879 + ], + [ + "▁read", + -8.258403778076172 + ], + [ + "▁\"", + -8.261567115783691 + ], + [ + "ite", + -8.272871017456055 + ], + [ + "▁Re", + -8.273717880249023 + ], + [ + "▁la", + -8.27470874786377 + ], + [ + "ience", + -8.276155471801758 + ], + [ + "▁mind", + -8.27617073059082 + ], + [ + "▁idea", + -8.277162551879883 + ], + [ + "▁hear", + -8.278149604797363 + ], + [ + "▁five", + -8.278955459594727 + ], + [ + "▁head", + -8.28038215637207 + ], + [ + "▁four", + -8.280485153198242 + ], + [ + "▁O", + -8.282163619995117 + ], + [ + "▁se", + -8.287294387817383 + ], + [ + "▁um", + -8.287642478942871 + ], + [ + "trying", + -8.298399925231934 + ], + [ + "▁th", + -8.29875373840332 + ], + [ + "▁Let", + -8.299449920654297 + ], + [ + "im", + -8.301044464111328 + ], + [ + "▁mo", + -8.30250072479248 + ], + [ + "▁Yes", + -8.306300163269043 + ], + [ + "▁N", + -8.311691284179688 + ], + [ + "▁few", + -8.31614875793457 + ], + [ + "called", + -8.317915916442871 + ], + [ + "no", + -8.319098472595215 + ], + [ + "gether", + -8.32050895690918 + ], + [ + "▁move", + -8.321401596069336 + ], + [ + "ie", + -8.322624206542969 + ], + [ + "coming", + -8.322659492492676 + ], + [ + "▁car", + -8.3236665725708 + ], + [ + "▁both", + -8.325749397277832 + ], + [ + "ously", + -8.327387809753418 + ], + [ + "ul", + -8.33009147644043 + ], + [ + "▁set", + -8.330425262451172 + ], + [ + "▁story", + -8.330855369567871 + ], + [ + "▁fuck", + -8.33165454864502 + ], + [ + "ction", + -8.348227500915527 + ], + [ + "▁open", + -8.348321914672852 + ], + [ + "▁old", + -8.350000381469727 + ], + [ + "C", + -8.350807189941406 + ], + [ + "▁ask", + -8.352201461791992 + ], + [ + "▁else", + -8.354040145874023 + ], + [ + "ortant", + -8.355762481689453 + ], + [ + "ward", + -8.356976509094238 + ], + [ + "▁power", + -8.359997749328613 + ], + [ + "lieve", + -8.362556457519531 + ], + [ + "ha", + -8.36269760131836 + ], + [ + "▁care", + -8.36302375793457 + ], + [ + "▁turn", + -8.363383293151855 + ], + [ + "▁Right", + -8.363404273986816 + ], + [ + "I", + -8.365367889404297 + ], + [ + "ies", + -8.367076873779297 + ], + [ + "é", + -8.367109298706055 + ], + [ + "▁ever", + -8.370315551757812 + ], + [ + "enough", + -8.371919631958008 + ], + [ + "▁used", + -8.375128746032715 + ], + [ + "ated", + -8.375615119934082 + ], + [ + "▁bring", + -8.375762939453125 + ], + [ + "▁fun", + -8.377237319946289 + ], + [ + "▁bad", + -8.378413200378418 + ], + [ + "▁found", + -8.381611824035645 + ], + [ + "▁each", + -8.385348320007324 + ], + [ + "▁book", + -8.388527870178223 + ], + [ + "co", + -8.391936302185059 + ], + [ + "▁fact", + -8.396342277526855 + ], + [ + "reason", + -8.397164344787598 + ], + [ + "P", + -8.400274276733398 + ], + [ + "▁live", + -8.400547981262207 + ], + [ + "▁per", + -8.400988578796387 + ], + [ + "▁When", + -8.401590347290039 + ], + [ + "▁while", + -8.402106285095215 + ], + [ + "▁Thank", + -8.404316902160645 + ], + [ + "times", + -8.404977798461914 + ], + [ + "um", + -8.405458450317383 + ], + [ + "ton", + -8.408987998962402 + ], + [ + "▁D", + -8.411992073059082 + ], + [ + "▁R", + -8.41552448272705 + ], + [ + "▁creat", + -8.420564651489258 + ], + [ + "ho", + -8.420775413513184 + ], + [ + "▁d", + -8.422122955322266 + ], + [ + "▁defin", + -8.422486305236816 + ], + [ + "▁line", + -8.425642967224121 + ], + [ + "▁guess", + -8.430755615234375 + ], + [ + "▁quite", + -8.432291030883789 + ], + [ + "pe", + -8.433984756469727 + ], + [ + "man", + -8.434449195861816 + ], + [ + "▁build", + -8.435030937194824 + ], + [ + "▁en", + -8.436271667480469 + ], + [ + "rself", + -8.441118240356445 + ], + [ + "pisode", + -8.443821907043457 + ], + [ + "moment", + -8.44384479522705 + ], + [ + "▁yes", + -8.444398880004883 + ], + [ + "▁basic", + -8.452391624450684 + ], + [ + "▁video", + -8.45334529876709 + ], + [ + "▁night", + -8.453591346740723 + ], + [ + "couple", + -8.455424308776855 + ], + [ + "making", + -8.455667495727539 + ], + [ + "▁month", + -8.456650733947754 + ], + [ + "▁run", + -8.460701942443848 + ], + [ + "lu", + -8.465465545654297 + ], + [ + "lready", + -8.46560001373291 + ], + [ + "▁To", + -8.465655326843262 + ], + [ + "da", + -8.466363906860352 + ], + [ + "▁goes", + -8.466663360595703 + ], + [ + "D", + -8.466976165771484 + ], + [ + "▁For", + -8.467130661010742 + ], + [ + "...", + -8.469169616699219 + ], + [ + "ga", + -8.469924926757812 + ], + [ + "▁ex", + -8.470888137817383 + ], + [ + "▁po", + -8.470900535583496 + ], + [ + "▁says", + -8.471372604370117 + ], + [ + "▁free", + -8.472845077514648 + ], + [ + "ut", + -8.474469184875488 + ], + [ + "ance", + -8.474730491638184 + ], + [ + "▁job", + -8.4752836227417 + ], + [ + "ng", + -8.477029800415039 + ], + [ + "ach", + -8.480815887451172 + ], + [ + "▁seen", + -8.481229782104492 + ], + [ + "و", + -8.481822967529297 + ], + [ + "up", + -8.482392311096191 + ], + [ + "un", + -8.482681274414062 + ], + [ + "tween", + -8.486188888549805 + ], + [ + "▁won", + -8.486241340637207 + ], + [ + "ant", + -8.488265037536621 + ], + [ + "▁such", + -8.488940238952637 + ], + [ + "▁hope", + -8.489032745361328 + ], + [ + "▁comes", + -8.489861488342285 + ], + [ + "▁cool", + -8.490562438964844 + ], + [ + "▁house", + -8.490806579589844 + ], + [ + "tional", + -8.492448806762695 + ], + [ + "▁pass", + -8.492779731750488 + ], + [ + "▁water", + -8.497396469116211 + ], + [ + "▁walk", + -8.499316215515137 + ], + [ + "▁light", + -8.502432823181152 + ], + [ + "esting", + -8.502674102783203 + ], + [ + "▁Ma", + -8.503865242004395 + ], + [ + "▁h", + -8.504929542541504 + ], + [ + "ations", + -8.507333755493164 + ], + [ + "ni", + -8.50997257232666 + ], + [ + "▁pa", + -8.511340141296387 + ], + [ + "minute", + -8.51488208770752 + ], + [ + "▁bo", + -8.515910148620605 + ], + [ + "▁times", + -8.516918182373047 + ], + [ + "les", + -8.516949653625488 + ], + [ + "ر", + -8.518157005310059 + ], + [ + "▁case", + -8.520344734191895 + ], + [ + "▁far", + -8.522486686706543 + ], + [ + "▁child", + -8.523870468139648 + ], + [ + "ca", + -8.526588439941406 + ], + [ + "▁bu", + -8.52804183959961 + ], + [ + "▁sound", + -8.529618263244629 + ], + [ + "ard", + -8.531852722167969 + ], + [ + "▁wasn", + -8.532976150512695 + ], + [ + "▁que", + -8.539838790893555 + ], + [ + "day", + -8.540030479431152 + ], + [ + "system", + -8.5426664352417 + ], + [ + "▁hand", + -8.54411506652832 + ], + [ + "vi", + -8.545309066772461 + ], + [ + "ish", + -8.546232223510742 + ], + [ + "ci", + -8.55002498626709 + ], + [ + "▁speci", + -8.551002502441406 + ], + [ + "T", + -8.552409172058105 + ], + [ + "school", + -8.553696632385254 + ], + [ + "son", + -8.55629825592041 + ], + [ + "▁gener", + -8.562760353088379 + ], + [ + "▁once", + -8.564325332641602 + ], + [ + "▁w", + -8.564567565917969 + ], + [ + "▁small", + -8.565271377563477 + ], + [ + "am", + -8.56930923461914 + ], + [ + "▁K", + -8.570649147033691 + ], + [ + "ture", + -8.576712608337402 + ], + [ + "M", + -8.577024459838867 + ], + [ + "▁trans", + -8.579708099365234 + ], + [ + "▁left", + -8.582918167114258 + ], + [ + "itely", + -8.583322525024414 + ], + [ + "all", + -8.58346939086914 + ], + [ + "ag", + -8.58770751953125 + ], + [ + "family", + -8.587883949279785 + ], + [ + "▁V", + -8.589518547058105 + ], + [ + "ous", + -8.592347145080566 + ], + [ + "om", + -8.593643188476562 + ], + [ + "▁movie", + -8.593978881835938 + ], + [ + "ice", + -8.594159126281738 + ], + [ + "ions", + -8.594797134399414 + ], + [ + "م", + -8.595993041992188 + ], + [ + "▁anyth", + -8.597857475280762 + ], + [ + "ian", + -8.59841251373291 + ], + [ + "follow", + -8.59969711303711 + ], + [ + "▁top", + -8.599716186523438 + ], + [ + "▁pay", + -8.600637435913086 + ], + [ + "be", + -8.604819297790527 + ], + [ + "▁nice", + -8.607076644897461 + ], + [ + "%", + -8.60915756225586 + ], + [ + "▁thank", + -8.614678382873535 + ], + [ + "▁saw", + -8.616087913513184 + ], + [ + "▁cur", + -8.617782592773438 + ], + [ + "he", + -8.619572639465332 + ], + [ + "ge", + -8.619688034057617 + ], + [ + "▁ba", + -8.620129585266113 + ], + [ + "▁half", + -8.621623039245605 + ], + [ + "▁lead", + -8.622058868408203 + ], + [ + "roblem", + -8.622466087341309 + ], + [ + "▁La", + -8.62397289276123 + ], + [ + "▁rec", + -8.632270812988281 + ], + [ + "em", + -8.63492202758789 + ], + [ + "va", + -8.63544750213623 + ], + [ + "dy", + -8.635499000549316 + ], + [ + "bility", + -8.638021469116211 + ], + [ + "▁ca", + -8.639488220214844 + ], + [ + "ten", + -8.642119407653809 + ], + [ + "▁exact", + -8.64391040802002 + ], + [ + "di", + -8.64551830291748 + ], + [ + "▁obvi", + -8.648545265197754 + ], + [ + "ey", + -8.64885425567627 + ], + [ + "▁deal", + -8.64913272857666 + ], + [ + "ssion", + -8.649358749389648 + ], + [ + "▁pick", + -8.653026580810547 + ], + [ + "▁sub", + -8.655430793762207 + ], + [ + "xample", + -8.658014297485352 + ], + [ + "become", + -8.658555030822754 + ], + [ + "▁na", + -8.659600257873535 + ], + [ + "▁My", + -8.659753799438477 + ], + [ + "market", + -8.660055160522461 + ], + [ + "che", + -8.661465644836426 + ], + [ + "where", + -8.662162780761719 + ], + [ + "tor", + -8.662221908569336 + ], + [ + "self", + -8.663838386535645 + ], + [ + "▁hit", + -8.663921356201172 + ], + [ + "▁posi", + -8.663960456848145 + ], + [ + "myself", + -8.66524600982666 + ], + [ + "ي", + -8.665751457214355 + ], + [ + "com", + -8.668530464172363 + ], + [ + "▁six", + -8.668973922729492 + ], + [ + "▁stand", + -8.670042991638184 + ], + [ + "▁check", + -8.670136451721191 + ], + [ + "▁U", + -8.672911643981934 + ], + [ + "▁Just", + -8.673397064208984 + ], + [ + "▁close", + -8.673858642578125 + ], + [ + "▁speak", + -8.674603462219238 + ], + [ + "▁state", + -8.675527572631836 + ], + [ + "▁fan", + -8.677284240722656 + ], + [ + "▁wait", + -8.681979179382324 + ], + [ + "▁break", + -8.682747840881348 + ], + [ + "ster", + -8.683348655700684 + ], + [ + "ive", + -8.68435001373291 + ], + [ + "ren", + -8.685545921325684 + ], + [ + "ka", + -8.685918807983398 + ], + [ + "▁W", + -8.68787670135498 + ], + [ + "R", + -8.688252449035645 + ], + [ + "▁music", + -8.691445350646973 + ], + [ + "▁super", + -8.69190788269043 + ], + [ + "▁level", + -8.69225788116455 + ], + [ + "almost", + -8.69260311126709 + ], + [ + "spect", + -8.696332931518555 + ], + [ + "▁space", + -8.697981834411621 + ], + [ + "tic", + -8.69830322265625 + ], + [ + "ak", + -8.702743530273438 + ], + [ + "▁sense", + -8.704197883605957 + ], + [ + "▁grow", + -8.704347610473633 + ], + [ + "▁As", + -8.704444885253906 + ], + [ + "▁n", + -8.706270217895508 + ], + [ + "ially", + -8.706563949584961 + ], + [ + "▁took", + -8.707412719726562 + ], + [ + "▁cons", + -8.707503318786621 + ], + [ + "B", + -8.709356307983398 + ], + [ + "selves", + -8.713006973266602 + ], + [ + "season", + -8.713068008422852 + ], + [ + "ig", + -8.713550567626953 + ], + [ + "ook", + -8.713750839233398 + ], + [ + "pa", + -8.715437889099121 + ], + [ + "▁L", + -8.71619701385498 + ], + [ + "utely", + -8.716267585754395 + ], + [ + "▁k", + -8.718250274658203 + ], + [ + "ments", + -8.726122856140137 + ], + [ + "os", + -8.727188110351562 + ], + [ + "▁Jesus", + -8.728135108947754 + ], + [ + "▁days", + -8.72931957244873 + ], + [ + "▁Ro", + -8.732542037963867 + ], + [ + "▁ago", + -8.73293685913086 + ], + [ + "land", + -8.732978820800781 + ], + [ + "▁heart", + -8.735259056091309 + ], + [ + "د", + -8.736091613769531 + ], + [ + "▁Chris", + -8.7373628616333 + ], + [ + "▁quick", + -8.737496376037598 + ], + [ + "cu", + -8.737568855285645 + ], + [ + "matter", + -8.738037109375 + ], + [ + "▁face", + -8.738126754760742 + ], + [ + "ted", + -8.738473892211914 + ], + [ + "▁Some", + -8.738590240478516 + ], + [ + "do", + -8.738718032836914 + ], + [ + "und", + -8.742030143737793 + ], + [ + "▁mi", + -8.74361515045166 + ], + [ + "▁true", + -8.750214576721191 + ], + [ + "▁stay", + -8.751595497131348 + ], + [ + "▁Pa", + -8.755435943603516 + ], + [ + "▁hold", + -8.755465507507324 + ], + [ + "O", + -8.75627326965332 + ], + [ + "▁type", + -8.756329536437988 + ], + [ + "▁tr", + -8.757129669189453 + ], + [ + "less", + -8.757354736328125 + ], + [ + "▁H", + -8.758471488952637 + ], + [ + "line", + -8.763260841369629 + ], + [ + "upport", + -8.765844345092773 + ], + [ + "gainst", + -8.766072273254395 + ], + [ + "▁least", + -8.7665433883667 + ], + [ + "▁word", + -8.767441749572754 + ], + [ + "answer", + -8.767786979675293 + ], + [ + "j", + -8.767943382263184 + ], + [ + "▁step", + -8.768478393554688 + ], + [ + "▁heard", + -8.768655776977539 + ], + [ + "▁hi", + -8.769448280334473 + ], + [ + "ling", + -8.769691467285156 + ], + [ + "mi", + -8.770326614379883 + ], + [ + "▁sa", + -8.77038860321045 + ], + [ + "either", + -8.771315574645996 + ], + [ + "▁buy", + -8.77165699005127 + ], + [ + "▁rest", + -8.77206802368164 + ], + [ + "▁St", + -8.774239540100098 + ], + [ + "▁comp", + -8.774499893188477 + ], + [ + "fic", + -8.775188446044922 + ], + [ + "ا", + -8.77776050567627 + ], + [ + "ت", + -8.77912712097168 + ], + [ + "▁makes", + -8.779967308044434 + ], + [ + "ically", + -8.781737327575684 + ], + [ + "ang", + -8.782662391662598 + ], + [ + "produc", + -8.783069610595703 + ], + [ + "E", + -8.784475326538086 + ], + [ + "▁vi", + -8.78581428527832 + ], + [ + "taking", + -8.788702011108398 + ], + [ + "form", + -8.792396545410156 + ], + [ + "ن", + -8.79869270324707 + ], + [ + "▁since", + -8.799327850341797 + ], + [ + "ot", + -8.79963493347168 + ], + [ + "▁count", + -8.800046920776367 + ], + [ + "▁shit", + -8.80013370513916 + ], + [ + "sh", + -8.80069637298584 + ], + [ + "resent", + -8.800989151000977 + ], + [ + "▁until", + -8.802389144897461 + ], + [ + "conver", + -8.80280590057373 + ], + [ + "▁young", + -8.809401512145996 + ], + [ + "▁Ha", + -8.809650421142578 + ], + [ + "▁kids", + -8.80980110168457 + ], + [ + "▁Or", + -8.810272216796875 + ], + [ + "▁New", + -8.814461708068848 + ], + [ + "▁leave", + -8.81515121459961 + ], + [ + "ground", + -8.815988540649414 + ], + [ + "▁miss", + -8.817492485046387 + ], + [ + "▁hey", + -8.81812858581543 + ], + [ + "▁l", + -8.820104598999023 + ], + [ + "▁told", + -8.82188606262207 + ], + [ + "▁song", + -8.82461929321289 + ], + [ + "▁means", + -8.8248291015625 + ], + [ + "ure", + -8.82526683807373 + ], + [ + "▁ho", + -8.825498580932617 + ], + [ + "▁full", + -8.827272415161133 + ], + [ + "▁cut", + -8.828385353088379 + ], + [ + "▁using", + -8.828414916992188 + ], + [ + "▁sc", + -8.828606605529785 + ], + [ + "▁body", + -8.829814910888672 + ], + [ + "▁group", + -8.830730438232422 + ], + [ + "▁Go", + -8.831496238708496 + ], + [ + "▁absol", + -8.831792831420898 + ], + [ + "▁room", + -8.832965850830078 + ], + [ + "onship", + -8.834259986877441 + ], + [ + "▁human", + -8.837621688842773 + ], + [ + "▁sur", + -8.839649200439453 + ], + [ + "▁allow", + -8.840558052062988 + ], + [ + "red", + -8.8445463180542 + ], + [ + "▁yet", + -8.848400115966797 + ], + [ + "lar", + -8.849067687988281 + ], + [ + "ries", + -8.849857330322266 + ], + [ + "▁One", + -8.84987735748291 + ], + [ + "au", + -8.85007381439209 + ], + [ + "▁Mo", + -8.8519287109375 + ], + [ + "search", + -8.853538513183594 + ], + [ + "е", + -8.85414981842041 + ], + [ + "▁sh", + -8.854841232299805 + ], + [ + "ه", + -8.854894638061523 + ], + [ + "▁Le", + -8.856083869934082 + ], + [ + "ل", + -8.857172012329102 + ], + [ + "hy", + -8.858010292053223 + ], + [ + "▁air", + -8.86134147644043 + ], + [ + "vo", + -8.861430168151855 + ], + [ + "▁Ba", + -8.864173889160156 + ], + [ + "▁area", + -8.865365982055664 + ], + [ + "ba", + -8.865779876708984 + ], + [ + "▁stop", + -8.866847038269043 + ], + [ + "ow", + -8.867715835571289 + ], + [ + "▁clear", + -8.868525505065918 + ], + [ + "▁wrong", + -8.868855476379395 + ], + [ + "mazing", + -8.870097160339355 + ], + [ + "▁lo", + -8.871426582336426 + ], + [ + "fully", + -8.873072624206543 + ], + [ + "per", + -8.873246192932129 + ], + [ + "go", + -8.874667167663574 + ], + [ + "wa", + -8.876036643981934 + ], + [ + "▁possi", + -8.876326560974121 + ], + [ + "ی", + -8.877270698547363 + ], + [ + "tro", + -8.879477500915527 + ], + [ + "▁past", + -8.884129524230957 + ], + [ + "▁da", + -8.886237144470215 + ], + [ + "tory", + -8.887429237365723 + ], + [ + "expect", + -8.892228126525879 + ], + [ + "and", + -8.89306354522705 + ], + [ + "ition", + -8.893683433532715 + ], + [ + "▁wheth", + -8.893691062927246 + ], + [ + "▁issue", + -8.894025802612305 + ], + [ + "▁writ", + -8.89453125 + ], + [ + "fi", + -8.894998550415039 + ], + [ + "▁Ja", + -8.895549774169922 + ], + [ + "▁piece", + -8.896522521972656 + ], + [ + "▁ha", + -8.896896362304688 + ], + [ + "ah", + -8.899083137512207 + ], + [ + "wi", + -8.899349212646484 + ], + [ + "▁order", + -8.901896476745605 + ], + [ + "▁espec", + -8.904784202575684 + ], + [ + "ner", + -8.904807090759277 + ], + [ + "relati", + -8.90532112121582 + ], + [ + "▁knew", + -8.9068603515625 + ], + [ + "▁isn", + -8.910194396972656 + ], + [ + "▁focus", + -8.910478591918945 + ], + [ + "tain", + -8.913580894470215 + ], + [ + "▁test", + -8.914241790771484 + ], + [ + "▁bi", + -8.917304039001465 + ], + [ + "▁Ho", + -8.918100357055664 + ], + [ + "gu", + -8.918365478515625 + ], + [ + "record", + -8.921841621398926 + ], + [ + "▁yn", + -8.925897598266602 + ], + [ + "▁food", + -8.925972938537598 + ], + [ + "▁later", + -8.928470611572266 + ], + [ + "ap", + -8.929786682128906 + ], + [ + "▁$", + -8.930574417114258 + ], + [ + "ompany", + -8.930594444274902 + ], + [ + "▁girl", + -8.931021690368652 + ], + [ + "▁must", + -8.932236671447754 + ], + [ + "▁excit", + -8.932639122009277 + ], + [ + "gg", + -8.933134078979492 + ], + [ + "▁Ca", + -8.933996200561523 + ], + [ + "od", + -8.934683799743652 + ], + [ + "▁Bo", + -8.934969902038574 + ], + [ + "▁plan", + -8.935236930847168 + ], + [ + "▁meet", + -8.93568229675293 + ], + [ + "▁sp", + -8.936230659484863 + ], + [ + "▁Se", + -8.94008731842041 + ], + [ + "▁succe", + -8.94096851348877 + ], + [ + "▁less", + -8.94432258605957 + ], + [ + "ize", + -8.944500923156738 + ], + [ + "icular", + -8.945056915283203 + ], + [ + "row", + -8.947490692138672 + ], + [ + "ld", + -8.947938919067383 + ], + [ + "▁teach", + -8.94893741607666 + ], + [ + "ip", + -8.954086303710938 + ], + [ + "ontrol", + -8.95534610748291 + ], + [ + "bo", + -8.95704174041748 + ], + [ + "▁star", + -8.95771598815918 + ], + [ + "▁share", + -8.95938491821289 + ], + [ + "light", + -8.961296081542969 + ], + [ + "ball", + -8.96158218383789 + ], + [ + "mation", + -8.96166706085205 + ], + [ + "▁rep", + -8.962485313415527 + ], + [ + "▁value", + -8.966387748718262 + ], + [ + "▁train", + -8.97018051147461 + ], + [ + "sive", + -8.970291137695312 + ], + [ + "rence", + -8.970890998840332 + ], + [ + "qui", + -8.97185230255127 + ], + [ + "▁techn", + -8.972455978393555 + ], + [ + "▁certa", + -8.9733247756958 + ], + [ + "▁Then", + -8.974882125854492 + ], + [ + "ities", + -8.975265502929688 + ], + [ + "po", + -8.976327896118164 + ], + [ + "way", + -8.976357460021973 + ], + [ + "▁Me", + -8.977892875671387 + ], + [ + "▁short", + -8.978824615478516 + ], + [ + "▁main", + -8.978894233703613 + ], + [ + "▁fall", + -8.982150077819824 + ], + [ + "wards", + -8.9824857711792 + ], + [ + "▁Sa", + -8.983622550964355 + ], + [ + "tra", + -8.984106063842773 + ], + [ + "L", + -8.985461235046387 + ], + [ + "▁often", + -8.985709190368652 + ], + [ + "bi", + -8.98623275756836 + ], + [ + "▁class", + -8.986233711242676 + ], + [ + "▁Li", + -8.986499786376953 + ], + [ + "ger", + -8.988371849060059 + ], + [ + "ep", + -8.989365577697754 + ], + [ + "▁win", + -8.989603996276855 + ], + [ + "iv", + -8.990174293518066 + ], + [ + "▁mu", + -8.992423057556152 + ], + [ + "▁along", + -8.994239807128906 + ], + [ + "å", + -8.99525260925293 + ], + [ + "▁OK", + -8.996599197387695 + ], + [ + "▁front", + -8.998224258422852 + ], + [ + "during", + -8.999367713928223 + ], + [ + "▁du", + -8.999914169311523 + ], + [ + "ating", + -9.000185012817383 + ], + [ + "led", + -9.00027847290039 + ], + [ + "ism", + -9.000290870666504 + ], + [ + "▁easy", + -9.001694679260254 + ], + [ + "ator", + -9.002311706542969 + ], + [ + "▁enjoy", + -9.004293441772461 + ], + [ + "ber", + -9.004940032958984 + ], + [ + "▁haven", + -9.006134033203125 + ], + [ + "▁situa", + -9.007049560546875 + ], + [ + "▁black", + -9.00743293762207 + ], + [ + "energy", + -9.007623672485352 + ], + [ + "▁det", + -9.010364532470703 + ], + [ + "▁women", + -9.01065444946289 + ], + [ + "N", + -9.011513710021973 + ], + [ + "▁Don", + -9.0160551071167 + ], + [ + "parent", + -9.017770767211914 + ], + [ + "▁Why", + -9.020317077636719 + ], + [ + "▁fire", + -9.020331382751465 + ], + [ + "der", + -9.021602630615234 + ], + [ + "▁le", + -9.024235725402832 + ], + [ + "▁fight", + -9.024870872497559 + ], + [ + "▁chang", + -9.025651931762695 + ], + [ + "ement", + -9.026383399963379 + ], + [ + "can", + -9.027878761291504 + ], + [ + "▁foot", + -9.029459953308105 + ], + [ + "▁Na", + -9.030130386352539 + ], + [ + "▁Ra", + -9.031861305236816 + ], + [ + "create", + -9.032488822937012 + ], + [ + "▁add", + -9.033122062683105 + ], + [ + "ron", + -9.034554481506348 + ], + [ + "F", + -9.03532600402832 + ], + [ + "stead", + -9.035615921020508 + ], + [ + "▁des", + -9.036656379699707 + ], + [ + "▁morn", + -9.0381498336792 + ], + [ + "▁Co", + -9.040934562683105 + ], + [ + "ab", + -9.04224681854248 + ], + [ + "▁eat", + -9.042732238769531 + ], + [ + "▁felt", + -9.042911529541016 + ], + [ + "▁runn", + -9.044498443603516 + ], + [ + "we", + -9.044965744018555 + ], + [ + "ning", + -9.048744201660156 + ], + [ + "▁post", + -9.04956340789795 + ], + [ + "pi", + -9.050450325012207 + ], + [ + ";", + -9.053049087524414 + ], + [ + "ny", + -9.05362319946289 + ], + [ + "▁Lord", + -9.05436897277832 + ], + [ + "racter", + -9.055581092834473 + ], + [ + "behind", + -9.060038566589355 + ], + [ + "▁Can", + -9.06406307220459 + ], + [ + "▁bea", + -9.064136505126953 + ], + [ + "▁favor", + -9.064603805541992 + ], + [ + "▁ready", + -9.066211700439453 + ], + [ + "ee", + -9.067480087280273 + ], + [ + "single", + -9.067906379699707 + ], + [ + "▁seven", + -9.075495719909668 + ], + [ + "▁war", + -9.076934814453125 + ], + [ + "health", + -9.077648162841797 + ], + [ + "social", + -9.079365730285645 + ], + [ + "▁happy", + -9.080602645874023 + ], + [ + "io", + -9.080713272094727 + ], + [ + "▁liter", + -9.081754684448242 + ], + [ + "▁bar", + -9.082159996032715 + ], + [ + "across", + -9.085174560546875 + ], + [ + "action", + -9.08549690246582 + ], + [ + "▁Not", + -9.086748123168945 + ], + [ + "▁mill", + -9.089664459228516 + ], + [ + "▁gone", + -9.091009140014648 + ], + [ + "▁huge", + -9.091429710388184 + ], + [ + "mp", + -9.092206954956055 + ], + [ + "▁early", + -9.093128204345703 + ], + [ + "ple", + -9.093631744384766 + ], + [ + "hu", + -9.095439910888672 + ], + [ + "▁Every", + -9.098529815673828 + ], + [ + "tract", + -9.099678039550781 + ], + [ + "▁die", + -9.099730491638184 + ], + [ + "rd", + -9.101494789123535 + ], + [ + "▁infor", + -9.102574348449707 + ], + [ + "ration", + -9.102784156799316 + ], + [ + "▁final", + -9.103464126586914 + ], + [ + "▁Da", + -9.10403060913086 + ], + [ + "ric", + -9.107940673828125 + ], + [ + "▁direc", + -9.109135627746582 + ], + [ + "▁gra", + -9.109183311462402 + ], + [ + "fo", + -9.109792709350586 + ], + [ + "▁Al", + -9.11117172241211 + ], + [ + "▁film", + -9.112849235534668 + ], + [ + "fe", + -9.113022804260254 + ], + [ + "▁large", + -9.115986824035645 + ], + [ + "gen", + -9.119293212890625 + ], + [ + "strong", + -9.119502067565918 + ], + [ + "▁disc", + -9.120002746582031 + ], + [ + "▁fa", + -9.121415138244629 + ], + [ + "▁boy", + -9.122583389282227 + ], + [ + "▁pull", + -9.12259292602539 + ], + [ + "ology", + -9.124296188354492 + ], + [ + "utiful", + -9.12459945678711 + ], + [ + "▁va", + -9.124744415283203 + ], + [ + "▁looks", + -9.127294540405273 + ], + [ + "sta", + -9.127840042114258 + ], + [ + "rocess", + -9.132211685180664 + ], + [ + "▁Y", + -9.132549285888672 + ], + [ + "nation", + -9.132739067077637 + ], + [ + "figure", + -9.139142990112305 + ], + [ + "honest", + -9.140530586242676 + ], + [ + "▁form", + -9.140780448913574 + ], + [ + "tri", + -9.142746925354004 + ], + [ + "▁Wa", + -9.143213272094727 + ], + [ + "ع", + -9.143847465515137 + ], + [ + "▁fine", + -9.145126342773438 + ], + [ + "▁door", + -9.147528648376465 + ], + [ + "▁cover", + -9.147950172424316 + ], + [ + "ious", + -9.150182723999023 + ], + [ + "▁land", + -9.151371955871582 + ], + [ + "▁words", + -9.151972770690918 + ], + [ + "▁list", + -9.152130126953125 + ], + [ + "▁sign", + -9.153017044067383 + ], + [ + "ris", + -9.154001235961914 + ], + [ + "gi", + -9.154319763183594 + ], + [ + "▁oppor", + -9.155195236206055 + ], + [ + "▁write", + -9.155582427978516 + ], + [ + "rate", + -9.158084869384766 + ], + [ + "living", + -9.159357070922852 + ], + [ + "▁crazy", + -9.159431457519531 + ], + [ + "▁seems", + -9.16001033782959 + ], + [ + "▁John", + -9.160340309143066 + ], + [ + "time", + -9.16416072845459 + ], + [ + "▁terms", + -9.164505004882812 + ], + [ + "val", + -9.166070938110352 + ], + [ + "ervice", + -9.166971206665039 + ], + [ + "▁appro", + -9.168490409851074 + ], + [ + "port", + -9.170356750488281 + ], + [ + "erfect", + -9.170977592468262 + ], + [ + "▁Di", + -9.174539566040039 + ], + [ + "▁har", + -9.176504135131836 + ], + [ + "▁plus", + -9.177936553955078 + ], + [ + "▁ال", + -9.178487777709961 + ], + [ + "giving", + -9.178792953491211 + ], + [ + "▁Maybe", + -9.181262969970703 + ], + [ + "ors", + -9.183570861816406 + ], + [ + "night", + -9.184505462646484 + ], + [ + "▁push", + -9.186213493347168 + ], + [ + "▁join", + -9.18624210357666 + ], + [ + "▁Mar", + -9.186954498291016 + ], + [ + "G", + -9.187036514282227 + ], + [ + "con", + -9.187541961669922 + ], + [ + "entire", + -9.189494132995605 + ], + [ + "munity", + -9.190133094787598 + ], + [ + "▁sit", + -9.190606117248535 + ], + [ + "▁reach", + -9.190654754638672 + ], + [ + "▁elect", + -9.191340446472168 + ], + [ + "▁data", + -9.192991256713867 + ], + [ + "▁agree", + -9.19411849975586 + ], + [ + "oo", + -9.198872566223145 + ], + [ + "▁asked", + -9.19937515258789 + ], + [ + "▁On", + -9.201492309570312 + ], + [ + "▁send", + -9.20433235168457 + ], + [ + "ide", + -9.204376220703125 + ], + [ + "future", + -9.204535484313965 + ], + [ + "▁sell", + -9.205377578735352 + ], + [ + "ft", + -9.206799507141113 + ], + [ + "jo", + -9.207775115966797 + ], + [ + "▁ni", + -9.207779884338379 + ], + [ + "ctor", + -9.209047317504883 + ], + [ + "ff", + -9.209071159362793 + ], + [ + "depend", + -9.212953567504883 + ], + [ + "▁self", + -9.213046073913574 + ], + [ + "▁ac", + -9.214652061462402 + ], + [ + "ub", + -9.215879440307617 + ], + [ + "▁phone", + -9.216513633728027 + ], + [ + "H", + -9.21670913696289 + ], + [ + "king", + -9.217204093933105 + ], + [ + "س", + -9.217345237731934 + ], + [ + "ou", + -9.21832275390625 + ], + [ + "amount", + -9.21908950805664 + ], + [ + "▁ja", + -9.220247268676758 + ], + [ + "▁fin", + -9.221041679382324 + ], + [ + "bb", + -9.222188949584961 + ], + [ + "end", + -9.2229585647583 + ], + [ + "▁coach", + -9.223636627197266 + ], + [ + "▁kid", + -9.224626541137695 + ], + [ + "ertain", + -9.224735260009766 + ], + [ + "letely", + -9.229743957519531 + ], + [ + "▁ball", + -9.233514785766602 + ], + [ + "moving", + -9.233748435974121 + ], + [ + "▁hours", + -9.234230041503906 + ], + [ + "▁manag", + -9.234424591064453 + ], + [ + "▁spend", + -9.23469066619873 + ], + [ + "▁usual", + -9.234749794006348 + ], + [ + "▁kill", + -9.23589038848877 + ], + [ + "ick", + -9.236687660217285 + ], + [ + "▁su", + -9.236882209777832 + ], + [ + "▁touch", + -9.237579345703125 + ], + [ + "result", + -9.238622665405273 + ], + [ + "ä", + -9.23870849609375 + ], + [ + "mb", + -9.238740921020508 + ], + [ + "▁diff", + -9.240494728088379 + ], + [ + "▁pictu", + -9.240958213806152 + ], + [ + "mother", + -9.241060256958008 + ], + [ + "ial", + -9.241181373596191 + ], + [ + "▁gen", + -9.243279457092285 + ], + [ + "▁arm", + -9.243325233459473 + ], + [ + "▁media", + -9.2437162399292 + ], + [ + "ear", + -9.243833541870117 + ], + [ + "anyway", + -9.24473762512207 + ], + [ + "▁At", + -9.244958877563477 + ], + [ + "▁lost", + -9.245104789733887 + ], + [ + "▁je", + -9.245494842529297 + ], + [ + "tinue", + -9.245914459228516 + ], + [ + "▁weird", + -9.246435165405273 + ], + [ + "source", + -9.24682331085205 + ], + [ + "▁Je", + -9.247194290161133 + ], + [ + "и", + -9.24785327911377 + ], + [ + "▁baby", + -9.247946739196777 + ], + [ + "ell", + -9.248261451721191 + ], + [ + "old", + -9.25293254852295 + ], + [ + "▁key", + -9.253212928771973 + ], + [ + "▁multi", + -9.253715515136719 + ], + [ + "с", + -9.255903244018555 + ], + [ + "▁deep", + -9.25597095489502 + ], + [ + "▁equal", + -9.258957862854004 + ], + [ + "▁given", + -9.2611083984375 + ], + [ + "▁white", + -9.261425971984863 + ], + [ + "а", + -9.261722564697266 + ], + [ + "▁pe", + -9.262528419494629 + ], + [ + "gram", + -9.265002250671387 + ], + [ + "▁Um", + -9.26501750946045 + ], + [ + "men", + -9.265767097473145 + ], + [ + "о", + -9.265829086303711 + ], + [ + "emp", + -9.266592979431152 + ], + [ + "▁gave", + -9.26683521270752 + ], + [ + "ك", + -9.267192840576172 + ], + [ + "▁low", + -9.267684936523438 + ], + [ + "ph", + -9.268341064453125 + ], + [ + "▁fast", + -9.26915168762207 + ], + [ + "wn", + -9.269535064697266 + ], + [ + "▁Man", + -9.269639015197754 + ], + [ + "▁eight", + -9.26994514465332 + ], + [ + "▁ahead", + -9.270735740661621 + ], + [ + "▁eyes", + -9.27210521697998 + ], + [ + "▁journ", + -9.272489547729492 + ], + [ + "י", + -9.27332592010498 + ], + [ + "ler", + -9.273736000061035 + ], + [ + "ient", + -9.274319648742676 + ], + [ + "ise", + -9.274370193481445 + ], + [ + "qu", + -9.274736404418945 + ], + [ + "▁law", + -9.27739143371582 + ], + [ + "▁Lo", + -9.277520179748535 + ], + [ + "▁Ameri", + -9.279704093933105 + ], + [ + "ized", + -9.280120849609375 + ], + [ + "▁funny", + -9.283392906188965 + ], + [ + "▁event", + -9.28357219696045 + ], + [ + "our", + -9.284185409545898 + ], + [ + "▁Here", + -9.284661293029785 + ], + [ + "wonder", + -9.28836441040039 + ], + [ + "▁app", + -9.288464546203613 + ], + [ + "▁offer", + -9.289196014404297 + ], + [ + "▁An", + -9.289769172668457 + ], + [ + "rogram", + -9.291475296020508 + ], + [ + "▁Hi", + -9.291528701782227 + ], + [ + "▁Hey", + -9.291892051696777 + ], + [ + ".\"", + -9.292503356933594 + ], + [ + "▁art", + -9.29272747039795 + ], + [ + "hi", + -9.29306697845459 + ], + [ + "cho", + -9.29344367980957 + ], + [ + "▁v", + -9.293703079223633 + ], + [ + "vision", + -9.293916702270508 + ], + [ + "design", + -9.295731544494629 + ], + [ + "▁takes", + -9.297224044799805 + ], + [ + "▁Ar", + -9.29771614074707 + ], + [ + "▁Po", + -9.298718452453613 + ], + [ + "▁woman", + -9.300501823425293 + ], + [ + "▁Mi", + -9.300869941711426 + ], + [ + "▁Who", + -9.303844451904297 + ], + [ + "lock", + -9.306121826171875 + ], + [ + "tioned", + -9.30648422241211 + ], + [ + "á", + -9.306832313537598 + ], + [ + "▁games", + -9.308627128601074 + ], + [ + "ac", + -9.309030532836914 + ], + [ + "▁throw", + -9.309338569641113 + ], + [ + "nder", + -9.309569358825684 + ], + [ + "view", + -9.310959815979004 + ], + [ + "▁ju", + -9.311022758483887 + ], + [ + "rought", + -9.313830375671387 + ], + [ + "og", + -9.314338684082031 + ], + [ + "ud", + -9.314417839050293 + ], + [ + "▁sent", + -9.3158597946167 + ], + [ + "icult", + -9.316699981689453 + ], + [ + "▁third", + -9.320051193237305 + ], + [ + "public", + -9.320467948913574 + ], + [ + "▁Sp", + -9.32242202758789 + ], + [ + "▁brain", + -9.322565078735352 + ], + [ + "▁decis", + -9.324774742126465 + ], + [ + "▁hundr", + -9.326744079589844 + ], + [ + "▁fit", + -9.327546119689941 + ], + [ + "dollar", + -9.328387260437012 + ], + [ + "▁neces", + -9.330474853515625 + ], + [ + "▁voice", + -9.330772399902344 + ], + [ + "▁J", + -9.331167221069336 + ], + [ + "ify", + -9.334321975708008 + ], + [ + "▁Con", + -9.336448669433594 + ], + [ + "ade", + -9.340838432312012 + ], + [ + "▁soon", + -9.341312408447266 + ], + [ + "▁Un", + -9.341906547546387 + ], + [ + "▁cap", + -9.341944694519043 + ], + [ + "lin", + -9.343247413635254 + ], + [ + "father", + -9.34326457977295 + ], + [ + "tter", + -9.34344482421875 + ], + [ + "lan", + -9.343685150146484 + ], + [ + "wesome", + -9.343770980834961 + ], + [ + "uck", + -9.34377670288086 + ], + [ + "▁dog", + -9.344061851501465 + ], + [ + "by", + -9.345102310180664 + ], + [ + "ob", + -9.345699310302734 + ], + [ + "ب", + -9.346378326416016 + ], + [ + "▁ten", + -9.34676456451416 + ], + [ + "▁si", + -9.346918106079102 + ], + [ + "den", + -9.347368240356445 + ], + [ + "▁track", + -9.349933624267578 + ], + [ + "ze", + -9.35116195678711 + ], + [ + "▁chi", + -9.351595878601074 + ], + [ + "appear", + -9.352456092834473 + ], + [ + "chance", + -9.35256290435791 + ], + [ + "llenge", + -9.35284423828125 + ], + [ + "▁shot", + -9.355164527893066 + ], + [ + "▁sorry", + -9.357897758483887 + ], + [ + "tories", + -9.358085632324219 + ], + [ + "▁nega", + -9.360742568969727 + ], + [ + "▁Te", + -9.3621244430542 + ], + [ + "▁para", + -9.362360000610352 + ], + [ + "▁god", + -9.363547325134277 + ], + [ + "▁view", + -9.364309310913086 + ], + [ + "rather", + -9.365878105163574 + ], + [ + "▁Mr", + -9.366072654724121 + ], + [ + "ו", + -9.367229461669922 + ], + [ + "▁drink", + -9.368313789367676 + ], + [ + "▁hour", + -9.369148254394531 + ], + [ + "ugh", + -9.370623588562012 + ], + [ + "▁These", + -9.370634078979492 + ], + [ + "lic", + -9.371176719665527 + ], + [ + "▁hot", + -9.371552467346191 + ], + [ + "late", + -9.373501777648926 + ], + [ + "▁incre", + -9.37368392944336 + ], + [ + "▁serv", + -9.374038696289062 + ], + [ + "please", + -9.376060485839844 + ], + [ + "terest", + -9.379517555236816 + ], + [ + "▁field", + -9.380144119262695 + ], + [ + "ently", + -9.380892753601074 + ], + [ + "▁dark", + -9.381349563598633 + ], + [ + "▁relat", + -9.381586074829102 + ], + [ + "stance", + -9.386783599853516 + ], + [ + "ai", + -9.388028144836426 + ], + [ + "ya", + -9.388672828674316 + ], + [ + "V", + -9.388874053955078 + ], + [ + "▁Ka", + -9.389886856079102 + ], + [ + "custom", + -9.39107608795166 + ], + [ + "church", + -9.392374038696289 + ], + [ + "▁expla", + -9.392638206481934 + ], + [ + "▁Ju", + -9.392658233642578 + ], + [ + "▁seem", + -9.392862319946289 + ], + [ + "ousand", + -9.393081665039062 + ], + [ + "ency", + -9.393359184265137 + ], + [ + "finish", + -9.393660545349121 + ], + [ + "▁pi", + -9.394664764404297 + ], + [ + "▁ways", + -9.395894050598145 + ], + [ + "ح", + -9.39594554901123 + ], + [ + "▁exist", + -9.39599323272705 + ], + [ + "nc", + -9.397337913513184 + ], + [ + "▁dead", + -9.39737606048584 + ], + [ + "▁Dr", + -9.398378372192383 + ], + [ + "client", + -9.398672103881836 + ], + [ + "▁Z", + -9.399008750915527 + ], + [ + "sider", + -9.400711059570312 + ], + [ + "▁trust", + -9.401521682739258 + ], + [ + "gest", + -9.401556015014648 + ], + [ + "gan", + -9.40227222442627 + ], + [ + "han", + -9.404308319091797 + ], + [ + "W", + -9.4043550491333 + ], + [ + "▁Are", + -9.406368255615234 + ], + [ + "ace", + -9.408614158630371 + ], + [ + "graph", + -9.410457611083984 + ], + [ + "travel", + -9.410662651062012 + ], + [ + "invest", + -9.4140043258667 + ], + [ + "▁news", + -9.41524600982666 + ], + [ + "ki", + -9.4153413772583 + ], + [ + "▁fear", + -9.415443420410156 + ], + [ + "nk", + -9.415628433227539 + ], + [ + "imilar", + -9.415858268737793 + ], + [ + "▁emo", + -9.41601848602295 + ], + [ + "suppos", + -9.417911529541016 + ], + [ + "cla", + -9.418488502502441 + ], + [ + "iz", + -9.419187545776367 + ], + [ + "▁flow", + -9.419829368591309 + ], + [ + "bu", + -9.420503616333008 + ], + [ + "load", + -9.420577049255371 + ], + [ + "▁based", + -9.420777320861816 + ], + [ + "▁weeks", + -9.421319007873535 + ], + [ + "lf", + -9.423846244812012 + ], + [ + "▁drive", + -9.425610542297363 + ], + [ + "roject", + -9.426355361938477 + ], + [ + "▁brand", + -9.427434921264648 + ], + [ + "dom", + -9.427870750427246 + ], + [ + "magine", + -9.428024291992188 + ], + [ + "▁link", + -9.429953575134277 + ], + [ + "sation", + -9.430054664611816 + ], + [ + "cast", + -9.430257797241211 + ], + [ + "▁worth", + -9.430852890014648 + ], + [ + "▁sleep", + -9.433205604553223 + ], + [ + "gar", + -9.434643745422363 + ], + [ + "simple", + -9.436808586120605 + ], + [ + "report", + -9.437264442443848 + ], + [ + "▁force", + -9.438314437866211 + ], + [ + "pp", + -9.438736915588379 + ], + [ + "▁mom", + -9.438970565795898 + ], + [ + "ccount", + -9.439584732055664 + ], + [ + "▁lives", + -9.44177532196045 + ], + [ + "ley", + -9.442485809326172 + ], + [ + "▁known", + -9.443177223205566 + ], + [ + "ountry", + -9.443464279174805 + ], + [ + "bsite", + -9.44507122039795 + ], + [ + "▁major", + -9.4465913772583 + ], + [ + "в", + -9.447555541992188 + ], + [ + "▁roll", + -9.447762489318848 + ], + [ + "▁jump", + -9.447834014892578 + ], + [ + "mber", + -9.45079231262207 + ], + [ + "▁red", + -9.453396797180176 + ], + [ + "▁city", + -9.456280708312988 + ], + [ + "set", + -9.45693588256836 + ], + [ + "rk", + -9.457257270812988 + ], + [ + "mental", + -9.457337379455566 + ], + [ + "av", + -9.460675239562988 + ], + [ + "▁death", + -9.460692405700684 + ], + [ + "ship", + -9.461281776428223 + ], + [ + "▁hands", + -9.461979866027832 + ], + [ + "ug", + -9.462041854858398 + ], + [ + "middle", + -9.462311744689941 + ], + [ + "▁sun", + -9.464760780334473 + ], + [ + "▁base", + -9.465078353881836 + ], + [ + "inside", + -9.465493202209473 + ], + [ + "▁road", + -9.466241836547852 + ], + [ + "velop", + -9.466629981994629 + ], + [ + "ight", + -9.46700382232666 + ], + [ + "▁ا", + -9.467375755310059 + ], + [ + "lly", + -9.467445373535156 + ], + [ + "í", + -9.4685640335083 + ], + [ + "▁rock", + -9.468690872192383 + ], + [ + "▁compa", + -9.469518661499023 + ], + [ + "▁Vi", + -9.469911575317383 + ], + [ + "▁needs", + -9.47014045715332 + ], + [ + "wo", + -9.471970558166504 + ], + [ + "▁por", + -9.473296165466309 + ], + [ + "▁dream", + -9.473833084106445 + ], + [ + "ack", + -9.473909378051758 + ], + [ + "▁works", + -9.473946571350098 + ], + [ + "▁With", + -9.475066184997559 + ], + [ + "▁Where", + -9.47530746459961 + ], + [ + "▁Jo", + -9.476038932800293 + ], + [ + "▁begin", + -9.476088523864746 + ], + [ + "ential", + -9.477075576782227 + ], + [ + "▁rule", + -9.477331161499023 + ], + [ + "▁compe", + -9.479007720947266 + ], + [ + "▁Look", + -9.479084968566895 + ], + [ + "▁Br", + -9.479087829589844 + ], + [ + "▁cost", + -9.480192184448242 + ], + [ + "▁board", + -9.480210304260254 + ], + [ + "▁goal", + -9.480402946472168 + ], + [ + "▁Ta", + -9.48093318939209 + ], + [ + "ker", + -9.48359203338623 + ], + [ + "ssage", + -9.484257698059082 + ], + [ + "tunity", + -9.484946250915527 + ], + [ + "▁taken", + -9.487295150756836 + ], + [ + "▁box", + -9.48802661895752 + ], + [ + "ative", + -9.48840618133545 + ], + [ + "▁match", + -9.489437103271484 + ], + [ + "if", + -9.490029335021973 + ], + [ + "lcome", + -9.490504264831543 + ], + [ + "▁sin", + -9.4909029006958 + ], + [ + "amp", + -9.491107940673828 + ], + [ + "▁State", + -9.491924285888672 + ], + [ + "▁extra", + -9.492724418640137 + ], + [ + "▁draw", + -9.493718147277832 + ], + [ + "nch", + -9.494184494018555 + ], + [ + "fu", + -9.498034477233887 + ], + [ + "scribe", + -9.499979972839355 + ], + [ + "▁proce", + -9.500555992126465 + ], + [ + "▁clean", + -9.500929832458496 + ], + [ + "sident", + -9.501543045043945 + ], + [ + "▁lose", + -9.502408981323242 + ], + [ + "▁card", + -9.503335952758789 + ], + [ + "tially", + -9.504414558410645 + ], + [ + "ق", + -9.50486946105957 + ], + [ + "breath", + -9.505594253540039 + ], + [ + "▁Ne", + -9.506277084350586 + ], + [ + "▁Har", + -9.506720542907715 + ], + [ + "▁tried", + -9.507039070129395 + ], + [ + "wy", + -9.508370399475098 + ], + [ + "▁decid", + -9.508963584899902 + ], + [ + "▁color", + -9.509623527526855 + ], + [ + "▁upon", + -9.510516166687012 + ], + [ + "▁mar", + -9.512139320373535 + ], + [ + "wood", + -9.512967109680176 + ], + [ + "▁bill", + -9.513838768005371 + ], + [ + "▁vers", + -9.513875007629395 + ], + [ + "bl", + -9.513968467712402 + ], + [ + "▁table", + -9.514453887939453 + ], + [ + "▁dad", + -9.514573097229004 + ], + [ + "ality", + -9.515373229980469 + ], + [ + "du", + -9.515666961669922 + ], + [ + "▁price", + -9.516672134399414 + ], + [ + "qua", + -9.51689338684082 + ], + [ + "▁qual", + -9.517621994018555 + ], + [ + "series", + -9.51822566986084 + ], + [ + "▁pain", + -9.51880931854248 + ], + [ + "impact", + -9.520054817199707 + ], + [ + "▁opera", + -9.52074909210205 + ], + [ + "fer", + -9.52340030670166 + ], + [ + "▁broth", + -9.525179862976074 + ], + [ + "▁sitt", + -9.525618553161621 + ], + [ + "▁nu", + -9.52674674987793 + ], + [ + "▁enter", + -9.526768684387207 + ], + [ + "▁stick", + -9.527190208435059 + ], + [ + "▁act", + -9.52748966217041 + ], + [ + "bor", + -9.528085708618164 + ], + [ + "▁conf", + -9.529946327209473 + ], + [ + "▁Car", + -9.530797004699707 + ], + [ + "ium", + -9.530998229980469 + ], + [ + "dic", + -9.531200408935547 + ], + [ + "anyone", + -9.531356811523438 + ], + [ + "▁Bu", + -9.532878875732422 + ], + [ + "▁terri", + -9.53305435180664 + ], + [ + "▁treat", + -9.533828735351562 + ], + [ + "common", + -9.534955978393555 + ], + [ + "▁Did", + -9.535140991210938 + ], + [ + "ercent", + -9.535236358642578 + ], + [ + "▁wife", + -9.535518646240234 + ], + [ + "▁ب", + -9.53854751586914 + ], + [ + "▁spot", + -9.539159774780273 + ], + [ + "ute", + -9.540247917175293 + ], + [ + "▁blood", + -9.540822982788086 + ], + [ + "▁polit", + -9.542089462280273 + ], + [ + "▁natur", + -9.543805122375488 + ], + [ + "▁atten", + -9.544745445251465 + ], + [ + "raight", + -9.54609203338623 + ], + [ + "tu", + -9.546097755432129 + ], + [ + "т", + -9.54770278930664 + ], + [ + "▁bunch", + -9.547865867614746 + ], + [ + "nment", + -9.547986030578613 + ], + [ + ",\"", + -9.548636436462402 + ], + [ + "▁cultu", + -9.549612045288086 + ], + [ + "▁truth", + -9.550116539001465 + ], + [ + "inly", + -9.55191707611084 + ], + [ + "▁round", + -9.552741050720215 + ], + [ + "▁wall", + -9.554384231567383 + ], + [ + "▁beat", + -9.555932998657227 + ], + [ + "▁safe", + -9.556070327758789 + ], + [ + "▁store", + -9.557929039001465 + ], + [ + "onnect", + -9.55828857421875 + ], + [ + "▁email", + -9.558768272399902 + ], + [ + "ف", + -9.56057071685791 + ], + [ + "▁den", + -9.56119441986084 + ], + [ + "▁gr", + -9.561676979064941 + ], + [ + "ray", + -9.563387870788574 + ], + [ + "planet", + -9.565610885620117 + ], + [ + "ה", + -9.565816879272461 + ], + [ + "actice", + -9.56598949432373 + ], + [ + "▁stage", + -9.569098472595215 + ], + [ + "ledge", + -9.570317268371582 + ], + [ + "ran", + -9.571011543273926 + ], + [ + "▁late", + -9.573040962219238 + ], + [ + "ش", + -9.5733003616333 + ], + [ + "▁Du", + -9.574204444885254 + ], + [ + "▁cont", + -9.576539993286133 + ], + [ + "rage", + -9.577510833740234 + ], + [ + "gy", + -9.577733039855957 + ], + [ + "que", + -9.579856872558594 + ], + [ + "pping", + -9.580473899841309 + ], + [ + "ja", + -9.58115291595459 + ], + [ + "her", + -9.581399917602539 + ], + [ + "ham", + -9.581884384155273 + ], + [ + "▁wear", + -9.583069801330566 + ], + [ + "degree", + -9.583757400512695 + ], + [ + "▁role", + -9.584003448486328 + ], + [ + "▁gover", + -9.584391593933105 + ], + [ + "▁prote", + -9.586602210998535 + ], + [ + "▁pot", + -9.587722778320312 + ], + [ + "orrect", + -9.587873458862305 + ], + [ + "▁scene", + -9.588967323303223 + ], + [ + "▁Ki", + -9.589038848876953 + ], + [ + "return", + -9.590897560119629 + ], + [ + "▁Mu", + -9.591329574584961 + ], + [ + "▁plant", + -9.591662406921387 + ], + [ + "len", + -9.592211723327637 + ], + [ + "▁nine", + -9.592400550842285 + ], + [ + "▁shoot", + -9.59289836883545 + ], + [ + "▁study", + -9.593313217163086 + ], + [ + "nymore", + -9.593546867370605 + ], + [ + "▁band", + -9.59363079071045 + ], + [ + "cle", + -9.59450626373291 + ], + [ + "▁son", + -9.594846725463867 + ], + [ + "tually", + -9.595165252685547 + ], + [ + "oc", + -9.596518516540527 + ], + [ + "ink", + -9.597445487976074 + ], + [ + "▁feed", + -9.59815502166748 + ], + [ + "▁rate", + -9.59876823425293 + ], + [ + "▁faith", + -9.600125312805176 + ], + [ + "▁cell", + -9.60013198852539 + ], + [ + "ec", + -9.600286483764648 + ], + [ + "▁earth", + -9.60041332244873 + ], + [ + "volved", + -9.601034164428711 + ], + [ + "ory", + -9.601201057434082 + ], + [ + "▁pri", + -9.601423263549805 + ], + [ + "ral", + -9.602245330810547 + ], + [ + "▁wa", + -9.603468894958496 + ], + [ + "period", + -9.60350513458252 + ], + [ + "▁paper", + -9.604124069213867 + ], + [ + "▁hole", + -9.604816436767578 + ], + [ + "office", + -9.60515022277832 + ], + [ + "▁audi", + -9.605591773986816 + ], + [ + "▁His", + -9.605901718139648 + ], + [ + "urpose", + -9.606229782104492 + ], + [ + "▁age", + -9.60710620880127 + ], + [ + "▁earli", + -9.607839584350586 + ], + [ + "▁wish", + -9.608660697937012 + ], + [ + "▁limit", + -9.608819007873535 + ], + [ + "eg", + -9.61091136932373 + ], + [ + "▁Of", + -9.611836433410645 + ], + [ + "normal", + -9.611957550048828 + ], + [ + "▁conne", + -9.612325668334961 + ], + [ + "cious", + -9.612701416015625 + ], + [ + "dustry", + -9.613129615783691 + ], + [ + "▁page", + -9.613688468933105 + ], + [ + "panies", + -9.615263938903809 + ], + [ + "zz", + -9.615419387817383 + ], + [ + "▁dude", + -9.61575984954834 + ], + [ + "▁fair", + -9.616605758666992 + ], + [ + "center", + -9.618453979492188 + ], + [ + "back", + -9.618600845336914 + ], + [ + "▁non", + -9.61951732635498 + ], + [ + "fa", + -9.620133399963379 + ], + [ + "istic", + -9.620843887329102 + ], + [ + "engine", + -9.621585845947266 + ], + [ + "▁local", + -9.621711730957031 + ], + [ + "became", + -9.621801376342773 + ], + [ + "cannot", + -9.62216567993164 + ], + [ + "direct", + -9.624130249023438 + ], + [ + "over", + -9.624938011169434 + ], + [ + "▁tree", + -9.625069618225098 + ], + [ + "▁mix", + -9.625258445739746 + ], + [ + "van", + -9.625259399414062 + ], + [ + "▁yo", + -9.626197814941406 + ], + [ + "▁model", + -9.626543045043945 + ], + [ + "wise", + -9.627309799194336 + ], + [ + "ex", + -9.628133773803711 + ], + [ + "▁quart", + -9.629525184631348 + ], + [ + "room", + -9.630119323730469 + ], + [ + "access", + -9.630171775817871 + ], + [ + "▁aren", + -9.63112735748291 + ], + [ + "▁Your", + -9.631990432739258 + ], + [ + "ile", + -9.63232421875 + ], + [ + "м", + -9.633732795715332 + ], + [ + "▁cy", + -9.63390064239502 + ], + [ + "▁slow", + -9.63414192199707 + ], + [ + "ective", + -9.635130882263184 + ], + [ + "tance", + -9.63584041595459 + ], + [ + "▁ident", + -9.636628150939941 + ], + [ + "low", + -9.637642860412598 + ], + [ + "▁Engl", + -9.638734817504883 + ], + [ + "▁li", + -9.639229774475098 + ], + [ + "▁feet", + -9.63999080657959 + ], + [ + "the", + -9.642289161682129 + ], + [ + "▁encou", + -9.642325401306152 + ], + [ + "▁Sha", + -9.642536163330078 + ], + [ + "forget", + -9.642633438110352 + ], + [ + "my", + -9.644803047180176 + ], + [ + "ko", + -9.646860122680664 + ], + [ + "▁pray", + -9.647466659545898 + ], + [ + "▁burn", + -9.649913787841797 + ], + [ + "ina", + -9.650321960449219 + ], + [ + "▁Ga", + -9.651046752929688 + ], + [ + "couldn", + -9.652205467224121 + ], + [ + "ek", + -9.653759002685547 + ], + [ + "▁drop", + -9.65405559539795 + ], + [ + "▁Even", + -9.655414581298828 + ], + [ + "▁War", + -9.6554594039917 + ], + [ + "ef", + -9.65590763092041 + ], + [ + "▁path", + -9.656214714050293 + ], + [ + "▁hang", + -9.656817436218262 + ], + [ + "▁Cha", + -9.656923294067383 + ], + [ + "ay", + -9.65809440612793 + ], + [ + "▁quote", + -9.659514427185059 + ], + [ + "▁hell", + -9.659610748291016 + ], + [ + "stress", + -9.6611967086792 + ], + [ + "▁Th", + -9.66137409210205 + ], + [ + "crease", + -9.661754608154297 + ], + [ + "rn", + -9.66263484954834 + ], + [ + "rticle", + -9.663130760192871 + ], + [ + "bottom", + -9.663515090942383 + ], + [ + "effect", + -9.663614273071289 + ], + [ + "▁Hu", + -9.66423511505127 + ], + [ + "▁marri", + -9.664422988891602 + ], + [ + "▁ext", + -9.664436340332031 + ], + [ + "▁hate", + -9.664773941040039 + ], + [ + "ier", + -9.666439056396484 + ], + [ + "▁alone", + -9.666513442993164 + ], + [ + "▁mine", + -9.667889595031738 + ], + [ + "ested", + -9.66866683959961 + ], + [ + "▁Good", + -9.669190406799316 + ], + [ + "alize", + -9.669462203979492 + ], + [ + "▁Pe", + -9.670099258422852 + ], + [ + "▁US", + -9.671167373657227 + ], + [ + "▁catch", + -9.671882629394531 + ], + [ + "sudden", + -9.672216415405273 + ], + [ + "choice", + -9.672410011291504 + ], + [ + "tar", + -9.673001289367676 + ], + [ + "▁X", + -9.673397064208984 + ], + [ + "▁explo", + -9.673413276672363 + ], + [ + "▁built", + -9.675474166870117 + ], + [ + "attack", + -9.677810668945312 + ], + [ + "▁risk", + -9.677962303161621 + ], + [ + "▁cross", + -9.677979469299316 + ], + [ + "ک", + -9.67825698852539 + ], + [ + "ju", + -9.678667068481445 + ], + [ + "lution", + -9.680370330810547 + ], + [ + "▁eye", + -9.680672645568848 + ], + [ + "▁gun", + -9.680837631225586 + ], + [ + "ash", + -9.6810941696167 + ], + [ + "▁Si", + -9.681462287902832 + ], + [ + "everal", + -9.681543350219727 + ], + [ + "▁pen", + -9.683480262756348 + ], + [ + "▁wanna", + -9.683747291564941 + ], + [ + "lor", + -9.684992790222168 + ], + [ + "U", + -9.68535327911377 + ], + [ + "beginn", + -9.685479164123535 + ], + [ + "▁grand", + -9.68637466430664 + ], + [ + "▁phys", + -9.68679141998291 + ], + [ + "▁photo", + -9.686860084533691 + ], + [ + "mit", + -9.688801765441895 + ], + [ + "ox", + -9.689840316772461 + ], + [ + "▁thro", + -9.69028091430664 + ], + [ + "ather", + -9.691633224487305 + ], + [ + "▁educa", + -9.692315101623535 + ], + [ + "tally", + -9.692428588867188 + ], + [ + "▁sport", + -9.694089889526367 + ], + [ + "д", + -9.695698738098145 + ], + [ + "▁El", + -9.696224212646484 + ], + [ + "▁avail", + -9.696478843688965 + ], + [ + "Christ", + -9.698997497558594 + ], + [ + "▁king", + -9.700275421142578 + ], + [ + "ó", + -9.701098442077637 + ], + [ + "idual", + -9.701898574829102 + ], + [ + "▁refer", + -9.702362060546875 + ], + [ + "▁near", + -9.702634811401367 + ], + [ + "struct", + -9.703474044799805 + ], + [ + "▁block", + -9.703927040100098 + ], + [ + "eth", + -9.70551872253418 + ], + [ + "▁size", + -9.706526756286621 + ], + [ + "ado", + -9.707244873046875 + ], + [ + "▁tu", + -9.70842456817627 + ], + [ + "▁blue", + -9.709330558776855 + ], + [ + "ians", + -9.70950984954834 + ], + [ + "▁hair", + -9.712701797485352 + ], + [ + "▁Wow", + -9.71353530883789 + ], + [ + "wouldn", + -9.713675498962402 + ], + [ + "turned", + -9.713834762573242 + ], + [ + "ants", + -9.714234352111816 + ], + [ + "▁books", + -9.715872764587402 + ], + [ + "л", + -9.716533660888672 + ], + [ + "kan", + -9.717462539672852 + ], + [ + "▁envir", + -9.717784881591797 + ], + [ + "terial", + -9.718120574951172 + ], + [ + "▁loved", + -9.718969345092773 + ], + [ + "▁York", + -9.719944953918457 + ], + [ + "▁town", + -9.720175743103027 + ], + [ + "▁fix", + -9.72035026550293 + ], + [ + "▁skill", + -9.721322059631348 + ], + [ + "ctions", + -9.72209644317627 + ], + [ + "▁party", + -9.722453117370605 + ], + [ + "▁Faceb", + -9.723325729370117 + ], + [ + "▁indiv", + -9.723444938659668 + ], + [ + "▁tend", + -9.723479270935059 + ], + [ + "tunate", + -9.72500228881836 + ], + [ + "tual", + -9.727933883666992 + ], + [ + "▁visit", + -9.72862720489502 + ], + [ + "א", + -9.73007583618164 + ], + [ + "▁hurt", + -9.73007583618164 + ], + [ + "rently", + -9.73192310333252 + ], + [ + "▁plat", + -9.732077598571777 + ], + [ + "ana", + -9.732902526855469 + ], + [ + "▁Gu", + -9.733174324035645 + ], + [ + "▁Uh", + -9.733319282531738 + ], + [ + "ontent", + -9.733403205871582 + ], + [ + "▁speed", + -9.73346996307373 + ], + [ + "weight", + -9.733762741088867 + ], + [ + "erhaps", + -9.735224723815918 + ], + [ + "ollege", + -9.73665714263916 + ], + [ + "ung", + -9.736949920654297 + ], + [ + "ky", + -9.739045143127441 + ], + [ + "▁argu", + -9.740375518798828 + ], + [ + "bigger", + -9.742036819458008 + ], + [ + "▁sea", + -9.742565155029297 + ], + [ + "cor", + -9.74303913116455 + ], + [ + "gular", + -9.74307918548584 + ], + [ + "▁aware", + -9.743123054504395 + ], + [ + "ib", + -9.74317455291748 + ], + [ + "merica", + -9.743760108947754 + ], + [ + "online", + -9.747056007385254 + ], + [ + "▁profe", + -9.7500581741333 + ], + [ + "▁Wi", + -9.750246047973633 + ], + [ + "atch", + -9.750496864318848 + ], + [ + "▁race", + -9.750602722167969 + ], + [ + "▁host", + -9.751294136047363 + ], + [ + "lation", + -9.752449989318848 + ], + [ + "▁Ri", + -9.754128456115723 + ], + [ + "option", + -9.75422191619873 + ], + [ + "hannel", + -9.754450798034668 + ], + [ + "dible", + -9.755760192871094 + ], + [ + "verage", + -9.756254196166992 + ], + [ + "cra", + -9.756361961364746 + ], + [ + "▁oppos", + -9.75643253326416 + ], + [ + "▁wrote", + -9.756532669067383 + ], + [ + "round", + -9.756579399108887 + ], + [ + "sarily", + -9.757035255432129 + ], + [ + "▁save", + -9.757048606872559 + ], + [ + "▁text", + -9.757234573364258 + ], + [ + "▁furth", + -9.757269859313965 + ], + [ + "ease", + -9.757889747619629 + ], + [ + "ة", + -9.758129119873047 + ], + [ + "▁cat", + -9.758264541625977 + ], + [ + "onment", + -9.758491516113281 + ], + [ + "ified", + -9.759445190429688 + ], + [ + "▁Ex", + -9.759817123413086 + ], + [ + "▁ship", + -9.760437965393066 + ], + [ + "accept", + -9.76142692565918 + ], + [ + "▁mis", + -9.761713027954102 + ], + [ + "atient", + -9.762151718139648 + ], + [ + "▁ele", + -9.762874603271484 + ], + [ + "▁By", + -9.763901710510254 + ], + [ + "uff", + -9.764801979064941 + ], + [ + "nybody", + -9.765146255493164 + ], + [ + "▁activ", + -9.766569137573242 + ], + [ + "work", + -9.767642974853516 + ], + [ + "Tube", + -9.76783561706543 + ], + [ + "off", + -9.768280982971191 + ], + [ + "▁bra", + -9.770877838134766 + ], + [ + "▁gotta", + -9.773221969604492 + ], + [ + "▁essen", + -9.773236274719238 + ], + [ + "▁organ", + -9.773520469665527 + ], + [ + "author", + -9.77517318725586 + ], + [ + "essure", + -9.777778625488281 + ], + [ + "▁devel", + -9.777955055236816 + ], + [ + "▁gas", + -9.778109550476074 + ], + [ + "review", + -9.778618812561035 + ], + [ + "hood", + -9.779032707214355 + ], + [ + "ganiza", + -9.779346466064453 + ], + [ + "▁kick", + -9.780259132385254 + ], + [ + "nter", + -9.780973434448242 + ], + [ + "▁radio", + -9.781001091003418 + ], + [ + "▁heat", + -9.781617164611816 + ], + [ + "▁fund", + -9.781694412231445 + ], + [ + "▁discu", + -9.781829833984375 + ], + [ + "ج", + -9.781956672668457 + ], + [ + "nefit", + -9.783215522766113 + ], + [ + "▁fra", + -9.783334732055664 + ], + [ + "mic", + -9.783900260925293 + ], + [ + "cro", + -9.785663604736328 + ], + [ + "▁mass", + -9.788065910339355 + ], + [ + "▁image", + -9.78815746307373 + ], + [ + "za", + -9.788511276245117 + ], + [ + "simply", + -9.788826942443848 + ], + [ + "▁Which", + -9.789216995239258 + ], + [ + "▁struc", + -9.789774894714355 + ], + [ + "affect", + -9.790105819702148 + ], + [ + "vel", + -9.79013442993164 + ], + [ + "▁laugh", + -9.790526390075684 + ], + [ + "achine", + -9.790694236755371 + ], + [ + "▁tax", + -9.791321754455566 + ], + [ + "charge", + -9.793266296386719 + ], + [ + "▁slash", + -9.794378280639648 + ], + [ + "▁bank", + -9.79457950592041 + ], + [ + "ban", + -9.794783592224121 + ], + [ + "▁See", + -9.795280456542969 + ], + [ + "nic", + -9.795687675476074 + ], + [ + "ל", + -9.796050071716309 + ], + [ + "▁cold", + -9.796278953552246 + ], + [ + "cul", + -9.797404289245605 + ], + [ + "▁apart", + -9.797872543334961 + ], + [ + "▁mess", + -9.801264762878418 + ], + [ + "▁Paul", + -9.801401138305664 + ], + [ + "People", + -9.801894187927246 + ], + [ + "▁scien", + -9.805109024047852 + ], + [ + "ties", + -9.80539608001709 + ], + [ + "come", + -9.80688762664795 + ], + [ + "screen", + -9.806979179382324 + ], + [ + "у", + -9.807251930236816 + ], + [ + "▁mid", + -9.810304641723633 + ], + [ + "iginal", + -9.814031600952148 + ], + [ + "ath", + -9.814257621765137 + ], + [ + "kin", + -9.814712524414062 + ], + [ + "▁langu", + -9.815145492553711 + ], + [ + "▁Fa", + -9.816030502319336 + ], + [ + "▁Su", + -9.816306114196777 + ], + [ + "▁shall", + -9.8181791305542 + ], + [ + "▁comm", + -9.819071769714355 + ], + [ + "includ", + -9.819738388061523 + ], + [ + "longer", + -9.820608139038086 + ], + [ + "ontinu", + -9.820806503295898 + ], + [ + "▁Col", + -9.821048736572266 + ], + [ + "α", + -9.82120132446289 + ], + [ + "▁term", + -9.821939468383789 + ], + [ + "ez", + -9.82217025756836 + ], + [ + "▁hat", + -9.82271957397461 + ], + [ + "▁kept", + -9.822925567626953 + ], + [ + "detail", + -9.82410717010498 + ], + [ + "ommend", + -9.824295043945312 + ], + [ + "ficial", + -9.825002670288086 + ], + [ + "lon", + -9.825498580932617 + ], + [ + "ew", + -9.825504302978516 + ], + [ + "▁topic", + -9.825815200805664 + ], + [ + "▁shar", + -9.826598167419434 + ], + [ + "use", + -9.826918601989746 + ], + [ + "▁Ru", + -9.827040672302246 + ], + [ + "▁TV", + -9.829479217529297 + ], + [ + "▁persp", + -9.829874992370605 + ], + [ + "sha", + -9.831364631652832 + ], + [ + "▁zero", + -9.831670761108398 + ], + [ + "▁trip", + -9.832266807556152 + ], + [ + "à", + -9.832454681396484 + ], + [ + "ik", + -9.832542419433594 + ], + [ + "▁press", + -9.83255386352539 + ], + [ + "dig", + -9.832793235778809 + ], + [ + "▁green", + -9.833504676818848 + ], + [ + "choose", + -9.833712577819824 + ], + [ + "etwork", + -9.835243225097656 + ], + [ + "▁Star", + -9.835770606994629 + ], + [ + "believ", + -9.835813522338867 + ], + [ + "remain", + -9.83581829071045 + ], + [ + "urface", + -9.836164474487305 + ], + [ + "▁fill", + -9.837203979492188 + ], + [ + "▁verse", + -9.837578773498535 + ], + [ + "▁pop", + -9.837593078613281 + ], + [ + "act", + -9.837939262390137 + ], + [ + "▁condi", + -9.838601112365723 + ], + [ + "United", + -9.838619232177734 + ], + [ + "▁bri", + -9.839584350585938 + ], + [ + "▁fish", + -9.839962005615234 + ], + [ + "▁Com", + -9.841276168823242 + ], + [ + "▁San", + -9.842772483825684 + ], + [ + "ect", + -9.84339427947998 + ], + [ + "▁sett", + -9.843775749206543 + ], + [ + "أ", + -9.84487533569336 + ], + [ + "bly", + -9.845298767089844 + ], + [ + "lease", + -9.846741676330566 + ], + [ + "▁soul", + -9.848713874816895 + ], + [ + "▁addi", + -9.84918212890625 + ], + [ + "▁opin", + -9.851396560668945 + ], + [ + "fense", + -9.853272438049316 + ], + [ + "▁featu", + -9.853583335876465 + ], + [ + "iment", + -9.853619575500488 + ], + [ + "for", + -9.853842735290527 + ], + [ + "▁wide", + -9.854438781738281 + ], + [ + "tian", + -9.856575965881348 + ], + [ + "ith", + -9.856717109680176 + ], + [ + "▁above", + -9.85675048828125 + ], + [ + "ue", + -9.858016014099121 + ], + [ + "ा", + -9.858460426330566 + ], + [ + "remind", + -9.858540534973145 + ], + [ + "▁Get", + -9.859617233276367 + ], + [ + "camera", + -9.860365867614746 + ], + [ + "street", + -9.86094856262207 + ], + [ + "▁partn", + -9.861285209655762 + ], + [ + "▁appr", + -9.86130142211914 + ], + [ + "▁sex", + -9.861745834350586 + ], + [ + "alized", + -9.862457275390625 + ], + [ + "▁worry", + -9.86273193359375 + ], + [ + "ible", + -9.863109588623047 + ], + [ + "ership", + -9.863546371459961 + ], + [ + "▁wh", + -9.863916397094727 + ], + [ + "tantly", + -9.864358901977539 + ], + [ + "▁wild", + -9.864824295043945 + ], + [ + "iverse", + -9.865660667419434 + ], + [ + "place", + -9.8657865524292 + ], + [ + "▁lower", + -9.866541862487793 + ], + [ + "▁inten", + -9.86776351928711 + ], + [ + "▁deliv", + -9.86785888671875 + ], + [ + "▁moved", + -9.868019104003906 + ], + [ + "▁är", + -9.868229866027832 + ], + [ + "▁separ", + -9.86890697479248 + ], + [ + "bra", + -9.869545936584473 + ], + [ + "▁Bar", + -9.86987590789795 + ], + [ + "Q", + -9.87055778503418 + ], + [ + "▁blow", + -9.871233940124512 + ], + [ + "▁tough", + -9.872020721435547 + ], + [ + "▁date", + -9.872201919555664 + ], + [ + "stream", + -9.872364044189453 + ], + [ + "▁fly", + -9.873198509216309 + ], + [ + "ome", + -9.873790740966797 + ], + [ + "ately", + -9.875539779663086 + ], + [ + "comput", + -9.876399040222168 + ], + [ + "▁bed", + -9.877206802368164 + ], + [ + "career", + -9.878140449523926 + ], + [ + "▁stra", + -9.878450393676758 + ], + [ + "press", + -9.878798484802246 + ], + [ + "▁Pr", + -9.879388809204102 + ], + [ + "je", + -9.879964828491211 + ], + [ + "▁sat", + -9.881732940673828 + ], + [ + "mar", + -9.88222599029541 + ], + [ + "pound", + -9.88345718383789 + ], + [ + "ssful", + -9.88361930847168 + ], + [ + "K", + -9.885997772216797 + ], + [ + "umb", + -9.886067390441895 + ], + [ + "▁tip", + -9.886817932128906 + ], + [ + "▁ended", + -9.887141227722168 + ], + [ + "▁gift", + -9.887568473815918 + ], + [ + "fr", + -9.88810920715332 + ], + [ + "ively", + -9.888528823852539 + ], + [ + "▁immed", + -9.890312194824219 + ], + [ + "sight", + -9.891072273254395 + ], + [ + "tmas", + -9.891350746154785 + ], + [ + "izing", + -9.892232894897461 + ], + [ + "▁dan", + -9.892439842224121 + ], + [ + "▁wind", + -9.894176483154297 + ], + [ + "▁par", + -9.895620346069336 + ], + [ + "cept", + -9.902034759521484 + ], + [ + "stant", + -9.902400016784668 + ], + [ + "mmer", + -9.902860641479492 + ], + [ + "▁wow", + -9.904047966003418 + ], + [ + "launch", + -9.905645370483398 + ], + [ + "▁folks", + -9.90599250793457 + ], + [ + "▁wave", + -9.906126976013184 + ], + [ + "log", + -9.906353950500488 + ], + [ + "▁leg", + -9.906429290771484 + ], + [ + "▁carry", + -9.907379150390625 + ], + [ + "easier", + -9.909173011779785 + ], + [ + "evious", + -9.911510467529297 + ], + [ + "tage", + -9.912466049194336 + ], + [ + "rength", + -9.912664413452148 + ], + [ + "fl", + -9.91328239440918 + ], + [ + "credit", + -9.914097785949707 + ], + [ + "▁total", + -9.914595603942871 + ], + [ + "▁pack", + -9.915796279907227 + ], + [ + "summer", + -9.917508125305176 + ], + [ + "▁En", + -9.917713165283203 + ], + [ + "▁Come", + -9.918974876403809 + ], + [ + "lip", + -9.919098854064941 + ], + [ + "▁minus", + -9.919347763061523 + ], + [ + "▁sy", + -9.919524192810059 + ], + [ + "▁micro", + -9.919597625732422 + ], + [ + "eciate", + -9.920467376708984 + ], + [ + "▁After", + -9.922719955444336 + ], + [ + "▁anti", + -9.923659324645996 + ], + [ + "▁weren", + -9.92367172241211 + ], + [ + "lk", + -9.924665451049805 + ], + [ + "rsity", + -9.926215171813965 + ], + [ + "▁First", + -9.926276206970215 + ], + [ + "▁King", + -9.927388191223145 + ], + [ + "pper", + -9.927520751953125 + ], + [ + "▁mark", + -9.928973197937012 + ], + [ + "▁birth", + -9.931220054626465 + ], + [ + "р", + -9.931319236755371 + ], + [ + "▁Jack", + -9.932406425476074 + ], + [ + "▁club", + -9.933923721313477 + ], + [ + "switch", + -9.93446159362793 + ], + [ + "▁code", + -9.935473442077637 + ], + [ + "▁Ge", + -9.93659496307373 + ], + [ + "af", + -9.936867713928223 + ], + [ + "lay", + -9.937019348144531 + ], + [ + "plain", + -9.937148094177246 + ], + [ + "itch", + -9.937833786010742 + ], + [ + "▁bro", + -9.938887596130371 + ], + [ + "gotten", + -9.938961029052734 + ], + [ + "plete", + -9.939597129821777 + ], + [ + "itself", + -9.940816879272461 + ], + [ + "player", + -9.941915512084961 + ], + [ + "letter", + -9.94326114654541 + ], + [ + "▁shift", + -9.943703651428223 + ], + [ + "▁celeb", + -9.944153785705566 + ], + [ + "▁Have", + -9.944408416748047 + ], + [ + "▁David", + -9.944523811340332 + ], + [ + "tastic", + -9.945112228393555 + ], + [ + "ddi", + -9.945878982543945 + ], + [ + "doctor", + -9.945939064025879 + ], + [ + "cently", + -9.94595718383789 + ], + [ + "▁bless", + -9.94775104522705 + ], + [ + "usband", + -9.948332786560059 + ], + [ + "lement", + -9.94902515411377 + ], + [ + "▁rid", + -9.94959545135498 + ], + [ + "▁effec", + -9.949902534484863 + ], + [ + "▁compl", + -9.950129508972168 + ], + [ + "▁guest", + -9.950855255126953 + ], + [ + "▁Joe", + -9.952438354492188 + ], + [ + "▁Our", + -9.953227996826172 + ], + [ + "craft", + -9.954538345336914 + ], + [ + "down", + -9.954874992370605 + ], + [ + "▁cook", + -9.95514965057373 + ], + [ + "nature", + -9.956022262573242 + ], + [ + "regard", + -9.95604419708252 + ], + [ + "▁loca", + -9.956280708312988 + ], + [ + "omfort", + -9.956418991088867 + ], + [ + "▁score", + -9.956646919250488 + ], + [ + "ekend", + -9.95778751373291 + ], + [ + "▁paid", + -9.958367347717285 + ], + [ + "ix", + -9.958584785461426 + ], + [ + "▁Gra", + -9.958686828613281 + ], + [ + "▁horse", + -9.959284782409668 + ], + [ + "▁stat", + -9.959702491760254 + ], + [ + "ternal", + -9.960081100463867 + ], + [ + "fresh", + -9.960569381713867 + ], + [ + "▁trade", + -9.961538314819336 + ], + [ + "▁colle", + -9.961553573608398 + ], + [ + "higher", + -9.96181583404541 + ], + [ + "▁spiri", + -9.962003707885742 + ], + [ + "▁sad", + -9.962324142456055 + ], + [ + "▁medic", + -9.962325096130371 + ], + [ + "▁sales", + -9.962642669677734 + ], + [ + "▁till", + -9.963162422180176 + ], + [ + "daught", + -9.963242530822754 + ], + [ + "league", + -9.963614463806152 + ], + [ + "meters", + -9.963699340820312 + ], + [ + "zi", + -9.963703155517578 + ], + [ + "pu", + -9.965476989746094 + ], + [ + "uc", + -9.965785026550293 + ], + [ + "ان", + -9.965865135192871 + ], + [ + "▁Insta", + -9.966387748718262 + ], + [ + "air", + -9.967612266540527 + ], + [ + "▁smart", + -9.967663764953613 + ], + [ + "ame", + -9.968646049499512 + ], + [ + "▁floor", + -9.96866226196289 + ], + [ + "▁Twitt", + -9.970035552978516 + ], + [ + "▁unc", + -9.97048282623291 + ], + [ + "rated", + -9.970625877380371 + ], + [ + "▁driv", + -9.97124195098877 + ], + [ + "▁peace", + -9.972053527832031 + ], + [ + "ino", + -9.972740173339844 + ], + [ + "▁dia", + -9.9753999710083 + ], + [ + "▁Ch", + -9.976434707641602 + ], + [ + "tribut", + -9.976941108703613 + ], + [ + "▁Ni", + -9.977397918701172 + ], + [ + "▁Ke", + -9.979132652282715 + ], + [ + "▁tour", + -9.980440139770508 + ], + [ + "uch", + -9.984177589416504 + ], + [ + "cted", + -9.984688758850098 + ], + [ + "▁flu", + -9.984945297241211 + ], + [ + "Europe", + -9.984996795654297 + ], + [ + "erious", + -9.985994338989258 + ], + [ + "window", + -9.987778663635254 + ], + [ + "secret", + -9.988755226135254 + ], + [ + "althy", + -9.989420890808105 + ], + [ + "double", + -9.989727020263672 + ], + [ + "▁print", + -9.989923477172852 + ], + [ + "▁album", + -9.994110107421875 + ], + [ + "stood", + -9.99413013458252 + ], + [ + "▁spent", + -9.994664192199707 + ], + [ + "Anyway", + -9.994829177856445 + ], + [ + "rouble", + -9.995696067810059 + ], + [ + "▁born", + -9.995747566223145 + ], + [ + "▁Absol", + -9.995820045471191 + ], + [ + "unk", + -9.99606990814209 + ], + [ + "▁chapt", + -9.997347831726074 + ], + [ + "ku", + -9.998237609863281 + ], + [ + "▁min", + -9.998613357543945 + ], + [ + "rity", + -9.99901008605957 + ], + [ + "provid", + -10.00113296508789 + ], + [ + "▁measu", + -10.001266479492188 + ], + [ + "rant", + -10.002668380737305 + ], + [ + "versus", + -10.003643989562988 + ], + [ + "police", + -10.004751205444336 + ], + [ + "factor", + -10.005300521850586 + ], + [ + "▁vote", + -10.005748748779297 + ], + [ + "bin", + -10.006091117858887 + ], + [ + "how", + -10.006845474243164 + ], + [ + "lright", + -10.006864547729492 + ], + [ + "hor", + -10.009695053100586 + ], + [ + "ov", + -10.010833740234375 + ], + [ + "effort", + -10.011256217956543 + ], + [ + "▁Earth", + -10.011488914489746 + ], + [ + "▁older", + -10.011619567871094 + ], + [ + "▁så", + -10.013259887695312 + ], + [ + "▁South", + -10.013427734375 + ], + [ + "▁lie", + -10.013652801513672 + ], + [ + "roduct", + -10.014901161193848 + ], + [ + "خ", + -10.015073776245117 + ], + [ + "rovide", + -10.01561450958252 + ], + [ + "▁Tom", + -10.015673637390137 + ], + [ + "▁Pre", + -10.01659870147705 + ], + [ + "ddress", + -10.016826629638672 + ], + [ + "tune", + -10.016863822937012 + ], + [ + "object", + -10.016952514648438 + ], + [ + "▁notes", + -10.018004417419434 + ], + [ + "▁shut", + -10.01895523071289 + ], + [ + "spread", + -10.020230293273926 + ], + [ + "▁Mon", + -10.020455360412598 + ], + [ + "master", + -10.02111530303955 + ], + [ + "▁began", + -10.021138191223145 + ], + [ + "▁mae", + -10.021315574645996 + ], + [ + "▁court", + -10.021502494812012 + ], + [ + "target", + -10.022071838378906 + ], + [ + "morrow", + -10.02212142944336 + ], + [ + "▁fat", + -10.023024559020996 + ], + [ + "▁shape", + -10.024040222167969 + ], + [ + "▁lay", + -10.02407169342041 + ], + [ + "derful", + -10.024131774902344 + ], + [ + "▁tempe", + -10.024544715881348 + ], + [ + "▁plann", + -10.024694442749023 + ], + [ + "▁Lu", + -10.025320053100586 + ], + [ + "▁truly", + -10.025788307189941 + ], + [ + "▁poor", + -10.025917053222656 + ], + [ + "▁oil", + -10.027077674865723 + ], + [ + "terday", + -10.027290344238281 + ], + [ + "dding", + -10.028390884399414 + ], + [ + "roduce", + -10.02885627746582 + ], + [ + "notice", + -10.029275894165039 + ], + [ + "ump", + -10.030826568603516 + ], + [ + "▁style", + -10.031364440917969 + ], + [ + "▁Am", + -10.031371116638184 + ], + [ + "ancial", + -10.031991958618164 + ], + [ + "▁sweet", + -10.032469749450684 + ], + [ + "ש", + -10.032954216003418 + ], + [ + "▁stopp", + -10.034064292907715 + ], + [ + "spond", + -10.034988403320312 + ], + [ + "gon", + -10.035453796386719 + ], + [ + "mate", + -10.035539627075195 + ], + [ + "▁ran", + -10.036310195922852 + ], + [ + "▁Fin", + -10.036548614501953 + ], + [ + "cha", + -10.03824520111084 + ], + [ + "gue", + -10.04012393951416 + ], + [ + "▁pour", + -10.040270805358887 + ], + [ + "▁Austr", + -10.040841102600098 + ], + [ + "refore", + -10.041781425476074 + ], + [ + "▁mouth", + -10.04231071472168 + ], + [ + "▁disa", + -10.04267692565918 + ], + [ + "▁Pro", + -10.043288230895996 + ], + [ + "▁pan", + -10.0436429977417 + ], + [ + "▁fell", + -10.04405403137207 + ], + [ + "ique", + -10.044717788696289 + ], + [ + "lie", + -10.04494571685791 + ], + [ + "spirit", + -10.046067237854004 + ], + [ + "▁Those", + -10.046113014221191 + ], + [ + "ock", + -10.046306610107422 + ], + [ + "ughout", + -10.046684265136719 + ], + [ + "advice", + -10.048791885375977 + ], + [ + "bought", + -10.048879623413086 + ], + [ + "sister", + -10.050151824951172 + ], + [ + "yard", + -10.05057144165039 + ], + [ + "▁click", + -10.051811218261719 + ], + [ + "▁milli", + -10.052291870117188 + ], + [ + "▁bag", + -10.0529203414917 + ], + [ + "board", + -10.05322265625 + ], + [ + "▁glad", + -10.05324935913086 + ], + [ + "े", + -10.053581237792969 + ], + [ + "▁expen", + -10.055469512939453 + ], + [ + "▁typic", + -10.055550575256348 + ], + [ + "beyond", + -10.056478500366211 + ], + [ + "clude", + -10.0567045211792 + ], + [ + "▁tele", + -10.056709289550781 + ], + [ + "corner", + -10.056848526000977 + ], + [ + "▁stock", + -10.057112693786621 + ], + [ + "▁goals", + -10.0572509765625 + ], + [ + "?\"", + -10.057263374328613 + ], + [ + "ontact", + -10.058298110961914 + ], + [ + "▁onto", + -10.05924129486084 + ], + [ + "nobody", + -10.060246467590332 + ], + [ + "▁ultim", + -10.060295104980469 + ], + [ + "▁skin", + -10.061016082763672 + ], + [ + "▁unit", + -10.061964988708496 + ], + [ + "▁signi", + -10.062111854553223 + ], + [ + "ö", + -10.062374114990234 + ], + [ + "▁gw", + -10.062604904174805 + ], + [ + "rick", + -10.062727928161621 + ], + [ + "lam", + -10.06441879272461 + ], + [ + "nge", + -10.064605712890625 + ], + [ + "idence", + -10.064746856689453 + ], + [ + "▁site", + -10.065878868103027 + ], + [ + "operty", + -10.066079139709473 + ], + [ + "losing", + -10.066174507141113 + ], + [ + "finger", + -10.06749153137207 + ], + [ + "▁due", + -10.067901611328125 + ], + [ + "▁Yep", + -10.06817626953125 + ], + [ + "▁tradi", + -10.068504333496094 + ], + [ + "oblems", + -10.069104194641113 + ], + [ + "▁died", + -10.069435119628906 + ], + [ + "▁note", + -10.070813179016113 + ], + [ + "ç", + -10.072113037109375 + ], + [ + "Google", + -10.07288646697998 + ], + [ + "▁root", + -10.07401180267334 + ], + [ + "cycl", + -10.074360847473145 + ], + [ + "▁Again", + -10.075601577758789 + ], + [ + "▁moon", + -10.07696533203125 + ], + [ + "actual", + -10.07731819152832 + ], + [ + "▁joy", + -10.077448844909668 + ], + [ + "icated", + -10.078292846679688 + ], + [ + "▁spin", + -10.078655242919922 + ], + [ + "▁Also", + -10.078779220581055 + ], + [ + "ز", + -10.081624984741211 + ], + [ + "ponent", + -10.0818510055542 + ], + [ + "cience", + -10.082518577575684 + ], + [ + "az", + -10.08304214477539 + ], + [ + "ev", + -10.083252906799316 + ], + [ + "ternet", + -10.08480453491211 + ], + [ + "▁digit", + -10.0865478515625 + ], + [ + "meter", + -10.087221145629883 + ], + [ + "▁title", + -10.087566375732422 + ], + [ + "hold", + -10.087953567504883 + ], + [ + "▁perfo", + -10.088130950927734 + ], + [ + "▁COVID", + -10.088394165039062 + ], + [ + "iately", + -10.088464736938477 + ], + [ + "sponse", + -10.089836120605469 + ], + [ + "ר", + -10.093354225158691 + ], + [ + "bur", + -10.093548774719238 + ], + [ + "battle", + -10.094565391540527 + ], + [ + "proper", + -10.095444679260254 + ], + [ + "unique", + -10.096156120300293 + ], + [ + "rie", + -10.09699821472168 + ], + [ + "▁Va", + -10.097256660461426 + ], + [ + "▁park", + -10.0979585647583 + ], + [ + "▁evid", + -10.098162651062012 + ], + [ + "udents", + -10.098260879516602 + ], + [ + "caught", + -10.099212646484375 + ], + [ + "▁Exact", + -10.099221229553223 + ], + [ + "ney", + -10.09926986694336 + ], + [ + "▁Mae", + -10.099285125732422 + ], + [ + "lap", + -10.099358558654785 + ], + [ + "▁auto", + -10.099874496459961 + ], + [ + "▁Bible", + -10.100229263305664 + ], + [ + "▁Tra", + -10.100936889648438 + ], + [ + "_", + -10.101971626281738 + ], + [ + "▁Trump", + -10.103963851928711 + ], + [ + "gri", + -10.104196548461914 + ], + [ + "rmance", + -10.10422420501709 + ], + [ + "▁secur", + -10.104419708251953 + ], + [ + "ong", + -10.104751586914062 + ], + [ + "▁expre", + -10.104999542236328 + ], + [ + "▁joke", + -10.106304168701172 + ], + [ + "▁World", + -10.107054710388184 + ], + [ + "▁May", + -10.108388900756836 + ], + [ + "ciety", + -10.108838081359863 + ], + [ + "▁paint", + -10.10971736907959 + ], + [ + "ão", + -10.109980583190918 + ], + [ + "▁warm", + -10.110660552978516 + ], + [ + "ission", + -10.110753059387207 + ], + [ + "specif", + -10.111449241638184 + ], + [ + "tz", + -10.111836433410645 + ], + [ + "ich", + -10.112691879272461 + ], + [ + "▁Dan", + -10.114439010620117 + ], + [ + "▁stuck", + -10.114778518676758 + ], + [ + "growth", + -10.115043640136719 + ], + [ + "neighb", + -10.115317344665527 + ], + [ + "▁lived", + -10.11565113067627 + ], + [ + "clock", + -10.116480827331543 + ], + [ + "▁James", + -10.116738319396973 + ], + [ + "observ", + -10.117159843444824 + ], + [ + "cri", + -10.11731243133545 + ], + [ + "termin", + -10.118310928344727 + ], + [ + "nu", + -10.119078636169434 + ], + [ + "titude", + -10.11942195892334 + ], + [ + "lum", + -10.120079040527344 + ], + [ + "н", + -10.121515274047852 + ], + [ + "▁dr", + -10.121786117553711 + ], + [ + "▁appar", + -10.122537612915039 + ], + [ + "▁sick", + -10.122579574584961 + ], + [ + "vin", + -10.122815132141113 + ], + [ + "sional", + -10.12287425994873 + ], + [ + "matic", + -10.123496055603027 + ], + [ + "win", + -10.12365436553955 + ], + [ + "prise", + -10.12387752532959 + ], + [ + "▁range", + -10.124391555786133 + ], + [ + "▁popul", + -10.124507904052734 + ], + [ + "suffer", + -10.124533653259277 + ], + [ + "▁Pi", + -10.124735832214355 + ], + [ + "▁farm", + -10.125048637390137 + ], + [ + "ari", + -10.125086784362793 + ], + [ + "▁vari", + -10.125594139099121 + ], + [ + "employ", + -10.12563705444336 + ], + [ + "▁Mc", + -10.125757217407227 + ], + [ + "ica", + -10.125856399536133 + ], + [ + "consum", + -10.125958442687988 + ], + [ + "▁volt", + -10.12724494934082 + ], + [ + "fla", + -10.1274995803833 + ], + [ + "▁viol", + -10.129425048828125 + ], + [ + "▁Her", + -10.129803657531738 + ], + [ + "rge", + -10.130268096923828 + ], + [ + "rating", + -10.130616188049316 + ], + [ + "▁och", + -10.130819320678711 + ], + [ + "מ", + -10.13093090057373 + ], + [ + "▁glass", + -10.131475448608398 + ], + [ + "nie", + -10.132033348083496 + ], + [ + "▁Sam", + -10.13340950012207 + ], + [ + "▁heavy", + -10.134556770324707 + ], + [ + "nny", + -10.134685516357422 + ], + [ + "▁rain", + -10.134969711303711 + ], + [ + "▁Ti", + -10.135025024414062 + ], + [ + "ssible", + -10.13542366027832 + ], + [ + "▁jag", + -10.13565444946289 + ], + [ + "ably", + -10.136154174804688 + ], + [ + "easily", + -10.13758373260498 + ], + [ + "▁edge", + -10.138204574584961 + ], + [ + "German", + -10.139602661132812 + ], + [ + "▁bear", + -10.139653205871582 + ], + [ + "▁loss", + -10.14111328125 + ], + [ + "ceive", + -10.141904830932617 + ], + [ + "ficant", + -10.14231014251709 + ], + [ + "nesses", + -10.142776489257812 + ], + [ + "▁trick", + -10.14448356628418 + ], + [ + "▁drug", + -10.14449691772461 + ], + [ + "▁worse", + -10.145928382873535 + ], + [ + "▁magic", + -10.146300315856934 + ], + [ + "tant", + -10.146368980407715 + ], + [ + "active", + -10.146578788757324 + ], + [ + "play", + -10.147043228149414 + ], + [ + "ada", + -10.147171020507812 + ], + [ + "grade", + -10.147198677062988 + ], + [ + "ane", + -10.148364067077637 + ], + [ + "square", + -10.148372650146484 + ], + [ + "▁Bill", + -10.150099754333496 + ], + [ + "▁Sc", + -10.150138854980469 + ], + [ + "▁adult", + -10.150797843933105 + ], + [ + "▁camp", + -10.151275634765625 + ], + [ + "▁heal", + -10.15200138092041 + ], + [ + "▁ski", + -10.15229320526123 + ], + [ + "▁ride", + -10.152454376220703 + ], + [ + "▁worst", + -10.152654647827148 + ], + [ + "▁below", + -10.154220581054688 + ], + [ + "▁doubt", + -10.154784202575684 + ], + [ + "Qu", + -10.15700626373291 + ], + [ + "sses", + -10.15715217590332 + ], + [ + "▁scale", + -10.158280372619629 + ], + [ + "▁tight", + -10.158331871032715 + ], + [ + "▁exc", + -10.159679412841797 + ], + [ + "▁dive", + -10.159948348999023 + ], + [ + "▁massi", + -10.160194396972656 + ], + [ + "cover", + -10.162125587463379 + ], + [ + "▁notic", + -10.162805557250977 + ], + [ + "ב", + -10.163251876831055 + ], + [ + "ichael", + -10.164036750793457 + ], + [ + "cument", + -10.165120124816895 + ], + [ + "▁gain", + -10.165534973144531 + ], + [ + "acy", + -10.16579818725586 + ], + [ + "lished", + -10.167566299438477 + ], + [ + "▁Wal", + -10.167583465576172 + ], + [ + "▁prior", + -10.167638778686523 + ], + [ + "pen", + -10.168702125549316 + ], + [ + "centr", + -10.169639587402344 + ], + [ + "afraid", + -10.170306205749512 + ], + [ + "accord", + -10.170836448669434 + ], + [ + "trange", + -10.17434310913086 + ], + [ + "▁Ben", + -10.174382209777832 + ], + [ + "▁impor", + -10.177399635314941 + ], + [ + "▁frame", + -10.178369522094727 + ], + [ + "dri", + -10.179755210876465 + ], + [ + "▁Black", + -10.180194854736328 + ], + [ + "lig", + -10.1827392578125 + ], + [ + "desire", + -10.182759284973145 + ], + [ + "Friday", + -10.183302879333496 + ], + [ + "rature", + -10.1835298538208 + ], + [ + "▁dance", + -10.183812141418457 + ], + [ + "▁grew", + -10.184454917907715 + ], + [ + "ε", + -10.18454360961914 + ], + [ + "▁resis", + -10.184781074523926 + ], + [ + "handle", + -10.185986518859863 + ], + [ + "▁Cor", + -10.1871976852417 + ], + [ + "ת", + -10.187911987304688 + ], + [ + "opment", + -10.189626693725586 + ], + [ + "▁på", + -10.189916610717773 + ], + [ + "▁North", + -10.191165924072266 + ], + [ + "J", + -10.191312789916992 + ], + [ + "physic", + -10.191861152648926 + ], + [ + "munica", + -10.191866874694824 + ], + [ + "plane", + -10.19206428527832 + ], + [ + "strict", + -10.192811965942383 + ], + [ + "ttle", + -10.193184852600098 + ], + [ + "▁rich", + -10.193625450134277 + ], + [ + "rtable", + -10.194138526916504 + ], + [ + "ppoint", + -10.194510459899902 + ], + [ + "Sunday", + -10.19520092010498 + ], + [ + "▁pract", + -10.195930480957031 + ], + [ + "▁pin", + -10.195956230163574 + ], + [ + "year", + -10.196964263916016 + ], + [ + "▁wrap", + -10.19772720336914 + ], + [ + "▁habit", + -10.200349807739258 + ], + [ + "unless", + -10.202239036560059 + ], + [ + "▁sold", + -10.202786445617676 + ], + [ + "я", + -10.202831268310547 + ], + [ + "ο", + -10.20422649383545 + ], + [ + "▁lift", + -10.205026626586914 + ], + [ + "▁West", + -10.205592155456543 + ], + [ + "leader", + -10.205972671508789 + ], + [ + "▁blah", + -10.206141471862793 + ], + [ + "overed", + -10.20697021484375 + ], + [ + "▁tie", + -10.207064628601074 + ], + [ + "τ", + -10.207271575927734 + ], + [ + "▁broke", + -10.20827865600586 + ], + [ + "alia", + -10.208494186401367 + ], + [ + "rotect", + -10.208569526672363 + ], + [ + "क", + -10.209202766418457 + ], + [ + "ension", + -10.209510803222656 + ], + [ + "▁suck", + -10.21154499053955 + ], + [ + "ι", + -10.211731910705566 + ], + [ + "length", + -10.212618827819824 + ], + [ + "▁flat", + -10.213066101074219 + ], + [ + "talent", + -10.213895797729492 + ], + [ + "rivate", + -10.215043067932129 + ], + [ + "car", + -10.21538257598877 + ], + [ + "▁grab", + -10.21570110321045 + ], + [ + "▁integ", + -10.216116905212402 + ], + [ + "▁cloth", + -10.217020034790039 + ], + [ + "lit", + -10.217181205749512 + ], + [ + "▁broad", + -10.218297004699707 + ], + [ + "▁From", + -10.219063758850098 + ], + [ + "то", + -10.219321250915527 + ], + [ + "▁confe", + -10.219537734985352 + ], + [ + "region", + -10.220168113708496 + ], + [ + "romise", + -10.222389221191406 + ], + [ + "ہ", + -10.22304916381836 + ], + [ + "famous", + -10.223468780517578 + ], + [ + "sphere", + -10.223977088928223 + ], + [ + "▁dropp", + -10.224064826965332 + ], + [ + "▁seat", + -10.22438907623291 + ], + [ + "▁eu", + -10.224940299987793 + ], + [ + "▁apply", + -10.225857734680176 + ], + [ + "▁daily", + -10.22654914855957 + ], + [ + "decide", + -10.22960376739502 + ], + [ + "oedd", + -10.229938507080078 + ], + [ + "coffee", + -10.2305908203125 + ], + [ + "untain", + -10.232213020324707 + ], + [ + "▁Ko", + -10.232414245605469 + ], + [ + "▁Mark", + -10.232718467712402 + ], + [ + "havior", + -10.232805252075195 + ], + [ + "ando", + -10.232845306396484 + ], + [ + "▁relig", + -10.232978820800781 + ], + [ + "▁City", + -10.23343563079834 + ], + [ + "▁dress", + -10.233484268188477 + ], + [ + "▁among", + -10.235831260681152 + ], + [ + "▁lack", + -10.235958099365234 + ], + [ + "heaven", + -10.236164093017578 + ], + [ + "▁comfo", + -10.237929344177246 + ], + [ + "ologic", + -10.237979888916016 + ], + [ + "▁Mor", + -10.2385835647583 + ], + [ + "▁Mike", + -10.238929748535156 + ], + [ + "▁hynny", + -10.239561080932617 + ], + [ + "▁mad", + -10.239850044250488 + ], + [ + "▁owner", + -10.241365432739258 + ], + [ + "igh", + -10.242547988891602 + ], + [ + "▁motiv", + -10.243488311767578 + ], + [ + "ubject", + -10.243498802185059 + ], + [ + "▁figur", + -10.24385929107666 + ], + [ + "▁requi", + -10.24590015411377 + ], + [ + "▁norm", + -10.245977401733398 + ], + [ + "east", + -10.248088836669922 + ], + [ + "▁var", + -10.248852729797363 + ], + [ + "▁Great", + -10.25070571899414 + ], + [ + "of", + -10.251041412353516 + ], + [ + "▁East", + -10.251087188720703 + ], + [ + "▁guide", + -10.251407623291016 + ], + [ + "▁unfor", + -10.251895904541016 + ], + [ + "hospit", + -10.251975059509277 + ], + [ + "circle", + -10.25223445892334 + ], + [ + "mark", + -10.252384185791016 + ], + [ + "▁file", + -10.25325870513916 + ], + [ + "attern", + -10.253447532653809 + ], + [ + "▁Any", + -10.25418472290039 + ], + [ + "broken", + -10.254358291625977 + ], + [ + "rio", + -10.255105018615723 + ], + [ + "spring", + -10.255717277526855 + ], + [ + "arch", + -10.255938529968262 + ], + [ + "▁wheel", + -10.256159782409668 + ], + [ + "▁wire", + -10.257328987121582 + ], + [ + "▁named", + -10.257999420166016 + ], + [ + "▁pitch", + -10.258216857910156 + ], + [ + "är", + -10.258387565612793 + ], + [ + "▁Min", + -10.258625030517578 + ], + [ + "▁Cause", + -10.25888729095459 + ], + [ + "▁Sure", + -10.259203910827637 + ], + [ + "iption", + -10.259281158447266 + ], + [ + "remely", + -10.260855674743652 + ], + [ + "▁taste", + -10.26113510131836 + ], + [ + "▁spoke", + -10.261466979980469 + ], + [ + "▁prop", + -10.261953353881836 + ], + [ + "origin", + -10.262539863586426 + ], + [ + "lia", + -10.26313304901123 + ], + [ + "▁advan", + -10.263461112976074 + ], + [ + "▁fail", + -10.263720512390137 + ], + [ + "▁cup", + -10.264583587646484 + ], + [ + "leased", + -10.264592170715332 + ], + [ + "ulous", + -10.265219688415527 + ], + [ + "▁sand", + -10.26540756225586 + ], + [ + "formed", + -10.265471458435059 + ], + [ + "▁avoid", + -10.26623249053955 + ], + [ + "▁Will", + -10.266727447509766 + ], + [ + "signed", + -10.266907691955566 + ], + [ + "▁cash", + -10.267955780029297 + ], + [ + "stitu", + -10.268619537353516 + ], + [ + "right", + -10.269811630249023 + ], + [ + "▁held", + -10.27235221862793 + ], + [ + "▁leav", + -10.272955894470215 + ], + [ + "lesson", + -10.27365779876709 + ], + [ + "ommand", + -10.273760795593262 + ], + [ + "▁damn", + -10.274020195007324 + ], + [ + "▁Bro", + -10.274316787719727 + ], + [ + "gress", + -10.274532318115234 + ], + [ + "▁metal", + -10.274605751037598 + ], + [ + "bright", + -10.27560043334961 + ], + [ + "▁gold", + -10.27669620513916 + ], + [ + "▁hum", + -10.277426719665527 + ], + [ + "▁batte", + -10.277641296386719 + ], + [ + "script", + -10.278361320495605 + ], + [ + "▁tiny", + -10.279881477355957 + ], + [ + "▁Does", + -10.280011177062988 + ], + [ + "▁Apple", + -10.280108451843262 + ], + [ + "die", + -10.280220031738281 + ], + [ + "▁flip", + -10.281213760375977 + ], + [ + "app", + -10.28146743774414 + ], + [ + "profit", + -10.281745910644531 + ], + [ + "▁Chi", + -10.285748481750488 + ], + [ + "▁Peter", + -10.28698444366455 + ], + [ + "modern", + -10.287701606750488 + ], + [ + "▁egg", + -10.28785514831543 + ], + [ + "▁wine", + -10.288445472717285 + ], + [ + "▁initi", + -10.288498878479004 + ], + [ + "ndemic", + -10.289203643798828 + ], + [ + "र", + -10.289246559143066 + ], + [ + "▁none", + -10.289361000061035 + ], + [ + "econom", + -10.289778709411621 + ], + [ + "▁Take", + -10.290480613708496 + ], + [ + "ctive", + -10.291156768798828 + ], + [ + "nimals", + -10.291191101074219 + ], + [ + "ruggle", + -10.291882514953613 + ], + [ + "combin", + -10.292776107788086 + ], + [ + "nected", + -10.2930326461792 + ], + [ + "▁defen", + -10.293577194213867 + ], + [ + "▁knock", + -10.293991088867188 + ], + [ + "▁Holy", + -10.295110702514648 + ], + [ + "▁mode", + -10.295365333557129 + ], + [ + "▁promo", + -10.295403480529785 + ], + [ + "damage", + -10.295553207397461 + ], + [ + "▁execu", + -10.296518325805664 + ], + [ + "animal", + -10.297408103942871 + ], + [ + "ose", + -10.297662734985352 + ], + [ + "▁engag", + -10.298096656799316 + ], + [ + "▁boat", + -10.298218727111816 + ], + [ + "▁capit", + -10.298249244689941 + ], + [ + "▁solid", + -10.29934024810791 + ], + [ + "wr", + -10.300436019897461 + ], + [ + "gra", + -10.301482200622559 + ], + [ + "former", + -10.30164909362793 + ], + [ + "word", + -10.302098274230957 + ], + [ + "field", + -10.303442001342773 + ], + [ + "ص", + -10.303881645202637 + ], + [ + "▁manu", + -10.30394458770752 + ], + [ + "п", + -10.303946495056152 + ], + [ + "oncern", + -10.304476737976074 + ], + [ + "▁China", + -10.304566383361816 + ], + [ + "▁Once", + -10.305680274963379 + ], + [ + "elle", + -10.307352066040039 + ], + [ + "receiv", + -10.30764389038086 + ], + [ + "▁impro", + -10.307794570922852 + ], + [ + "к", + -10.309157371520996 + ], + [ + "fri", + -10.30974006652832 + ], + [ + "▁milit", + -10.310173034667969 + ], + [ + "▁sum", + -10.310725212097168 + ], + [ + "ط", + -10.31165885925293 + ], + [ + "ال", + -10.311917304992676 + ], + [ + "▁influ", + -10.311981201171875 + ], + [ + "نا", + -10.312116622924805 + ], + [ + "device", + -10.312660217285156 + ], + [ + "▁alive", + -10.313037872314453 + ], + [ + "venue", + -10.313447952270508 + ], + [ + "▁frequ", + -10.314738273620605 + ], + [ + "▁holi", + -10.314741134643555 + ], + [ + "uk", + -10.314929962158203 + ], + [ + "▁weak", + -10.315044403076172 + ], + [ + "▁pet", + -10.315558433532715 + ], + [ + "demand", + -10.316438674926758 + ], + [ + "▁busy", + -10.316550254821777 + ], + [ + "belief", + -10.31677532196045 + ], + [ + "murder", + -10.31706428527832 + ], + [ + "▁suit", + -10.31763744354248 + ], + [ + "well", + -10.317720413208008 + ], + [ + "▁Ken", + -10.317827224731445 + ], + [ + "▁worri", + -10.318060874938965 + ], + [ + "▁Very", + -10.318219184875488 + ], + [ + "give", + -10.318511009216309 + ], + [ + "▁sky", + -10.319262504577637 + ], + [ + "ppy", + -10.31951904296875 + ], + [ + "creas", + -10.319866180419922 + ], + [ + "law", + -10.319890022277832 + ], + [ + "troduc", + -10.322098731994629 + ], + [ + "▁hunt", + -10.322113990783691 + ], + [ + "▁descr", + -10.322768211364746 + ], + [ + "än", + -10.325050354003906 + ], + [ + "ette", + -10.32651138305664 + ], + [ + "reneur", + -10.326530456542969 + ], + [ + "▁Charl", + -10.327561378479004 + ], + [ + "opular", + -10.328202247619629 + ], + [ + "urance", + -10.3295316696167 + ], + [ + "miliar", + -10.329680442810059 + ], + [ + "repeat", + -10.329800605773926 + ], + [ + "Please", + -10.329964637756348 + ], + [ + "ese", + -10.330385208129883 + ], + [ + "▁seri", + -10.330881118774414 + ], + [ + "ection", + -10.330907821655273 + ], + [ + "ase", + -10.332832336425781 + ], + [ + "quest", + -10.332897186279297 + ], + [ + "▁added", + -10.333372116088867 + ], + [ + "▁quiet", + -10.334416389465332 + ], + [ + "alance", + -10.334856986999512 + ], + [ + "X", + -10.336480140686035 + ], + [ + "iff", + -10.337349891662598 + ], + [ + "ush", + -10.337737083435059 + ], + [ + "shirt", + -10.33786392211914 + ], + [ + "ession", + -10.340350151062012 + ], + [ + "▁rough", + -10.341065406799316 + ], + [ + "variou", + -10.341761589050293 + ], + [ + "method", + -10.342447280883789 + ], + [ + "▁Love", + -10.342713356018066 + ], + [ + "female", + -10.343212127685547 + ], + [ + "▁gosh", + -10.34376335144043 + ], + [ + "hedule", + -10.3438081741333 + ], + [ + "▁kilo", + -10.344195365905762 + ], + [ + "aurant", + -10.34428882598877 + ], + [ + "istent", + -10.344655990600586 + ], + [ + "ercise", + -10.3452730178833 + ], + [ + "า", + -10.3452787399292 + ], + [ + "stupid", + -10.34537410736084 + ], + [ + "▁shock", + -10.345527648925781 + ], + [ + "▁Big", + -10.345941543579102 + ], + [ + "decade", + -10.3471097946167 + ], + [ + "▁rise", + -10.347153663635254 + ], + [ + "▁pleas", + -10.348209381103516 + ], + [ + "б", + -10.348535537719727 + ], + [ + "▁core", + -10.348880767822266 + ], + [ + "▁motor", + -10.35004711151123 + ], + [ + "▁Satur", + -10.350918769836426 + ], + [ + "▁horri", + -10.35107707977295 + ], + [ + "▁func", + -10.352418899536133 + ], + [ + "▁squar", + -10.353026390075684 + ], + [ + "▁estab", + -10.353150367736816 + ], + [ + "▁dry", + -10.353479385375977 + ], + [ + "▁fasci", + -10.354162216186523 + ], + [ + "▁Super", + -10.356266021728516 + ], + [ + "▁angle", + -10.357160568237305 + ], + [ + "associ", + -10.35777759552002 + ], + [ + "sexual", + -10.35788631439209 + ], + [ + "▁Per", + -10.358599662780762 + ], + [ + "bri", + -10.358744621276855 + ], + [ + "▁raise", + -10.359526634216309 + ], + [ + "▁fruit", + -10.359601974487305 + ], + [ + "▁layer", + -10.360115051269531 + ], + [ + "▁impo", + -10.36052417755127 + ], + [ + "▁Cali", + -10.360697746276855 + ], + [ + "▁contr", + -10.361104965209961 + ], + [ + "▁earn", + -10.36202621459961 + ], + [ + "rategy", + -10.362475395202637 + ], + [ + "entrep", + -10.36298942565918 + ], + [ + "▁Day", + -10.364472389221191 + ], + [ + "▁smell", + -10.364921569824219 + ], + [ + "Israel", + -10.365646362304688 + ], + [ + "▁psych", + -10.36768627166748 + ], + [ + "▁regul", + -10.368138313293457 + ], + [ + "▁expos", + -10.368258476257324 + ], + [ + "book", + -10.36933422088623 + ], + [ + "▁poten", + -10.370589256286621 + ], + [ + "truggl", + -10.371207237243652 + ], + [ + "ч", + -10.37166976928711 + ], + [ + "▁wake", + -10.37192153930664 + ], + [ + "oducts", + -10.37213134765625 + ], + [ + "pho", + -10.374445915222168 + ], + [ + "▁famil", + -10.37486457824707 + ], + [ + "з", + -10.375049591064453 + ], + [ + "▁chem", + -10.375420570373535 + ], + [ + "Y", + -10.375645637512207 + ], + [ + "phone", + -10.375679969787598 + ], + [ + "mechan", + -10.376375198364258 + ], + [ + "chi", + -10.37645435333252 + ], + [ + "commun", + -10.376762390136719 + ], + [ + "ceived", + -10.377518653869629 + ], + [ + "ν", + -10.37844181060791 + ], + [ + "vestig", + -10.378660202026367 + ], + [ + "verse", + -10.378804206848145 + ], + [ + "aries", + -10.380276679992676 + ], + [ + "▁Miss", + -10.38051700592041 + ], + [ + "sponsi", + -10.38076400756836 + ], + [ + "eign", + -10.381153106689453 + ], + [ + "▁plug", + -10.381563186645508 + ], + [ + "▁plate", + -10.382637977600098 + ], + [ + "closer", + -10.383174896240234 + ], + [ + "omplex", + -10.383233070373535 + ], + [ + "▁honor", + -10.38416862487793 + ], + [ + "tific", + -10.384209632873535 + ], + [ + "▁cloud", + -10.384445190429688 + ], + [ + "Number", + -10.384780883789062 + ], + [ + "wel", + -10.384953498840332 + ], + [ + "ydd", + -10.386923789978027 + ], + [ + "ы", + -10.387308120727539 + ], + [ + "▁scar", + -10.387858390808105 + ], + [ + "▁odd", + -10.387920379638672 + ], + [ + "mula", + -10.389065742492676 + ], + [ + "power", + -10.389567375183105 + ], + [ + "▁carri", + -10.39055347442627 + ], + [ + "fault", + -10.3911714553833 + ], + [ + "▁Matt", + -10.391336441040039 + ], + [ + "▁lady", + -10.391399383544922 + ], + [ + "rupt", + -10.391622543334961 + ], + [ + "nating", + -10.391939163208008 + ], + [ + "▁shout", + -10.39336109161377 + ], + [ + "tudent", + -10.393732070922852 + ], + [ + "▁crowd", + -10.394848823547363 + ], + [ + "▁Most", + -10.394868850708008 + ], + [ + "Monday", + -10.394881248474121 + ], + [ + "Spirit", + -10.39541244506836 + ], + [ + "▁Hello", + -10.395712852478027 + ], + [ + "▁Two", + -10.396139144897461 + ], + [ + "▁vol", + -10.396318435668945 + ], + [ + "urious", + -10.39686107635498 + ], + [ + "ria", + -10.398846626281738 + ], + [ + "▁tea", + -10.399087905883789 + ], + [ + "prised", + -10.399940490722656 + ], + [ + "tively", + -10.401432037353516 + ], + [ + "random", + -10.402119636535645 + ], + [ + "▁Mary", + -10.402578353881836 + ], + [ + "chieve", + -10.402902603149414 + ], + [ + "fornia", + -10.403154373168945 + ], + [ + "▁cheap", + -10.403660774230957 + ], + [ + "▁trail", + -10.404324531555176 + ], + [ + "string", + -10.405130386352539 + ], + [ + "seemed", + -10.4053373336792 + ], + [ + "▁staff", + -10.405632972717285 + ], + [ + "motion", + -10.405681610107422 + ], + [ + "ما", + -10.405778884887695 + ], + [ + "ttempt", + -10.406728744506836 + ], + [ + "stern", + -10.408076286315918 + ], + [ + "▁giant", + -10.410100936889648 + ], + [ + "alist", + -10.410428047180176 + ], + [ + "lated", + -10.410513877868652 + ], + [ + "▁Cu", + -10.410820960998535 + ], + [ + "▁Bri", + -10.410855293273926 + ], + [ + "uppose", + -10.411734580993652 + ], + [ + "น", + -10.412208557128906 + ], + [ + "▁Japan", + -10.412674903869629 + ], + [ + "oncept", + -10.413311958312988 + ], + [ + "flight", + -10.413581848144531 + ], + [ + "cting", + -10.413604736328125 + ], + [ + "▁guard", + -10.41398811340332 + ], + [ + "▁emerg", + -10.414094924926758 + ], + [ + "house", + -10.415946006774902 + ], + [ + "▁Rob", + -10.416576385498047 + ], + [ + "▁spa", + -10.416584014892578 + ], + [ + "imagin", + -10.417609214782715 + ], + [ + "ontext", + -10.417627334594727 + ], + [ + "ji", + -10.418004989624023 + ], + [ + "▁vill", + -10.418045043945312 + ], + [ + "▁freak", + -10.418855667114258 + ], + [ + "▁winn", + -10.420063018798828 + ], + [ + "recent", + -10.420376777648926 + ], + [ + "lish", + -10.42226791381836 + ], + [ + "อ", + -10.423386573791504 + ], + [ + "improv", + -10.424089431762695 + ], + [ + "â", + -10.42435359954834 + ], + [ + "lac", + -10.424939155578613 + ], + [ + "spite", + -10.426549911499023 + ], + [ + "▁snow", + -10.427319526672363 + ], + [ + "▁Today", + -10.431661605834961 + ], + [ + "memory", + -10.431905746459961 + ], + [ + "▁loud", + -10.434752464294434 + ], + [ + "dinner", + -10.435269355773926 + ], + [ + "Listen", + -10.435942649841309 + ], + [ + "pha", + -10.436092376708984 + ], + [ + "▁vehic", + -10.436552047729492 + ], + [ + "neath", + -10.436726570129395 + ], + [ + "phi", + -10.437564849853516 + ], + [ + "expand", + -10.437565803527832 + ], + [ + "critic", + -10.437944412231445 + ], + [ + "nnounc", + -10.438097953796387 + ], + [ + "▁Ste", + -10.438520431518555 + ], + [ + "lastic", + -10.439834594726562 + ], + [ + "ware", + -10.441667556762695 + ], + [ + "▁Mil", + -10.441893577575684 + ], + [ + "▁assum", + -10.442011833190918 + ], + [ + "▁screw", + -10.442163467407227 + ], + [ + "taught", + -10.442934036254883 + ], + [ + "scope", + -10.44521713256836 + ], + [ + "tists", + -10.445639610290527 + ], + [ + "estate", + -10.446050643920898 + ], + [ + "▁alter", + -10.447250366210938 + ], + [ + "bol", + -10.447257995605469 + ], + [ + "▁proud", + -10.448360443115234 + ], + [ + "▁chain", + -10.448383331298828 + ], + [ + "prison", + -10.448729515075684 + ], + [ + "▁retir", + -10.449239730834961 + ], + [ + "▁crit", + -10.449372291564941 + ], + [ + "except", + -10.449457168579102 + ], + [ + "▁fuel", + -10.44961166381836 + ], + [ + "▁era", + -10.449820518493652 + ], + [ + "raised", + -10.450366020202637 + ], + [ + "▁unive", + -10.45085620880127 + ], + [ + "lliant", + -10.452255249023438 + ], + [ + "وا", + -10.4528169631958 + ], + [ + "ndo", + -10.453935623168945 + ], + [ + "נ", + -10.454459190368652 + ], + [ + "▁tear", + -10.454872131347656 + ], + [ + "tuniti", + -10.455232620239258 + ], + [ + "ronic", + -10.455567359924316 + ], + [ + "supply", + -10.455918312072754 + ], + [ + "illa", + -10.456318855285645 + ], + [ + "▁adver", + -10.456907272338867 + ], + [ + "ymlaen", + -10.457372665405273 + ], + [ + "head", + -10.457986831665039 + ], + [ + "tretch", + -10.458090782165527 + ], + [ + "safety", + -10.45943832397461 + ], + [ + "▁Give", + -10.460541725158691 + ], + [ + "dibly", + -10.46263313293457 + ], + [ + "▁copy", + -10.46278190612793 + ], + [ + "fourth", + -10.463149070739746 + ], + [ + "ensive", + -10.463319778442383 + ], + [ + "ognize", + -10.463507652282715 + ], + [ + "mpaign", + -10.463955879211426 + ], + [ + "▁judge", + -10.464116096496582 + ], + [ + "▁Make", + -10.464287757873535 + ], + [ + "like", + -10.465320587158203 + ], + [ + "▁trial", + -10.465615272521973 + ], + [ + "▁firm", + -10.465628623962402 + ], + [ + "▁för", + -10.465923309326172 + ], + [ + "▁math", + -10.466601371765137 + ], + [ + "▁medit", + -10.468025207519531 + ], + [ + "revent", + -10.46819019317627 + ], + [ + "▁tank", + -10.469042778015137 + ], + [ + "hicken", + -10.46916675567627 + ], + [ + "artist", + -10.469673156738281 + ], + [ + "tellig", + -10.471722602844238 + ], + [ + "▁bone", + -10.471781730651855 + ], + [ + "applic", + -10.47205924987793 + ], + [ + "commit", + -10.472993850708008 + ], + [ + "▁happi", + -10.473943710327148 + ], + [ + "bottle", + -10.474286079406738 + ], + [ + "▁House", + -10.4756441116333 + ], + [ + "▁virtu", + -10.475837707519531 + ], + [ + "▁seek", + -10.476916313171387 + ], + [ + "▁zone", + -10.47781753540039 + ], + [ + "mitted", + -10.478500366210938 + ], + [ + "▁Steve", + -10.479057312011719 + ], + [ + "ör", + -10.479596138000488 + ], + [ + "larger", + -10.480365753173828 + ], + [ + "▁mynd", + -10.480977058410645 + ], + [ + "ے", + -10.482012748718262 + ], + [ + "▁evil", + -10.483011245727539 + ], + [ + "▁Ze", + -10.483084678649902 + ], + [ + "istake", + -10.48376750946045 + ], + [ + "ê", + -10.483776092529297 + ], + [ + "vy", + -10.483809471130371 + ], + [ + "—", + -10.484145164489746 + ], + [ + "▁While", + -10.486205101013184 + ], + [ + "stall", + -10.486210823059082 + ], + [ + "reveal", + -10.486494064331055 + ], + [ + "timate", + -10.486796379089355 + ], + [ + "▁grace", + -10.487192153930664 + ], + [ + "▁crack", + -10.487761497497559 + ], + [ + "▁Tu", + -10.488630294799805 + ], + [ + "▁dda", + -10.488792419433594 + ], + [ + "weapon", + -10.488797187805176 + ], + [ + "▁equa", + -10.489641189575195 + ], + [ + "▁male", + -10.49160385131836 + ], + [ + "▁agent", + -10.492493629455566 + ], + [ + "prayer", + -10.493410110473633 + ], + [ + "quence", + -10.495377540588379 + ], + [ + "▁tired", + -10.496092796325684 + ], + [ + "▁relax", + -10.496841430664062 + ], + [ + "ponsor", + -10.496971130371094 + ], + [ + "George", + -10.4976224899292 + ], + [ + "covery", + -10.49836540222168 + ], + [ + "▁Tell", + -10.498878479003906 + ], + [ + "▁Men", + -10.498988151550293 + ], + [ + "burg", + -10.499070167541504 + ], + [ + "▁smile", + -10.499298095703125 + ], + [ + "▁Hall", + -10.499497413635254 + ], + [ + "▁drama", + -10.501380920410156 + ], + [ + "▁twice", + -10.501627922058105 + ], + [ + "▁basis", + -10.502786636352539 + ], + [ + "لا", + -10.502843856811523 + ], + [ + "phe", + -10.503973007202148 + ], + [ + "▁shoe", + -10.504322052001953 + ], + [ + "▁trigg", + -10.504695892333984 + ], + [ + "studio", + -10.504886627197266 + ], + [ + "▁في", + -10.505846977233887 + ], + [ + "update", + -10.50634479522705 + ], + [ + "▁Time", + -10.506872177124023 + ], + [ + "cc", + -10.507535934448242 + ], + [ + "▁pair", + -10.507863998413086 + ], + [ + "ifica", + -10.508064270019531 + ], + [ + "conven", + -10.508258819580078 + ], + [ + "▁legal", + -10.508530616760254 + ], + [ + "instru", + -10.50997257232666 + ], + [ + "biliti", + -10.510305404663086 + ], + [ + "oli", + -10.510422706604004 + ], + [ + "ticket", + -10.511330604553223 + ], + [ + "▁draft", + -10.511741638183594 + ], + [ + "Really", + -10.513991355895996 + ], + [ + "▁crime", + -10.514695167541504 + ], + [ + "!\"", + -10.515329360961914 + ], + [ + "▁calm", + -10.516790390014648 + ], + [ + "zo", + -10.517613410949707 + ], + [ + "▁react", + -10.517938613891602 + ], + [ + "▁edit", + -10.518258094787598 + ], + [ + "▁audio", + -10.520017623901367 + ], + [ + "▁legi", + -10.520306587219238 + ], + [ + "▁Gar", + -10.520743370056152 + ], + [ + "adjust", + -10.52094841003418 + ], + [ + "▁truck", + -10.521756172180176 + ], + [ + "French", + -10.523289680480957 + ], + [ + "ville", + -10.523869514465332 + ], + [ + "▁orbit", + -10.524456977844238 + ], + [ + "Amazon", + -10.524479866027832 + ], + [ + "▁Han", + -10.525338172912598 + ], + [ + "visual", + -10.526057243347168 + ], + [ + "▁Alex", + -10.527277946472168 + ], + [ + "flect", + -10.527393341064453 + ], + [ + "ddy", + -10.527634620666504 + ], + [ + "▁UK", + -10.527807235717773 + ], + [ + "▁task", + -10.528364181518555 + ], + [ + "afford", + -10.528529167175293 + ], + [ + "▁gear", + -10.529349327087402 + ], + [ + "slowly", + -10.529826164245605 + ], + [ + "basket", + -10.53076171875 + ], + [ + "mwneud", + -10.53091049194336 + ], + [ + "▁iron", + -10.53134822845459 + ], + [ + "è", + -10.531427383422852 + ], + [ + "▁thick", + -10.531728744506836 + ], + [ + "teen", + -10.531805992126465 + ], + [ + "▁syn", + -10.532490730285645 + ], + [ + "▁pump", + -10.53270435333252 + ], + [ + "▁Sorry", + -10.533560752868652 + ], + [ + "▁purch", + -10.5335693359375 + ], + [ + "bby", + -10.533907890319824 + ], + [ + "▁lucky", + -10.534551620483398 + ], + [ + "▁Rich", + -10.535158157348633 + ], + [ + "▁net", + -10.535989761352539 + ], + [ + "▁Next", + -10.536017417907715 + ], + [ + "London", + -10.53614616394043 + ], + [ + "carbon", + -10.536320686340332 + ], + [ + "▁meat", + -10.536458969116211 + ], + [ + "point", + -10.536999702453613 + ], + [ + "▁sir", + -10.537755012512207 + ], + [ + "bodies", + -10.539311408996582 + ], + [ + "muscle", + -10.539559364318848 + ], + [ + "etic", + -10.540706634521484 + ], + [ + "▁mask", + -10.542184829711914 + ], + [ + "lusion", + -10.542276382446289 + ], + [ + "▁caus", + -10.542645454406738 + ], + [ + "▁equip", + -10.542759895324707 + ], + [ + "eating", + -10.545016288757324 + ], + [ + "▁Bob", + -10.545037269592285 + ], + [ + "▁effic", + -10.54571533203125 + ], + [ + "ار", + -10.546808242797852 + ], + [ + "▁chair", + -10.549225807189941 + ], + [ + "▁bath", + -10.550677299499512 + ], + [ + "minist", + -10.550804138183594 + ], + [ + "histor", + -10.551228523254395 + ], + [ + "▁frust", + -10.551602363586426 + ], + [ + "global", + -10.552578926086426 + ], + [ + "signal", + -10.552810668945312 + ], + [ + "▁judg", + -10.553674697875977 + ], + [ + ":", + -10.555337905883789 + ], + [ + "orship", + -10.558013916015625 + ], + [ + "▁Would", + -10.558074951171875 + ], + [ + "ם", + -10.558472633361816 + ], + [ + "▁Ah", + -10.558693885803223 + ], + [ + "▁blog", + -10.558697700500488 + ], + [ + "▁river", + -10.56008243560791 + ], + [ + "▁prepa", + -10.560263633728027 + ], + [ + "ü", + -10.560462951660156 + ], + [ + "entral", + -10.560786247253418 + ], + [ + "्", + -10.562368392944336 + ], + [ + "filled", + -10.562837600708008 + ], + [ + "native", + -10.563020706176758 + ], + [ + "depart", + -10.564236640930176 + ], + [ + "hoping", + -10.564261436462402 + ], + [ + "ritish", + -10.567056655883789 + ], + [ + "▁India", + -10.567224502563477 + ], + [ + "▁ocean", + -10.568563461303711 + ], + [ + "▁luck", + -10.569059371948242 + ], + [ + "▁sugar", + -10.569547653198242 + ], + [ + "gather", + -10.569740295410156 + ], + [ + "▁quant", + -10.570094108581543 + ], + [ + "▁rare", + -10.570295333862305 + ], + [ + "vous", + -10.570857048034668 + ], + [ + "vid", + -10.571303367614746 + ], + [ + "faster", + -10.57193374633789 + ], + [ + "mail", + -10.572575569152832 + ], + [ + "▁gradu", + -10.572944641113281 + ], + [ + "manage", + -10.574678421020508 + ], + [ + "▁map", + -10.576395988464355 + ], + [ + "nxiety", + -10.57663345336914 + ], + [ + "volve", + -10.577160835266113 + ], + [ + "न", + -10.577399253845215 + ], + [ + "nearly", + -10.57783317565918 + ], + [ + "ercial", + -10.578917503356934 + ], + [ + "blo", + -10.579207420349121 + ], + [ + "dio", + -10.580347061157227 + ], + [ + "▁Adam", + -10.581408500671387 + ], + [ + "cancer", + -10.583087921142578 + ], + [ + "▁survi", + -10.583209037780762 + ], + [ + "ambi", + -10.58334732055664 + ], + [ + "bridge", + -10.58411979675293 + ], + [ + "killed", + -10.58425521850586 + ], + [ + "▁Che", + -10.584539413452148 + ], + [ + "▁noise", + -10.587294578552246 + ], + [ + "▁Unive", + -10.58737850189209 + ], + [ + "▁north", + -10.588005065917969 + ], + [ + "▁vent", + -10.58834171295166 + ], + [ + "ون", + -10.588705062866211 + ], + [ + "่", + -10.589229583740234 + ], + [ + "▁Texas", + -10.589641571044922 + ], + [ + "▁commu", + -10.589668273925781 + ], + [ + "ienc", + -10.590213775634766 + ], + [ + "garden", + -10.591288566589355 + ], + [ + "Father", + -10.591609954833984 + ], + [ + "sembl", + -10.591828346252441 + ], + [ + "▁invit", + -10.591954231262207 + ], + [ + "hand", + -10.593727111816406 + ], + [ + "▁AI", + -10.594749450683594 + ], + [ + "asi", + -10.594850540161133 + ], + [ + "▁Roman", + -10.595535278320312 + ], + [ + "▁grate", + -10.595959663391113 + ], + [ + "prepar", + -10.596010208129883 + ], + [ + "▁hook", + -10.59783935546875 + ], + [ + "на", + -10.598081588745117 + ], + [ + "▁Obvi", + -10.598943710327148 + ], + [ + "eithio", + -10.599019050598145 + ], + [ + "limate", + -10.599762916564941 + ], + [ + "ublish", + -10.60051441192627 + ], + [ + "▁charg", + -10.60097885131836 + ], + [ + "▁Wait", + -10.60296630859375 + ], + [ + "cia", + -10.603226661682129 + ], + [ + "▁Scott", + -10.603611946105957 + ], + [ + "quired", + -10.603901863098145 + ], + [ + "pared", + -10.607038497924805 + ], + [ + "ford", + -10.607675552368164 + ], + [ + "excuse", + -10.608179092407227 + ], + [ + "▁south", + -10.608502388000488 + ], + [ + "▁rush", + -10.608954429626465 + ], + [ + "entury", + -10.60957145690918 + ], + [ + "▁occur", + -10.611124038696289 + ], + [ + "idered", + -10.61162281036377 + ], + [ + "▁tech", + -10.611693382263184 + ], + [ + "ellent", + -10.612112998962402 + ], + [ + "ú", + -10.614253997802734 + ], + [ + "▁minim", + -10.614343643188477 + ], + [ + "ร", + -10.614816665649414 + ], + [ + "lcohol", + -10.616278648376465 + ], + [ + "▁Dar", + -10.617920875549316 + ], + [ + "policy", + -10.618968963623047 + ], + [ + "strike", + -10.619135856628418 + ], + [ + "vince", + -10.619328498840332 + ], + [ + "style", + -10.620047569274902 + ], + [ + "wan", + -10.620744705200195 + ], + [ + "ustain", + -10.621456146240234 + ], + [ + "ftware", + -10.622318267822266 + ], + [ + "winter", + -10.624390602111816 + ], + [ + "▁label", + -10.624944686889648 + ], + [ + "rack", + -10.626117706298828 + ], + [ + "▁hotel", + -10.626368522644043 + ], + [ + "Russia", + -10.626593589782715 + ], + [ + "Before", + -10.62663459777832 + ], + [ + "▁dear", + -10.627157211303711 + ], + [ + "Willia", + -10.627374649047852 + ], + [ + "lorida", + -10.627778053283691 + ], + [ + "circum", + -10.627978324890137 + ], + [ + "▁divid", + -10.628945350646973 + ], + [ + "▁Ya", + -10.629300117492676 + ], + [ + "▁categ", + -10.629781723022461 + ], + [ + "▁ridic", + -10.629817962646484 + ], + [ + "år", + -10.630229949951172 + ], + [ + "ella", + -10.631875038146973 + ], + [ + "danger", + -10.63260555267334 + ], + [ + "▁salt", + -10.6332368850708 + ], + [ + "▁pure", + -10.633991241455078 + ], + [ + "logist", + -10.63420295715332 + ], + [ + "ات", + -10.63503360748291 + ], + [ + "▁rap", + -10.635194778442383 + ], + [ + "▁tap", + -10.635924339294434 + ], + [ + "button", + -10.639406204223633 + ], + [ + "▁Tim", + -10.639606475830078 + ], + [ + "▁bomb", + -10.64001750946045 + ], + [ + "▁Rock", + -10.643566131591797 + ], + [ + "▁upset", + -10.645363807678223 + ], + [ + "escape", + -10.64555549621582 + ], + [ + "▁Jeff", + -10.64624309539795 + ], + [ + "bel", + -10.646703720092773 + ], + [ + "▁lean", + -10.647820472717285 + ], + [ + "▁accur", + -10.648143768310547 + ], + [ + "▁adapt", + -10.65062427520752 + ], + [ + "closed", + -10.6517972946167 + ], + [ + "bil", + -10.652249336242676 + ], + [ + "prefer", + -10.65286922454834 + ], + [ + "&", + -10.653299331665039 + ], + [ + "▁Other", + -10.653401374816895 + ], + [ + "liquid", + -10.654008865356445 + ], + [ + "budget", + -10.655657768249512 + ], + [ + "▁respo", + -10.655940055847168 + ], + [ + "nciple", + -10.656023025512695 + ], + [ + "स", + -10.656610488891602 + ], + [ + "▁Park", + -10.65776538848877 + ], + [ + "noon", + -10.657962799072266 + ], + [ + "stroy", + -10.658618927001953 + ], + [ + "promis", + -10.658881187438965 + ], + [ + "erback", + -10.659444808959961 + ], + [ + "▁Janua", + -10.659446716308594 + ], + [ + "assume", + -10.659655570983887 + ], + [ + "▁angry", + -10.659663200378418 + ], + [ + "fairly", + -10.66165542602539 + ], + [ + "▁storm", + -10.661924362182617 + ], + [ + "flict", + -10.662686347961426 + ], + [ + "▁trend", + -10.66327953338623 + ], + [ + "▁hall", + -10.663591384887695 + ], + [ + "▁Actu", + -10.663599014282227 + ], + [ + "▁Over", + -10.663866996765137 + ], + [ + "▁queen", + -10.663979530334473 + ], + [ + "▁bod", + -10.665416717529297 + ], + [ + "indeed", + -10.666727066040039 + ], + [ + "stinct", + -10.66693115234375 + ], + [ + "plenty", + -10.666949272155762 + ], + [ + "▁failu", + -10.667309761047363 + ], + [ + "illion", + -10.667485237121582 + ], + [ + "tended", + -10.667510986328125 + ], + [ + "▁bond", + -10.6676607131958 + ], + [ + "▁bitch", + -10.668707847595215 + ], + [ + "▁phase", + -10.669278144836426 + ], + [ + "▁scary", + -10.6696138381958 + ], + [ + "ía", + -10.670365333557129 + ], + [ + "▁chip", + -10.670655250549316 + ], + [ + "box", + -10.671211242675781 + ], + [ + "threat", + -10.67174243927002 + ], + [ + "▁shame", + -10.67233657836914 + ], + [ + "▁climb", + -10.672908782958984 + ], + [ + "ь", + -10.673310279846191 + ], + [ + "▁Yo", + -10.673399925231934 + ], + [ + "▁admit", + -10.675646781921387 + ], + [ + "volume", + -10.676674842834473 + ], + [ + "▁Think", + -10.678500175476074 + ], + [ + "▁saved", + -10.678580284118652 + ], + [ + "ogress", + -10.678776741027832 + ], + [ + "▁Radio", + -10.680113792419434 + ], + [ + "cerned", + -10.680306434631348 + ], + [ + "uggest", + -10.680525779724121 + ], + [ + "ذ", + -10.681251525878906 + ], + [ + "hmm", + -10.681805610656738 + ], + [ + "▁flash", + -10.681939125061035 + ], + [ + "▁dar", + -10.682701110839844 + ], + [ + "mocrat", + -10.683390617370605 + ], + [ + "▁loan", + -10.683489799499512 + ], + [ + "▁flag", + -10.683565139770508 + ], + [ + "▁grav", + -10.68416976928711 + ], + [ + "▁bread", + -10.684388160705566 + ], + [ + "▁White", + -10.684408187866211 + ], + [ + "remove", + -10.685150146484375 + ], + [ + "▁debt", + -10.685603141784668 + ], + [ + "▁Wash", + -10.685881614685059 + ], + [ + "▁align", + -10.686711311340332 + ], + [ + "trauma", + -10.686873435974121 + ], + [ + "serva", + -10.68724250793457 + ], + [ + "cident", + -10.687265396118164 + ], + [ + "ע", + -10.691047668457031 + ], + [ + "▁refle", + -10.691572189331055 + ], + [ + "ddle", + -10.691900253295898 + ], + [ + "icipat", + -10.69223403930664 + ], + [ + "▁hur", + -10.692465782165527 + ], + [ + "ד", + -10.693521499633789 + ], + [ + "▁Frank", + -10.6940336227417 + ], + [ + "▁brief", + -10.694040298461914 + ], + [ + "▁gym", + -10.694724082946777 + ], + [ + "▁occas", + -10.695362091064453 + ], + [ + "▁split", + -10.696383476257324 + ], + [ + "▁Sun", + -10.698915481567383 + ], + [ + "▁crew", + -10.699298858642578 + ], + [ + "vestor", + -10.700345039367676 + ], + [ + "▁mount", + -10.700827598571777 + ], + [ + "▁Tri", + -10.702634811401367 + ], + [ + "saving", + -10.702791213989258 + ], + [ + "▁suppl", + -10.703216552734375 + ], + [ + "fectly", + -10.703566551208496 + ], + [ + "▁boss", + -10.704200744628906 + ], + [ + "BC", + -10.704440116882324 + ], + [ + "▁Bur", + -10.70581340789795 + ], + [ + "driver", + -10.705901145935059 + ], + [ + "▁Shi", + -10.707297325134277 + ], + [ + "Canada", + -10.707464218139648 + ], + [ + "й", + -10.707540512084961 + ], + [ + "▁bike", + -10.707961082458496 + ], + [ + "Disney", + -10.708860397338867 + ], + [ + "spira", + -10.70908260345459 + ], + [ + "▁knee", + -10.709210395812988 + ], + [ + "▁neck", + -10.70927619934082 + ], + [ + "pocket", + -10.709699630737305 + ], + [ + "speech", + -10.709813117980957 + ], + [ + "▁route", + -10.710102081298828 + ], + [ + "▁facil", + -10.710370063781738 + ], + [ + "istry", + -10.710509300231934 + ], + [ + "belong", + -10.711322784423828 + ], + [ + "surpri", + -10.711699485778809 + ], + [ + "▁varie", + -10.712739944458008 + ], + [ + "▁gap", + -10.713200569152832 + ], + [ + "▁civil", + -10.713313102722168 + ], + [ + "▁March", + -10.71369457244873 + ], + [ + "▁smoke", + -10.714314460754395 + ], + [ + "▁gate", + -10.714974403381348 + ], + [ + "▁loop", + -10.716251373291016 + ], + [ + "fellow", + -10.717053413391113 + ], + [ + "▁tape", + -10.717557907104492 + ], + [ + "ी", + -10.717962265014648 + ], + [ + "aptain", + -10.718921661376953 + ], + [ + "▁elev", + -10.719413757324219 + ], + [ + "▁clip", + -10.719504356384277 + ], + [ + "▁sharp", + -10.719542503356934 + ], + [ + "forgot", + -10.719751358032227 + ], + [ + "با", + -10.721407890319824 + ], + [ + "▁cael", + -10.722020149230957 + ], + [ + "wrestl", + -10.723108291625977 + ], + [ + "▁optim", + -10.72352409362793 + ], + [ + "▁champ", + -10.723821640014648 + ], + [ + "ature", + -10.724418640136719 + ], + [ + "defend", + -10.724947929382324 + ], + [ + "guitar", + -10.726400375366211 + ], + [ + "berg", + -10.726605415344238 + ], + [ + "ppreci", + -10.72754955291748 + ], + [ + "▁enemy", + -10.728711128234863 + ], + [ + "ffe", + -10.729398727416992 + ], + [ + "▁remov", + -10.729968070983887 + ], + [ + "egment", + -10.730751991271973 + ], + [ + "what", + -10.73118782043457 + ], + [ + "nervou", + -10.731405258178711 + ], + [ + "phrase", + -10.731864929199219 + ], + [ + "fident", + -10.73208236694336 + ], + [ + "ها", + -10.733819007873535 + ], + [ + "▁corre", + -10.734114646911621 + ], + [ + "gerous", + -10.734747886657715 + ], + [ + "▁studi", + -10.73479175567627 + ], + [ + "▁minor", + -10.735052108764648 + ], + [ + "▁blind", + -10.735089302062988 + ], + [ + "▁Talk", + -10.735300064086914 + ], + [ + "▁reduc", + -10.735514640808105 + ], + [ + "scared", + -10.736498832702637 + ], + [ + "useful", + -10.738395690917969 + ], + [ + "▁Green", + -10.739567756652832 + ], + [ + "financ", + -10.740066528320312 + ], + [ + "ico", + -10.740817070007324 + ], + [ + "shadow", + -10.741120338439941 + ], + [ + "ं", + -10.7415771484375 + ], + [ + "▁Brown", + -10.741870880126953 + ], + [ + "League", + -10.742010116577148 + ], + [ + "mirror", + -10.743651390075684 + ], + [ + "propos", + -10.743803024291992 + ], + [ + "▁pool", + -10.74498462677002 + ], + [ + "ρ", + -10.746208190917969 + ], + [ + "Robert", + -10.747621536254883 + ], + [ + "ridge", + -10.747870445251465 + ], + [ + "▁fake", + -10.74846076965332 + ], + [ + "но", + -10.748626708984375 + ], + [ + "posit", + -10.749181747436523 + ], + [ + "▁capac", + -10.749898910522461 + ], + [ + "▁hide", + -10.750333786010742 + ], + [ + "wisdom", + -10.750779151916504 + ], + [ + "▁pursu", + -10.750940322875977 + ], + [ + "whelm", + -10.750971794128418 + ], + [ + "▁iawn", + -10.751581192016602 + ], + [ + "▁pub", + -10.751888275146484 + ], + [ + "▁steal", + -10.752155303955078 + ], + [ + "war", + -10.75255298614502 + ], + [ + "cribed", + -10.752699851989746 + ], + [ + "▁Nick", + -10.753357887268066 + ], + [ + "nsible", + -10.75343132019043 + ], + [ + "▁scrip", + -10.754422187805176 + ], + [ + "σ", + -10.754457473754883 + ], + [ + "enario", + -10.754537582397461 + ], + [ + "factur", + -10.754936218261719 + ], + [ + "▁bio", + -10.754964828491211 + ], + [ + "vor", + -10.755305290222168 + ], + [ + "▁hip", + -10.756375312805176 + ], + [ + "scream", + -10.756481170654297 + ], + [ + "▁kit", + -10.757012367248535 + ], + [ + "▁yw", + -10.757450103759766 + ], + [ + "▁Jim", + -10.757746696472168 + ], + [ + "raffic", + -10.760311126708984 + ], + [ + "beauty", + -10.76134204864502 + ], + [ + "AC", + -10.762918472290039 + ], + [ + "▁Show", + -10.762977600097656 + ], + [ + "crimin", + -10.763059616088867 + ], + [ + "▁Thurs", + -10.763647079467773 + ], + [ + "▁awful", + -10.76436996459961 + ], + [ + "thetic", + -10.765739440917969 + ], + [ + "▁inspi", + -10.766145706176758 + ], + [ + "▁rank", + -10.767035484313965 + ], + [ + "▁erupt", + -10.76734733581543 + ], + [ + "▁prime", + -10.767800331115723 + ], + [ + "deeper", + -10.768885612487793 + ], + [ + "▁من", + -10.768996238708496 + ], + [ + "August", + -10.770344734191895 + ], + [ + "ollect", + -10.770415306091309 + ], + [ + "ि", + -10.770562171936035 + ], + [ + "▁gig", + -10.770957946777344 + ], + [ + "smooth", + -10.77111530303955 + ], + [ + "▁volca", + -10.7711820602417 + ], + [ + "ains", + -10.77159595489502 + ], + [ + "▁demo", + -10.771601676940918 + ], + [ + "Donald", + -10.772075653076172 + ], + [ + "▁Carol", + -10.772480010986328 + ], + [ + "▁depth", + -10.774382591247559 + ], + [ + "▁Mac", + -10.775834083557129 + ], + [ + "▁Phil", + -10.776392936706543 + ], + [ + "aunt", + -10.779007911682129 + ], + [ + "▁bli", + -10.779083251953125 + ], + [ + "Africa", + -10.779634475708008 + ], + [ + "compos", + -10.780667304992676 + ], + [ + "fast", + -10.781500816345215 + ], + [ + "bwysig", + -10.782716751098633 + ], + [ + "▁gene", + -10.78337287902832 + ], + [ + "ourage", + -10.783553123474121 + ], + [ + "▁simpl", + -10.783772468566895 + ], + [ + "xtreme", + -10.785308837890625 + ], + [ + "▁crash", + -10.786928176879883 + ], + [ + "▁Keep", + -10.787826538085938 + ], + [ + "▁Luc", + -10.78905963897705 + ], + [ + "encies", + -10.790239334106445 + ], + [ + "lican", + -10.790282249450684 + ], + [ + "▁drag", + -10.790680885314941 + ], + [ + "▁coast", + -10.7911376953125 + ], + [ + "त", + -10.791498184204102 + ], + [ + "ician", + -10.791775703430176 + ], + [ + "▁lunch", + -10.79233455657959 + ], + [ + "witnes", + -10.793160438537598 + ], + [ + "▁sensi", + -10.793693542480469 + ], + [ + "▁slide", + -10.793717384338379 + ], + [ + "▁pheno", + -10.794388771057129 + ], + [ + "cluded", + -10.794565200805664 + ], + [ + "yellow", + -10.794814109802246 + ], + [ + "stakes", + -10.795548439025879 + ], + [ + "▁guar", + -10.795769691467285 + ], + [ + "engage", + -10.796321868896484 + ], + [ + "▁här", + -10.796483993530273 + ], + [ + "כ", + -10.796703338623047 + ], + [ + "▁hydro", + -10.796770095825195 + ], + [ + "ो", + -10.79737663269043 + ], + [ + "▁depre", + -10.797636985778809 + ], + [ + "rapy", + -10.79797649383545 + ], + [ + "nounce", + -10.798117637634277 + ], + [ + "▁tube", + -10.798859596252441 + ], + [ + "▁liber", + -10.799055099487305 + ], + [ + "▁empty", + -10.799363136291504 + ], + [ + "▁sacri", + -10.800439834594727 + ], + [ + "▁swim", + -10.800854682922363 + ], + [ + "▁Check", + -10.801629066467285 + ], + [ + "flavor", + -10.801759719848633 + ], + [ + "گ", + -10.801935195922852 + ], + [ + "▁Qui", + -10.802206993103027 + ], + [ + "lash", + -10.80372428894043 + ], + [ + "income", + -10.803894996643066 + ], + [ + "▁milk", + -10.803975105285645 + ], + [ + "▁drill", + -10.804010391235352 + ], + [ + "filter", + -10.805081367492676 + ], + [ + "govern", + -10.805580139160156 + ], + [ + "प", + -10.80679702758789 + ], + [ + "▁rob", + -10.808521270751953 + ], + [ + "▁acid", + -10.80915641784668 + ], + [ + "arian", + -10.810784339904785 + ], + [ + "placed", + -10.811027526855469 + ], + [ + "victim", + -10.812843322753906 + ], + [ + "▁Hope", + -10.814870834350586 + ], + [ + "▁blame", + -10.815279006958008 + ], + [ + "▁panel", + -10.815288543701172 + ], + [ + "shine", + -10.815549850463867 + ], + [ + "▁narra", + -10.815962791442871 + ], + [ + "cheese", + -10.816854476928711 + ], + [ + "▁lab", + -10.81775951385498 + ], + [ + "served", + -10.817997932434082 + ], + [ + "▁kiss", + -10.81813907623291 + ], + [ + "lick", + -10.818584442138672 + ], + [ + "cancel", + -10.81859016418457 + ], + [ + "▁June", + -10.820411682128906 + ], + [ + "town", + -10.820478439331055 + ], + [ + "proof", + -10.821663856506348 + ], + [ + "front", + -10.822224617004395 + ], + [ + "لي", + -10.822355270385742 + ], + [ + "ก", + -10.824640274047852 + ], + [ + "▁tweet", + -10.825393676757812 + ], + [ + "▁teeth", + -10.82551383972168 + ], + [ + "print", + -10.826305389404297 + ], + [ + "▁neutr", + -10.826570510864258 + ], + [ + "▁Smith", + -10.826850891113281 + ], + [ + "Daniel", + -10.82708740234375 + ], + [ + "detect", + -10.827676773071289 + ], + [ + "ม", + -10.828977584838867 + ], + [ + "▁fash", + -10.829408645629883 + ], + [ + "surviv", + -10.82983684539795 + ], + [ + "▁quit", + -10.82984733581543 + ], + [ + "rocket", + -10.830109596252441 + ], + [ + "▁posse", + -10.83021354675293 + ], + [ + "▁fool", + -10.830740928649902 + ], + [ + "▁Brad", + -10.831032752990723 + ], + [ + "▁grant", + -10.831756591796875 + ], + [ + "writer", + -10.832123756408691 + ], + [ + "▁Octob", + -10.832169532775879 + ], + [ + "▁brown", + -10.832206726074219 + ], + [ + "cceler", + -10.83284854888916 + ], + [ + "spired", + -10.833824157714844 + ], + [ + "▁ideal", + -10.835405349731445 + ], + [ + "▁kitch", + -10.836577415466309 + ], + [ + "▁Nice", + -10.836885452270508 + ], + [ + "▁não", + -10.837200164794922 + ], + [ + "hinese", + -10.837967872619629 + ], + [ + "magnet", + -10.838642120361328 + ], + [ + "▁Last", + -10.839577674865723 + ], + [ + "▁angel", + -10.8400297164917 + ], + [ + "conomy", + -10.84192943572998 + ], + [ + "based", + -10.843270301818848 + ], + [ + "▁NFL", + -10.843395233154297 + ], + [ + "▁float", + -10.845415115356445 + ], + [ + "▁Brian", + -10.845653533935547 + ], + [ + "uesday", + -10.846219062805176 + ], + [ + "▁false", + -10.846965789794922 + ], + [ + "▁Out", + -10.847525596618652 + ], + [ + "เ", + -10.847990989685059 + ], + [ + "▁trad", + -10.848758697509766 + ], + [ + "ض", + -10.848877906799316 + ], + [ + "border", + -10.849693298339844 + ], + [ + "▁danc", + -10.849804878234863 + ], + [ + "▁Queen", + -10.849884033203125 + ], + [ + "dinary", + -10.851078987121582 + ], + [ + "▁Back", + -10.8512601852417 + ], + [ + "▁delic", + -10.851305961608887 + ], + [ + "onsult", + -10.851909637451172 + ], + [ + "▁nutri", + -10.853232383728027 + ], + [ + "cular", + -10.854483604431152 + ], + [ + "tense", + -10.854982376098633 + ], + [ + "▁labor", + -10.855551719665527 + ], + [ + "priate", + -10.855647087097168 + ], + [ + "▁Septe", + -10.856303215026855 + ], + [ + "π", + -10.856354713439941 + ], + [ + "▁Wedne", + -10.85671329498291 + ], + [ + "▁dust", + -10.85749340057373 + ], + [ + "▁adven", + -10.858049392700195 + ], + [ + "▁gut", + -10.859704971313477 + ], + [ + "enging", + -10.860471725463867 + ], + [ + "extend", + -10.860929489135742 + ], + [ + "▁April", + -10.862089157104492 + ], + [ + "▁glory", + -10.864315032958984 + ], + [ + "的", + -10.865930557250977 + ], + [ + "▁shake", + -10.866453170776367 + ], + [ + "ين", + -10.866634368896484 + ], + [ + "utomat", + -10.866637229919434 + ], + [ + "crisis", + -10.866888046264648 + ], + [ + "▁Op", + -10.866888999938965 + ], + [ + "▁innov", + -10.866888999938965 + ], + [ + "insane", + -10.867361068725586 + ], + [ + "oxygen", + -10.868640899658203 + ], + [ + "▁bowl", + -10.870083808898926 + ], + [ + "не", + -10.87057113647461 + ], + [ + "wealth", + -10.871017456054688 + ], + [ + "stairs", + -10.872329711914062 + ], + [ + "▁accid", + -10.873066902160645 + ], + [ + "▁dynam", + -10.873315811157227 + ], + [ + "▁Angel", + -10.873762130737305 + ], + [ + "rimary", + -10.873791694641113 + ], + [ + "▁aggre", + -10.873941421508789 + ], + [ + "▁dumb", + -10.873950958251953 + ], + [ + "▁philo", + -10.874820709228516 + ], + [ + "hicago", + -10.874903678894043 + ], + [ + "ж", + -10.875112533569336 + ], + [ + "μ", + -10.875536918640137 + ], + [ + "▁roof", + -10.876570701599121 + ], + [ + "▁cream", + -10.879012107849121 + ], + [ + "▁Life", + -10.879152297973633 + ], + [ + "▁Mel", + -10.881063461303711 + ], + [ + "▁bang", + -10.881121635437012 + ], + [ + "▁Fuck", + -10.881489753723145 + ], + [ + "▁pipe", + -10.882229804992676 + ], + [ + "cogniz", + -10.882920265197754 + ], + [ + "Decemb", + -10.884922981262207 + ], + [ + "▁magn", + -10.885581970214844 + ], + [ + "▁robot", + -10.88658618927002 + ], + [ + "Street", + -10.88681697845459 + ], + [ + "vulner", + -10.887882232666016 + ], + [ + "known", + -10.888412475585938 + ], + [ + "▁Port", + -10.889223098754883 + ], + [ + "▁Repub", + -10.890266418457031 + ], + [ + "κ", + -10.890893936157227 + ], + [ + "lik", + -10.891523361206055 + ], + [ + "gazine", + -10.89182186126709 + ], + [ + "dvance", + -10.89209270477295 + ], + [ + "▁crap", + -10.892509460449219 + ], + [ + "▁ackno", + -10.893021583557129 + ], + [ + "erform", + -10.89340591430664 + ], + [ + "▁epi", + -10.893449783325195 + ], + [ + "▁input", + -10.893498420715332 + ], + [ + "▁ghost", + -10.893983840942383 + ], + [ + "France", + -10.894094467163086 + ], + [ + "▁trem", + -10.894696235656738 + ], + [ + "sample", + -10.895881652832031 + ], + [ + "▁loose", + -10.895973205566406 + ], + [ + "monstr", + -10.896265983581543 + ], + [ + "ث", + -10.897016525268555 + ], + [ + "levant", + -10.897186279296875 + ], + [ + "ncient", + -10.898689270019531 + ], + [ + "▁Three", + -10.898987770080566 + ], + [ + "facing", + -10.900247573852539 + ], + [ + "ח", + -10.90062141418457 + ], + [ + "ั", + -10.903738975524902 + ], + [ + "▁pause", + -10.903752326965332 + ], + [ + "lawyer", + -10.904776573181152 + ], + [ + "ल", + -10.904987335205078 + ], + [ + "▁chose", + -10.905000686645508 + ], + [ + "hopify", + -10.90624713897705 + ], + [ + "▁annoy", + -10.90634536743164 + ], + [ + "reduce", + -10.908594131469727 + ], + [ + "▁Franc", + -10.908727645874023 + ], + [ + "▁decla", + -10.909989356994629 + ], + [ + "▁punch", + -10.910679817199707 + ], + [ + "ô", + -10.911134719848633 + ], + [ + "ladies", + -10.91143798828125 + ], + [ + "▁simul", + -10.911808013916016 + ], + [ + "ول", + -10.91224193572998 + ], + [ + "▁rose", + -10.912528991699219 + ], + [ + "preach", + -10.912839889526367 + ], + [ + "sugges", + -10.913009643554688 + ], + [ + "ह", + -10.913655281066895 + ], + [ + "alysis", + -10.913741111755371 + ], + [ + "▁Tony", + -10.914107322692871 + ], + [ + "▁acco", + -10.915043830871582 + ], + [ + "▁circu", + -10.915997505187988 + ], + [ + "*", + -10.916940689086914 + ], + [ + "▁Kevin", + -10.917048454284668 + ], + [ + "latest", + -10.91810417175293 + ], + [ + "υ", + -10.918713569641113 + ], + [ + "dustri", + -10.918869018554688 + ], + [ + "pret", + -10.91940975189209 + ], + [ + "▁embar", + -10.92151165008545 + ], + [ + "▁feder", + -10.922131538391113 + ], + [ + "▁admin", + -10.9244384765625 + ], + [ + "apolog", + -10.924995422363281 + ], + [ + "▁virus", + -10.925063133239746 + ], + [ + "injury", + -10.92536449432373 + ], + [ + "rtists", + -10.925671577453613 + ], + [ + "▁radia", + -10.928077697753906 + ], + [ + "▁drum", + -10.931766510009766 + ], + [ + "debate", + -10.932255744934082 + ], + [ + "▁heavi", + -10.93384838104248 + ], + [ + "ست", + -10.934065818786621 + ], + [ + "▁licen", + -10.934349060058594 + ], + [ + "▁Biden", + -10.934447288513184 + ], + [ + "corpor", + -10.934536933898926 + ], + [ + "▁CEO", + -10.934947967529297 + ], + [ + "fluenc", + -10.935134887695312 + ], + [ + "char", + -10.935479164123535 + ], + [ + "▁Ryan", + -10.935934066772461 + ], + [ + "reserv", + -10.93640422821045 + ], + [ + "▁steel", + -10.937424659729004 + ], + [ + "▁DJ", + -10.93751335144043 + ], + [ + "▁adopt", + -10.937623977661133 + ], + [ + "ام", + -10.937653541564941 + ], + [ + "ק", + -10.938431739807129 + ], + [ + "▁swing", + -10.940691947937012 + ], + [ + "▁geo", + -10.941317558288574 + ], + [ + "▁blank", + -10.941427230834961 + ], + [ + "▁corpo", + -10.941861152648926 + ], + [ + "▁Since", + -10.942475318908691 + ], + [ + "ribute", + -10.944482803344727 + ], + [ + "ferred", + -10.945115089416504 + ], + [ + "nicate", + -10.94517707824707 + ], + [ + "▁Jones", + -10.94521713256836 + ], + [ + "▁assis", + -10.946941375732422 + ], + [ + "attach", + -10.947190284729004 + ], + [ + "llabor", + -10.947715759277344 + ], + [ + "▁chart", + -10.948652267456055 + ], + [ + "satisf", + -10.949557304382324 + ], + [ + "▁wound", + -10.94998550415039 + ], + [ + "output", + -10.950337409973145 + ], + [ + "้", + -10.950465202331543 + ], + [ + "loyees", + -10.950650215148926 + ], + [ + "height", + -10.951047897338867 + ], + [ + "ircuit", + -10.95324420928955 + ], + [ + "strain", + -10.953304290771484 + ], + [ + "▁gay", + -10.954068183898926 + ], + [ + "ldier", + -10.95488452911377 + ], + [ + "▁strip", + -10.955124855041504 + ], + [ + "▁excep", + -10.955567359924316 + ], + [ + "▁ruin", + -10.955862998962402 + ], + [ + "▁permi", + -10.955880165100098 + ], + [ + "ко", + -10.95655345916748 + ], + [ + "▁mouse", + -10.956609725952148 + ], + [ + "titute", + -10.957381248474121 + ], + [ + "▁spell", + -10.960101127624512 + ], + [ + "clinic", + -10.962820053100586 + ], + [ + "▁High", + -10.963903427124023 + ], + [ + "▁unusu", + -10.96397876739502 + ], + [ + "ية", + -10.964470863342285 + ], + [ + "म", + -10.965099334716797 + ], + [ + "\"", + -10.96533489227295 + ], + [ + "▁fifth", + -10.965564727783203 + ], + [ + "ро", + -10.96642780303955 + ], + [ + "highly", + -10.966802597045898 + ], + [ + "tural", + -10.967851638793945 + ], + [ + "▁objec", + -10.969690322875977 + ], + [ + "▁Could", + -10.96976375579834 + ], + [ + "ñ", + -10.972229957580566 + ], + [ + "antasy", + -10.973527908325195 + ], + [ + "fessor", + -10.974627494812012 + ], + [ + "▁abuse", + -10.97529411315918 + ], + [ + "▁maxim", + -10.97618579864502 + ], + [ + "▁alien", + -10.977004051208496 + ], + [ + "▁Steph", + -10.977204322814941 + ], + [ + "forced", + -10.977270126342773 + ], + [ + "redict", + -10.977640151977539 + ], + [ + "▁victo", + -10.97801399230957 + ], + [ + "▁Ukrai", + -10.97880744934082 + ], + [ + "ı", + -10.979412078857422 + ], + [ + "▁está", + -10.979948997497559 + ], + [ + "▁snap", + -10.980480194091797 + ], + [ + "secure", + -10.981690406799316 + ], + [ + "decent", + -10.982316017150879 + ], + [ + "ojects", + -10.982415199279785 + ], + [ + "▁vibe", + -10.982423782348633 + ], + [ + "▁ymddy", + -10.982519149780273 + ], + [ + "praise", + -10.983177185058594 + ], + [ + "▁slave", + -10.986868858337402 + ], + [ + "balanc", + -10.98690414428711 + ], + [ + "▁Play", + -10.988106727600098 + ], + [ + "▁plot", + -10.988462448120117 + ], + [ + "▁sweat", + -10.988475799560547 + ], + [ + "mobile", + -10.99018383026123 + ], + [ + "horror", + -10.991141319274902 + ], + [ + "uthent", + -10.992399215698242 + ], + [ + "rofile", + -10.993474960327148 + ], + [ + "▁nigga", + -10.995172500610352 + ], + [ + "▁anima", + -10.99706745147705 + ], + [ + "▁Blue", + -10.997095108032227 + ], + [ + "arrest", + -10.99715518951416 + ], + [ + "bubble", + -10.997528076171875 + ], + [ + "tempor", + -10.997678756713867 + ], + [ + "acting", + -10.997917175292969 + ], + [ + "▁Jason", + -10.998682975769043 + ], + [ + "rystal", + -11.00022029876709 + ], + [ + "▁twist", + -11.000696182250977 + ], + [ + "▁devil", + -11.001209259033203 + ], + [ + "▁الم", + -11.001358032226562 + ], + [ + "twenty", + -11.002650260925293 + ], + [ + "ymptom", + -11.002705574035645 + ], + [ + "▁vaca", + -11.002714157104492 + ], + [ + "crying", + -11.002969741821289 + ], + [ + "▁Wh", + -11.003202438354492 + ], + [ + "diagno", + -11.005369186401367 + ], + [ + "cruit", + -11.006766319274902 + ], + [ + "▁sauce", + -11.007676124572754 + ], + [ + "Martin", + -11.007824897766113 + ], + [ + "▁hypo", + -11.00908374786377 + ], + [ + "branch", + -11.009202003479004 + ], + [ + "آ", + -11.031463623046875 + ], + [ + "q", + -11.039170265197754 + ], + [ + "پ", + -11.054652214050293 + ], + [ + "ш", + -11.056396484375 + ], + [ + "ז", + -11.068325996398926 + ], + [ + "г", + -11.068893432617188 + ], + [ + "ς", + -11.073197364807129 + ], + [ + "ง", + -11.080995559692383 + ], + [ + "х", + -11.096549987792969 + ], + [ + "ว", + -11.12403678894043 + ], + [ + "η", + -11.132715225219727 + ], + [ + "ں", + -11.151740074157715 + ], + [ + "ี", + -11.159049034118652 + ], + [ + "פ", + -11.163168907165527 + ], + [ + "إ", + -11.16495132446289 + ], + [ + "ø", + -11.16624927520752 + ], + [ + "ί", + -11.172032356262207 + ], + [ + "ย", + -11.18651008605957 + ], + [ + "ى", + -11.202125549316406 + ], + [ + "đ", + -11.214404106140137 + ], + [ + "ס", + -11.219958305358887 + ], + [ + "λ", + -11.236371994018555 + ], + [ + "ค", + -11.246789932250977 + ], + [ + "غ", + -11.251553535461426 + ], + [ + "ئ", + -11.255985260009766 + ], + [ + "ю", + -11.269397735595703 + ], + [ + "ज", + -11.285001754760742 + ], + [ + "य", + -11.31639575958252 + ], + [ + "Z", + -11.333300590515137 + ], + [ + "š", + -11.3670015335083 + ], + [ + "ג", + -11.36729907989502 + ], + [ + "¿", + -11.36859130859375 + ], + [ + "ά", + -11.374673843383789 + ], + [ + "ό", + -11.393960952758789 + ], + [ + "บ", + -11.408028602600098 + ], + [ + "我", + -11.414668083190918 + ], + [ + "是", + -11.430194854736328 + ], + [ + "ล", + -11.434857368469238 + ], + [ + "ब", + -11.449085235595703 + ], + [ + "έ", + -11.458937644958496 + ], + [ + "і", + -11.460898399353027 + ], + [ + "–", + -11.472521781921387 + ], + [ + "ן", + -11.475947380065918 + ], + [ + "ด", + -11.476611137390137 + ], + [ + "چ", + -11.496973037719727 + ], + [ + "ھ", + -11.500029563903809 + ], + [ + "ต", + -11.50856876373291 + ], + [ + "ě", + -11.512234687805176 + ], + [ + "द", + -11.527382850646973 + ], + [ + "ग", + -11.53557300567627 + ], + [ + "ž", + -11.544540405273438 + ], + [ + "ء", + -11.553231239318848 + ], + [ + "ิ", + -11.553351402282715 + ], + [ + "ט", + -11.555865287780762 + ], + [ + "व", + -11.578168869018555 + ], + [ + "。", + -11.57915210723877 + ], + [ + "ท", + -11.590256690979004 + ], + [ + "ă", + -11.592244148254395 + ], + [ + "צ", + -11.595108032226562 + ], + [ + "γ", + -11.60841178894043 + ], + [ + "ব", + -12.348573684692383 + ], + [ + "ã", + -12.596792221069336 + ], + [ + "ै", + -12.822904586791992 + ], + [ + "$", + -13.87006664276123 + ] + ], + "byte_fallback": true + } +} \ No newline at end of file diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs new file mode 100644 index 00000000..e5ef1897 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs @@ -0,0 +1,70 @@ +use candle_core::{DType, Device, Tensor}; +use candle_nn::VarBuilder; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use pocket_tts::modules::{attention::StreamingMultiheadAttention, rope::RotaryEmbedding}; +use std::collections::HashMap; + +fn bench_attention_scaling(c: &mut Criterion) { + let device = Device::Cpu; + let dim = 512; + let heads = 8; + let dim_head = dim / heads; + let max_period = 10000.0; + + // Setup model + let vb = VarBuilder::zeros(DType::F32, &device); + let rope = RotaryEmbedding::new(max_period as f32, dim_head, &device).unwrap(); + let attention = + StreamingMultiheadAttention::new(dim, heads, rope, None, "bench_attn", vb).unwrap(); + + let mut group = c.benchmark_group("Attention_Forward_Step_Time"); + + // Measure time for a SINGLE step at various context lengths + for start_len in [0, 500, 1000, 1500, 2000, 3000].iter() { + group.bench_with_input( + BenchmarkId::new("ctx_len", start_len), + start_len, + |b, &len| { + let q = Tensor::randn(0f32, 1.0, (1, 1, dim), &device).unwrap(); + + b.iter_custom(|iters| { + let mut total_time = std::time::Duration::ZERO; + + for _ in 0..iters { + let mut state = HashMap::new(); + let mut module_state = HashMap::new(); + + if len > 0 { + let k = Tensor::zeros( + (1, heads, len.max(64), dim_head), + DType::F32, + &device, + ) + .unwrap(); + let v = Tensor::zeros( + (1, heads, len.max(64), dim_head), + DType::F32, + &device, + ) + .unwrap(); + + module_state.insert("k_buf".to_string(), k); + module_state.insert("v_buf".to_string(), v); + } + + state.insert("bench_attn".to_string(), module_state); + + let start = std::time::Instant::now(); + let _ = attention.forward(&q, &mut state, len, len).unwrap(); + total_time += start.elapsed(); + } + total_time + }); + }, + ); + } + group.finish(); +} + +criterion_group!(benches, bench_attention_scaling); +criterion_main!(benches); diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs new file mode 100644 index 00000000..346b18ed --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs @@ -0,0 +1,96 @@ +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use pocket_tts::TTSModel; +use std::time::Duration; + +fn bench_full_generation(c: &mut Criterion) { + let mut model = TTSModel::load("b6369a24").expect("Failed to load model"); + model.temp = 0.0; // Deterministic + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + + if !ref_wav.exists() { + eprintln!("Skipping benchmark: ref.wav not found at {:?}", ref_wav); + return; + } + + let state = model + .get_voice_state(&ref_wav) + .expect("Failed to get voice state"); + + let mut group = c.benchmark_group("generation"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(20)); + + // Short + let short_text = "Hello world"; + group.throughput(Throughput::Bytes(short_text.len() as u64)); + group.bench_function("short", |b| { + b.iter(|| { + let _ = model.generate(short_text, &state).expect("Failed"); + }) + }); + + // Medium + let medium_text = + "This is a medium length sentence for benchmarking the text to speech system."; + group.throughput(Throughput::Bytes(medium_text.len() as u64)); + group.bench_function("medium", |b| { + b.iter(|| { + let _ = model.generate(medium_text, &state).expect("Failed"); + }) + }); + + // Long + let long_text = "The quick brown fox jumps over the lazy dog. ".repeat(10); + group.throughput(Throughput::Bytes(long_text.len() as u64)); + group.bench_function("long", |b| { + b.iter(|| { + let _ = model.generate(&long_text, &state).expect("Failed"); + }) + }); + + group.finish(); +} + +fn bench_first_chunk(c: &mut Criterion) { + let mut model = TTSModel::load("b6369a24").expect("Failed to load model"); + model.temp = 0.0; + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + + if !ref_wav.exists() { + return; + } + + let state = model + .get_voice_state(&ref_wav) + .expect("Failed to get voice state"); + let text = "Start"; + + let mut group = c.benchmark_group("latency"); + // We care about latency, so 50 samples is good for distribution + group.sample_size(20); + + group.bench_function("first_chunk", |b| { + b.iter(|| { + let mut stream = model.generate_stream(text, &state); + let _ = stream.next().expect("No chunks").expect("Error"); + }) + }); + group.finish(); +} + +criterion_group!(benches, bench_full_generation, bench_first_chunk); +criterion_main!(benches); diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs new file mode 100644 index 00000000..e6484a1e --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs @@ -0,0 +1,75 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use pocket_tts::TTSModel; +use std::time::Instant; + +fn bench_streaming_latency(c: &mut Criterion) { + let mut model = TTSModel::load("b6369a24").expect("Failed to load model"); + model.temp = 0.0; + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + if !ref_wav.exists() { + eprintln!("Skipping benchmark: ref.wav not found"); + return; + } + + let state = model + .get_voice_state(&ref_wav) + .expect("Failed to get voice state"); + let text = "Hello, this is a test for latency."; + + c.bench_function("first_chunk_latency", |b| { + b.iter(|| { + let _start = Instant::now(); + let mut stream = model.generate_stream(text, &state); + let _first_chunk = stream + .next() + .expect("Stream empty") + .expect("Generation failed"); + // We want to measure time to first chunk, iter() handles timing? + // constant overhead of iter() might be small. + // Actually, Criterion defaults to measuring the whole closure. + // But we only want first chunk. + + // For correct measurement, we should probably output the duration manually or structure the test differently, + // but for a simple "is it fast enough" check: + }) + }); +} + +fn bench_streaming_throughput(c: &mut Criterion) { + let mut model = TTSModel::load("b6369a24").expect("Failed to load model"); + model.temp = 0.0; + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + if !ref_wav.exists() { + return; + } + + let state = model + .get_voice_state(&ref_wav) + .expect("Failed to get voice state"); + let text = "This is a longer sentence to measure the throughput of the system over time."; + + c.bench_function("streaming_throughput", |b| { + b.iter(|| { + for chunk in model.generate_stream(text, &state) { + let _ = chunk.expect("Generation failed"); + } + }) + }); +} + +criterion_group!(benches, bench_streaming_latency, bench_streaming_throughput); +criterion_main!(benches); diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/config/b6369a24.yaml b/plugins/native/pocket-tts/vendor/pocket-tts/config/b6369a24.yaml new file mode 100644 index 00000000..68c01ceb --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/config/b6369a24.yaml @@ -0,0 +1,57 @@ +# sig: b6369a24 + +weights_path: hf://kyutai/pocket-tts/tts_b6369a24.safetensors@427e3d61b276ed69fdd03de0d185fa8a8d97fc5b +weights_path_without_voice_cloning: hf://kyutai/pocket-tts-without-voice-cloning/tts_b6369a24.safetensors@d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3 + +flow_lm: + dtype: float32 + flow: + depth: 6 + dim: 512 + transformer: + d_model: 1024 + hidden_scale: 4 + max_period: 10000 + num_heads: 16 + num_layers: 6 + lookup_table: + dim: 1024 + n_bins: 4000 + tokenizer: sentencepiece + tokenizer_path: hf://kyutai/pocket-tts-without-voice-cloning/tokenizer.model@d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3 + #weights_path: flow_lm_b6369a24.safetensors + +mimi: + dtype: float32 + sample_rate: 24000 + channels: 1 + frame_rate: 12.5 + seanet: + dimension: 512 + channels: 1 + n_filters: 64 + n_residual_layers: 1 + ratios: + - 6 + - 5 + - 4 + kernel_size: 7 + residual_kernel_size: 3 + last_kernel_size: 3 + dilation_base: 2 + pad_mode: constant + compress: 2 + transformer: + d_model: 512 + num_heads: 8 + num_layers: 2 + layer_scale: 0.01 + context: 250 + dim_feedforward: 2048 + input_dimension: 512 + output_dimensions: + - 512 + quantizer: + dimension: 32 + output_dimension: 512 + #weights_path: mimi_b6369a24.safetensors diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs new file mode 100644 index 00000000..8d141ffd --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs @@ -0,0 +1,91 @@ +use candle_core::{Device, Tensor}; +use pocket_tts::modules::sdpa::sdpa; +use std::time::Instant; + +fn naive_sdpa(q: &Tensor, k: &Tensor, v: &Tensor, scale: f64) -> candle_core::Result { + let k_t = k.transpose(2, 3)?; + let att = (q.matmul(&k_t)? * scale)?; + let att = candle_nn::ops::softmax(&att, candle_core::D::Minus1)?; + att.matmul(v) +} + +fn run_bench_case(device: &Device, q_len: usize, kv_len: usize, iter: u32) -> anyhow::Result<()> { + // Dimensions + let b = 1; + let h = 8; + let d = 128; + + // Create tensors (random data) + let q = Tensor::randn(0f32, 1f32, (b, h, q_len, d), device)?; + let k = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), device)?; + let v = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), device)?; + + // --- Tiled SDPA --- + // Warmup + let _ = sdpa(&q, &k, &v, 0.1, false, None)?; + + let start = Instant::now(); + for _ in 0..iter { + // Enforce computation with sum() to avoid lazy evaluation shortcuts if any + let out = sdpa(&q, &k, &v, 0.1, false, None)?; + let _ = out.sum_all()?.to_scalar::()?; + } + let dur_tiled = start.elapsed(); + + // --- Naive SDPA --- + // Warmup + let _ = naive_sdpa(&q, &k, &v, 0.1)?; + + let start = Instant::now(); + for _ in 0..iter { + let out = naive_sdpa(&q, &k, &v, 0.1)?; + let _ = out.sum_all()?.to_scalar::()?; + } + let dur_naive = start.elapsed(); + + // Stats + let tiled_avg_ms = dur_tiled.as_secs_f64() * 1000.0 / iter as f64; + let naive_avg_ms = dur_naive.as_secs_f64() * 1000.0 / iter as f64; + let ratio = tiled_avg_ms / naive_avg_ms; + + // Status + let winner = if ratio < 0.95 { + "Tiled" + } else if ratio > 1.05 { + "Naive" + } else { + "Tie" + }; + + println!( + "Q={:<4} KV={:<6} | Tiled: {:>7.3}ms | Naive: {:>7.3}ms | Ratio: {:>4.2}x | Winner: {}", + q_len, kv_len, tiled_avg_ms, naive_avg_ms, ratio, winner + ); + + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let device = Device::Cpu; + + println!("Running Comprehensive SDPA Benchmark..."); + println!("-----------------------------------------------------------------------------"); + println!("Config: Batch=1, Heads=8, HeadDim=128"); + println!("Ratio > 1.0 means Naive is faster. Ratio < 1.0 means Tiled is faster."); + println!("-----------------------------------------------------------------------------"); + + let kv_lengths = [128, 1024, 4096, 8192]; + let q_lengths = [1, 64, 128, 256]; + + for &q_len in &q_lengths { + println!("\n--- Testing Q_LEN = {} ---", q_len); + for &kv_len in &kv_lengths { + let iterations = if kv_len > 4096 { 10 } else { 50 }; + if let Err(e) = run_bench_case(&device, q_len, kv_len, iterations) { + println!("Q={} KV={} | Error: {}", q_len, kv_len, e); + } + } + } + + Ok(()) +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs new file mode 100644 index 00000000..a649543d --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs @@ -0,0 +1,12 @@ +use pocket_tts::config::load_config; +use std::path::PathBuf; + +fn main() { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.pop(); // crates + path.pop(); // candle + let config_path = path.join("pocket_tts").join("config").join("b6369a24.yaml"); + println!("Loading config from {:?}", config_path); + let config = load_config(&config_path).unwrap(); + println!("Ratios: {:?}", config.mimi.seanet.ratios); +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs new file mode 100644 index 00000000..ceb33f28 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs @@ -0,0 +1,8 @@ +fn main() { + let cursor = std::io::Cursor::new(vec![]); + let reader = hound::WavReader::new(cursor); + match reader { + Ok(_) => println!("Unexpected success"), + Err(e) => println!("Error Variant Debug: {:?}", e), + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs new file mode 100644 index 00000000..b2d341ae --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs @@ -0,0 +1,73 @@ +//! Quantization example for Pocket TTS +//! +//! Demonstrates the quantization module by simulating int8 quantization +//! on random tensors and measuring SNR quality. +//! +//! Run with: `cargo run --example quantize_demo --features quantized` + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn main() -> Result<()> { + println!("=== Pocket TTS Quantization Demo ===\n"); + + let device = Device::Cpu; + + // Simulate various tensor sizes and shapes + let test_cases = vec![ + ("Small tensor (100 elements)", 100), + ("Medium tensor (10,000 elements)", 10_000), + ("Large tensor (1,000,000 elements)", 1_000_000), + ]; + + for (name, size) in test_cases { + println!("📊 {}", name); + + // Create tensor with values similar to neural network weights + let values: Vec = (0..size).map(|i| (i as f32 * 0.001).sin() * 2.0).collect(); + let tensor = Tensor::new(&values[..], &device)?; + + // Quantize + let quantized = pocket_tts::QuantizedTensor::quantize(&tensor, 256)?; + + // Calculate quality metrics + let snr = pocket_tts::quantize::calculate_snr(&tensor, quantized.data())?; + let savings = quantized.theoretical_memory_savings(); + + println!(" Scale: {:.6}", quantized.scale()); + println!(" SNR: {:.2} dB", snr); + println!(" Theoretical memory savings: {:.1}x", savings); + println!(); + } + + // Test specific weight patterns + println!("📊 Testing weight distribution patterns:"); + + // Uniform distribution + let uniform: Vec = (0..10000) + .map(|i| (i as f32 / 10000.0) * 2.0 - 1.0) + .collect(); + let uniform_tensor = Tensor::new(&uniform[..], &device)?; + let uniform_q = pocket_tts::QuantizedTensor::quantize(&uniform_tensor, 256)?; + let uniform_snr = pocket_tts::quantize::calculate_snr(&uniform_tensor, uniform_q.data())?; + println!(" Uniform [-1, 1]: SNR = {:.2} dB", uniform_snr); + + // Normal-like distribution (using sin to approximate) + let normal: Vec = (0..10000) + .map(|i| { + let x = (i as f32 / 1000.0).sin(); + let y = (i as f32 / 500.0).cos(); + x * y * 0.5 + }) + .collect(); + let normal_tensor = Tensor::new(&normal[..], &device)?; + let normal_q = pocket_tts::QuantizedTensor::quantize(&normal_tensor, 256)?; + let normal_snr = pocket_tts::quantize::calculate_snr(&normal_tensor, normal_q.data())?; + println!(" Normal-like: SNR = {:.2} dB", normal_snr); + + println!("\n✅ All quantization tests passed!"); + println!("\nNote: For production use, enable the 'quantized' feature:"); + println!(" cargo build --release --features quantized"); + + Ok(()) +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs new file mode 100644 index 00000000..0c2da390 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs @@ -0,0 +1,32 @@ +use pocket_tts::TTSModel; +use std::time::Instant; + +fn main() -> anyhow::Result<()> { + let model = TTSModel::load("b6369a24")?; + let hf_path = "hf://kyutai/pocket-tts-without-voice-cloning/embeddings/cosette.safetensors"; + let local_path = pocket_tts::weights::download_if_necessary(hf_path)?; + let voice_state = model.get_voice_state_from_prompt_file(&local_path)?; + + let text_short = "Short text example."; + let text_medium = "This is a medium length text example that has more words but is still relatively short for a transformer model to handle without much sweat."; + let text_long = "Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, 'and what is the use of a book,' thought Alice 'without pictures or conversations?' So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her."; + + for text in &[text_short, text_medium, text_long] { + let start = Instant::now(); + let mut count = 0; + for chunk in model.generate_stream(text, &voice_state) { + let _ = chunk?; + count += 1; + } + let duration = start.elapsed(); + println!( + "Text length: {}, Frames: {}, Time: {:?}, ms/frame: {:.2}", + text.len(), + count, + duration, + duration.as_millis() as f64 / count as f64 + ); + } + + Ok(()) +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs new file mode 100644 index 00000000..d720f619 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs @@ -0,0 +1,84 @@ +use candle_core::{Device, Tensor}; +use pocket_tts::modules::sdpa::sdpa; + +fn naive_sdpa_masked_reference( + q: &Tensor, + k: &Tensor, + v: &Tensor, + scale: f64, + is_causal: bool, +) -> anyhow::Result { + let (_b, _h, q_len, _d) = q.dims4()?; + let kv_len = k.dims()[2]; + + let k_t = k.transpose(2, 3)?; + let scores = (q.matmul(&k_t)? * scale)?; + + let scores = if is_causal { + let mut mask_vals = vec![]; + // Simple causal mask generation: M[i, j] = 0 if j <= i + shift else -inf + // shift = kv_len - q_len + let shift = kv_len.saturating_sub(q_len); + + for i in 0..q_len { + for j in 0..kv_len { + if j > i + shift { + mask_vals.push(f32::NEG_INFINITY); + } else { + mask_vals.push(0.0); + } + } + } + let mask = Tensor::from_vec(mask_vals, (1, 1, q_len, kv_len), q.device())?; + scores.broadcast_add(&mask)? + } else { + scores + }; + + let probs = candle_nn::ops::softmax(&scores, candle_core::D::Minus1)?; + Ok(probs.matmul(v)?) +} + +fn main() -> anyhow::Result<()> { + let device = Device::Cpu; + let b = 1; + let h = 4; + let d = 32; + + println!("Verifying SDPA Correctness..."); + + // Test 1: Small Q (hits Naive path) + let q_len = 64; // < 512 + let kv_len = 128; + println!("Test 1: Q={} (Naive Path), Causal=true", q_len); + + let q = Tensor::randn(0f32, 1f32, (b, h, q_len, d), &device)?; + let k = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), &device)?; + let v = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), &device)?; + + let out_opt = sdpa(&q, &k, &v, 0.1, true, None)?; + let out_ref = naive_sdpa_masked_reference(&q, &k, &v, 0.1, true)?; + + let diff = (out_opt - out_ref)?.abs()?.max_all()?.to_scalar::()?; + println!(" Max Diff: {:.6}", diff); + assert!(diff < 1e-4, "Naive path divergent!"); + + // Test 2: Large Q (hits Tiled path) + let q_len = 600; // > 512 + let kv_len = 600; + println!("Test 2: Q={} (Tiled Path), Causal=true", q_len); + + let q2 = Tensor::randn(0f32, 1f32, (b, h, q_len, d), &device)?; + let k2 = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), &device)?; + let v2 = Tensor::randn(0f32, 1f32, (b, h, kv_len, d), &device)?; + + let out_opt = sdpa(&q2, &k2, &v2, 0.1, true, None)?; + let out_ref = naive_sdpa_masked_reference(&q2, &k2, &v2, 0.1, true)?; + + let diff = (out_opt - out_ref)?.abs()?.max_all()?.to_scalar::()?; + println!(" Max Diff: {:.6}", diff); + assert!(diff < 1e-4, "Tiled path divergent!"); + + println!("SUCCESS: Both paths verified against reference."); + Ok(()) +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html b/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html new file mode 100644 index 00000000..93b820b2 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html @@ -0,0 +1,766 @@ + + + + + + + Pocket TTS - WebAssembly Demo + + + + + + + +
+
+

🎤 Pocket TTS

+

Next-gen Text-to-Speech via WebAssembly

+
+ +
+ +
+

✨ Synthesis

+ + + +
+ + +
+ + + + + + + +
+ Local Static assets from /pkg + HF Fetched via Hugging Face Hub +
+
+ + + +
+ +
+

Pocket TTS 🦀 Pure Rust & WASM Machine Learning

+

No servers. No data tracking. Pure privacy.

+
+
+ + + + + \ No newline at end of file diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs new file mode 100644 index 00000000..5c409343 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs @@ -0,0 +1,301 @@ +use candle_core::Tensor; + +use hound::{Error as HoundError, WavReader}; +#[cfg(not(target_arch = "wasm32"))] +use hound::{WavSpec, WavWriter}; + +use std::io; +#[cfg(not(target_arch = "wasm32"))] +use std::path::Path; + +#[cfg(not(target_arch = "wasm32"))] +pub fn read_wav>(path: P) -> anyhow::Result<(Tensor, u32)> { + let reader = WavReader::open(path)?; + read_wav_internal(reader) +} + +pub fn read_wav_from_bytes(bytes: &[u8]) -> anyhow::Result<(Tensor, u32)> { + let reader = WavReader::new(std::io::Cursor::new(bytes))?; + read_wav_internal(reader) +} + +fn read_wav_internal( + mut reader: WavReader, +) -> anyhow::Result<(Tensor, u32)> { + let spec = reader.spec(); + let sample_rate = spec.sample_rate; + let channels = spec.channels as usize; + + let samples: Vec = match spec.sample_format { + hound::SampleFormat::Int => { + let max_val = (1 << (spec.bits_per_sample - 1)) as f32; + let mut samples = Vec::new(); + for s in reader.samples::() { + match s { + Ok(v) => samples.push(v as f32 / max_val), + Err(e) => { + // If we hit an unexpected EOF but have read valid samples, we accept it. + if let HoundError::IoError(ref io_err) = e { + // Check for UnexpectedEof OR "Failed to read enough bytes" (which is Other in standard hound) + let is_unexpected_eof = io_err.kind() == io::ErrorKind::UnexpectedEof; + // Check string representation for the specific hound error message + let is_truncated_msg = io_err.kind() == io::ErrorKind::Other + && io_err.to_string().contains("enough bytes"); + + if (is_unexpected_eof || is_truncated_msg) && !samples.is_empty() { + break; + } + } + return Err(anyhow::Error::from(e)); + } + } + } + samples + } + hound::SampleFormat::Float => { + let mut samples = Vec::new(); + for s in reader.samples::() { + match s { + Ok(v) => samples.push(v), + Err(e) => { + if let HoundError::IoError(ref io_err) = e { + let is_unexpected_eof = io_err.kind() == io::ErrorKind::UnexpectedEof; + let is_truncated_msg = io_err.kind() == io::ErrorKind::Other + && io_err.to_string().contains("enough bytes"); + + if (is_unexpected_eof || is_truncated_msg) && !samples.is_empty() { + break; + } + } + return Err(anyhow::Error::from(e)); + } + } + } + samples + } + }; + + let device = if cfg!(target_arch = "wasm32") { + &candle_core::Device::Cpu + } else { + #[cfg(not(target_arch = "wasm32"))] + { + &candle_core::Device::Cpu + } + #[cfg(target_arch = "wasm32")] + { + &candle_core::Device::Cpu + } + }; + + let tensor = if channels > 1 { + // Interleaved to [channels, samples] + let num_total_samples = samples.len(); + let num_samples = num_total_samples / channels; + let mut reshaped = vec![0.0f32; num_total_samples]; + for c in 0..channels { + for i in 0..num_samples { + reshaped[c * num_samples + i] = samples[i * channels + c]; + } + } + Tensor::from_vec(reshaped, (channels, num_samples), device)? + } else { + let n = samples.len(); + Tensor::from_vec(samples, (1, n), device)? + }; + + Ok((tensor, sample_rate)) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn write_wav>(path: P, audio: &Tensor, sample_rate: u32) -> anyhow::Result<()> { + let mut writer = std::fs::File::create(path)?; + write_wav_to_writer(&mut writer, audio, sample_rate) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn write_wav_to_writer( + writer: W, + audio: &Tensor, + sample_rate: u32, +) -> anyhow::Result<()> { + let shape = audio.dims(); + if shape.len() != 2 { + anyhow::bail!( + "Expected audio tensor with shape [channels, samples], got {:?}", + shape + ); + } + let channels = shape[0] as u16; + let _num_samples = shape[1]; + + let spec = WavSpec { + channels, + sample_rate, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + + let mut wav_writer = WavWriter::new(writer, spec)?; + let data = audio.to_vec2::()?; + + // Interleave channels if more than 1 + if !data.is_empty() { + for (i, _) in data[0].iter().enumerate() { + for channel_data in &data { + // Hard clamp to [-1, 1] to match Python's behavior + let val = channel_data[i].clamp(-1.0, 1.0); + let val = (val * 32767.0) as i16; + wav_writer.write_sample(val)?; + } + } + } + wav_writer.finalize()?; + Ok(()) +} + +pub fn normalize_peak(audio: &Tensor) -> anyhow::Result { + let max_abs = audio.abs()?.max_all()?.to_scalar::()?; + if max_abs > 0.0 { + Ok(audio.affine(1.0 / max_abs as f64, 0.0)?) + } else { + Ok(audio.clone()) + } +} + +// Matches Python's scipy.signal.resample_poly behavior +pub fn resample(audio: &Tensor, from_rate: u32, to_rate: u32) -> anyhow::Result { + if from_rate == to_rate { + return Ok(audio.clone()); + } + + let shape = audio.dims(); + let channels = shape[0]; + let num_samples = shape[1]; + + if num_samples == 0 { + return Ok(audio.clone()); + } + + use rubato::{FastFixedIn, Resampler}; + + // Calculate output size + let ratio = to_rate as f64 / from_rate as f64; + let _new_num_samples = (num_samples as f64 * ratio) as usize; + + // Convert candle Tensor to Vec> for rubato + // Rubato expects [channel][sample] + let audio_vec = audio.to_vec2::()?; + + // Create resampler + // FastFixedIn is synchronous and suitable for full-file resampling + let mut resampler = FastFixedIn::::new( + ratio, + 1.0, // max_resample_ratio_relative (1.0 for fixed) + rubato::PolynomialDegree::Septic, // High quality interpolation + num_samples, // block_size_in + channels, + )?; + + // Resample + let resampled_vec = resampler.process(&audio_vec, None)?; + + // Truncate or pad to exact expected length if necessary (rubato might return slightly more/less due to block/filter delay) + // But FastFixedIn with fixed block size should be mainly correct. + // We'll trust rubato's output but sanity check dimensions in the Tensor creation would be good. + // Actually, rubato might return a slightly different number of samples than naive calculation. + // Let's use whatever rubato returned. + + let out_channels = resampled_vec.len(); + let out_samples = resampled_vec[0].len(); + + // Flatten back to column-major (or whatever candle expects for from_vec) + // Candle from_vec takes a flat vector and shape. + // If we have [C][T], we need to flatten to C*T. + let mut flat_data = Vec::with_capacity(out_channels * out_samples); + for channel in resampled_vec { + flat_data.extend(channel); + } + + Ok(Tensor::from_vec( + flat_data, + (out_channels, out_samples), + audio.device(), + )?) +} + +#[deprecated(note = "Use resample() instead which provides higher quality.")] +pub fn resample_linear(audio: &Tensor, from_rate: u32, to_rate: u32) -> anyhow::Result { + resample(audio, from_rate, to_rate) +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::{Device, Tensor}; + + #[test] + fn test_normalize_peak() -> anyhow::Result<()> { + let device = Device::Cpu; + let t = Tensor::from_vec(vec![-0.5f32, 0.2, 0.5], (1, 3), &device)?; + let normalized = normalize_peak(&t)?; + let data = normalized.to_vec2::()?; + assert_eq!(data[0], vec![-1.0, 0.4, 1.0]); + Ok(()) + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn test_resample() -> anyhow::Result<()> { + let device = Device::Cpu; + // rubato works best with reasonable block sizes. + // Let's use a larger sample count to be safe. + let input_samples = 1024; + let data: Vec = (0..input_samples).map(|i| (i as f32 * 0.1).sin()).collect(); + let t = Tensor::from_vec(data, (1, input_samples), &device)?; + + // Resample 100Hz to 200Hz (Ratio 2.0) + let resampled = resample(&t, 100, 200)?; + let out_samples = resampled.dims()[1]; + + println!("Resample test: In={}, Out={}", input_samples, out_samples); + + // Expect approx double + let expected = 2048; + let diff = (out_samples as i64 - expected as i64).abs(); + + assert!( + diff <= 50, + "Output samples {} deviates too much from expected {}", + out_samples, + expected + ); + Ok(()) + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn test_wav_io() -> anyhow::Result<()> { + let device = Device::Cpu; + // Use small values to avoid clipping + // write_wav applies clamp(-1, 1) to match Python's behavior + let t = Tensor::from_vec(vec![0.0f32, 0.5, -0.5, 0.1], (1, 4), &device)?; + let path = "test_io.wav"; + write_wav(path, &t, 16000)?; + + let (read_t, sr) = read_wav(path)?; + assert_eq!(sr, 16000); + assert_eq!(read_t.dims(), t.dims()); + + // Pre-calculate expected values (clamp doesn't change values in [-1, 1]) + let expected_data: Vec = vec![0.0, 0.5, -0.5, 0.1]; + let expected = Tensor::from_vec(expected_data, (1, 4), &device)?; + + // Tolerance for 16-bit quantization (1/32768 ~= 3e-5) plus float error + let diff = (read_t - expected)?.abs()?.max_all()?.to_scalar::()?; + assert!(diff < 1e-3, "Diff was {}", diff); + + std::fs::remove_file(path)?; + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs new file mode 100644 index 00000000..481c63ac --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs @@ -0,0 +1 @@ +pub mod text; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs new file mode 100644 index 00000000..768ce7bd --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs @@ -0,0 +1,161 @@ +use candle_core::Tensor; +use candle_nn::{Embedding, Module, VarBuilder}; + +#[cfg(not(target_arch = "wasm32"))] +use sentencepiece::SentencePieceProcessor; + +#[cfg(target_arch = "wasm32")] +use tokenizers::Tokenizer; + +use anyhow::Result; +use std::path::Path; + +use std::sync::Arc; + +#[derive(Clone)] +pub struct LUTConditioner { + #[cfg(not(target_arch = "wasm32"))] + sp: Arc, + #[cfg(target_arch = "wasm32")] + tokenizer: Arc, + embed: Embedding, +} + +impl LUTConditioner { + pub fn new( + n_bins: usize, + tokenizer_path: &Path, + dim: usize, + _output_dim: usize, + vb: VarBuilder, + ) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let sp = SentencePieceProcessor::open(tokenizer_path) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer: {:?}", e))?; + + // Verify vocab size matches + let vocab_size = sp.len(); + if vocab_size != n_bins { + anyhow::bail!( + "Tokenizer vocab size {} doesn't match n_bins {}", + vocab_size, + n_bins + ); + } + + // n_bins + 1 for padding + let embed = candle_nn::embedding(n_bins + 1, dim, vb.pp("embed"))?; + + Ok(Self { + sp: Arc::new(sp), + embed, + }) + } + + #[cfg(target_arch = "wasm32")] + { + // Note: Tokenizer::from_file on WASM might have issues with local paths. + // In a real WASM app, you'd likely load from bytes. + let tokenizer = Tokenizer::from_file(tokenizer_path) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer from file: {:?}", e))?; + + // n_bins + 1 for padding + let embed = candle_nn::embedding(n_bins + 1, dim, vb.pp("embed"))?; + + Ok(Self { tokenizer, embed }) + } + } + + /// Create LUTConditioner from pre-loaded tokenizer bytes (useful for WASM) + pub fn new_from_bytes( + n_bins: usize, + tokenizer_bytes: &[u8], + dim: usize, + _output_dim: usize, + vb: VarBuilder, + ) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let sp = SentencePieceProcessor::from_serialized_proto(tokenizer_bytes) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer: {:?}", e))?; + + let vocab_size = sp.len(); + if vocab_size != n_bins { + anyhow::bail!( + "Tokenizer vocab size {} doesn't match n_bins {}", + vocab_size, + n_bins + ); + } + + let embed = candle_nn::embedding(n_bins + 1, dim, vb.pp("embed"))?; + + Ok(Self { + sp: Arc::new(sp), + embed, + }) + } + + #[cfg(target_arch = "wasm32")] + { + let tokenizer = Tokenizer::from_bytes(tokenizer_bytes) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer from bytes: {:?}", e))?; + + // n_bins + 1 for padding + let embed = candle_nn::embedding(n_bins + 1, dim, vb.pp("embed"))?; + + Ok(Self { tokenizer, embed }) + } + } + + pub fn prepare(&self, text: &str, device: &candle_core::Device) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let pieces = self + .sp + .encode(text) + .map_err(|e| anyhow::anyhow!("Failed to encode text: {:?}", e))?; + + let ids: Vec = pieces.iter().map(|p| p.id).collect(); + Ok(Tensor::from_vec(ids.clone(), (1, ids.len()), device)?) + } + + #[cfg(target_arch = "wasm32")] + { + let encoding = self + .tokenizer + .encode(text, true) + .map_err(|e| anyhow::anyhow!("Failed to encode text: {:?}", e))?; + + let ids = encoding.get_ids(); + Ok(Tensor::from_vec(ids.to_vec(), (1, ids.len()), device)?) + } + } + + pub fn forward(&self, tokens: &Tensor) -> Result { + Ok(self.embed.forward(tokens)?) + } + + /// Count tokens in a text string without creating tensors. + /// Used for accurate text splitting to avoid oversized chunks. + pub fn count_tokens(&self, text: &str) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let pieces = self + .sp + .encode(text) + .map_err(|e| anyhow::anyhow!("Failed to encode text: {:?}", e))?; + Ok(pieces.len()) + } + + #[cfg(target_arch = "wasm32")] + { + let encoding = self + .tokenizer + .encode(text, true) + .map_err(|e| anyhow::anyhow!("Failed to encode text: {:?}", e))?; + Ok(encoding.get_ids().len()) + } + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs new file mode 100644 index 00000000..70c199c1 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs @@ -0,0 +1,168 @@ +//! Configuration types for pocket-tts, matching Python's utils/config.py + +use serde::Deserialize; +use std::path::Path; + +/// Flow network configuration +#[derive(Debug, Clone, Deserialize)] +pub struct FlowConfig { + pub dim: usize, + pub depth: usize, +} + +/// Transformer configuration for FlowLM +#[derive(Debug, Clone, Deserialize)] +pub struct FlowLMTransformerConfig { + pub hidden_scale: usize, + pub max_period: usize, + pub d_model: usize, + pub num_heads: usize, + pub num_layers: usize, +} + +/// Lookup table (text conditioner) configuration +#[derive(Debug, Clone, Deserialize)] +pub struct LookupTableConfig { + pub dim: usize, + pub n_bins: usize, + pub tokenizer: String, + pub tokenizer_path: String, +} + +/// FlowLM model configuration +#[derive(Debug, Clone, Deserialize)] +pub struct FlowLMConfig { + pub dtype: String, + pub flow: FlowConfig, + pub transformer: FlowLMTransformerConfig, + pub lookup_table: LookupTableConfig, + #[serde(default)] + pub weights_path: Option, +} + +/// SEANet encoder/decoder configuration +#[derive(Debug, Clone, Deserialize)] +pub struct SEANetConfig { + pub dimension: usize, + pub channels: usize, + pub n_filters: usize, + pub n_residual_layers: usize, + pub ratios: Vec, + pub kernel_size: usize, + pub residual_kernel_size: usize, + pub last_kernel_size: usize, + pub dilation_base: usize, + pub pad_mode: String, + pub compress: usize, +} + +/// Transformer configuration for Mimi +#[derive(Debug, Clone, Deserialize)] +pub struct MimiTransformerConfig { + pub d_model: usize, + pub input_dimension: usize, + pub output_dimensions: Vec, + pub num_heads: usize, + pub num_layers: usize, + pub layer_scale: f64, + pub context: usize, + #[serde(default = "default_max_period")] + pub max_period: f64, + pub dim_feedforward: usize, +} + +fn default_max_period() -> f64 { + 10000.0 +} + +/// Quantizer configuration +#[derive(Debug, Clone, Deserialize)] +pub struct QuantizerConfig { + pub dimension: usize, + pub output_dimension: usize, +} + +/// Mimi model configuration +#[derive(Debug, Clone, Deserialize)] +pub struct MimiConfig { + pub dtype: String, + pub sample_rate: usize, + pub channels: usize, + pub frame_rate: f64, + pub seanet: SEANetConfig, + pub transformer: MimiTransformerConfig, + pub quantizer: QuantizerConfig, + #[serde(default)] + pub weights_path: Option, +} + +/// Root configuration +#[derive(Debug, Clone, Deserialize)] +pub struct Config { + pub flow_lm: FlowLMConfig, + pub mimi: MimiConfig, + #[serde(default)] + pub weights_path: Option, + #[serde(default)] + pub weights_path_without_voice_cloning: Option, +} + +/// Load configuration from a YAML file +pub fn load_config>(path: P) -> anyhow::Result { + let contents = std::fs::read_to_string(path)?; + let config: Config = serde_yaml::from_str(&contents)?; + Ok(config) +} + +/// Default generation parameters (matching Python's default_parameters.py) +pub mod defaults { + pub const TEMPERATURE: f32 = 0.7; + pub const LSD_DECODE_STEPS: usize = 1; + pub const NOISE_CLAMP: Option = None; + pub const EOS_THRESHOLD: f32 = -4.0; + pub const DEFAULT_VARIANT: &str = "b6369a24"; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn get_config_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("pocket_tts") + .join("config") + .join("b6369a24.yaml") + } + + #[test] + fn test_load_config() { + let path = get_config_path(); + if path.exists() { + let config = load_config(&path).expect("Failed to load config"); + + // Verify FlowLM config + assert_eq!(config.flow_lm.transformer.d_model, 1024); + assert_eq!(config.flow_lm.transformer.num_heads, 16); + assert_eq!(config.flow_lm.transformer.num_layers, 6); + assert_eq!(config.flow_lm.flow.dim, 512); + assert_eq!(config.flow_lm.flow.depth, 6); + assert_eq!(config.flow_lm.lookup_table.n_bins, 4000); + + // Verify Mimi config + assert_eq!(config.mimi.sample_rate, 24000); + assert_eq!(config.mimi.channels, 1); + assert!((config.mimi.frame_rate - 12.5).abs() < 1e-6); + assert_eq!(config.mimi.seanet.dimension, 512); + assert_eq!(config.mimi.seanet.ratios, vec![6, 5, 4]); + assert_eq!(config.mimi.transformer.num_layers, 2); + assert_eq!(config.mimi.quantizer.dimension, 32); + } else { + eprintln!("Config file not found at {:?}, skipping test", path); + } + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs new file mode 100644 index 00000000..7f750e03 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs @@ -0,0 +1,18 @@ +pub mod audio; +pub mod conditioners; +pub mod config; +pub mod models; +pub mod modules; +pub mod pause; +pub mod quantize; +pub mod tts_model; +pub mod voice_state; +pub mod weights; + +#[cfg(target_arch = "wasm32")] +pub mod wasm; + +pub use pause::{ParsedText, PauseMarker, parse_text_with_pauses}; +pub use quantize::{QuantizeConfig, QuantizedTensor}; +pub use tts_model::TTSModel; +pub use voice_state::ModelState; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs new file mode 100644 index 00000000..f4f5481d --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs @@ -0,0 +1,233 @@ +use crate::ModelState; +use crate::modules::rope::RotaryEmbedding; +use candle_core::{DType, Result, Tensor}; +use candle_nn::{Linear, Module, VarBuilder}; +use std::collections::HashMap; + +#[derive(Clone)] +pub struct StreamingMultiheadAttention { + embed_dim: usize, + num_heads: usize, + rope: RotaryEmbedding, + in_proj: Linear, + out_proj: Linear, + context: Option, + name: String, +} + +impl StreamingMultiheadAttention { + pub fn new( + embed_dim: usize, + num_heads: usize, + rope: RotaryEmbedding, + context: Option, + name: &str, + vb: VarBuilder, + ) -> Result { + // out_dim = embed_dim + 2 * kv_dim (GQA/MHA logic in original) + // Original code: + // out_dim = embed_dim + // num_kv = num_heads + // kv_dim = (embed_dim // num_heads) * num_kv -> so embed_dim + // out_dim += 2 * kv_dim -> so 3 * embed_dim + let in_proj = candle_nn::linear_no_bias(embed_dim, 3 * embed_dim, vb.pp("in_proj"))?; + let out_proj = candle_nn::linear_no_bias(embed_dim, embed_dim, vb.pp("out_proj"))?; + + Ok(Self { + embed_dim, + num_heads, + rope, + in_proj, + out_proj, + context, + name: name.to_string(), + }) + } + + pub fn init_state( + &self, + batch_size: usize, + _sequence_length: usize, + device: &candle_core::Device, + ) -> Result> { + let dim_per_head = self.embed_dim / self.num_heads; + let mut state = HashMap::new(); + state.insert("pos".to_string(), Tensor::zeros((), DType::U32, device)?); + + // Initial capacity: match context if windowed, otherwise reasonable default + let cap = self.context.unwrap_or(64); + state.insert( + "k_buf".to_string(), + Tensor::zeros( + (batch_size, self.num_heads, cap, dim_per_head), + DType::F32, + device, + )?, + ); + state.insert( + "v_buf".to_string(), + Tensor::zeros( + (batch_size, self.num_heads, cap, dim_per_head), + DType::F32, + device, + )?, + ); + state.insert("l".to_string(), Tensor::zeros((), DType::I64, device)?); + Ok(state) + } + + pub fn forward( + &self, + query: &Tensor, + model_state: &mut ModelState, + current_pos: usize, + current_len: usize, + ) -> Result { + let (b, t, _) = query.dims3()?; + let d = self.embed_dim / self.num_heads; + let window_size = self.context; + + // Auto-initialize state if missing + if !model_state.contains_key(&self.name) { + model_state.insert(self.name.clone(), self.init_state(b, 0, query.device())?); + } + + let module_state = model_state.get_mut(&self.name).unwrap(); + + let projected = self.in_proj.forward(query)?; + + // Reshape to (b, t, 3, h, d) + let packed = projected.reshape((b, t, 3, self.num_heads, d))?; + let mut q = packed.narrow(2, 0, 1)?.squeeze(2)?; // (b, t, h, d) + let mut k = packed.narrow(2, 1, 1)?.squeeze(2)?; // (b, t, h, d) + let mut v = packed.narrow(2, 2, 1)?.squeeze(2)?; // (b, t, h, d) + + // current_pos passed as argument + + // Apply RoPE + // RoPE expects (B, T, H, D) + (q, k) = self.rope.forward(&q, &k, current_pos)?; + + // Transpose q, k, v to (B, H, T, D) for SDPA and KV cache + q = q.transpose(1, 2)?; + k = k.transpose(1, 2)?; + v = v.transpose(1, 2)?; + + // KV Cache Management with Doubling Buffer + // We take ownership from the state to avoid clones and ensure uniqueness for slice_set + let (mut k_buf, mut v_buf, current_len) = + match (module_state.remove("k_buf"), module_state.remove("v_buf")) { + (Some(kb), Some(vb)) => (kb, vb, current_len), + _ => { + let initial_cap = window_size.unwrap_or(64); + let kb = + Tensor::zeros((b, self.num_heads, initial_cap, d), q.dtype(), q.device())?; + let vb = + Tensor::zeros((b, self.num_heads, initial_cap, d), q.dtype(), q.device())?; + (kb, vb, 0) + } + }; + + let cap = k_buf.dim(2)?; // Current capacity of the buffer + + let (kc, vc, k_buf, v_buf, current_len) = if let Some(window_size) = self.context { + // Windowed Attention (Mimi) + // If the current batch is larger than or equal to the window size, + // we can't just slice_set. We needs to handle the overflow by + // producing a concatenated KV for the current call and a truncated + // buffer for the next call. + if t >= window_size { + let kc = if current_len > 0 { + Tensor::cat(&[&k_buf.narrow(2, 0, current_len)?, &k], 2)? + } else { + k.clone() + }; + let vc = if current_len > 0 { + Tensor::cat(&[&v_buf.narrow(2, 0, current_len)?, &v], 2)? + } else { + v.clone() + }; + + // For the next state, we only keep the last window_size + let next_kb = kc + .narrow(2, kc.dim(2)? - window_size, window_size)? + .contiguous()?; + let next_vb = vc + .narrow(2, vc.dim(2)? - window_size, window_size)? + .contiguous()?; + (kc, vc, next_kb, next_vb, window_size) + } else { + // Standard streaming / small batch path + let mut current_len = current_len; + if current_len + t > window_size { + let shift = (current_len + t).saturating_sub(window_size); + let to_move = current_len.saturating_sub(shift); + if to_move > 0 { + let k_to_move = k_buf.narrow(2, shift, to_move)?; + let v_to_move = v_buf.narrow(2, shift, to_move)?; + k_buf.slice_set(&k_to_move.contiguous()?, 2, 0)?; + v_buf.slice_set(&v_to_move.contiguous()?, 2, 0)?; + current_len = to_move; + } else { + current_len = 0; + } + } + k_buf.slice_set(&k.contiguous()?, 2, current_len)?; + v_buf.slice_set(&v.contiguous()?, 2, current_len)?; + let next_len = current_len + t; + + // Get current KV for attention + let kc = k_buf.narrow(2, 0, next_len)?; + let vc = v_buf.narrow(2, 0, next_len)?; + (kc, vc, k_buf, v_buf, next_len) + } + } else { + // Linear Attention (FlowLM) with Doubling Buffer + if current_len + t > cap { + let new_cap = (current_len + t).next_power_of_two(); + let zeros_shape = (b, self.num_heads, new_cap - cap, d); + let k_zeros = Tensor::zeros(zeros_shape, q.dtype(), q.device())?; + let v_zeros = Tensor::zeros(zeros_shape, q.dtype(), q.device())?; + k_buf = Tensor::cat(&[k_buf, k_zeros], 2)?; + v_buf = Tensor::cat(&[v_buf, v_zeros], 2)?; + } + k_buf.slice_set(&k.contiguous()?, 2, current_len)?; + v_buf.slice_set(&v.contiguous()?, 2, current_len)?; + let next_len = current_len + t; + + // Get current KV for attention + let kc = k_buf.narrow(2, 0, next_len)?; + let vc = v_buf.narrow(2, 0, next_len)?; + (kc, vc, k_buf, v_buf, next_len) + }; + + // Update state + module_state.insert("k_buf".to_string(), k_buf); + module_state.insert("v_buf".to_string(), v_buf); + module_state.insert( + "l".to_string(), + Tensor::new(current_len as i64, q.device())?, + ); + module_state.insert( + "pos".to_string(), + Tensor::new((current_pos + t) as u32, q.device())?, + ); + + // Scaled dot-product attention + let scale = 1.0 / (d as f64).sqrt(); + let x = crate::modules::sdpa::sdpa( + &q, + &kc, + &vc, + scale, + true, // is_causal + self.context, + )?; + + // Transpose back to [B, T, H, D] and project out + let x = x.transpose(1, 2)?.reshape((b, t, self.embed_dim))?; + let x = self.out_proj.forward(&x)?; + + Ok(x) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs new file mode 100644 index 00000000..302f3936 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs @@ -0,0 +1,346 @@ +use crate::ModelState; +use candle_core::{DType, Result, Tensor}; +use candle_nn::{Conv1d, Conv1dConfig, ConvTranspose1d, ConvTranspose1dConfig, Module, VarBuilder}; +use std::collections::HashMap; + +#[derive(Clone)] +pub struct StreamingConv1d { + conv: Conv1d, + padding_mode: String, + stride: usize, + kernel_size: usize, + dilation: usize, + in_channels: usize, + name: String, +} + +impl StreamingConv1d { + #[allow(clippy::too_many_arguments)] + pub fn new( + in_channels: usize, + out_channels: usize, + kernel_size: usize, + stride: usize, + dilation: usize, + groups: usize, + bias: bool, + padding_mode: &str, + name: &str, + vb: VarBuilder, + ) -> Result { + let config = Conv1dConfig { + stride, + padding: 0, + dilation, + groups, + ..Default::default() + }; + let conv = if bias { + candle_nn::conv1d( + in_channels, + out_channels, + kernel_size, + config, + vb.pp("conv"), + )? + } else { + candle_nn::conv1d_no_bias( + in_channels, + out_channels, + kernel_size, + config, + vb.pp("conv"), + )? + }; + + Ok(Self { + conv, + padding_mode: padding_mode.to_string(), + stride, + kernel_size, + dilation, + in_channels, + name: name.to_string(), + }) + } + + pub fn effective_kernel_size(&self) -> usize { + (self.kernel_size - 1) * self.dilation + 1 + } + + pub fn init_state( + &self, + batch_size: usize, + _sequence_length: usize, + device: &candle_core::Device, + ) -> Result> { + let kernel = self.effective_kernel_size(); + let mut state = HashMap::new(); + if kernel > self.stride { + let previous = Tensor::zeros( + (batch_size, self.in_channels, kernel - self.stride), + DType::F32, + device, + )?; + state.insert("previous".to_string(), previous); + } + Ok(state) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + let (b, c, t) = x.dims3()?; + let s = self.stride; + if t == 0 || t % s != 0 { + return Err(candle_core::Error::Msg(format!( + "Steps must be multiple of stride {}, got {}", + s, t + ))); + } + + // Auto-initialize state if missing + if !model_state.contains_key(&self.name) { + let init = self.init_state(b, t, x.device())?; + model_state.insert(self.name.clone(), init); + } + + let module_state = model_state.get_mut(&self.name).unwrap(); + let kernel = self.effective_kernel_size(); + let pad_left = kernel.saturating_sub(s); + + if pad_left > 0 { + let previous = module_state + .remove("previous") + .ok_or_else(|| candle_core::Error::Msg("previous state not found".to_string()))?; + let is_first = step == 0; + + let x_with_padding = if is_first && self.padding_mode == "replicate" { + // Replicate the first frame for the initial padding + let first_frame = x.narrow(2, 0, 1)?; + let replicated_padding = first_frame.broadcast_as((b, c, pad_left))?; + Tensor::cat(&[replicated_padding, x.clone()], 2)? + } else { + Tensor::cat(&[previous, x.clone()], 2)? + }; + + let y = self.conv.forward(&x_with_padding)?; + + // Update previous state for next call + let total_len = x_with_padding.dims()[2]; + let new_previous = x_with_padding.narrow(2, total_len - pad_left, pad_left)?; + module_state.insert("previous".to_string(), new_previous); + + Ok(y) + } else { + self.conv.forward(x) + } + } + + pub fn weight(&self) -> &Tensor { + self.conv.weight() + } + + pub fn bias(&self) -> Option<&Tensor> { + self.conv.bias() + } +} + +#[derive(Clone)] +pub struct StreamingConvTranspose1d { + convtr: ConvTranspose1d, + stride: usize, + kernel_size: usize, + out_channels: usize, + name: String, +} + +impl StreamingConvTranspose1d { + #[allow(clippy::too_many_arguments)] + pub fn new( + in_channels: usize, + out_channels: usize, + kernel_size: usize, + stride: usize, + groups: usize, + bias: bool, + name: &str, + vb: VarBuilder, + ) -> Result { + let config = ConvTranspose1dConfig { + stride, + padding: 0, + output_padding: 0, + dilation: 1, + groups, + }; + let convtr = if bias { + candle_nn::conv_transpose1d( + in_channels, + out_channels, + kernel_size, + config, + vb.pp("convtr"), + )? + } else { + candle_nn::conv_transpose1d_no_bias( + in_channels, + out_channels, + kernel_size, + config, + vb.pp("convtr"), + )? + }; + + Ok(Self { + convtr, + stride, + kernel_size, + out_channels, + name: name.to_string(), + }) + } + + pub fn init_state( + &self, + batch_size: usize, + _sequence_length: usize, + device: &candle_core::Device, + ) -> Result> { + let mut state = HashMap::new(); + let k = self.kernel_size; + let s = self.stride; + if k > s { + let partial = + Tensor::zeros((batch_size, self.out_channels, k - s), DType::F32, device)?; + state.insert("partial".to_string(), partial); + } + Ok(state) + } + + pub fn forward( + &self, + x: &Tensor, + model_state: &mut ModelState, + _step: usize, + ) -> Result { + let (b, _c, t) = x.dims3()?; + let k = self.kernel_size; + let s = self.stride; + let trim = k.saturating_sub(s); + + // Auto-initialize state if missing + if !model_state.contains_key(&self.name) { + let init = self.init_state(b, t, x.device())?; + model_state.insert(self.name.clone(), init); + } + + let module_state = model_state.get_mut(&self.name).unwrap(); + + let mut y = self.convtr.forward(x)?; + + if trim > 0 { + if let Some(partial) = module_state.remove("partial") { + // y is (B, C, S*T + trim) + // We add partial to the start of y + let y_head = y.narrow(2, 0, trim)?; + let y_sum = (y_head + partial)?; + let y_tail = y.narrow(2, trim, y.dims()[2] - trim)?; + y = Tensor::cat(&[y_sum, y_tail], 2)?; + } + + // The last `trim` elements of `y` become the next `partial` + let len = y.dims()[2]; + let mut next_partial = y.narrow(2, len - trim, trim)?; + + // If bias exists, we need to subtract it from the partial state + // because it will be added again when we run the next forward pass. + if let Some(bias) = self.convtr.bias() { + let b_reshaped = bias.reshape((self.out_channels, 1))?; + next_partial = next_partial.broadcast_sub(&b_reshaped)?; + } + module_state.insert("partial".to_string(), next_partial); + + // The output we actually return is y MINUS the new partial tail + y = y.narrow(2, 0, len - trim)?; + } + + Ok(y) + } + + pub fn weight(&self) -> &Tensor { + self.convtr.weight() + } + + pub fn bias(&self) -> Option<&Tensor> { + self.convtr.bias() + } +} + +#[derive(Clone)] +pub struct ConvDownsample1d { + conv: StreamingConv1d, +} + +impl ConvDownsample1d { + pub fn new(stride: usize, dimension: usize, name: &str, vb: VarBuilder) -> Result { + let conv = StreamingConv1d::new( + dimension, + dimension, + 2 * stride, + stride, + 1, + 1, + false, + "replicate", + &format!("{}.conv", name), + vb.pp("conv"), + )?; + Ok(Self { conv }) + } + + pub fn init_state( + &self, + batch_size: usize, + sequence_length: usize, + device: &candle_core::Device, + ) -> Result> { + self.conv.init_state(batch_size, sequence_length, device) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.conv.forward(x, model_state, step) + } +} + +#[derive(Clone)] +pub struct ConvTrUpsample1d { + convtr: StreamingConvTranspose1d, +} + +impl ConvTrUpsample1d { + pub fn new(stride: usize, dimension: usize, name: &str, vb: VarBuilder) -> Result { + let convtr = StreamingConvTranspose1d::new( + dimension, + dimension, + 2 * stride, + stride, + dimension, + false, + &format!("{}.convtr", name), + vb.pp("convtr"), + )?; + Ok(Self { convtr }) + } + + pub fn init_state( + &self, + batch_size: usize, + sequence_length: usize, + device: &candle_core::Device, + ) -> Result> { + self.convtr.init_state(batch_size, sequence_length, device) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.convtr.forward(x, model_state, step) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs new file mode 100644 index 00000000..c6c65e1e --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs @@ -0,0 +1,418 @@ +use candle_core::{DType, Result, Tensor}; +use candle_nn::{Linear, Module, VarBuilder}; + +pub type StepFn = Box Result + Send + Sync>; + +#[derive(Clone)] +pub struct RMSNorm { + alpha: Tensor, + eps: f64, +} + +impl RMSNorm { + pub fn new(dim: usize, eps: f64, vb: VarBuilder) -> Result { + let alpha = vb.get(dim, "alpha")?; + Ok(Self { alpha, eps }) + } + + pub fn forward(&self, x: &Tensor) -> Result { + let x_dtype = x.dtype(); + // Python's "RMSNorm" uses x.var() which IS mean((x - mean)²), NOT standard RMSNorm + // We must match Python exactly for parity + let var = x.var_keepdim(candle_core::D::Minus1)?; + let inv_rms = (var + self.eps)?.sqrt()?.recip()?; + let normalized = x.broadcast_mul(&inv_rms)?; + normalized.broadcast_mul(&self.alpha)?.to_dtype(x_dtype) + } +} + +#[derive(Clone)] +pub struct LayerNorm { + inner: candle_nn::LayerNorm, +} + +impl LayerNorm { + pub fn new(dim: usize, eps: f64, affine: bool, vb: VarBuilder) -> Result { + let (weight, bias) = if affine { + let weight = vb.get(dim, "weight")?; + let bias = vb.get(dim, "bias")?; + (Some(weight), Some(bias)) + } else { + (None, None) + }; + // candle_nn::LayerNorm::new takes (weight, bias, eps) + // If not affine, we can pass None for weight/bias but candle_nn::LayerNorm expects Tensor if present. + // Actually candle_nn has a layer_norm function. + // Let's use it. + let weight = weight.unwrap_or_else(|| Tensor::ones(dim, vb.dtype(), vb.device()).unwrap()); + let bias = bias.unwrap_or_else(|| Tensor::zeros(dim, vb.dtype(), vb.device()).unwrap()); + + Ok(Self { + inner: candle_nn::LayerNorm::new(weight, bias, eps), + }) + } + + pub fn forward(&self, x: &Tensor) -> Result { + self.inner.forward(x) + } +} + +#[derive(Clone)] +pub struct LayerScale { + scale: Tensor, +} + +impl LayerScale { + pub fn new(channels: usize, _init: f32, vb: VarBuilder) -> Result { + let scale = vb.get(channels, "scale")?; + Ok(Self { scale }) + } + + pub fn forward(&self, x: &Tensor) -> Result { + x.broadcast_mul(&self.scale) + } +} + +#[derive(Clone)] +pub struct TimestepEmbedder { + lin1: Linear, + lin2: Linear, + norm: RMSNorm, + freqs: Tensor, +} + +impl TimestepEmbedder { + pub fn new( + hidden_size: usize, + frequency_embedding_size: usize, + max_period: f32, + vb: VarBuilder, + ) -> Result { + let lin1 = candle_nn::linear(frequency_embedding_size, hidden_size, vb.pp("mlp.0"))?; + let lin2 = candle_nn::linear(hidden_size, hidden_size, vb.pp("mlp.2"))?; + let norm = RMSNorm::new(hidden_size, 1e-5, vb.pp("mlp.3"))?; + + let half = frequency_embedding_size / 2; + let ds = Tensor::arange(0u32, half as u32, vb.device())?.to_dtype(DType::F32)?; + let freqs = ds + .affine(-(max_period.ln() as f64) / half as f64, 0.0)? + .exp()? + .to_dtype(vb.dtype())?; // Pre-convert to model dtype + + Ok(Self { + lin1, + lin2, + norm, + freqs, + }) + } + + pub fn forward(&self, t: &Tensor) -> Result { + // t is [B], freqs is [half] + // We need args to be [B, half] for MLP to process + let t = if t.dims().len() == 1 { + t.unsqueeze(1)? // [B] -> [B, 1] + } else { + t.clone() + }; + // args = t * freqs: [B, 1] * [half] -> [B, half] + let args = t.broadcast_mul(&self.freqs)?; + let cos = args.cos()?; + let sin = args.sin()?; + // [B, half] cat [B, half] -> [B, frequency_embedding_size] + let mut x = Tensor::cat(&[cos, sin], candle_core::D::Minus1)?; + + // Forward through MLP sequence: lin1 -> silu -> lin2 -> norm + x = self.lin1.forward(&x)?; + x = x.silu()?; + x = self.lin2.forward(&x)?; + x = self.norm.forward(&x)?; + + Ok(x) + } +} + +pub fn modulate(x: &Tensor, shift: &Tensor, scale: &Tensor) -> Result { + x.broadcast_mul(&(scale + 1.0)?)?.broadcast_add(shift) +} + +#[derive(Clone)] +pub struct ModulationParams { + pub shift: Tensor, + pub scale: Tensor, + pub gate: Option, +} + +#[derive(Clone)] +pub struct ResBlock { + in_ln: LayerNorm, + mlp_lin1: Linear, + mlp_lin2: Linear, + ada_ln_lin: Linear, +} + +impl ResBlock { + pub fn new(channels: usize, vb: VarBuilder) -> Result { + let in_ln = LayerNorm::new(channels, 1e-6, true, vb.pp("in_ln"))?; + let mlp_lin1 = candle_nn::linear(channels, channels, vb.pp("mlp.0"))?; + let mlp_lin2 = candle_nn::linear(channels, channels, vb.pp("mlp.2"))?; + let ada_ln_lin = candle_nn::linear(channels, 3 * channels, vb.pp("adaLN_modulation.1"))?; + Ok(Self { + in_ln, + mlp_lin1, + mlp_lin2, + ada_ln_lin, + }) + } + + pub fn forward(&self, x: &Tensor, modulation: &ModulationParams) -> Result { + let mut h = self.in_ln.forward(x)?; + h = modulate(&h, &modulation.shift, &modulation.scale)?; + h = self.mlp_lin1.forward(&h)?.silu()?; + h = self.mlp_lin2.forward(&h)?; + + if let Some(gate) = &modulation.gate { + x + h.broadcast_mul(gate) + } else { + x + h + } + } +} + +#[derive(Clone)] +pub struct FinalLayer { + norm_final: LayerNorm, + linear: Linear, + ada_ln_lin: Linear, +} + +impl FinalLayer { + pub fn new(model_channels: usize, out_channels: usize, vb: VarBuilder) -> Result { + let norm_final = LayerNorm::new(model_channels, 1e-6, false, vb.pp("norm_final"))?; + let linear = candle_nn::linear(model_channels, out_channels, vb.pp("linear"))?; + let ada_ln_lin = candle_nn::linear( + model_channels, + 2 * model_channels, + vb.pp("adaLN_modulation.1"), + )?; + Ok(Self { + norm_final, + linear, + ada_ln_lin, + }) + } + + pub fn forward(&self, x: &Tensor, modulation: &ModulationParams) -> Result { + let h = modulate( + &self.norm_final.forward(x)?, + &modulation.shift, + &modulation.scale, + )?; + self.linear.forward(&h) + } +} + +#[derive(Clone)] +pub struct SimpleMLPAdaLN { + time_embeds: Vec, + cond_embed: Linear, + input_proj: Linear, + res_blocks: Vec, + final_layer: FinalLayer, + num_time_conds: usize, +} + +impl SimpleMLPAdaLN { + #[allow(clippy::too_many_arguments)] + pub fn new( + in_channels: usize, + model_channels: usize, + out_channels: usize, + cond_channels: usize, + num_res_blocks: usize, + num_time_conds: usize, + max_period: f32, + vb: VarBuilder, + ) -> Result { + let mut time_embeds = Vec::new(); + for i in 0..num_time_conds { + time_embeds.push(TimestepEmbedder::new( + model_channels, + 256, + max_period, + vb.pp(format!("time_embed.{}", i)), + )?); + } + + let cond_embed = candle_nn::linear(cond_channels, model_channels, vb.pp("cond_embed"))?; + let input_proj = candle_nn::linear(in_channels, model_channels, vb.pp("input_proj"))?; + + let mut res_blocks = Vec::new(); + for i in 0..num_res_blocks { + res_blocks.push(ResBlock::new( + model_channels, + vb.pp(format!("res_blocks.{}", i)), + )?); + } + + let final_layer = FinalLayer::new(model_channels, out_channels, vb.pp("final_layer"))?; + + Ok(Self { + time_embeds, + cond_embed, + input_proj, + res_blocks, + final_layer, + num_time_conds, + }) + } + + pub fn forward(&self, c: &Tensor, s: &Tensor, t: &Tensor, x: &Tensor) -> Result { + let c_emb = self.embed_condition(c)?; + self.forward_step(x, &c_emb, s, t) + } + + pub fn embed_condition(&self, c: &Tensor) -> Result { + self.cond_embed.forward(c) + } + + pub fn forward_step( + &self, + x: &Tensor, + c_emb: &Tensor, + s: &Tensor, + t: &Tensor, + ) -> Result { + let y = (self.time_embeds[0].forward(s)? + self.time_embeds[1].forward(t)?)?; + let t_combined = (y / self.num_time_conds as f64)?; + + // Compute modulations on the fly for non-cached call + let mod_vec = self.precompute_modulations(c_emb, &t_combined)?; + self.forward_step_cached(x, &mod_vec[0]) + } +} + +impl SimpleMLPAdaLN { + pub fn compute_time_embeddings( + &self, + num_steps: usize, + device: &candle_core::Device, + dtype: DType, + ) -> Result { + let mut embeddings = Vec::with_capacity(num_steps); + for i in 0..num_steps { + let s = i as f64 / num_steps as f64; + let t = (i + 1) as f64 / num_steps as f64; + + // 1D Tensors [1] + let s_tensor = Tensor::new(&[s as f32], device)?.to_dtype(dtype)?; + let t_tensor = Tensor::new(&[t as f32], device)?.to_dtype(dtype)?; + + let t0 = self.time_embeds[0].forward(&s_tensor)?; + let t1 = self.time_embeds[1].forward(&t_tensor)?; + let t_combined = ((t0 + t1)? / self.num_time_conds as f64)?; + embeddings.push(t_combined); + } + // stack of [1, 512] -> [num_steps, 1, 512] + // squeeze(1) -> [num_steps, 512] + Tensor::stack(&embeddings, 0)?.squeeze(1) + } + + #[allow(clippy::needless_range_loop)] + pub fn precompute_modulations( + &self, + c_emb: &Tensor, + time_embeddings: &Tensor, + ) -> Result>> { + // c_emb: [1, 512], time_embeddings: [8, 512] + let num_steps = time_embeddings.dim(0)?; + let y = time_embeddings.broadcast_add(c_emb)?; // [8, 512] + let y_silu = y.silu()?; + + let mut all_step_modulations = + vec![Vec::with_capacity(self.res_blocks.len() + 1); num_steps]; + + // ResBlocks + for block in &self.res_blocks { + let mod_batch = block.ada_ln_lin.forward(&y_silu)?; // [8, 1536] + let dim = mod_batch.dim(candle_core::D::Minus1)? / 3; + + for s in 0..num_steps { + let modulation = mod_batch.narrow(0, s, 1)?; // [1, 1536] + let shift = modulation.narrow(candle_core::D::Minus1, 0, dim)?; + let scale = modulation.narrow(candle_core::D::Minus1, dim, dim)?; + let gate = modulation.narrow(candle_core::D::Minus1, 2 * dim, dim)?; + all_step_modulations[s].push(ModulationParams { + shift, + scale, + gate: Some(gate), + }); + } + } + + // Final layer + let mod_batch = self.final_layer.ada_ln_lin.forward(&y_silu)?; // [8, 1024] + let dim = mod_batch.dim(candle_core::D::Minus1)? / 2; + for s in 0..num_steps { + let modulation = mod_batch.narrow(0, s, 1)?; // [1, 1024] + let shift = modulation.narrow(candle_core::D::Minus1, 0, dim)?; + let scale = modulation.narrow(candle_core::D::Minus1, dim, dim)?; + all_step_modulations[s].push(ModulationParams { + shift, + scale, + gate: None, + }); + } + + Ok(all_step_modulations) + } + + pub fn forward_step_cached( + &self, + x: &Tensor, + modulations: &[ModulationParams], + ) -> Result { + let mut x = self.input_proj.forward(x)?; + + for (i, block) in self.res_blocks.iter().enumerate() { + x = block.forward(&x, &modulations[i])?; + } + + self.final_layer + .forward(&x, &modulations[self.res_blocks.len()]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::{Device, Tensor}; + use candle_nn::VarBuilder; + use std::collections::HashMap; + + #[test] + fn test_rmsnorm_parity() -> Result<()> { + let device = Device::Cpu; + let mut map = HashMap::new(); + map.insert( + "alpha".to_string(), + Tensor::ones((4,), DType::F32, &device)?, + ); + let vb = VarBuilder::from_tensors(map, DType::F32, &device); + let norm = RMSNorm::new(4, 1e-5, vb)?; + + // Input: [[1.0, 2.0, 3.0, 4.0]] + let x = Tensor::new(&[[1.0f32, 2.0, 3.0, 4.0]], &device)?; + let y = norm.forward(&x)?; + + // Python's "RMSNorm" uses x.var() = mean((x - mean)²) + // mean = 2.5, var = ((1-2.5)² + (2-2.5)² + (3-2.5)² + (4-2.5)²) / 3 = 1.6667 (Bessel) + // rsqrt(1.6667 + 1e-5) ≈ 0.7746 + // output = x * 0.7746 = [0.7746, 1.5492, 2.3238, 3.0984] + let expected = Tensor::new(&[[0.7746f32, 1.5492, 2.3238, 3.0984]], &device)?; + + let diff = (y - expected)?.abs()?.max_all()?.to_scalar::()?; + assert!(diff < 1e-3, "RMSNorm parity failed: diff={}", diff); + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs new file mode 100644 index 00000000..718234c4 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs @@ -0,0 +1,5 @@ +pub mod attention; +pub mod conv; +pub mod mlp; +pub mod rope; +pub mod sdpa; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs new file mode 100644 index 00000000..dd635b75 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs @@ -0,0 +1,79 @@ +use candle_core::{DType, Result, Tensor}; + +#[derive(Debug, Clone)] +pub struct RotaryEmbedding { + inv_freq: Tensor, +} + +impl RotaryEmbedding { + pub fn new(max_period: f32, head_dim: usize, device: &candle_core::Device) -> Result { + let d = head_dim / 2; + let ds = Tensor::arange(0u32, d as u32, device)?.to_dtype(DType::F32)?; + let inv_freq = ds + .affine((-max_period.ln() * 2.0 / head_dim as f32) as f64, 0.0)? + .exp()?; + Ok(Self { inv_freq }) + } + + pub fn forward(&self, q: &Tensor, k: &Tensor, offset: usize) -> Result<(Tensor, Tensor)> { + let (b, t, h, d_full) = q.dims4()?; + let (_bk, _tk, hk, _dk) = k.dims4()?; + let d = d_full / 2; + let dev = q.device(); + + // ts = (arange(T) + offset).view(-1, 1, 1) + let ts = if t == 1 { + Tensor::new(&[offset as f32], dev)? + } else { + Tensor::arange(0u32, t as u32, dev)? + .to_dtype(DType::F32)? + .affine(1.0, offset as f64)? + } + .reshape((t, 1, 1))?; + + // freqs * ts -> shape (t, 1, d) + let freqs_ts = self.inv_freq.reshape((1, 1, d))?.broadcast_mul(&ts)?; + let cos = freqs_ts.cos()?; + let sin = freqs_ts.sin()?; + + // Reshape q and k to (b, t, h, d, 2) + let q = q.reshape((b, t, h, d, 2))?; + let k = k.reshape((b, t, hk, d, 2))?; + + let qr = q.narrow(4, 0, 1)?.squeeze(4)?; + let qi = q.narrow(4, 1, 1)?.squeeze(4)?; + let kr = k.narrow(4, 0, 1)?.squeeze(4)?; + let ki = k.narrow(4, 1, 1)?.squeeze(4)?; + + // qor = qr * cos - qi * sin + // qoi = qr * sin + qi * cos + let qor = (qr.broadcast_mul(&cos)? - qi.broadcast_mul(&sin)?)?; + let qoi = (qr.broadcast_mul(&sin)? + qi.broadcast_mul(&cos)?)?; + + let kor = (kr.broadcast_mul(&cos)? - ki.broadcast_mul(&sin)?)?; + let koi = (kr.broadcast_mul(&sin)? + ki.broadcast_mul(&cos)?)?; + + let qo = Tensor::stack(&[qor, qoi], 4)?.reshape((b, t, h, d_full))?; + let ko = Tensor::stack(&[kor, koi], 4)?.reshape((b, t, hk, d_full))?; + + Ok((qo, ko)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::{Device, Tensor}; + + #[test] + fn test_rope_shape() -> Result<()> { + let device = Device::Cpu; + let q = Tensor::zeros((1, 10, 4, 32), DType::F32, &device)?; + let k = Tensor::zeros((1, 10, 4, 32), DType::F32, &device)?; + let rope = RotaryEmbedding::new(10000.0, 32, &device)?; + let (qo, ko) = rope.forward(&q, &k, 0)?; + assert_eq!(qo.dims(), &[1, 10, 4, 32]); + assert_eq!(ko.dims(), &[1, 10, 4, 32]); + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs new file mode 100644 index 00000000..605ec976 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs @@ -0,0 +1,312 @@ +use candle_core::{D, Result, Tensor}; + +/// Memory-efficient Scaled Dot Product Attention +/// +/// Computes `softmax(Q @ K.T / sqrt(d) + mask) @ V` using tiling on the query dimension +/// to avoid materializing the full N x N attention matrix. +/// +/// # Arguments +/// * `q` - Query tensor of shape [Batch, Heads, Q_Len, Dim] +/// * `k` - Key tensor of shape [Batch, Heads, KV_Len, Dim] +/// * `v` - Value tensor of shape [Batch, Heads, KV_Len, Dim] +/// * `scale` - Scaling factor (usually 1 / sqrt(dim)) +/// * `is_causal` - Whether to apply causal masking +/// * `context_window` - Optional context window size for local attention +/// +/// # Returns +/// * Tensor of shape [Batch, Heads, Q_Len, Dim] +#[inline] +pub fn sdpa( + q: &Tensor, + k: &Tensor, + v: &Tensor, + scale: f64, + is_causal: bool, + context_window: Option, +) -> Result { + let q = q.contiguous()?; + let (_b, _h, q_len, _dim) = q.dims4()?; + let kv_len = k.dims()[2]; + + // Adaptive strategy: + // For small Q (decoding, chunked prefill), tiling overhead hurts performance. + // Use naive implementation if Q is small enough. + // Benchmark showed naive is faster for Q=1 and comparable for Q=50/64. + const TILING_THRESHOLD: usize = 512; + + let k_t = k.transpose(2, 3)?; // [B, H, D, S] + + if q_len < TILING_THRESHOLD { + // Naive path (no tiling) + let scores = (q.matmul(&k_t)? * scale)?; + + // Generate mask + // Optimization: If q_len (t) is 1 and it's causal, everything is visible, + // so we can skip mask generation. + // Even with a context window, the attention.rs logic now prunes the KV cache + // to that window, so we can skip the windowed mask here as well. + let scores = if is_causal || context_window.is_some() { + let mask = generate_mask_chunk( + 0, + q_len, + kv_len, + q_len, + is_causal, + context_window, + q.device(), + )?; + scores.broadcast_add(&mask)? + } else { + scores + }; + + let probs = candle_nn::ops::softmax(&scores, D::Minus1)?; + return probs.matmul(v); + } + + // Tiled path for large Q + // Always tile if sequence length is significant to avoid N^2 mask allocation + let block_size = 128; // Tiling size for Q dimension. + + let mut outputs = Vec::new(); + + for start in (0..q_len).step_by(block_size) { + let end = std::cmp::min(start + block_size, q_len); + let len = end - start; + + // Slice Q: [B, H, Block, D] + let q_chunk = q.narrow(2, start, len)?; + + // Compute scores: [B, H, Block, S] = [B, H, Block, D] @ [B, H, D, S] + let scores = (q_chunk.matmul(&k_t)? * scale)?; + + // Generate and apply mask on-the-fly for this chunk + let scores = if is_causal || context_window.is_some() { + let mask_chunk = generate_mask_chunk( + start, + len, + kv_len, + q_len, + is_causal, + context_window, + q.device(), + )?; + scores.broadcast_add(&mask_chunk)? + } else { + scores + }; + + // Softmax + let probs = candle_nn::ops::softmax(&scores, D::Minus1)?; + + // Output chunk: [B, H, Block, D] = [B, H, Block, S] @ [B, H, S, D] + let out_chunk = probs.matmul(v)?; + + outputs.push(out_chunk); + } + + // Cat along Q dimension (dim 2) + Tensor::cat(&outputs, 2) +} + +/// Helper to generate a mask chunk for a specific query range using vectorized operations +fn generate_mask_chunk( + start_q: usize, + num_q: usize, + k_len: usize, + total_q_len: usize, + is_causal: bool, + context_window: Option, + device: &candle_core::Device, +) -> Result { + let shift = k_len.saturating_sub(total_q_len); + + // pos_q: [num_q, 1] + let pos_q = (Tensor::arange(0u32, num_q as u32, device)? + .to_dtype(candle_core::DType::F32)? + .affine(1.0, (start_q + shift) as f64)? + .reshape((num_q, 1)))?; + + // pos_k: [1, k_len] + let pos_k = Tensor::arange(0u32, k_len as u32, device)? + .to_dtype(candle_core::DType::F32)? + .reshape((1, k_len))?; + + let mut mask = Tensor::zeros((num_q, k_len), candle_core::DType::F32, device)?; + + if is_causal { + let is_future = pos_k.broadcast_gt(&pos_q)?; + mask = is_future.where_cond( + &Tensor::full(f32::NEG_INFINITY, (num_q, k_len), device)?, + &mask, + )?; + } + + if let Some(ctx) = context_window { + let limit = pos_q.broadcast_sub(&Tensor::full(ctx as f32, (num_q, 1), device)?)?; + let is_out = pos_k.broadcast_le(&limit)?; + mask = is_out.where_cond( + &Tensor::full(f32::NEG_INFINITY, (num_q, k_len), device)?, + &mask, + )?; + } + + mask.reshape((1, 1, num_q, k_len)) +} + +/// Chunked version of SDPA that accepts a list of Key/Value pointers +/// to avoid concatenating the full KV cache. +pub fn sdpa_chunked( + q: &Tensor, + k_chunks: &[Tensor], + v_chunks: &[Tensor], + scale: f64, + is_causal: bool, + context_window: Option, +) -> Result { + if k_chunks.is_empty() { + let (_b, h, _q, d) = q.dims4()?; + return Tensor::zeros((_b, h, _q, d), q.dtype(), q.device()); + } + + let device = q.device(); + let dtype = q.dtype(); + let q = q.contiguous()?; + let (b, h, q_len, d) = q.dims4()?; + + // Fast path for single chunk + if k_chunks.len() == 1 { + let k_t = k_chunks[0].transpose(2, 3)?; + let scores = (q.matmul(&k_t)? * scale)?; + + let masked_scores = if is_causal || context_window.is_some() { + if q_len == 1 && context_window.is_none() { + scores + } else { + let mask = generate_mask_chunk( + 0, + q_len, + k_chunks[0].dims()[2], + q_len, + is_causal, + context_window, + device, + )?; + scores.broadcast_add(&mask)? + } + } else { + scores + }; + + let probs = candle_nn::ops::softmax(&masked_scores, D::Minus1)?; + return probs.matmul(&v_chunks[0]); + } + + // 1. Compute scores against all K chunks + let mut score_chunks = Vec::with_capacity(k_chunks.len()); + let mut total_kv_len = 0; + + for k_chunk in k_chunks { + total_kv_len += k_chunk.dims()[2]; + let k_t = k_chunk.transpose(2, 3)?; + let score_chunk = (q.matmul(&k_t)? * scale)?; + score_chunks.push(score_chunk); + } + + // 2. Concatenate scores to apply global Softmax + let all_scores = Tensor::cat(&score_chunks, 3)?; + + // 3. Apply masking + let masked_scores = if is_causal || context_window.is_some() { + if q_len == 1 && context_window.is_none() { + all_scores + } else { + let mask = generate_mask_chunk( + 0, + q_len, + total_kv_len, + q_len, + is_causal, + context_window, + device, + )?; + all_scores.broadcast_add(&mask)? + } + } else { + all_scores + }; + + // 4. Softmax + let probs = candle_nn::ops::softmax(&masked_scores, D::Minus1)?; + + // 5. Compute Weighted Sum: Probs @ V + let mut output = Tensor::zeros((b, h, q_len, d), dtype, device)?; + + let mut offset = 0; + for v_chunk in v_chunks { + let chunk_len = v_chunk.dims()[2]; + let probs_chunk = probs.narrow(3, offset, chunk_len)?; + let out_chunk = probs_chunk.matmul(v_chunk)?; + output = (output + out_chunk)?; + offset += chunk_len; + } + + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[test] + fn test_generate_mask_chunk_causal() -> Result<()> { + let device = Device::Cpu; + // q_len = 1, k_len = 5, total_q = 1 + // shift = 4. pos_q = 4. pos_k = 0..5. + // is_future = j > 4. No futures. + let mask = generate_mask_chunk(0, 1, 5, 1, true, None, &device)?; + let mask_data = mask.flatten_all()?.to_vec1::()?; + assert_eq!(mask_data, vec![0.0, 0.0, 0.0, 0.0, 0.0]); + + // q_len = 3, k_len = 3, total_q = 3 (prefill) + // shift = 0. pos_q = 0..3. pos_k = 0..3. + let mask = generate_mask_chunk(0, 3, 3, 3, true, None, &device)?; + let mask_data = mask.reshape((3, 3))?.to_vec2::()?; + // Row 0: pos_q=0. k=0 ok, k=1 future, k=2 future + assert_eq!( + mask_data[0], + vec![0.0, f32::NEG_INFINITY, f32::NEG_INFINITY] + ); + // Row 1: pos_q=1. k=0,1 ok, k=2 future + assert_eq!(mask_data[1], vec![0.0, 0.0, f32::NEG_INFINITY]); + // Row 2: pos_q=2. k=0,1,2 ok + assert_eq!(mask_data[2], vec![0.0, 0.0, 0.0]); + + Ok(()) + } + + #[test] + fn test_generate_mask_chunk_window() -> Result<()> { + let device = Device::Cpu; + // ctx = 2. pos_q = 5. k_len = 6. total_q = 1. + // shift = 5. pos_q = 5. pos_k = 0..6. + // limit = 5 - 2 = 3. + // is_out = j <= 3 -> 0,1,2,3 masked. 4,5 ok. + let mask = generate_mask_chunk(0, 1, 6, 1, false, Some(2), &device)?; + let mask_data = mask.flatten_all()?.to_vec1::()?; + assert_eq!( + mask_data, + vec![ + f32::NEG_INFINITY, + f32::NEG_INFINITY, + f32::NEG_INFINITY, + f32::NEG_INFINITY, + 0.0, + 0.0 + ] + ); + + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs new file mode 100644 index 00000000..94ac2b01 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs @@ -0,0 +1,249 @@ +//! Pause/silence handling for text-to-speech +//! +//! Supports: +//! - Explicit pause markers: `[pause:Xms]` or `[pause:Xs]` +//! - Natural pauses from punctuation: `...`, `,` + +use regex::Regex; +use std::sync::LazyLock; + +/// Pause marker found in text +#[derive(Debug, Clone, PartialEq)] +pub struct PauseMarker { + /// Original text that was matched + pub original: String, + /// Duration in milliseconds + pub duration_ms: u32, + /// Position in the original text (byte offset) + pub position: usize, +} + +/// Default pause durations (in milliseconds) for punctuation +pub mod defaults { + /// Ellipsis "..." pause duration + pub const ELLIPSIS_MS: u32 = 500; + /// Comma pause duration + pub const COMMA_MS: u32 = 200; + /// Period/sentence end pause duration + pub const PERIOD_MS: u32 = 400; + /// Semicolon pause duration + pub const SEMICOLON_MS: u32 = 300; +} + +// Regex patterns for pause parsing +static EXPLICIT_PAUSE_REGEX: LazyLock = LazyLock::new(|| { + // Matches [pause:500ms] or [pause:1s] or [pause:1.5s] + Regex::new(r"\[pause:(\d+(?:\.\d+)?)(ms|s)\]").unwrap() +}); + +static ELLIPSIS_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"\.{3,}").unwrap()); + +/// Parse explicit pause markers from text +/// +/// # Example +/// ``` +/// use pocket_tts::pause::parse_explicit_pauses; +/// +/// let pauses = parse_explicit_pauses("Hello [pause:500ms] world [pause:1s] done"); +/// assert_eq!(pauses.len(), 2); +/// assert_eq!(pauses[0].duration_ms, 500); +/// assert_eq!(pauses[1].duration_ms, 1000); +/// ``` +pub fn parse_explicit_pauses(text: &str) -> Vec { + EXPLICIT_PAUSE_REGEX + .captures_iter(text) + .filter_map(|cap| { + let full_match = cap.get(0)?; + let value: f64 = cap.get(1)?.as_str().parse().ok()?; + let unit = cap.get(2)?.as_str(); + + let duration_ms = match unit { + "ms" => value as u32, + "s" => (value * 1000.0) as u32, + _ => return None, + }; + + Some(PauseMarker { + original: full_match.as_str().to_string(), + duration_ms, + position: full_match.start(), + }) + }) + .collect() +} + +/// Parse natural pauses from punctuation +pub fn parse_natural_pauses(text: &str) -> Vec { + let mut pauses = Vec::new(); + + // Find ellipses + for cap in ELLIPSIS_REGEX.find_iter(text) { + pauses.push(PauseMarker { + original: cap.as_str().to_string(), + duration_ms: defaults::ELLIPSIS_MS, + position: cap.start(), + }); + } + + // Find commas (but not inside numbers like "1,000") + for (i, c) in text.char_indices() { + if c == ',' { + // Check if it's not surrounded by digits + let prev_is_digit = + i > 0 && text[..i].chars().last().is_some_and(|c| c.is_ascii_digit()); + let next_is_digit = text[(i + 1)..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()); + + if !prev_is_digit || !next_is_digit { + pauses.push(PauseMarker { + original: ",".to_string(), + duration_ms: defaults::COMMA_MS, + position: i, + }); + } + } + } + + // Sort by position + pauses.sort_by_key(|p| p.position); + pauses +} + +/// Remove pause markers from text, returning clean text for TTS +pub fn strip_pause_markers(text: &str) -> String { + EXPLICIT_PAUSE_REGEX.replace_all(text, " ").to_string() +} + +/// Parsed text with pause information +#[derive(Debug, Clone)] +pub struct ParsedText { + /// Text with pause markers removed + pub clean_text: String, + /// All pause markers (explicit + natural) with adjusted positions + pub pauses: Vec, +} + +/// Parse text for all pause markers (explicit and natural) +pub fn parse_text_with_pauses(text: &str) -> ParsedText { + // First, find explicit pauses in original text + let mut all_pauses = parse_explicit_pauses(text); + + // Strip explicit markers to get clean text + let clean_text = strip_pause_markers(text); + + // Find natural pauses in clean text + let natural_pauses = parse_natural_pauses(&clean_text); + + // Note: positions in all_pauses are relative to original text + // We need to adjust them to the clean text + // For simplicity, we'll recalculate based on clean text positions + + // Clear and rebuild with correct positions + all_pauses.clear(); + all_pauses.extend(natural_pauses); + + // Re-parse explicit pauses and calculate where they would be in clean text + let mut offset = 0; + for cap in EXPLICIT_PAUSE_REGEX.captures_iter(text) { + let full_match = cap.get(0).unwrap(); + let original_pos = full_match.start(); + let adjusted_pos = original_pos.saturating_sub(offset); + let value: f64 = cap.get(1).unwrap().as_str().parse().unwrap_or(0.0); + let unit = cap.get(2).unwrap().as_str(); + + let duration_ms = match unit { + "ms" => value as u32, + "s" => (value * 1000.0) as u32, + _ => 0, + }; + + if duration_ms > 0 { + all_pauses.push(PauseMarker { + original: full_match.as_str().to_string(), + duration_ms, + position: adjusted_pos, + }); + } + + offset += full_match.len() - 1; // -1 for the space we replace with + } + + // Sort by position + all_pauses.sort_by_key(|p| p.position); + + ParsedText { + clean_text, + pauses: all_pauses, + } +} + +/// Calculate the number of silence samples for a given duration +pub fn silence_samples(duration_ms: u32, sample_rate: u32) -> usize { + ((duration_ms as u64 * sample_rate as u64) / 1000) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_explicit_pause_ms() { + let pauses = parse_explicit_pauses("Hello [pause:500ms] world"); + assert_eq!(pauses.len(), 1); + assert_eq!(pauses[0].duration_ms, 500); + assert_eq!(pauses[0].original, "[pause:500ms]"); + } + + #[test] + fn test_parse_explicit_pause_seconds() { + let pauses = parse_explicit_pauses("Test [pause:1s] and [pause:1.5s]"); + assert_eq!(pauses.len(), 2); + assert_eq!(pauses[0].duration_ms, 1000); + assert_eq!(pauses[1].duration_ms, 1500); + } + + #[test] + fn test_parse_ellipsis() { + let pauses = parse_natural_pauses("Hello... world"); + assert_eq!(pauses.len(), 1); + assert_eq!(pauses[0].duration_ms, defaults::ELLIPSIS_MS); + } + + #[test] + fn test_parse_comma() { + let pauses = parse_natural_pauses("Hello, world"); + assert_eq!(pauses.len(), 1); + assert_eq!(pauses[0].duration_ms, defaults::COMMA_MS); + } + + #[test] + fn test_comma_in_number_ignored() { + let pauses = parse_natural_pauses("That costs 1,000 dollars"); + // The comma in 1,000 should be ignored + assert_eq!(pauses.len(), 0); + } + + #[test] + fn test_strip_pause_markers() { + let clean = strip_pause_markers("Hello [pause:500ms] world [pause:1s] done"); + assert_eq!(clean, "Hello world done"); + } + + #[test] + fn test_parse_text_with_pauses() { + let parsed = parse_text_with_pauses("Hello... [pause:500ms] world, done"); + assert_eq!(parsed.clean_text, "Hello... world, done"); + // Should have: ellipsis, explicit pause, comma + assert_eq!(parsed.pauses.len(), 3); + } + + #[test] + fn test_silence_samples() { + // 500ms at 24kHz = 12000 samples + assert_eq!(silence_samples(500, 24000), 12000); + // 1s at 24kHz = 24000 samples + assert_eq!(silence_samples(1000, 24000), 24000); + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs new file mode 100644 index 00000000..8c663838 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs @@ -0,0 +1,219 @@ +//! Quantization support for Pocket TTS +//! +//! This module provides quantization utilities for reduced memory footprint +//! and potentially faster inference on CPU. +//! +//! Note: Candle doesn't natively support int8 tensor operations, so we use +//! a simulated quantization approach that stores quantized values as f32 but +//! represents them using only 256 discrete levels (mimicking int8 range). +//! +//! For true int8 acceleration, use GGML/GGUF format weights with candle-quantized. + +use anyhow::Result; +use candle_core::{DType, Tensor}; +use std::collections::HashMap; + +/// Quantization configuration +#[derive(Debug, Clone)] +pub struct QuantizeConfig { + /// Layers to skip quantization (keep in full precision) + pub skip_layers: Vec, + /// Minimum tensor size to quantize (smaller tensors stay full precision) + pub min_size: usize, + /// Number of quantization levels (256 for int8-like behavior) + pub num_levels: usize, +} + +impl Default for QuantizeConfig { + fn default() -> Self { + Self { + skip_layers: vec![ + // Embeddings often benefit from staying in full precision + "embed".to_string(), + "lut".to_string(), + // Final output projections + "out_proj".to_string(), + "eos_head".to_string(), + ], + min_size: 1024, // Don't bother quantizing small tensors + num_levels: 256, // int8-like + } + } +} + +/// Quantized tensor wrapper that stores scale for dequantization +/// +/// This uses simulated quantization - values are stored as f32 but discretized +/// to num_levels distinct values (256 for int8-equivalent). +#[derive(Debug, Clone)] +pub struct QuantizedTensor { + /// Quantized data (stored as f32 but with discrete values) + pub data: Tensor, + /// Scale factor for dequantization + pub scale: f32, + /// Zero point for asymmetric quantization + pub zero_point: f32, + /// Number of quantization levels used + pub num_levels: usize, +} + +impl QuantizedTensor { + /// Quantize a tensor using symmetric per-tensor quantization + /// + /// This discretizes values to num_levels distinct values centered around 0, + /// simulating int8 quantization behavior while using f32 storage. + pub fn quantize(tensor: &Tensor, num_levels: usize) -> Result { + // Convert to f32 if needed + let tensor_f32 = tensor.to_dtype(DType::F32)?; + + // Find max absolute value for symmetric quantization + let abs_max = tensor_f32.abs()?.max_all()?.to_scalar::()?; + + // Calculate scale (half the range for symmetric) + let half_levels = (num_levels / 2) as f32; + let scale = if abs_max > 0.0 { + abs_max / (half_levels - 1.0) + } else { + 1.0 + }; + + // Quantize: q = round(x / scale), then dequantize back: x' = q * scale + // This simulates quantization while staying in f32 + let scale_tensor = Tensor::new(&[scale], tensor.device())?; + let quantized = tensor_f32.broadcast_div(&scale_tensor)?; + let quantized = quantized.round()?; + let clamped = quantized.clamp(-(half_levels - 1.0) as f64, (half_levels - 1.0) as f64)?; + let data = clamped.broadcast_mul(&scale_tensor)?; + + Ok(Self { + data, + scale, + zero_point: 0.0, // Symmetric quantization + num_levels, + }) + } + + /// Get the quantized tensor data + pub fn data(&self) -> &Tensor { + &self.data + } + + /// Get the scale value + pub fn scale(&self) -> f32 { + self.scale + } + + /// Get theoretical memory savings ratio compared to f32 + /// (In practice, data is still stored as f32, but this shows potential savings) + pub fn theoretical_memory_savings(&self) -> f32 { + match self.num_levels { + 256 => 4.0, // int8 would be 4x smaller than f32 + 65536 => 2.0, // int16 would be 2x smaller + _ => 1.0, + } + } +} + +/// Check if a layer name should skip quantization +fn should_skip_layer(name: &str, config: &QuantizeConfig) -> bool { + config.skip_layers.iter().any(|skip| name.contains(skip)) +} + +/// Quantize a collection of weights according to config +/// +/// Returns quantized weights. Layers in skip_layers or smaller than min_size +/// are returned unchanged. +pub fn quantize_weights( + weights: &HashMap, + config: &QuantizeConfig, +) -> Result> { + let mut quantized = HashMap::new(); + + for (name, tensor) in weights { + // Skip small tensors and excluded layers + if tensor.elem_count() < config.min_size || should_skip_layer(name, config) { + // Keep unquantized (scale=1, no discretization) + quantized.insert( + name.clone(), + QuantizedTensor { + data: tensor.clone(), + scale: 1.0, + zero_point: 0.0, + num_levels: 0, // Indicates not actually quantized + }, + ); + } else { + quantized.insert( + name.clone(), + QuantizedTensor::quantize(tensor, config.num_levels)?, + ); + } + } + + Ok(quantized) +} + +/// Calculate signal-to-noise ratio between original and quantized tensors +pub fn calculate_snr(original: &Tensor, quantized: &Tensor) -> Result { + let original_f32 = original.to_dtype(DType::F32)?; + let quantized_f32 = quantized.to_dtype(DType::F32)?; + + // SNR = 10 * log10(signal_power / noise_power) + let signal_power = original_f32.sqr()?.mean_all()?.to_scalar::()?; + let noise = (&original_f32 - &quantized_f32)?; + let noise_power = noise.sqr()?.mean_all()?.to_scalar::()?; + + if noise_power <= 0.0 { + return Ok(f32::INFINITY); // Perfect reconstruction + } + + Ok(10.0 * (signal_power / noise_power).log10()) +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[test] + fn test_quantize_tensor() { + let device = Device::Cpu; + let tensor = Tensor::new(&[1.0f32, 2.0, -3.0, 4.5, -2.1], &device).unwrap(); + + let quantized = QuantizedTensor::quantize(&tensor, 256).unwrap(); + + // Check SNR - should be good for int8-like quantization + let snr = calculate_snr(&tensor, &quantized.data).unwrap(); + assert!(snr > 30.0, "SNR {} is too low", snr); + } + + #[test] + fn test_quantize_large_tensor() { + let device = Device::Cpu; + // Create a larger tensor with varying values + let values: Vec = (0..10000).map(|i| (i as f32 * 0.01).sin() * 10.0).collect(); + let tensor = Tensor::new(&values[..], &device).unwrap(); + + let quantized = QuantizedTensor::quantize(&tensor, 256).unwrap(); + let snr = calculate_snr(&tensor, &quantized.data).unwrap(); + + // For larger tensors with varied values, expect good SNR + assert!(snr > 30.0, "SNR {} is too low", snr); + } + + #[test] + fn test_quantize_config_skip_layers() { + let config = QuantizeConfig::default(); + assert!(should_skip_layer("model.embed_tokens", &config)); + assert!(should_skip_layer("decoder.out_proj", &config)); + assert!(!should_skip_layer("encoder.layers.0.linear", &config)); + } + + #[test] + fn test_theoretical_savings() { + let device = Device::Cpu; + let tensor = Tensor::new(&[1.0f32, 2.0, 3.0], &device).unwrap(); + let quantized = QuantizedTensor::quantize(&tensor, 256).unwrap(); + assert_eq!(quantized.theoretical_memory_savings(), 4.0); + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs new file mode 100644 index 00000000..1048ddcd --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs @@ -0,0 +1,1237 @@ +//! Main TTSModel struct - orchestrates the TTS pipeline +//! +//! This is the high-level API for text-to-speech generation, +//! matching Python's `pocket_tts/models/tts_model.py`. + +use crate::ModelState; +use crate::conditioners::text::LUTConditioner; +use crate::config::{Config, defaults, load_config}; +use crate::models::flow_lm::FlowLMModel; +use crate::models::mimi::MimiModel; +use crate::models::seanet::{SEANetDecoder, SEANetEncoder}; +use crate::models::transformer::{ProjectedTransformer, StreamingTransformer}; +use crate::modules::mlp::SimpleMLPAdaLN; +use crate::voice_state::{increment_steps, init_states}; + +use anyhow::Result; +use candle_core::{DType, Device, Tensor}; +use candle_nn::VarBuilder; + +/// Main TTS model that orchestrates the entire pipeline +#[derive(Clone)] +pub struct TTSModel { + /// Flow language model for latent generation + pub flow_lm: FlowLMModel, + /// Mimi neural audio codec + pub mimi: MimiModel, + /// Text conditioner (tokenizer + embeddings) + pub conditioner: LUTConditioner, + /// Speaker projection weight for voice cloning + pub speaker_proj_weight: Tensor, + /// Generation temperature + pub temp: f32, + /// Number of LSD decode steps + pub lsd_decode_steps: usize, + /// End-of-sequence threshold + pub eos_threshold: f32, + pub noise_clamp: Option, + /// Sample rate + pub sample_rate: usize, + /// Model dimension + pub dim: usize, + /// Latent dimension + pub ldim: usize, + /// Device + pub device: Device, +} + +impl TTSModel { + /// Load a pre-trained TTS model from HuggingFace + /// + /// # Arguments + /// * `variant` - Model variant (e.g., "b6369a24") + /// + /// # Returns + /// Fully initialized TTSModel ready for generation + pub fn load(variant: &str) -> Result { + Self::load_with_params( + variant, + defaults::TEMPERATURE, + defaults::LSD_DECODE_STEPS, + defaults::EOS_THRESHOLD, + ) + } + + /// Load with custom generation parameters + pub fn load_with_params( + variant: &str, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + ) -> Result { + Self::load_with_params_device( + variant, + temp, + lsd_decode_steps, + eos_threshold, + None, + &Device::Cpu, + ) + } + + /// Load with custom generation parameters and specific device + pub fn load_with_params_device( + variant: &str, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + noise_clamp: Option, + device: &Device, + ) -> Result { + // Find config file - look relative to the Rust crate, then fall back to Python location + let config_path = find_config_path(variant)?; + let config = load_config(&config_path)?; + + Self::from_config( + config, + temp, + lsd_decode_steps, + eos_threshold, + noise_clamp, + device, + ) + } + + /// Load model with quantized weights for reduced memory footprint + /// + /// This applies simulated int8 quantization to applicable layers, + /// reducing memory usage while maintaining acceptable quality. + /// + /// # Arguments + /// * `variant` - Model variant (e.g., "b6369a24") + /// + /// # Returns + /// TTSModel with quantized weights + /// + /// # Note + /// Quantization uses 256 discrete levels (int8-equivalent). + /// Some layers (embeddings, output projections) are kept in full precision. + #[cfg(feature = "quantized")] + pub fn load_quantized(variant: &str) -> Result { + Self::load_quantized_with_params( + variant, + defaults::TEMPERATURE, + defaults::LSD_DECODE_STEPS, + defaults::EOS_THRESHOLD, + ) + } + + /// Load quantized model with custom generation parameters + #[cfg(feature = "quantized")] + pub fn load_quantized_with_params( + variant: &str, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + ) -> Result { + Self::load_quantized_with_params_device( + variant, + temp, + lsd_decode_steps, + eos_threshold, + None, + &Device::Cpu, + ) + } + + /// Load quantized model with custom generation parameters and specific device + #[cfg(feature = "quantized")] + pub fn load_quantized_with_params_device( + variant: &str, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + noise_clamp: Option, + device: &Device, + ) -> Result { + // Load model normally first + let model = Self::load_with_params_device( + variant, + temp, + lsd_decode_steps, + eos_threshold, + noise_clamp, + device, + )?; + // ... (quantization placeholder logic remains same) + Ok(model) + } + + /// Check if this model was loaded with quantization + #[cfg(feature = "quantized")] + pub fn is_quantized(&self) -> bool { + // In current implementation, we don't actually store quantized weights + // This is a placeholder for future implementation + false + } + + /// Create model from configuration + fn from_config( + config: Config, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + noise_clamp: Option, + device: &Device, + ) -> Result { + let dtype = DType::F32; + + // Download weights + #[cfg(not(target_arch = "wasm32"))] + { + let weights_path = config + .weights_path + .as_ref() + .ok_or_else(|| anyhow::anyhow!("weights_path not specified in config"))?; + let weights_file = crate::weights::download_if_necessary(weights_path)?; + + // Load safetensors with VarBuilder + let vb = + unsafe { VarBuilder::from_mmaped_safetensors(&[weights_file], dtype, device)? }; + + // Download tokenizer + let tokenizer_path = + crate::weights::download_if_necessary(&config.flow_lm.lookup_table.tokenizer_path)?; + + // Build conditioner + let conditioner = LUTConditioner::new( + config.flow_lm.lookup_table.n_bins, + &tokenizer_path, + config.flow_lm.lookup_table.dim, + config.flow_lm.transformer.d_model, + vb.pp("flow_lm.conditioner"), + )?; + + Self::from_config_and_vb( + config, + temp, + lsd_decode_steps, + eos_threshold, + noise_clamp, + conditioner, + vb, + ) + } + + #[cfg(target_arch = "wasm32")] + { + let _ = (config, temp, lsd_decode_steps, eos_threshold, device, dtype); + anyhow::bail!( + "WASM requires from_bytes or providing a pre-built VarBuilder. Use load_from_bytes instead." + ); + } + } + + /// Load model from byte slices (useful for WASM) + pub fn load_from_bytes( + config_yaml: &[u8], + weights_bytes: &[u8], + tokenizer_bytes: &[u8], + ) -> Result { + let config: Config = serde_yaml::from_slice(config_yaml)?; + let device = Device::Cpu; + let dtype = DType::F32; + + let tensors = candle_core::safetensors::load_buffer(weights_bytes, &device)?; + let vb = VarBuilder::from_tensors(tensors, dtype, &device); + + // On WASM, LUTConditioner::new needs a path, but we've updated it to + // eventually support bytes. For now, we'll need to adapt it. + // Actually, my recent change to conditioners/text.rs still uses Tokenizer::from_file on WASM. + // I should probably fix that to support bytes too. + + // For now, let's keep it simple and assume we have a path for the tokenizer or a way to load it. + // This is a placeholder for real WASM loading. + + let conditioner = LUTConditioner::new_from_bytes( + config.flow_lm.lookup_table.n_bins, + tokenizer_bytes, + config.flow_lm.lookup_table.dim, + config.flow_lm.transformer.d_model, + vb.pp("flow_lm.conditioner"), + )?; + + Self::from_config_and_vb( + config, + defaults::TEMPERATURE, + defaults::LSD_DECODE_STEPS, + defaults::EOS_THRESHOLD, + None, + conditioner, + vb, + ) + } + + /// Internal helper to build model from config and VarBuilder + fn from_config_and_vb( + config: Config, + temp: f32, + lsd_decode_steps: usize, + eos_threshold: f32, + noise_clamp: Option, + conditioner: LUTConditioner, + vb: VarBuilder, + ) -> Result { + let device = vb.device().clone(); + + // Build FlowLM components + let dim = config.flow_lm.transformer.d_model; + let ldim = config.mimi.quantizer.dimension; + let hidden_dim = dim * config.flow_lm.transformer.hidden_scale; + + // SimpleMLPAdaLN::new(in_channels, model_channels, out_channels, cond_channels, num_res_blocks, num_time_conds, max_period, vb) + let flow_net = SimpleMLPAdaLN::new( + ldim, // in_channels (input is latent dim) + config.flow_lm.flow.dim, // model_channels + ldim, // out_channels (output is also latent dim) + dim, // cond_channels (conditioning from transformer) + config.flow_lm.flow.depth, // num_res_blocks + 2, // num_time_conds (s and t) + config.flow_lm.transformer.max_period as f32, + vb.pp("flow_lm.flow_net"), + )?; + + // StreamingTransformer::new(d_model, num_heads, num_layers, layer_scale, dim_feedforward, context, max_period, kind, name, vb) + let transformer = StreamingTransformer::new( + dim, + config.flow_lm.transformer.num_heads, + config.flow_lm.transformer.num_layers, + None, // layer_scale + hidden_dim, // dim_feedforward + None, // context (causal) + config.flow_lm.transformer.max_period as f32, + "kv", + "flow_lm.transformer", + vb.pp("flow_lm.transformer"), + )?; + + let mut flow_lm = FlowLMModel::new(flow_net, transformer, ldim, dim, vb.pp("flow_lm"))?; + flow_lm.noise_clamp = noise_clamp; + + // Build Mimi components + let seanet_cfg = &config.mimi.seanet; + let encoder = SEANetEncoder::new( + seanet_cfg.channels, + seanet_cfg.dimension, + seanet_cfg.n_filters, + seanet_cfg.n_residual_layers, + &seanet_cfg.ratios, + seanet_cfg.kernel_size, + seanet_cfg.residual_kernel_size, + seanet_cfg.last_kernel_size, + seanet_cfg.dilation_base, + &seanet_cfg.pad_mode, + seanet_cfg.compress, + "mimi.encoder", + vb.pp("mimi.encoder"), + )?; + + let decoder = SEANetDecoder::new( + seanet_cfg.channels, + seanet_cfg.dimension, + seanet_cfg.n_filters, + seanet_cfg.n_residual_layers, + &seanet_cfg.ratios, + seanet_cfg.kernel_size, + seanet_cfg.residual_kernel_size, + seanet_cfg.last_kernel_size, + seanet_cfg.dilation_base, + &seanet_cfg.pad_mode, + seanet_cfg.compress, + "mimi.decoder", + vb.pp("mimi.decoder"), + )?; + + let mimi_tr_cfg = &config.mimi.transformer; + // ProjectedTransformer::new(input_dimension, output_dimensions, d_model, num_heads, num_layers, layer_scale, context, max_period, dim_feedforward, name, vb) + let encoder_transformer = ProjectedTransformer::new( + mimi_tr_cfg.input_dimension, + mimi_tr_cfg.output_dimensions.clone(), + mimi_tr_cfg.d_model, + mimi_tr_cfg.num_heads, + mimi_tr_cfg.num_layers, + mimi_tr_cfg.layer_scale as f32, + mimi_tr_cfg.context, + mimi_tr_cfg.max_period as f32, + mimi_tr_cfg.dim_feedforward, + "mimi.encoder_transformer", + vb.pp("mimi.encoder_transformer"), + )?; + + let decoder_transformer = ProjectedTransformer::new( + mimi_tr_cfg.input_dimension, + mimi_tr_cfg.output_dimensions.clone(), + mimi_tr_cfg.d_model, + mimi_tr_cfg.num_heads, + mimi_tr_cfg.num_layers, + mimi_tr_cfg.layer_scale as f32, + mimi_tr_cfg.context, + mimi_tr_cfg.max_period as f32, + mimi_tr_cfg.dim_feedforward, + "mimi.decoder_transformer", + vb.pp("mimi.decoder_transformer"), + )?; + + // Calculate encoder frame rate from SEANet ratios + let hop_length: usize = seanet_cfg.ratios.iter().product(); + let encoder_frame_rate = config.mimi.sample_rate as f64 / hop_length as f64; + + let mimi = MimiModel::new( + encoder, + decoder, + encoder_transformer, + decoder_transformer, + config.mimi.frame_rate, + encoder_frame_rate, + config.mimi.sample_rate, + config.mimi.channels, + config.mimi.quantizer.dimension, + config.mimi.quantizer.output_dimension, + "mimi", + vb.pp("mimi"), + )?; + + // Load speaker projection weight - uses mimi output dimension, not internal ldim + let mimi_out_dim = config.mimi.quantizer.output_dimension; + let speaker_proj_weight = vb.get((dim, mimi_out_dim), "flow_lm.speaker_proj_weight")?; + + Ok(Self { + flow_lm, + mimi, + conditioner, + speaker_proj_weight, + temp, + lsd_decode_steps, + eos_threshold, + noise_clamp, + sample_rate: config.mimi.sample_rate, + dim, + ldim, + device, + }) + } + + /// Create voice state from audio prompt bytes for voice cloning + pub fn get_voice_state_from_bytes(&self, bytes: &[u8]) -> Result { + let (audio, sample_rate) = crate::audio::read_wav_from_bytes(bytes)?; + + // Resample to model sample rate if needed + let audio = if sample_rate != self.sample_rate as u32 { + crate::audio::resample(&audio, sample_rate, self.sample_rate as u32)? + } else { + audio + }; + + // Add batch dimension: [C, T] -> [B, C, T] + let audio = audio.unsqueeze(0)?; + + self.get_voice_state_from_tensor(&audio) + } + + /// Create voice state from audio prompt for voice cloning + /// + /// Encodes the audio through Mimi and projects to flow model space. + #[cfg(not(target_arch = "wasm32"))] + pub fn get_voice_state>(&self, audio_path: P) -> Result { + let (audio, sample_rate) = crate::audio::read_wav(audio_path)?; + + // Resample to model sample rate if needed + let audio = if sample_rate != self.sample_rate as u32 { + crate::audio::resample(&audio, sample_rate, self.sample_rate as u32)? + } else { + audio + }; + + // Add batch dimension: [C, T] -> [B, C, T] + let audio = audio.unsqueeze(0)?; + + self.get_voice_state_from_tensor(&audio) + } + + /// Create voice state from a pre-calculated latent prompt file (.safetensors) + #[cfg(not(target_arch = "wasm32"))] + pub fn get_voice_state_from_prompt_file>( + &self, + path: P, + ) -> Result { + let tensors = candle_core::safetensors::load(path, &self.device)?; + let prompt = tensors + .get("audio_prompt") + .ok_or_else(|| anyhow::anyhow!("'audio_prompt' not found in safetensors file"))?; + + self.get_voice_state_from_prompt_tensor(prompt) + } + + /// Create voice state from pre-calculated latent prompt bytes (.safetensors) + pub fn get_voice_state_from_prompt_bytes(&self, bytes: &[u8]) -> Result { + let tensors = candle_core::safetensors::load_buffer(bytes, &self.device)?; + let prompt = tensors + .get("audio_prompt") + .ok_or_else(|| anyhow::anyhow!("'audio_prompt' not found in safetensors bytes"))?; + + self.get_voice_state_from_prompt_tensor(prompt) + } + + /// Create voice state from a pre-calculated latent prompt tensor + pub fn get_voice_state_from_prompt_tensor(&self, prompt: &Tensor) -> Result { + let mut flow_state = init_states(1, 1000); + self.run_flow_lm_prompt(prompt, &mut flow_state)?; + Ok(flow_state) + } + + /// Create voice state from audio tensor + pub fn get_voice_state_from_tensor(&self, audio: &Tensor) -> Result { + let mut model_state = init_states(1, 1000); + + // Pad audio to a multiple of frame size for streaming conv stride alignment + let frame_size = self.mimi.frame_size(); + let (b, c, t) = audio.dims3()?; + let pad_len = if t % frame_size != 0 { + frame_size - (t % frame_size) + } else { + 0 + }; + let audio = if pad_len > 0 { + let pad = Tensor::zeros((b, c, pad_len), audio.dtype(), audio.device())?; + Tensor::cat(&[audio, &pad], 2)? + } else { + audio.clone() + }; + + // Encode audio through Mimi in chunks to avoid OOM in SEANet Conv1d layers + // (A 5-minute audio at 24kHz creates ~1GB feature maps if processed at once) + let chunk_size = frame_size * 100; // ~100 frames per chunk (reduced from 500 to safe mem) + let mut encoded_chunks = Vec::new(); + let (_b, _c, total_samples) = audio.dims3()?; + + for start in (0..total_samples).step_by(chunk_size) { + let end = std::cmp::min(start + chunk_size, total_samples); + let chunk = audio.narrow(2, start, end - start)?; + let code = self.mimi.encode_to_latent(&chunk, &mut model_state, 0)?; + encoded_chunks.push(code); + } + let encoded = Tensor::cat(&encoded_chunks, 2)?; + + // Transpose from [B, D, T] to [B, T, D] + let latents = encoded.transpose(1, 2)?.to_dtype(DType::F32)?; + + // Project to flow model space: [B, T, ldim] @ [dim, ldim].T -> [B, T, dim] + // Candle needs 2D @ 2D for matmul, so reshape + let (b, t, d) = latents.dims3()?; + let latents_2d = latents.reshape((b * t, d))?; + let conditioning_2d = latents_2d.matmul(&self.speaker_proj_weight.t()?)?; + let conditioning = conditioning_2d.reshape((b, t, self.dim))?; + + // Run flow_lm with audio conditioning to update state + let mut flow_state = init_states(1, 1000); + self.run_flow_lm_prompt(&conditioning, &mut flow_state)?; + + Ok(flow_state) + } + + /// Run flow LM with audio conditioning (used during prompting) + fn run_flow_lm_prompt(&self, conditioning: &Tensor, state: &mut ModelState) -> Result<()> { + // Empty text tokens and backbone input + let empty_text = Tensor::zeros((1, 0), DType::I64, &self.device)?; + let text_embeddings = self.conditioner.forward(&empty_text)?; + + // Concatenate text embeddings and audio conditioning + // Match Python/reference order: audio conditioning comes before text embeddings. + let input = Tensor::cat(&[conditioning, &text_embeddings], 1)?; + + // Run through transformer (no generation, just prompting) + // With custom SDPA, this is now memory efficient + let _ = self.flow_lm.transformer.forward(&input, state, 0)?; + + // Increment FlowLM state after prompting (critical for RoPE positioning) + // Python: increment_steps(self.flow_lm, model_state, increment=audio_conditioning.shape[1]) + let increment_by = conditioning.dims()[1]; + increment_steps(state, "offset", increment_by); + + Ok(()) + } + + /// Split text into optimal chunks for generation, matching Python's logic exactly. + /// Uses actual tokenization to ensure chunks never exceed MAX_TOKENS_PER_CHUNK (50). + /// This prevents O(N²) attention complexity for long texts. + pub fn split_into_best_sentences(&self, text: &str) -> Vec { + const MAX_TOKENS_PER_CHUNK: usize = 50; + + let prepared_text = prepare_text_prompt(text); + + // 1. Initial split by punctuation to respect sentence boundaries + let raw_sentences: Vec<&str> = prepared_text + .split_inclusive(&['.', '!', '?', ';', ':']) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + + if raw_sentences.is_empty() { + return vec![prepared_text]; + } + + let mut chunks = Vec::new(); + let mut current_chunk = String::new(); + let mut current_token_count = 0; + + for sentence in raw_sentences { + let sentence_tokens = self + .conditioner + .count_tokens(sentence) + .unwrap_or(MAX_TOKENS_PER_CHUNK); + + // If a single sentence exceeds max tokens, split it by words + if sentence_tokens > MAX_TOKENS_PER_CHUNK { + // Flush pending chunk first + if !current_chunk.is_empty() { + chunks.push(current_chunk); + current_chunk = String::new(); + current_token_count = 0; + } + + // Split long sentence using word-batch estimation (~1.3 tokens per word average) + // This avoids calling count_tokens for every word (expensive!) + let words: Vec<&str> = sentence.split_whitespace().collect(); + const WORDS_PER_BATCH: usize = 35; // ~45 tokens, safe margin under 50 + + for word_batch in words.chunks(WORDS_PER_BATCH) { + let chunk_str = word_batch.join(" "); + // Verify this batch is actually under limit (should almost always pass) + let actual_tokens = self + .conditioner + .count_tokens(&chunk_str) + .unwrap_or(MAX_TOKENS_PER_CHUNK); + + if actual_tokens <= MAX_TOKENS_PER_CHUNK { + chunks.push(chunk_str); + } else { + // Rare case: batch still too big, split in half recursively + let mid = word_batch.len() / 2; + chunks.push(word_batch[..mid].join(" ")); + chunks.push(word_batch[mid..].join(" ")); + } + } + continue; + } + + // Normal accumulation logic + if current_chunk.is_empty() { + current_chunk = sentence.to_string(); + current_token_count = sentence_tokens; + } else if current_token_count + sentence_tokens > MAX_TOKENS_PER_CHUNK { + chunks.push(current_chunk); + current_chunk = sentence.to_string(); + current_token_count = sentence_tokens; + } else { + current_chunk.push(' '); + current_chunk.push_str(sentence); + current_token_count += sentence_tokens; + } + } + + if !current_chunk.is_empty() { + chunks.push(current_chunk); + } + + chunks + } + + /// Generate audio from text with voice state + pub fn generate(&self, text: &str, voice_state: &ModelState) -> Result { + let mut audio_chunks = Vec::new(); + + for chunk in self.generate_stream(text, voice_state) { + audio_chunks.push(chunk?); + } + + // Concatenate all audio chunks + if audio_chunks.is_empty() { + anyhow::bail!("No audio generated"); + } + let audio = Tensor::cat(&audio_chunks, 2)?; + // Remove batch dimension + let audio = audio.squeeze(0)?; + + Ok(audio) + } + + // ========================================================================= + // EXPERIMENTAL: Parallel FlowLM + Mimi decoding + // ========================================================================= + // + // This method was an experiment to decode Mimi audio in a separate thread + // while FlowLM generates the next latent. However, benchmarks showed it's + // ~21% SLOWER than sequential due to: + // - Thread spawning and channel synchronization overhead + // - CPU contention (MKL already parallelizes internally) + // - Bounded channel backpressure when Mimi can't keep up + // + // Keeping this commented out for future exploration with GPU acceleration + // where FlowLM and Mimi could run on different hardware. + // + // To re-enable: uncomment the method below and test with: + // model.generate_parallel(text, &voice_state) + // + /* + /// Generate audio with parallel FlowLM + Mimi decoding + /// + /// Uses std::thread to decode Mimi audio in parallel with FlowLM generation. + /// While Mimi decodes frame N, FlowLM generates frame N+1. + /// + /// NOTE: Currently slower than sequential due to thread overhead on CPU. + /// May be useful for GPU acceleration in the future. + pub fn generate_parallel(&self, text: &str, voice_state: &ModelState) -> Result { + use std::sync::mpsc; + use std::thread; + + // Channel for sending latents from FlowLM to Mimi decoder + let (latent_tx, latent_rx) = mpsc::sync_channel::>(4); + // Channel for receiving decoded audio from Mimi + let (audio_tx, audio_rx) = mpsc::channel::>(); + + // Clone what the decoder thread needs + let mimi = self.mimi.clone(); + let emb_mean = self.flow_lm.emb_mean.clone(); + let emb_std = self.flow_lm.emb_std.clone(); + + // Spawn Mimi decoder thread + let decoder_handle = thread::spawn(move || { + let mut mimi_state = init_states(1, 1000); + + while let Ok(Some((next_latent, step))) = latent_rx.recv() { + let result = (|| -> Result { + let next_latent_denorm = next_latent + .broadcast_mul(&emb_std)? + .broadcast_add(&emb_mean)?; + + let mimi_input = next_latent_denorm.unsqueeze(1)?.transpose(1, 2)?; + let quantized = mimi.quantize(&mimi_input)?; + let audio = mimi + .decode_from_latent(&quantized, &mut mimi_state, step) + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(audio) + })(); + + if audio_tx.send(result).is_err() { + break; // Receiver dropped + } + } + }); + + // Main thread: generate latents with FlowLM + let chunks = self.split_into_best_sentences(text); + let mut expected_frames = 0; + + for chunk_text in chunks { + let mut state = voice_state.clone(); + let prepared_text = prepare_text_prompt(&chunk_text); + + let tokens = self.conditioner.prepare(&prepared_text, &self.device)?; + let text_embeddings = self.conditioner.forward(&tokens)?; + + // Initial text prompt + self.flow_lm + .transformer + .forward(&text_embeddings, &mut state, 0)?; + + let max_gen_len = (prepared_text.split_whitespace().count() + 2) * 13; + let frames_after_eos = estimate_frames_after_eos(&chunk_text); + + let mut backbone_input = self.flow_lm.bos_emb.clone().reshape((1, 1, self.ldim))?; + let mut eos_step: Option = None; + + let time_embeddings = self.flow_lm.flow_net.compute_time_embeddings( + self.lsd_decode_steps, + &self.device, + DType::F32, + )?; + + let empty_text_embeddings = Tensor::zeros((1, 0, self.dim), DType::F32, &self.device)?; + + for step in 0..max_gen_len { + let (next_latent, is_eos) = self.flow_lm.forward( + &backbone_input, + &empty_text_embeddings, + &mut state, + &time_embeddings, + self.temp, + self.eos_threshold, + step, + )?; + + // Send latent to decoder thread (non-blocking with bounded channel) + if latent_tx.send(Some((next_latent.clone(), step))).is_err() { + break; + } + expected_frames += 1; + + if is_eos && eos_step.is_none() { + eos_step = Some(step); + } + + if let Some(e_step) = eos_step { + if step >= e_step + frames_after_eos { + break; + } + } + + backbone_input = next_latent.unsqueeze(1)?; + } + } + + // Signal decoder thread to finish + let _ = latent_tx.send(None); + + // Collect all decoded audio frames + let mut audio_chunks = Vec::with_capacity(expected_frames); + for _ in 0..expected_frames { + match audio_rx.recv() { + Ok(Ok(frame)) => audio_chunks.push(frame), + Ok(Err(e)) => return Err(e), + Err(_) => break, + } + } + + // Wait for decoder thread + let _ = decoder_handle.join(); + + if audio_chunks.is_empty() { + anyhow::bail!("No audio generated"); + } + + let audio = Tensor::cat(&audio_chunks, 2)?; + let audio = audio.squeeze(0)?; + Ok(audio) + } + */ + + /// Generate audio from text with pause handling + /// + /// This method parses pause markers in the text and inserts silence + /// at appropriate positions. Supports: + /// - Explicit pauses: `[pause:500ms]` or `[pause:1s]` + /// - Natural pauses from punctuation are handled during generation + /// + /// # Example + /// ```ignore + /// let audio = model.generate_with_pauses("Hello... [pause:500ms] world", &voice_state)?; + /// ``` + pub fn generate_with_pauses(&self, text: &str, voice_state: &ModelState) -> Result { + let mut audio_chunks = Vec::new(); + + for chunk in self.generate_stream_long(text, voice_state) { + audio_chunks.push(chunk?); + } + + // Concatenate all audio chunks + if audio_chunks.is_empty() { + anyhow::bail!("No audio generated"); + } + let audio = Tensor::cat(&audio_chunks, 2)?; + // Remove batch dimension + let audio = audio.squeeze(0)?; + + Ok(audio) + } + + /// Generate audio stream from text with voice state + /// + /// Returns an iterator that yields audio chunks (one per Mimi frame). + /// Generate audio stream from text with voice state + /// + /// Returns an iterator that yields audio chunks (one per Mimi frame). + /// + /// This method splits the text into optimal sentences and generates each independently, + /// matching Python's behavior to maintain O(N) complexity for long texts. + pub fn generate_stream<'a, 'b, 'c>( + &'a self, + text: &'b str, + voice_state: &'c ModelState, + ) -> Box> + 'a> { + // Split text into chunks to avoid quadratic complexity scaling + let chunks = self.split_into_best_sentences(text); + + // Clone voice state so the iterator owns a copy, untied from lifetime 'c + let voice_state_owned = voice_state.clone(); + + // Create an iterator that processes each chunk sequentially + let iterator = chunks.into_iter().flat_map(move |chunk_text| { + // We need to return an iterator for each chunk. + // We pass a reference to the owned voice state captured by the closure. + self.generate_stream_segment(chunk_text, &voice_state_owned) + }); + + Box::new(iterator) + } + + /// Internal helper to generate a single segment (short text) matching Python's _generate + fn generate_stream_segment( + &self, + text: String, + voice_state: &ModelState, + ) -> Box>> { + let mut state = voice_state.clone(); + let mut mimi_state = init_states(1, 1000); + + // Prepare text + let prepared_text = prepare_text_prompt(&text); + + // Error handling for preparation failures inside the iterator + let tokens = match self.conditioner.prepare(&prepared_text, &self.device) { + Ok(t) => t, + Err(e) => return Box::new(std::iter::once(Err(e))), + }; + + let text_embeddings = match self.conditioner.forward(&tokens) { + Ok(e) => e, + Err(e) => return Box::new(std::iter::once(Err(e))), + }; + + // Initial text prompt + if let Err(e) = self + .flow_lm + .transformer + .forward(&text_embeddings, &mut state, 0) + { + return Box::new(std::iter::once(Err(anyhow::Error::from(e)))); + } + + // Removed redundant increment_steps("offset") - handled internally by RoPE/Attention with current_end_len + + let max_gen_len = (prepared_text.split_whitespace().count() + 2) * 13; + let frames_after_eos = estimate_frames_after_eos(&text); + + let mut backbone_input = match self.flow_lm.bos_emb.clone().reshape((1, 1, self.ldim)) { + Ok(t) => t, + Err(e) => return Box::new(std::iter::once(Err(anyhow::Error::from(e)))), + }; + + let mut eos_step: Option = None; + let mut finished = false; + + // We need to move 'self' (reference) and owned data into the closure + // But 'self' is in `generate_stream` lifetime? + // We clone needed cheap things or use references. + // `flow_lm`, `mimi` are part of self. + // The closure will borrow `self`. + + // To make the iterator valid 'static or bound to self, we use move. + // But we need access to self inside. + // We can clone `self` if cheap? No, TTSModel is large (holds models). + // But TTSModel derives Clone! And models are wrappers around Arcs (Candle tensors/vars). + // So cloning TTSModel is CHEAP (shallow copy of Arc pointers). + let model = self.clone(); + + // Pre-compute time embeddings for the entire segment to avoid re-computing every frame + // Now returns a single batched Tensor [num_steps, channels] + let time_embeddings = match model.flow_lm.flow_net.compute_time_embeddings( + model.lsd_decode_steps, + &model.device, + DType::F32, + ) { + Ok(te) => te, + Err(e) => return Box::new(std::iter::once(Err(anyhow::Error::from(e)))), + }; + + let empty_text_embeddings = + Tensor::zeros((1, 0, model.dim), DType::F32, &model.device).unwrap(); + + Box::new((0..max_gen_len).map_while(move |step| { + if finished { + return None; + } + + // Text embeddings are already processed into state during initialization (line 752-757), + // so we always pass empty text embeddings during autoregressive generation. + // Passing text_embeddings again would cause duplicate/repeated speech. + let text_tokens_to_pass = &empty_text_embeddings; + + let (next_latent, is_eos) = match model.flow_lm.forward( + &backbone_input, + text_tokens_to_pass, + &mut state, + &time_embeddings, + model.temp, + model.eos_threshold, + step, + ) { + Ok(res) => res, + Err(e) => return Some(Err(anyhow::anyhow!(e))), + }; + + let audio_frame = match (|| -> Result { + let next_latent_denorm = next_latent + .broadcast_mul(&model.flow_lm.emb_std)? + .broadcast_add(&model.flow_lm.emb_mean)?; + + let mimi_input = next_latent_denorm.unsqueeze(1)?.transpose(1, 2)?; + let quantized = model.mimi.quantize(&mimi_input)?; + let audio = model + .mimi + .decode_from_latent(&quantized, &mut mimi_state, step) + .map_err(|e| anyhow::anyhow!(e))?; + + // Removed redundant increment_steps("offset") for mimi + + Ok(audio) + })() { + Ok(frame) => frame, + Err(e) => return Some(Err(e)), + }; + + if is_eos && eos_step.is_none() { + eos_step = Some(step); + } + + if let Some(e_step) = eos_step + && step >= e_step + frames_after_eos + { + finished = true; + } + + backbone_input = next_latent.unsqueeze(1).unwrap(); + + // Removed redundant increment_steps("offset") for FlowLM - handled by attention state + + Some(Ok(audio_frame)) + })) + } + + /// Generate audio stream from long text by segmenting it + pub fn generate_stream_long<'a>( + &'a self, + text: &str, + voice_state: &'a ModelState, + ) -> impl Iterator> + 'a { + use crate::pause::{parse_text_with_pauses, silence_samples}; + + let parsed = parse_text_with_pauses(text); + let mut segments = Vec::new(); + + // Interleave text chunks and pauses + let mut last_pos = 0; + for pause in &parsed.pauses { + if pause.position > last_pos { + let text_seg = &parsed.clean_text[last_pos..pause.position]; + if !text_seg.trim().is_empty() { + segments.push(Segment::Text(text_seg.to_string())); + } + } + segments.push(Segment::Pause(pause.duration_ms)); + + // Explicit pauses were replaced by a single space in clean_text + // Natural pauses (commas, ellipses) are still in clean_text + if pause.original.starts_with("[pause:") { + last_pos = pause.position + 1; + } else { + last_pos = pause.position + pause.original.len(); + } + } + if last_pos < parsed.clean_text.len() { + let text_seg = &parsed.clean_text[last_pos..]; + if !text_seg.trim().is_empty() { + segments.push(Segment::Text(text_seg.to_string())); + } + } + + let model = self; + segments.into_iter().flat_map(move |seg| match seg { + Segment::Text(s) => { + let iter = model.generate_stream(&s, voice_state); + Box::new(iter) as Box>> + } + Segment::Pause(ms) => { + let n_samples = silence_samples(ms, model.sample_rate as u32); + let silence_res = Tensor::zeros( + (1, model.mimi.channels, n_samples), + DType::F32, + &model.device, + ); + Box::new(std::iter::once(silence_res.map_err(anyhow::Error::from))) + as Box>> + } + }) + } + pub fn estimate_generation_steps(&self, text: &str) -> usize { + let prepared = prepare_text_prompt(text); + (prepared.split_whitespace().count() + 2) * 13 + } +} + +/// Internal segment type for interleaving text and pauses +enum Segment { + Text(String), + Pause(u32), +} + +/// Find the config file path for a variant +fn find_config_path(variant: &str) -> Result { + let filename = format!("{}.yaml", variant); + + // 1. Try relative to Rust crate (crates/pocket-tts/config) + let crate_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let crate_config = crate_path.join("config").join(&filename); + if crate_config.exists() { + return Ok(crate_config); + } + + // 2. Try relative to workspace root (for tests/cli) + // Go up 2 levels if in crates/pocket-tts or crates/pocket-tts-cli + let mut current = crate_path.as_path(); + for _ in 0..3 { + let python_config = current + .join("python-reference") + .join("pocket_tts") + .join("config") + .join(&filename); + if python_config.exists() { + return Ok(python_config); + } + + // Also try new crates structure if running from cli + let crates_config = current + .join("crates") + .join("pocket-tts") + .join("config") + .join(&filename); + if crates_config.exists() { + return Ok(crates_config); + } + + if let Some(parent) = current.parent() { + current = parent; + } else { + break; + } + } + + // 3. Try current directory + let local_path = std::path::PathBuf::from("config").join(&filename); + if local_path.exists() { + return Ok(local_path); + } + + anyhow::bail!( + "Config file {} not found. Checked crate-relative, workspace, and current directory.", + filename + ) +} + +/// Prepare text for generation, stripping pause markers for TTS processing +fn prepare_text_prompt(text: &str) -> String { + // First strip any explicit pause markers + let text = crate::pause::strip_pause_markers(text); + + let mut text = text.trim().to_string(); + if text.is_empty() { + return ".".to_string(); // Or handle error + } + + text = text.replace(['\n', '\r'], " ").replace(" ", " "); + + let word_count = text.split_whitespace().count(); + + // Ensure first character is uppercase + if let Some(first) = text.chars().next() + && !first.is_uppercase() + { + text = format!("{}{}", first.to_uppercase(), &text[first.len_utf8()..]); + } + + // Ensure ends with punctuation + if let Some(last) = text.chars().last() + && last.is_alphanumeric() + { + text.push('.'); + } + + // Python logic: prepend spaces if too short + if word_count < 5 { + text = format!("{}{}", " ".repeat(8), text); + } + + text +} + +/// Estimate frames after EOS based on text length +pub fn estimate_frames_after_eos(text: &str) -> usize { + let word_count = text.split_whitespace().count(); + if word_count <= 4 { + 3 + 2 // prepare_text_prompt guess + 2 + } else { + 1 + 2 // prepare_text_prompt guess + 2 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_prepare_text_prompt() { + // Short texts (<5 words) get 8 spaces prepended + assert_eq!(prepare_text_prompt("hello world"), " Hello world."); + assert_eq!(prepare_text_prompt("Hello world."), " Hello world."); + assert_eq!(prepare_text_prompt(" hello "), " Hello."); + // Long texts don't get spaces + assert_eq!( + prepare_text_prompt("one two three four five"), + "One two three four five." + ); + } + + #[test] + fn test_find_config_path() { + // This MUST pass now that we've moved the config into the crate + let result = find_config_path("b6369a24"); + assert!(result.is_ok(), "Config file should be found"); + let path = result.unwrap(); + assert!(path.exists(), "Config file path should exist"); + } + + #[test] + fn test_prepare_text_prompt_strips_pause_markers() { + // Pause markers should be stripped from text + let result = prepare_text_prompt("Hello [pause:500ms] world"); + // The pause marker should be gone, replaced with space + assert!(!result.contains("[pause:")); + assert!(result.contains("Hello")); + assert!(result.contains("world")); + } + + #[test] + fn test_prepare_text_prompt_handles_multiple_pauses() { + let result = prepare_text_prompt("One [pause:100ms] two [pause:1s] three"); + assert!(!result.contains("[pause:")); + assert!(result.contains("One")); + assert!(result.contains("two")); + assert!(result.contains("three")); + } + + #[test] + fn test_estimate_frames_after_eos() { + // Short text (<= 4 words) + assert_eq!(estimate_frames_after_eos("Hello world"), 5); + // Longer text (> 4 words) + assert_eq!(estimate_frames_after_eos("One two three four five"), 3); + } + + #[test] + #[cfg(feature = "quantized")] + fn test_load_quantized_requires_feature() { + // This test only runs with --features quantized + // It verifies the load_quantized method exists and compiles + // Actual model loading requires HF_TOKEN + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs new file mode 100644 index 00000000..a90b7bcd --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs @@ -0,0 +1,93 @@ +//! Voice state management for streaming generation and voice cloning + +use candle_core::{Result, Tensor}; +use std::collections::HashMap; + +/// Model state type for stateful modules +pub type ModelState = HashMap>; + +/// Initialize empty model state for all stateful modules +/// +/// Creates a nested HashMap structure that will be populated +/// as modules run their forward passes. +pub fn init_states(_batch_size: usize, _seq_len: usize) -> ModelState { + // Start with empty state - modules will populate as needed + HashMap::new() +} + +/// Get or create a module's state entry +pub fn get_or_create_state<'a>( + state: &'a mut ModelState, + module_name: &str, +) -> &'a mut HashMap { + state.entry(module_name.to_string()).or_default() +} + +/// Increment step counters in model state for all modules +/// +/// This is used after processing tokens to update position information +/// for streaming generation. +pub fn increment_steps(state: &mut ModelState, key: &str, increment: usize) { + for (_module_name, module_state) in state.iter_mut() { + if let Some(step_tensor) = module_state.get_mut(key) + && let Ok(current) = step_tensor.to_scalar::() + && let Ok(new_tensor) = Tensor::new(current + increment as i64, step_tensor.device()) + { + *step_tensor = new_tensor; + } + } +} + +/// Get the current step/offset for a module +pub fn get_offset(state: &ModelState, module_name: &str) -> usize { + state + .get(module_name) + .and_then(|s| s.get("offset")) + .and_then(|t| t.to_scalar::().ok()) + .unwrap_or(0) as usize +} + +/// Set the offset for a module +pub fn set_offset(state: &mut ModelState, module_name: &str, offset: usize) -> Result<()> { + let module_state = get_or_create_state(state, module_name); + let device = module_state + .values() + .next() + .map(|t| t.device().clone()) + .unwrap_or(candle_core::Device::Cpu); + module_state.insert("offset".to_string(), Tensor::new(offset as i64, &device)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_init_states() { + let state = init_states(1, 100); + assert!(state.is_empty()); + } + + #[test] + fn test_get_or_create_state() { + let mut state = init_states(1, 100); + let module_state = get_or_create_state(&mut state, "test_module"); + assert!(module_state.is_empty()); + assert!(state.contains_key("test_module")); + } + + #[test] + fn test_offset_operations() -> Result<()> { + let mut state = init_states(1, 100); + + // Initially offset is 0 + assert_eq!(get_offset(&state, "test"), 0); + + // Set offset + set_offset(&mut state, "test", 42)?; + assert_eq!(get_offset(&state, "test"), 42); + + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs new file mode 100644 index 00000000..0b228ed3 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs @@ -0,0 +1,235 @@ +//! WebAssembly bindings for Pocket TTS +//! +//! This module provides WASM-compatible entry points for browser usage. +//! Build with: `wasm-pack build --target web --features wasm` + +#![cfg(target_arch = "wasm32")] + +use crate::tts_model::TTSModel; +use js_sys::Float32Array; +use wasm_bindgen::prelude::*; + +/// Initialize console_error_panic_hook for better error messages in browser +#[wasm_bindgen(start)] +pub fn init() { + console_error_panic_hook::set_once(); + web_sys::console::log_1(&"Pocket TTS WASM initialized".into()); +} + +/// WASM-compatible TTS model wrapper +#[wasm_bindgen] +pub struct WasmTTSModel { + model: Option, + voice_state: Option, + sample_rate: u32, +} + +#[wasm_bindgen] +impl WasmTTSModel { + /// Create a new WASM TTS model + #[wasm_bindgen(constructor)] + pub fn new() -> WasmTTSModel { + Self { + model: None, + voice_state: None, + sample_rate: 24000, + } + } + + /// Load model from ArrayBuffers + /// + /// # Arguments + /// * `config_yaml` - ArrayBuffer containing config.yaml + /// * `weights_data` - ArrayBuffer containing safetensors model weights + /// * `tokenizer_bytes` - Optional ArrayBuffer containing tokenizer.json (falls back to embedded) + #[wasm_bindgen] + pub fn load_from_buffer( + &mut self, + config_yaml: &[u8], + weights_data: &[u8], + tokenizer_bytes: &[u8], + ) -> Result<(), JsValue> { + let tok_bytes = if tokenizer_bytes.is_empty() { + include_bytes!("../assets/tokenizer.json") + } else { + tokenizer_bytes + }; + + let model = TTSModel::load_from_bytes(config_yaml, weights_data, tok_bytes) + .map_err(|e| JsValue::from_str(&format!("Model loading failed: {:?}", e)))?; + + self.sample_rate = model.sample_rate as u32; + self.model = Some(model); + + web_sys::console::log_1( + &"Model loaded successfully (using embedded or provided tokenizer)".into(), + ); + Ok(()) + } + + /// Check if model is ready for generation + #[wasm_bindgen] + pub fn is_ready(&self) -> bool { + self.model.is_some() + } + + /// Load voice from WAV audio buffer for voice cloning + #[wasm_bindgen] + pub fn load_voice_from_buffer(&mut self, wav_bytes: &[u8]) -> Result<(), JsValue> { + let model = self + .model + .as_ref() + .ok_or_else(|| JsValue::from_str("Model not loaded. Call load_from_buffer first."))?; + + let voice_state = model + .get_voice_state_from_bytes(wav_bytes) + .map_err(|e| JsValue::from_str(&format!("Voice cloning failed: {:?}", e)))?; + + self.voice_state = Some(voice_state); + web_sys::console::log_1(&"Voice loaded successfully (from audio)".into()); + Ok(()) + } + + /// Load voice from safetensors buffer (pre-calculated embedding) + #[wasm_bindgen] + pub fn load_voice_from_safetensors(&mut self, bytes: &[u8]) -> Result<(), JsValue> { + let model = self + .model + .as_ref() + .ok_or_else(|| JsValue::from_str("Model not loaded. Call load_from_buffer first."))?; + + let voice_state = model + .get_voice_state_from_prompt_bytes(bytes) + .map_err(|e| JsValue::from_str(&format!("Voice loading failed: {:?}", e)))?; + + self.voice_state = Some(voice_state); + web_sys::console::log_1(&"Voice loaded successfully (from embedding)".into()); + Ok(()) + } + + /// Get the sample rate of generated audio + #[wasm_bindgen(getter)] + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + /// Generate audio from text + /// + /// # Arguments + /// * `text` - Text to synthesize + /// + /// # Returns + /// Float32Array containing audio samples at 24kHz mono + #[wasm_bindgen] + pub fn generate(&self, text: &str) -> Result { + let model = self + .model + .as_ref() + .ok_or_else(|| JsValue::from_str("Model not loaded. Call load_from_buffer first."))?; + + let voice_state = self + .voice_state + .clone() + .unwrap_or_else(|| crate::voice_state::init_states(1, 0)); + + let audio_tensor = model + .generate(text, &voice_state) + .map_err(|e| JsValue::from_str(&format!("Generation failed: {:?}", e)))?; + + // Flatten tensor to Vec + let samples = audio_tensor + .to_vec2::() + .map_err(|e| JsValue::from_str(&format!("Failed to extract samples: {:?}", e)))?[0] + .clone(); + + let array = Float32Array::new_with_length(samples.len() as u32); + array.copy_from(&samples); + + Ok(array) + } + + /// Generate audio and return as base64-encoded WAV + #[wasm_bindgen] + pub fn generate_wav_base64(&self, text: &str) -> Result { + let samples = self.generate(text)?; + let mut sample_vec = vec![0.0f32; samples.length() as usize]; + samples.copy_to(&mut sample_vec); + + let mut buffer = std::io::Cursor::new(Vec::new()); + { + // Simple WAV header creation since hound is gated + write_wav_header(&mut buffer, self.sample_rate, sample_vec.len() as u32) + .map_err(|e| JsValue::from_str(&format!("WAV header error: {:?}", e)))?; + + for &sample in &sample_vec { + let s = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + buffer.get_mut().extend_from_slice(&s.to_le_bytes()); + } + } + + let base64 = base64_encode(buffer.get_ref()); + Ok(format!("data:audio/wav;base64,{}", base64)) + } +} + +/// Helper to write a basic 16-bit PCM WAV header +fn write_wav_header( + w: &mut dyn std::io::Write, + sample_rate: u32, + num_samples: u32, +) -> std::io::Result<()> { + let subchunk2_size = num_samples * 2; // 2 bytes per sample (16-bit) + let chunk_size = 36 + subchunk2_size; + + w.write_all(b"RIFF")?; + w.write_all(&chunk_size.to_le_bytes())?; + w.write_all(b"WAVE")?; + w.write_all(b"fmt ")?; + w.write_all(&16u32.to_le_bytes())?; // Subchunk1Size + w.write_all(&1u16.to_le_bytes())?; // AudioFormat (PCM) + w.write_all(&1u16.to_le_bytes())?; // NumChannels (Mono) + w.write_all(&sample_rate.to_le_bytes())?; + w.write_all(&(sample_rate * 2).to_le_bytes())?; // ByteRate + w.write_all(&2u16.to_le_bytes())?; // BlockAlign + w.write_all(&16u16.to_le_bytes())?; // BitsPerSample + w.write_all(b"data")?; + w.write_all(&subchunk2_size.to_le_bytes())?; + Ok(()) +} + +/// Simple base64 encoder to avoid external dependencies in WASM build +fn base64_encode(input: &[u8]) -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut output = String::with_capacity((input.len() + 2) / 3 * 4); + + for chunk in input.chunks(3) { + let b = match chunk.len() { + 3 => (chunk[0] as u32) << 16 | (chunk[1] as u32) << 8 | (chunk[2] as u32), + 2 => (chunk[0] as u32) << 16 | (chunk[1] as u32) << 8, + 1 => (chunk[0] as u32) << 16, + _ => unreachable!(), + }; + + output.push(CHARSET[(b >> 18 & 0x3F) as usize] as char); + output.push(CHARSET[(b >> 12 & 0x3F) as usize] as char); + + if chunk.len() > 1 { + output.push(CHARSET[(b >> 6 & 0x3F) as usize] as char); + } else { + output.push('='); + } + + if chunk.len() > 2 { + output.push(CHARSET[(b & 0x3F) as usize] as char); + } else { + output.push('='); + } + } + output +} + +impl Default for WasmTTSModel { + fn default() -> Self { + Self::new() + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs new file mode 100644 index 00000000..9ff9246d --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs @@ -0,0 +1,108 @@ +use anyhow::Result; +use std::path::PathBuf; + +#[cfg(not(target_arch = "wasm32"))] +use candle_core::Device; + +#[cfg(not(target_arch = "wasm32"))] +use hf_hub::api::sync::ApiBuilder; +#[cfg(not(target_arch = "wasm32"))] +use hf_hub::{Repo, RepoType}; + +/// Download a file from HuggingFace Hub if necessary. +/// +/// Supports the format: `hf://owner/repo/filename@revision` +/// where `@revision` is optional. +/// +/// Note: Not available on wasm32 targets (use local file loading instead). +#[cfg(not(target_arch = "wasm32"))] +pub fn download_if_necessary(file_path: &str) -> Result { + if file_path.starts_with("hf://") { + let path = file_path.trim_start_matches("hf://"); + let parts: Vec<&str> = path.split('/').collect(); + if parts.len() < 3 { + anyhow::bail!( + "Invalid hf:// path: {}. Expected hf://repo_owner/repo_name/filename[@revision]", + file_path + ); + } + let repo_id = format!("{}/{}", parts[0], parts[1]); + let filename_with_revision = parts[2..].join("/"); + + // Parse optional revision from filename (e.g., "file.safetensors@abc123") + let (filename, revision) = if let Some(at_pos) = filename_with_revision.rfind('@') { + let (f, r) = filename_with_revision.split_at(at_pos); + (f.to_string(), Some(r[1..].to_string())) // Skip the '@' + } else { + (filename_with_revision, None) + }; + + // Use ApiBuilder to support HF_TOKEN from environment + let token = std::env::var("HF_TOKEN").ok(); + + let api = ApiBuilder::new().with_token(token).build()?; + + // Create repo with or without revision + let repo = if let Some(rev) = revision { + Repo::with_revision(repo_id, RepoType::Model, rev) + } else { + Repo::model(repo_id) + }; + + let api_repo = api.repo(repo); + let path = api_repo.get(&filename)?; + Ok(path) + } else { + Ok(PathBuf::from(file_path)) + } +} + +/// WASM version: Only supports local file paths +#[cfg(target_arch = "wasm32")] +pub fn download_if_necessary(file_path: &str) -> Result { + if file_path.starts_with("hf://") { + anyhow::bail!("HuggingFace Hub downloads not supported on WASM. Use local file paths."); + } + Ok(PathBuf::from(file_path)) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn load_weights( + file_path: &str, + _device: &Device, +) -> Result { + let path = download_if_necessary(file_path)?; + let safetensors = unsafe { candle_core::safetensors::MmapedSafetensors::new(path)? }; + Ok(safetensors) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_download_if_necessary_local() { + let path = "test.safetensors"; + let res = download_if_necessary(path).unwrap(); + assert_eq!(res, PathBuf::from(path)); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn test_invalid_hf_path() { + let path = "hf://invalid"; + let res = download_if_necessary(path); + assert!(res.is_err()); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn test_parse_revision() { + // Test parsing logic (doesn't actually download) + let path = "hf://kyutai/pocket-tts/file.safetensors@abc123def"; + // This will fail to download but we're testing the parsing + let res = download_if_necessary(path); + // We expect a network error, not a parsing error + assert!(res.is_err()); + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs new file mode 100644 index 00000000..7b5aad67 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs @@ -0,0 +1,320 @@ +//! Integration tests for TTSModel with real weights +//! +//! These tests require the HF_TOKEN environment variable to be set +//! for downloading model weights from HuggingFace. + +use pocket_tts::audio::{read_wav, write_wav}; +use pocket_tts::voice_state::init_states; +use pocket_tts::weights::download_if_necessary; + +use std::path::PathBuf; + +fn get_ref_wav_path() -> PathBuf { + // pocket-tts -> crates -> project_root + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("assets") + .join("ref.wav") +} + +#[test] +fn test_download_non_gated_tokenizer() { + // Test downloading from non-gated repo (pocket-tts-without-voice-cloning) + let path = "hf://kyutai/pocket-tts-without-voice-cloning/tokenizer.model@d4fdd22ae8c8e1cb3634e150ebeff1dab2d16df3"; + let result = download_if_necessary(path); + assert!(result.is_ok(), "Failed to download: {:?}", result.err()); + let local_path = result.unwrap(); + assert!( + local_path.exists(), + "Downloaded file does not exist: {:?}", + local_path + ); + println!("Downloaded to: {:?}", local_path); +} + +#[test] +// #[ignore = "requires HF_TOKEN and gated model access"] +fn test_download_gated_weights() { + // Test downloading from gated repo (pocket-tts) + let path = + "hf://kyutai/pocket-tts/tts_b6369a24.safetensors@427e3d61b276ed69fdd03de0d185fa8a8d97fc5b"; + let result = download_if_necessary(path); + assert!(result.is_ok(), "Failed to download: {:?}", result.err()); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +fn test_tts_model_load() { + use pocket_tts::TTSModel; + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + assert_eq!(model.sample_rate, 24000); + assert_eq!(model.dim, 1024); + assert_eq!(model.ldim, 32); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +fn test_voice_cloning_from_ref_wav() { + use pocket_tts::TTSModel; + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + + let ref_wav_path = get_ref_wav_path(); + if !ref_wav_path.exists() { + eprintln!("ref.wav not found at {:?}, skipping test", ref_wav_path); + return; + } + + let voice_state = model + .get_voice_state(&ref_wav_path) + .expect("Failed to get voice state"); + + // Voice state should have entries from running through the transformer + assert!(!voice_state.is_empty(), "Voice state should not be empty"); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +fn test_audio_generation_produces_valid_output() { + use pocket_tts::TTSModel; + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + + let ref_wav_path = get_ref_wav_path(); + if !ref_wav_path.exists() { + eprintln!("ref.wav not found at {:?}, skipping test", ref_wav_path); + return; + } + + let voice_state = model + .get_voice_state(&ref_wav_path) + .expect("Failed to get voice state"); + + let audio = model + .generate("Hello world.", &voice_state) + .expect("Failed to generate audio"); + + // Check output shape + let dims = audio.dims(); + assert_eq!(dims.len(), 2, "Audio should be [channels, samples]"); + assert_eq!(dims[0], 1, "Should have 1 channel"); + assert!(dims[1] > 0, "Should have some samples"); + + // Audio should be reasonable length (at least 0.1 seconds for "Hello world") + let duration_seconds = dims[1] as f32 / model.sample_rate as f32; + assert!( + duration_seconds > 0.1, + "Audio should be at least 0.1 seconds, got {}", + duration_seconds + ); + + // Optional: save to file for manual inspection + let output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test_output.wav"); + write_wav(&output_path, &audio, model.sample_rate as u32).expect("Failed to write audio"); + println!("Test audio written to {:?}", output_path); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +fn test_mimi_encode_decode_roundtrip() { + use pocket_tts::TTSModel; + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + + let ref_wav_path = get_ref_wav_path(); + if !ref_wav_path.exists() { + eprintln!("ref.wav not found at {:?}, skipping test", ref_wav_path); + return; + } + + let (audio, sample_rate) = read_wav(&ref_wav_path).expect("Failed to read ref.wav"); + + // Resample if needed + let audio = if sample_rate != model.sample_rate as u32 { + pocket_tts::audio::resample(&audio, sample_rate, model.sample_rate as u32) + .expect("Failed to resample") + } else { + audio + }; + + // Add batch dimension: [C, T] -> [B, C, T] + let audio = audio.unsqueeze(0).expect("Failed to add batch dim"); + + // Pad audio to a multiple of frame size + let frame_size = model.mimi.frame_size(); + let (b, c, t) = audio.dims3().expect("dims3"); + let pad_len = if t % frame_size != 0 { + frame_size - (t % frame_size) + } else { + 0 + }; + let audio = if pad_len > 0 { + let pad = + candle_core::Tensor::zeros((b, c, pad_len), audio.dtype(), audio.device()).unwrap(); + candle_core::Tensor::cat(&[&audio, &pad], 2).unwrap() + } else { + audio + }; + + // Encode + let mut encode_state = init_states(1, 1000); + let latent = model + .mimi + .encode_to_latent(&audio, &mut encode_state, 0) + .expect("Failed to encode"); + + println!("Encoded latent shape: {:?}", latent.dims()); + + // Decode + let mut decode_state = init_states(1, 1000); + let decoded = model + .mimi + .decode_from_latent(&latent, &mut decode_state, 0) + .expect("Failed to decode"); + + println!("Decoded audio shape: {:?}", decoded.dims()); + + // The decoded audio should have similar length (within a frame) + let original_len = audio.dims()[2]; + let decoded_len = decoded.dims()[2]; + + // Allow for some length difference due to framing + let max_diff = 1920 * 2; // ~2 frames at 24kHz + let len_diff = (original_len as i64 - decoded_len as i64).unsigned_abs() as usize; + assert!( + len_diff < max_diff, + "Audio length mismatch: original={}, decoded={}, diff={}", + original_len, + decoded_len, + len_diff + ); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +fn test_generate_with_pauses_adds_silence() { + use pocket_tts::TTSModel; + let model = TTSModel::load_with_params( + "b6369a24", + 0.0, + pocket_tts::config::defaults::LSD_DECODE_STEPS, + pocket_tts::config::defaults::EOS_THRESHOLD, + ) + .expect("Failed to load model"); + + let ref_wav_path = get_ref_wav_path(); + if !ref_wav_path.exists() { + eprintln!("ref.wav not found at {:?}, skipping test", ref_wav_path); + return; + } + + let voice_state = model + .get_voice_state(&ref_wav_path) + .expect("Failed to get voice state"); + + // Generate with pause (should be longer due to silence) + let text_with_pause = "Hello [pause:500ms] world."; + let audio_with_pause = model + .generate_with_pauses(text_with_pause, &voice_state) + .expect("Failed to generate audio with pauses"); + + // Get the clean text and generate audio for it directly + let clean_text = pocket_tts::pause::strip_pause_markers(text_with_pause); + let audio_base = model + .generate(&clean_text, &voice_state) + .expect("Failed to generate audio base"); + + let no_pause_samples = audio_base.dims()[1]; + let with_pause_samples = audio_with_pause.dims()[1]; + + // Audio with pause should be exactly 500ms longer (12000 samples at 24kHz) + // PLUS one extra EOS-tail (since we split into two segments, and each has a tail) + let expected_extra_samples = 12000; + + // Each segment in generate_stream_long gets its own EOS tail. + // The baseline generate() call has 1 tail. + // Our generate_with_pauses() call has 2 segments, thus 2 tails. + let mimi_frame_size = 1920; + let frames_after_eos = pocket_tts::tts_model::estimate_frames_after_eos("Hello"); + let extra_tail_samples = mimi_frame_size * frames_after_eos; + + let diff = with_pause_samples.saturating_sub(no_pause_samples); + + assert!( + diff >= expected_extra_samples + extra_tail_samples, + "Audio with pause should be at least {} samples longer (including extra tail), got {}", + expected_extra_samples + extra_tail_samples, + diff + ); + + // Should be very close to the expected extra + extra tail + // Allow for one Mimi frame of jitter (+/- 1920 samples) which can happen due to + // segment-level termination differences or model noise at the EOS boundary. + assert!( + diff <= expected_extra_samples + extra_tail_samples + mimi_frame_size + 10, + "Pause duration too long: got {} samples, expected ~{}", + diff, + expected_extra_samples + extra_tail_samples + ); +} + +#[test] +// #[ignore = "requires HF_TOKEN and model download"] +#[cfg(feature = "quantized")] +fn test_load_quantized_model() { + use pocket_tts::TTSModel; + let model = TTSModel::load_quantized("b6369a24").expect("Failed to load quantized model"); + + // Verify model loaded + assert_eq!(model.sample_rate, 24000); + assert_eq!(model.dim, 1024); + + // Verify is_quantized flag (currently returns false as placeholder) + // When real quantization is implemented, this should return true + assert!(!model.is_quantized()); // Placeholder behavior +} + +// Tests that don't require model download +#[test] +fn test_pause_module_integration() { + use pocket_tts::parse_text_with_pauses; + + let parsed = parse_text_with_pauses("Hello [pause:500ms] world... [pause:1s] done"); + + // Should have clean text without pause markers + assert!(!parsed.clean_text.contains("[pause:")); + + // Should have detected pauses + assert!(parsed.pauses.len() >= 2, "Should have at least 2 pauses"); + + // Check pause durations + let has_500ms = parsed.pauses.iter().any(|p| p.duration_ms == 500); + let has_1000ms = parsed.pauses.iter().any(|p| p.duration_ms == 1000); + assert!(has_500ms, "Should have 500ms pause"); + assert!(has_1000ms, "Should have 1000ms (1s) pause"); +} + +#[test] +fn test_quantize_module_integration() { + use candle_core::{Device, Tensor}; + use pocket_tts::{QuantizeConfig, QuantizedTensor}; + + let device = Device::Cpu; + let tensor = Tensor::new(&[1.0f32, 2.0, -3.0, 4.5, -2.1, 0.5, -0.5, 1.5], &device).unwrap(); + + // Test quantization + let quantized = QuantizedTensor::quantize(&tensor, 256).unwrap(); + + // Verify scale is reasonable + let scale = quantized.scale(); + assert!(scale > 0.0, "Scale should be positive"); + + // Verify memory savings + let savings = quantized.theoretical_memory_savings(); + assert_eq!(savings, 4.0, "int8 should give 4x memory savings"); + + // Test config + let config = QuantizeConfig::default(); + assert_eq!(config.num_levels, 256); + assert!(config.skip_layers.contains(&"embed".to_string())); +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs new file mode 100644 index 00000000..43add0da --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs @@ -0,0 +1,40 @@ +#[cfg(test)] +mod tests { + use candle_core::Tensor; + use pocket_tts::TTSModel; + use std::time::Instant; + + #[test] + #[ignore] // Ignore by default as it requires weights and takes time + fn test_long_audio_prompt_memory() -> anyhow::Result<()> { + println!("Loading model..."); + // Assuming weights are available for this variant. + // User's context implies "b6369a24" is relevant. + let model = TTSModel::load("b6369a24")?; + + println!("Creating long audio input (5 minutes @ 24kHz)..."); + // 5 * 60 * 24000 = 7,200,000 samples + let sample_rate = 24000; + let duration_secs = 60; + let num_samples = sample_rate * duration_secs; + + // Ensure we are using the device the model is on (CPU for now as per context) + let device = &model.device; + + // Random audio or silence. Random is better to avoid strict silence optimization if any. + let audio = Tensor::randn(0f32, 1f32, (1, 1, num_samples), device)?; + + let start = Instant::now(); + println!("Starting voice state processing..."); + + // This call was the one causing O(N^2) memory explosion + // It runs mimi encoding, projection, and then `run_flow_lm_prompt` + let _state = model.get_voice_state_from_tensor(&audio)?; + + let duration = start.elapsed(); + println!("Processed long audio in {:.2}s", duration.as_secs_f64()); + println!("Memory check passed (no panic/OOM)."); + + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs new file mode 100644 index 00000000..09999f93 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs @@ -0,0 +1,612 @@ +use candle_core::Tensor; +use pocket_tts::TTSModel; +use pocket_tts::audio::read_wav; +use pocket_tts::voice_state::init_states; +use std::path::PathBuf; + +fn get_project_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf() +} + +fn get_assets_root() -> PathBuf { + get_project_root().join("assets") +} + +fn get_ref_wav_path() -> PathBuf { + get_assets_root().join("ref.wav") +} + +fn assert_tensors_approx_eq(t1: &Tensor, t2: &Tensor, tolerance: f32) { + let diff = (t1 - t2).expect("sub failed").abs().expect("abs failed"); + let max_diff = diff + .max_all() + .expect("max_all failed") + .to_scalar::() + .expect("to_scalar failed"); + + let t1_max = t1 + .abs() + .unwrap() + .max_all() + .unwrap() + .to_scalar::() + .unwrap(); + let t2_max = t2 + .abs() + .unwrap() + .max_all() + .unwrap() + .to_scalar::() + .unwrap(); + println!( + "Max Diff: {}, T1 Max: {}, T2 Max: {}", + max_diff, t1_max, t2_max + ); + + assert!( + max_diff < tolerance, + "Tensors differ by max {}, which is > tolerance {}", + max_diff, + tolerance + ); +} + +#[test] +fn test_voice_conditioning_parity() { + let ref_path = get_assets_root().join("ref_voice_conditioning.safetensors"); + if !ref_path.exists() { + eprintln!("Skipping parity test: {:?} not found", ref_path); + return; + } + + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + // Use reference input if available to isolate speaker projection from resampler drift + let input_ref_path = get_assets_root().join("ref_mimi_input.safetensors"); + let audio = if input_ref_path.exists() { + let tensors = candle_core::safetensors::load(input_ref_path, &candle_core::Device::Cpu) + .expect("failed to load ref input"); + tensors + .get("mimi_input") + .expect("mimi_input not found") + .clone() + } else { + let (audio, sr) = read_wav(get_ref_wav_path()).expect("Failed to read ref.wav"); + + let audio = if sr != model.sample_rate as u32 { + pocket_tts::audio::resample(&audio, sr, model.sample_rate as u32) + .expect("Failed to resample") + } else { + audio + }; + audio.unsqueeze(0).expect("unsqueeze failed") + }; + + // Pad audio + let frame_size = model.mimi.frame_size(); + let (b, c, t) = audio.dims3().expect("dims3"); + let pad_len = if t % frame_size != 0 { + frame_size - (t % frame_size) + } else { + 0 + }; + let audio = if pad_len > 0 { + let pad = + candle_core::Tensor::zeros((b, c, pad_len), audio.dtype(), audio.device()).unwrap(); + candle_core::Tensor::cat(&[&audio, &pad], 2).unwrap() + } else { + audio + }; + + let mut state = init_states(1, 1000); + let latents = model + .mimi + .encode_to_latent(&audio, &mut state, 0) + .expect("encode failed"); + + // Calculate conditioning: latents * speaker_proj.t() + // Rust: latents is [B, C, T] ? Wait, Mimi output is [B, D, T] (32 dim) -> No, now 512 dim! + + let latents_t = latents.permute((0, 2, 1)).expect("permute failed"); + // latents_t: [B, T, 512] + + let (b, t, d) = latents_t.dims3().expect("dims3"); + let latents_flat = latents_t.reshape((b * t, d)).expect("reshape failed"); + let speaker_proj_t = model.speaker_proj_weight.t().expect("transpose failed"); + + let conditioning_flat = latents_flat.matmul(&speaker_proj_t).expect("matmul failed"); + let conditioning = conditioning_flat + .reshape((b, t, 1024)) + .expect("reshape back failed"); + + // Load ref + let tensors = candle_core::safetensors::load(ref_path, &candle_core::Device::Cpu) + .expect("failed to load ref tensors"); + let ref_cond = tensors + .get("voice_conditioning") + .expect("voice_conditioning not found"); + + // Python output might have different length due to padding? + // ref_cond same length as latents? + assert_eq!( + conditioning.dims(), + ref_cond.dims(), + "Conditioning shape mismatch" + ); + + assert_tensors_approx_eq(&conditioning, ref_cond, 2e-2); +} + +#[test] +fn test_mimi_latents_parity() { + let ref_path = get_assets_root().join("ref_mimi_latents.safetensors"); + if !ref_path.exists() { + eprintln!("Skipping parity test: {:?} not found", ref_path); + return; + } + + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + + /* + // Check initial layers weight stats + { + println!("Rust Mimi Encoder weight stats:"); + let vb = &model.vb; + + let layers = [ + ("mimi.encoder.model.0.conv", vec![64, 1, 7], true), + ("mimi.encoder.model.1.block.1.conv", vec![32, 64, 3], true), + ("mimi.encoder.model.1.block.3.conv", vec![64, 32, 1], true), + ]; + + for (name, shape, has_bias) in layers { + let vb_l = vb.pp(name); + let w = vb_l + .get(shape.clone(), "weight") + .expect(&format!("failed to get {}", name)); + println!( + "{}.weight: shape={:?}, mean={:.6}, min={:.6}, max={:.6}, abs_max={:.6}", + name, + w.dims(), + w.mean_all().unwrap().to_scalar::().unwrap(), + w.min_all().unwrap().to_scalar::().unwrap(), + w.max_all().unwrap().to_scalar::().unwrap(), + w.abs() + .unwrap() + .max_all() + .unwrap() + .to_scalar::() + .unwrap() + ); + + if has_bias { + let b_shape = shape[0]; + let b = vb_l + .get(b_shape, "bias") + .expect(&format!("failed to get {} bias", name)); + println!( + "{}.bias: shape={:?}, mean={:.6}, min={:.6}, max={:.6}, abs_max={:.6}", + name, + b.dims(), + b.mean_all().unwrap().to_scalar::().unwrap(), + b.min_all().unwrap().to_scalar::().unwrap(), + b.max_all().unwrap().to_scalar::().unwrap(), + b.abs() + .unwrap() + .max_all() + .unwrap() + .to_scalar::() + .unwrap() + ); + } + } + } + */ + + // Use reference input if available to isolate Mimi layers from resampler drift + let input_ref_path = get_assets_root().join("ref_mimi_input.safetensors"); + let audio = if input_ref_path.exists() { + let tensors = candle_core::safetensors::load(input_ref_path, &candle_core::Device::Cpu) + .expect("failed to load ref input"); + tensors + .get("mimi_input") + .expect("mimi_input not found") + .clone() + } else { + let (audio, sr) = read_wav(get_ref_wav_path()).expect("Failed to read ref.wav"); + let audio = if sr != model.sample_rate as u32 { + pocket_tts::audio::resample(&audio, sr, model.sample_rate as u32) + .expect("Failed to resample") + } else { + audio + }; + audio.unsqueeze(0).expect("unsqueeze failed") + }; + + // Encode with Mimi + // Note: Rust definition of encode_to_latent takes (x, state) + // But python script calls model.mimi.encode_to_latent(mimi_input) + // We must ensure we use the same state initialization or stateless if applicable. + // streaming convs need state. + // Pad audio to a multiple of frame size + let frame_size = model.mimi.frame_size(); + let (b, c, t) = audio.dims3().expect("dims3"); + let pad_len = if t % frame_size != 0 { + frame_size - (t % frame_size) + } else { + 0 + }; + let audio = if pad_len > 0 { + let pad = + candle_core::Tensor::zeros((b, c, pad_len), audio.dtype(), audio.device()).unwrap(); + candle_core::Tensor::cat(&[&audio, &pad], 2).unwrap() + } else { + audio + }; + + let mut state = init_states(1, 1000); + let latents = model + .mimi + .encode_to_latent(&audio, &mut state, 0) + .expect("encode failed"); + + // Load ref + let tensors = candle_core::safetensors::load(ref_path, &candle_core::Device::Cpu) + .expect("failed to load ref tensors"); + let ref_latents = tensors.get("mimi_latents").expect("mimi_latents not found"); + + // Compare intermediates + let mut state = init_states(1, 1000); + + // Layer 0: Conv + let layer0_out = if let Some(layer0_ref) = tensors.get("layer0_out") { + let out = model.mimi.encoder.layers[0] + .forward(&audio, &mut state, 0) + .expect("layer0 failed"); + println!( + "Layer0 Parity check... Rust: {:?}, Ref: {:?}", + out.dims(), + layer0_ref.dims() + ); + assert_tensors_approx_eq(&out, layer0_ref, 2e-2); + out + } else { + model.mimi.encoder.layers[0] + .forward(&audio, &mut state, 0) + .expect("layer0 failed") + }; + + // Layer 1: Resnet + let layer1_out = if let Some(layer1_ref) = tensors.get("layer1_out") { + let out = model.mimi.encoder.layers[1] + .forward(&layer0_out, &mut state, 0) + .expect("layer1 failed"); + println!( + "Layer1 (Resnet) Parity check... Rust: {:?}, Ref: {:?}", + out.dims(), + layer1_ref.dims() + ); + assert_tensors_approx_eq(&out, layer1_ref, 2e-2); + out + } else { + model.mimi.encoder.layers[1] + .forward(&layer0_out, &mut state, 0) + .expect("layer1 failed") + }; + + // Layer 2: ELU + let layer2_out = if let Some(layer2_ref) = tensors.get("layer2_out") { + let out = model.mimi.encoder.layers[2] + .forward(&layer1_out, &mut state, 0) + .expect("layer2 failed"); + println!( + "Layer2 (ELU) Parity check... Rust: {:?}, Ref: {:?}", + out.dims(), + layer2_ref.dims() + ); + assert_tensors_approx_eq(&out, layer2_ref, 2e-2); + out + } else { + model.mimi.encoder.layers[2] + .forward(&layer1_out, &mut state, 0) + .expect("layer2 failed") + }; + + // Layer 3: Downsample (ratio 4) + let _layer3_out = if let Some(layer3_ref) = tensors.get("layer3_out") { + let out = model.mimi.encoder.layers[3] + .forward(&layer2_out, &mut state, 0) + .expect("layer3 failed"); + println!( + "Layer3 (Downsample) Parity check... Rust: {:?}, Ref: {:?}", + out.dims(), + layer3_ref.dims() + ); + assert_tensors_approx_eq(&out, layer3_ref, 2e-2); + out + } else { + model.mimi.encoder.layers[3] + .forward(&layer2_out, &mut state, 0) + .expect("layer3 failed") + }; + + if let Some(seanet_ref) = tensors.get("seanet_out") { + // Run the full encoder to get seanet_out + let mut state = init_states(1, 1000); + let seanet_out = model + .mimi + .encoder + .forward(&audio, &mut state, 0) + .expect("seanet failed"); + println!( + "SEANet Parity check... Rust: {:?}, Ref: {:?}", + seanet_out.dims(), + seanet_ref.dims() + ); + assert_tensors_approx_eq(&seanet_out, seanet_ref, 2e-2); + } + + if let Some(tr_ref) = tensors.get("transformer_out") { + let mut state = init_states(1, 1000); + let seanet_out = model + .mimi + .encoder + .forward(&audio, &mut state, 0) + .expect("seanet failed"); + let mut tr_state = init_states(1, 1000); + let mut embs = model + .mimi + .encoder_transformer + .forward(&seanet_out, &mut tr_state, 0) + .expect("tr failed"); + let tr_out = embs.remove(0); + println!( + "Transformer Parity check... Rust: {:?}, Ref: {:?}", + tr_out.dims(), + tr_ref.dims() + ); + assert_tensors_approx_eq(&tr_out, tr_ref, 2e-2); + } + + assert_tensors_approx_eq(&latents, ref_latents, 0.1); +} + +#[test] +fn test_input_parity() { + let ref_path = get_assets_root().join("ref_mimi_input.safetensors"); + if !ref_path.exists() { + eprintln!("Skipping parity test: {:?} not found", ref_path); + return; + } + + let model = TTSModel::load("b6369a24").expect("Failed to load model"); + let (audio, sr) = read_wav(get_ref_wav_path()).expect("Failed to read ref.wav"); + + // Rust preprocessing + let audio = if sr != model.sample_rate as u32 { + pocket_tts::audio::resample(&audio, sr, model.sample_rate as u32) + .expect("Failed to resample") + } else { + audio + }; + // Ensure mono/batch as in parity tests + // Python mimi_input was: wav.unsqueeze(0) -> [1, C, T] ? + // In extract_refs.py: + // audio, sr = audio_read(...) -> [C, T] + // wav = convert_audio(...) -> [C, T] (mono) + // mimi_input = wav.unsqueeze(0) -> [1, C, T] + + // Rust audio is [C, T] after read (mono=1) + let b = 1; + let c = 1; + let x = audio.unsqueeze(0).expect("unsqueeze failed"); + let t = x.dims()[2]; + let hop = model.mimi.frame_size(); + let audio = if t % hop != 0 { + let padding = hop - (t % hop); + let pad = Tensor::zeros((b, c, padding), x.dtype(), x.device()).expect("pad zeros failed"); + Tensor::cat(&[x, pad], 2).expect("pad cat failed") + } else { + x + }; + + // Load ref + let tensors = candle_core::safetensors::load(ref_path, &candle_core::Device::Cpu) + .expect("failed to load ref tensors"); + let ref_input = tensors.get("mimi_input").expect("mimi_input not found"); + + // We expect ref_input to be [1, 1, T] + // And audio to be [1, 1, T] + println!( + "Input parity check... Rust: {:?}, Ref: {:?}", + audio.dims(), + ref_input.dims() + ); + assert_eq!(audio.dims(), ref_input.dims(), "Input shape mismatch"); + + // Check parity + assert_tensors_approx_eq(&audio, ref_input, 0.3); +} + +#[test] +fn test_audio_generation_parity() { + let ref_path = get_assets_root().join("ref_output.wav"); + if !ref_path.exists() { + eprintln!("Skipping parity test: {:?} not found", ref_path); + return; + } + + // Load Rust model with temperature = 0.0 for deterministic output + let model = TTSModel::load_with_params( + "b6369a24", + 0.0, // temp + pocket_tts::config::defaults::LSD_DECODE_STEPS, + pocket_tts::config::defaults::EOS_THRESHOLD, + ) + .expect("Failed to load model"); + + let voice_state = model + .get_voice_state(get_ref_wav_path()) + .expect("get_voice_state failed"); + + // Rust generation + let text = "Hello world."; + let generated = model + .generate(text, &voice_state) + .expect("generation failed"); + + // Load ref output for comparison + let (ref_audio, _) = read_wav(&ref_path).expect("Failed to read ref_output.wav"); + + println!( + "Generated dims: {:?}, Ref dims: {:?}", + generated.dims(), + ref_audio.dims() + ); + + // Save Rust output for manual listening + let output_path = get_project_root().join("test_output_parity.wav"); + pocket_tts::audio::write_wav(&output_path, &generated, model.sample_rate as u32) + .expect("failed to write output"); + println!("Saved Rust output to {:?}", output_path); + + // Basic sanity checks + let gen_samples = generated.dims()[1]; + let ref_samples = ref_audio.dims()[1]; + + // Audio should be in a reasonable range (generation is somewhat probabilistic even at temp=0) + // Due to accumulated state differences, allow wider tolerance + let length_ratio = gen_samples as f64 / ref_samples as f64; + assert!( + length_ratio > 0.2 && length_ratio < 5.0, + "Audio length significantly different: {} vs {} samples (ratio: {:.2})", + gen_samples, + ref_samples, + length_ratio + ); + + // Audio should have reasonable amplitude (not silent, not clipping) + let gen_max = generated + .abs() + .unwrap() + .max_all() + .unwrap() + .to_scalar::() + .unwrap(); + assert!( + gen_max > 0.01, + "Generated audio appears silent (max: {})", + gen_max + ); + assert!( + gen_max < 2.0, + "Generated audio appears clipped (max: {})", + gen_max + ); + + println!( + "Audio generation parity test passed (manual listening required for quality verification)" + ); + println!( + "Listen to: {:?} and compare with {:?}", + output_path, ref_path + ); +} + +#[test] +fn test_decoder_parity() { + let ref_path = get_assets_root().join("ref_decoder_intermediates.safetensors"); + if !ref_path.exists() { + eprintln!("Skipping decoder parity test: {:?} not found", ref_path); + eprintln!("Run: uv run python extract_decoder_refs.py"); + return; + } + + // Load reference intermediates + let tensors = candle_core::safetensors::load(&ref_path, &candle_core::Device::Cpu) + .expect("Failed to load ref_decoder_intermediates.safetensors"); + + let ref_quantized = tensors.get("quantized").expect("missing quantized"); + let ref_after_upsample = tensors + .get("after_upsample") + .expect("missing after_upsample"); + let ref_after_decoder_tr = tensors + .get("after_decoder_transformer") + .expect("missing after_decoder_transformer"); + let ref_final_audio = tensors.get("final_audio").expect("missing final_audio"); + + println!("Loaded reference decoder intermediates:"); + println!(" quantized: {:?}", ref_quantized.dims()); + println!(" after_upsample: {:?}", ref_after_upsample.dims()); + println!( + " after_decoder_transformer: {:?}", + ref_after_decoder_tr.dims() + ); + println!(" final_audio: {:?}", ref_final_audio.dims()); + + // Load model + let model = TTSModel::load_with_params( + "b6369a24", + 0.0, + pocket_tts::config::defaults::LSD_DECODE_STEPS, + pocket_tts::config::defaults::EOS_THRESHOLD, + ) + .expect("Failed to load model"); + + // Use the same quantized input as Python (to isolate decoder issues) + let mut mimi_state = init_states(1, 1000); + + // Test upsample + let after_upsample = if let Some(ref up) = model.mimi.upsample { + let out = up + .forward(ref_quantized, &mut mimi_state, 0) + .expect("upsample failed"); + println!( + "\nUpsample Rust: {:?}, Ref: {:?}", + out.dims(), + ref_after_upsample.dims() + ); + assert_tensors_approx_eq(&out, ref_after_upsample, 0.05); + println!("✓ Upsample PASSED"); + out + } else { + ref_quantized.clone() + }; + + // Test decoder transformer + let mut after_decoder_tr_vec = model + .mimi + .decoder_transformer + .forward(&after_upsample, &mut mimi_state, 0) + .expect("decoder_transformer failed"); + let after_decoder_tr = after_decoder_tr_vec.remove(0); + + println!( + "\nDecoder Transformer Rust: {:?}, Ref: {:?}", + after_decoder_tr.dims(), + ref_after_decoder_tr.dims() + ); + assert_tensors_approx_eq(&after_decoder_tr, ref_after_decoder_tr, 0.05); + println!("✓ Decoder Transformer PASSED"); + + // Test SEANet decoder + let final_audio = model + .mimi + .decoder + .forward(&after_decoder_tr, &mut mimi_state, 0) + .expect("decoder failed"); + + println!( + "\nFinal Audio Rust: {:?}, Ref: {:?}", + final_audio.dims(), + ref_final_audio.dims() + ); + assert_tensors_approx_eq(&final_audio, ref_final_audio, 0.1); + println!("✓ SEANet Decoder PASSED"); + + println!("\n=== ALL DECODER STAGES PASS ==="); +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs new file mode 100644 index 00000000..66927220 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs @@ -0,0 +1,156 @@ +#[cfg(test)] +mod tests { + use anyhow::Result; + use candle_core::Tensor; + use pocket_tts::TTSModel; + + // Helper to compare tensors + fn assert_tensors_close(t1: &Tensor, t2: &Tensor, tolerance: f64) -> Result<()> { + let diff = (t1 - t2)?.abs()?; + let max_diff = diff.max_all()?.to_scalar::()? as f64; + assert!( + max_diff < tolerance, + "Tensors differ by max {}. Tolerance: {}", + max_diff, + tolerance + ); + Ok(()) + } + + #[test] + fn test_streaming_matches_batch() -> Result<()> { + // Load model (assumes weights are present from previous phases) + let mut model = TTSModel::load("b6369a24")?; + + // Use deterministic settings for comparison + model.temp = 0.0; + + // Get voice state (using ref.wav as standard) + // Adjust path to point to project root ref.wav if running from crate dir + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + + if !ref_wav.exists() { + println!("Skipping test: ref.wav not found at {:?}", ref_wav); + return Ok(()); + } + + let state = model.get_voice_state(&ref_wav)?; + + let text = "Hello world, this is a test."; + + // 1. Batch generation + let batch_audio = model.generate(text, &state)?; + + // 2. Streaming generation + let stream_chunks: Vec = model + .generate_stream(text, &state) + .collect::>>()?; + + assert!( + !stream_chunks.is_empty(), + "Stream should yield at least one chunk" + ); + + // Concatenate chunks + let stream_audio = Tensor::cat(&stream_chunks, 2)?.squeeze(0)?; + + // 3. Compare + // Note: Floating point non-associativity might cause slight differences + // between batch (one big matmul mostly) and stream (step-by-step). + // Using a slightly loose tolerance. + assert_tensors_close(&batch_audio, &stream_audio, 1e-4)?; + + Ok(()) + } + + #[test] + fn test_streaming_yields_multiple_chunks() -> Result<()> { + let mut model = TTSModel::load("b6369a24")?; + model.temp = 0.0; + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + + if !ref_wav.exists() { + return Ok(()); + } + + let state = model.get_voice_state(&ref_wav)?; + + // Use a long enough text to ensure multiple chunks + let text = "This is a longer sentence to ensure that we get multiple chunks from the streaming generator."; + + let chunks: Vec<_> = model + .generate_stream(text, &state) + .collect::>>()?; + + println!("Generated {} chunks", chunks.len()); + assert!( + chunks.len() > 1, + "Should generate multiple chunks for long text" + ); + + // Verify chunk shapes (should be [1, 1, 1024] or similar depending on Mimi config) + // First chunk might be different or last chunk might be padding, but generally checking correctness. + for chunk in chunks.iter().take(chunks.len() - 1) { + let dims = chunk.dims(); + assert_eq!(dims.len(), 3, "Chunk should be [B, C, T]"); + assert_eq!(dims[0], 1); + assert_eq!(dims[1], 1); + // In Mimi, one frame is 1024 samples (approx 80ms at 12.5Hz frame rate)? + // Actually check expected size from config if strict, but just checking properties here. + } + + Ok(()) + } + + #[test] + #[ignore] + fn test_long_text_handling() -> Result<()> { + let mut model = TTSModel::load("b6369a24")?; + model.temp = 0.0; + + let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let ref_wav = root_dir.join("assets").join("ref.wav"); + + if !ref_wav.exists() { + return Ok(()); + } + + let state = model.get_voice_state(&ref_wav)?; + + // Create a long text input (approx 50 sentences) + let sentence = "This is a sentence that is repeated to simulate a long text input for the TTS model testing purposes."; + let long_text = (0..50).map(|_| sentence).collect::>().join(" "); + + // This effectively also tests memory usage if we monitor it, but here just checking it completes + // and yields chunks without error. + let mut chunks_count = 0; + for chunk in model.generate_stream_long(&long_text, &state) { + let _ = chunk?; + chunks_count += 1; + } + + assert!( + chunks_count > 10, + "Should generate many chunks for long text" + ); + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/wasm-pack.toml b/plugins/native/pocket-tts/vendor/pocket-tts/wasm-pack.toml new file mode 100644 index 00000000..a748ca82 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/wasm-pack.toml @@ -0,0 +1,22 @@ +# Pocket TTS WASM Build Configuration +# +# Build with: wasm-pack build --target web --features wasm +# +# This generates a `pkg/` directory with: +# - pocket_tts.js - ES module wrapper +# - pocket_tts_bg.wasm - WebAssembly binary +# - pocket_tts.d.ts - TypeScript definitions + +[package] +name = "pocket-tts" + +[package.metadata.wasm-pack.profile.release] +# Enable optimizations for smaller WASM binary +wasm-opt = ["-Oz", "--enable-mutable-globals"] + +[package.metadata.wasm-pack.profile.dev] +# Faster builds during development +wasm-opt = false + +[package.metadata.wasm-pack.profile.profiling] +wasm-opt = ["-O", "--enable-mutable-globals"] diff --git a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml new file mode 100644 index 00000000..095714e9 --- /dev/null +++ b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# +# SPDX-License-Identifier: MPL-2.0 + +# skit:input_formats=voice:wav + +name: Text-to-Speech (Pocket TTS Voice Clone) +description: Synthesizes speech from text using Pocket TTS with a voice prompt WAV +mode: oneshot +nodes: + uploads: + kind: streamkit::http_input + params: + fields: + - name: voice + required: false + - name: text + + text_chunker: + kind: core::text_chunker + params: + min_length: 10 + needs: uploads.text + + pocket_tts: + kind: plugin::native::pocket-tts + params: + weights_path: "models/pocket-tts/tts_b6369a24.safetensors" + tokenizer_path: "models/pocket-tts/tokenizer.model" + voice_embeddings_dir: "models/pocket-tts/embeddings" + temperature: 0.7 + lsd_decode_steps: 1 + eos_threshold: -4.0 + needs: + - text_chunker + - uploads.voice + + resampler: + kind: audio::resampler + params: + chunk_frames: 960 + output_frame_size: 960 + target_sample_rate: 48000 + needs: pocket_tts + + opus_encoder: + kind: audio::opus::encoder + needs: resampler + + webm_muxer: + kind: containers::webm::muxer + params: + channels: 1 + chunk_size: 65536 + sample_rate: 48000 + streaming_mode: live + needs: opus_encoder + + http_output: + kind: streamkit::http_output + needs: webm_muxer diff --git a/samples/pipelines/oneshot/pocket-tts.yml b/samples/pipelines/oneshot/pocket-tts.yml new file mode 100644 index 00000000..ae088dd7 --- /dev/null +++ b/samples/pipelines/oneshot/pocket-tts.yml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# +# SPDX-License-Identifier: MPL-2.0 + +name: Text-to-Speech (Pocket TTS) +description: Synthesizes speech from text using Pocket TTS +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: core::text_chunker + params: + min_length: 10 + - kind: plugin::native::pocket-tts + params: + weights_path: "models/pocket-tts/tts_b6369a24.safetensors" + tokenizer_path: "models/pocket-tts/tokenizer.model" + voice_embeddings_dir: "models/pocket-tts/embeddings" + voice: alba + temperature: 0.7 + lsd_decode_steps: 1 + eos_threshold: -4.0 + - kind: audio::resampler + params: + chunk_frames: 960 + output_frame_size: 960 + target_sample_rate: 48000 + - kind: audio::opus::encoder + - kind: containers::webm::muxer + params: + channels: 1 + chunk_size: 65536 + sample_rate: 48000 + streaming_mode: live + - kind: streamkit::http_output diff --git a/ui/src/components/converter/FileUpload.tsx b/ui/src/components/converter/FileUpload.tsx index 0b29b641..5758fa87 100644 --- a/ui/src/components/converter/FileUpload.tsx +++ b/ui/src/components/converter/FileUpload.tsx @@ -92,15 +92,18 @@ interface FileUploadProps { file: File | null; onFileSelect: (file: File | null) => void; accept?: string; + hint?: string; } export const FileUpload: React.FC = ({ file, onFileSelect, accept = 'audio/*,.ogg,.opus,.mp3,.wav,.flac', + hint, }) => { const [isDragActive, setIsDragActive] = useState(false); const fileInputRef = React.useRef(null); + const hintText = hint ?? 'Supports audio files (Ogg/Opus, MP3, WAV, FLAC)'; const handleDrag = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -176,7 +179,7 @@ export const FileUpload: React.FC = ({ {isDragActive ? 'Drop audio file here' : 'Click to upload or drag and drop'} - Supports audio files (Ogg/Opus, MP3, WAV, FLAC) + {hintText} diff --git a/ui/src/views/ConvertView.tsx b/ui/src/views/ConvertView.tsx index a9a285f6..3cf49f1e 100644 --- a/ui/src/views/ConvertView.tsx +++ b/ui/src/views/ConvertView.tsx @@ -6,8 +6,6 @@ import styled from '@emotion/styled'; import { load as loadYaml } from 'js-yaml'; import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -/* eslint-disable max-depth */ - import { AssetSelector } from '@/components/converter/AssetSelector'; import { ConversionProgress } from '@/components/converter/ConversionProgress'; import { FileUpload } from '@/components/converter/FileUpload'; @@ -33,6 +31,10 @@ import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering' import { injectFileReadNode } from '@/utils/yamlPipeline'; type HttpInputField = { name: string; required: boolean }; +type InputFormatSpec = { all: string[]; perField: Record }; + +const resolveUploadFields = (httpInputFields: HttpInputField[]): HttpInputField[] => + httpInputFields.length > 0 ? httpInputFields : [{ name: 'media', required: true }]; const normalizeHttpInputField = (entry: unknown): HttpInputField | null => { if (typeof entry === 'string' && entry.trim()) { @@ -513,6 +515,193 @@ const checkIfTTSPipeline = (yaml: string): boolean => { return hasTTS && !hasAudioDemuxer; }; +const resolveTextField = (fields: HttpInputField[]): HttpInputField | null => { + if (fields.length === 0) { + return null; + } + const textField = fields.find((field) => field.name.toLowerCase() === 'text'); + return textField ?? fields[0]; +}; + +const splitTtsFields = ( + fields: HttpInputField[] +): { textField: HttpInputField | null; extraFields: HttpInputField[] } => { + const textField = resolveTextField(fields); + if (!textField) { + return { textField: null, extraFields: [] }; + } + return { + textField, + extraFields: fields.filter((field) => field.name !== textField.name), + }; +}; + +const FORMAT_ACCEPT_MAP: Record = { + ogg: '.ogg', + opus: '.opus', + mp3: '.mp3', + wav: '.wav', + flac: '.flac', + txt: '.txt', + text: '.txt', + json: '.json', +}; + +const detectInputFormatSpec = (yaml: string): InputFormatSpec | null => { + const match = yaml.match(/^\s*#\s*skit:input_formats\s*=\s*([^\n#]+)\s*$/im); + if (!match?.[1]) return null; + + const spec: InputFormatSpec = { all: [], perField: {} }; + + for (const entry of match[1].split(',')) { + const token = entry.trim(); + if (!token) continue; + + const parts = token.split(':'); + if (parts.length > 1) { + const field = parts[0].trim().toLowerCase(); + const formats = parts + .slice(1) + .join(':') + .split(/[|+]/) + .map((format) => format.trim().toLowerCase()) + .filter(Boolean); + + if (!field || formats.length === 0) continue; + + const existing = spec.perField[field] ?? []; + for (const format of formats) { + if (!existing.includes(format)) { + existing.push(format); + } + } + spec.perField[field] = existing; + } else { + const format = token.toLowerCase(); + if (!spec.all.includes(format)) { + spec.all.push(format); + } + } + } + + if (spec.all.length === 0 && Object.keys(spec.perField).length === 0) { + return null; + } + + return spec; +}; + +const resolveFormatsForField = ( + fieldName: string, + formatSpec: InputFormatSpec | null, + fallbackFormats: string[] | null +): string[] | null => { + const key = fieldName.toLowerCase(); + const perField = formatSpec?.perField?.[key]; + if (perField && perField.length > 0) { + return perField; + } + if (formatSpec?.all && formatSpec.all.length > 0) { + return formatSpec.all; + } + return fallbackFormats; +}; + +const buildNoInputUploads = (fields: HttpInputField[]): UploadField[] => { + const blob = new Blob([''], { type: 'application/octet-stream' }); + const file = new File([blob], 'empty', { type: 'application/octet-stream' }); + return [{ field: fields[0].name, file }]; +}; + +const buildTtsUploads = ( + fields: HttpInputField[], + textInput: string, + fieldUploads: Record +): UploadField[] | null => { + if (!textInput.trim()) { + return null; + } + + const { textField, extraFields } = splitTtsFields(fields); + const textFieldName = textField?.name ?? fields[0]?.name; + if (!textFieldName) { + return null; + } + + const uploads: UploadField[] = []; + const blob = new Blob([textInput], { type: 'text/plain' }); + const file = new File([blob], 'input.txt', { type: 'text/plain' }); + uploads.push({ field: textFieldName, file }); + + for (const field of extraFields) { + const fieldFile = fieldUploads[field.name]; + if (!fieldFile) { + if (field.required) return null; + continue; + } + uploads.push({ field: field.name, file: fieldFile }); + } + + return uploads; +}; + +const buildUploadModeUploads = ( + fields: HttpInputField[], + fieldUploads: Record, + selectedFile: File | null +): UploadField[] | null => { + if (fields.length > 1) { + const uploads: UploadField[] = []; + for (const field of fields) { + const file = fieldUploads[field.name]; + if (!file) { + if (field.required) return null; + continue; + } + uploads.push({ field: field.name, file }); + } + return uploads; + } + + if (!selectedFile) { + return null; + } + return [{ field: fields[0].name, file: selectedFile }]; +}; + +const formatHintForField = ( + field: HttpInputField, + formatSpec: InputFormatSpec | null, + fallbackFormats: string[] | null, + isTts: boolean +): { accept?: string; hint?: string } => { + const name = field.name.toLowerCase(); + const fieldOverrides = + (formatSpec?.all?.length ?? 0) > 0 || (formatSpec?.perField?.[name]?.length ?? 0) > 0; + + if (isTts && name.includes('voice') && !fieldOverrides) { + return { accept: 'audio/wav,.wav,.wave', hint: 'Expected format: WAV audio' }; + } + + const formats = resolveFormatsForField(field.name, formatSpec, fallbackFormats); + if (!formats || formats.length === 0) { + return {}; + } + + const unique = Array.from(new Set(formats.map((format) => format.toLowerCase()))); + const accept = unique + .map((format) => FORMAT_ACCEPT_MAP[format]) + .filter(Boolean) + .join(','); + const label = unique.map((format) => format.toUpperCase()).join(', '); + const hint = `Expected format${unique.length > 1 ? 's' : ''}: ${label}`; + + return { + accept: accept || undefined, + hint, + }; +}; + /** * Generates a CLI command for running the pipeline with curl + ffplay */ @@ -537,10 +726,16 @@ const generateCliCommand = ( if (isTTS) { // TTS pipeline - pipe text input + const { textField, extraFields } = splitTtsFields(activeFields); + const textFieldName = textField?.name ?? 'media'; + const extraLines = extraFields + .map((field) => ` -F ${field.name}=@your-${field.name}.wav \\`) + .join('\n'); + const extraBlock = extraLines ? `${extraLines}\n` : ''; return `echo "Your text here" | curl --no-buffer \\ -F config=@${configPath} \\ - -F 'media=@-;type=text/plain' \\ - ${serverUrl}/api/v1/process -o - | ffplay -f webm -i -`; + -F '${textFieldName}=@-;type=text/plain' \\ +${extraBlock} ${serverUrl}/api/v1/process -o - | ffplay -f webm -i -`; } // Multi-upload pipelines @@ -773,31 +968,37 @@ const ConvertView: React.FC = () => { }; // Filter assets based on pipeline's expected format; for multi-field uploads, allow all assets so fields can mix + const inputFormatSpec = React.useMemo(() => detectInputFormatSpec(pipelineYaml), [pipelineYaml]); + const inferredFormats = React.useMemo(() => detectExpectedFormats(pipelineYaml), [pipelineYaml]); + const assetFieldName = httpInputFields.length > 0 ? httpInputFields[0].name : 'media'; + const assetFormats = React.useMemo( + () => resolveFormatsForField(assetFieldName, inputFormatSpec, inferredFormats), + [assetFieldName, inputFormatSpec, inferredFormats] + ); const filteredAssets = React.useMemo(() => { if (!pipelineYaml) { return audioAssets; } - const expectedFormats = detectExpectedFormats(pipelineYaml); const inputAssetTags = detectInputAssetTags(pipelineYaml); // Multi-field pipelines: only filter by format (avoid tag-based narrowing so users can mix content) if (httpInputFields.length > 1) { - if (!expectedFormats) return audioAssets; - return audioAssets.filter((asset) => expectedFormats.includes(asset.format.toLowerCase())); + if (!assetFormats) return audioAssets; + return audioAssets.filter((asset) => assetFormats.includes(asset.format.toLowerCase())); } // Single-field pipelines: apply both format and tag filters if present - if (!expectedFormats && !inputAssetTags) { + if (!assetFormats && !inputAssetTags) { viewsLogger.debug('No specific format required, showing all assets'); return audioAssets; } - viewsLogger.debug('Expected formats:', expectedFormats, 'Total assets:', audioAssets.length); + viewsLogger.debug('Expected formats:', assetFormats, 'Total assets:', audioAssets.length); // Filter assets to only those with compatible formats - const formatFiltered = expectedFormats - ? audioAssets.filter((asset) => expectedFormats.includes(asset.format.toLowerCase())) + const formatFiltered = assetFormats + ? audioAssets.filter((asset) => assetFormats.includes(asset.format.toLowerCase())) : audioAssets; const tagFiltered = inputAssetTags @@ -809,7 +1010,7 @@ const ConvertView: React.FC = () => { viewsLogger.debug('Filtered to', tagFiltered.length, 'compatible assets'); return tagFiltered; - }, [audioAssets, pipelineYaml, httpInputFields.length]); + }, [audioAssets, pipelineYaml, httpInputFields.length, assetFormats]); // Clear selected asset if it's no longer in the filtered list useEffect(() => { @@ -939,43 +1140,18 @@ const ConvertView: React.FC = () => { }; const prepareUploads = useCallback(async (): Promise => { - const fields = - httpInputFields.length > 0 ? httpInputFields : [{ name: 'media', required: true }]; + const fields = resolveUploadFields(httpInputFields); if (isNoInputPipeline) { - // No input needed - create empty placeholder for the first field - const blob = new Blob([''], { type: 'application/octet-stream' }); - const file = new File([blob], 'empty', { type: 'application/octet-stream' }); - return [{ field: fields[0].name, file }]; + return buildNoInputUploads(fields); } if (isTTSPipeline) { - if (!textInput.trim()) { - return null; - } - const blob = new Blob([textInput], { type: 'text/plain' }); - const file = new File([blob], 'input.txt', { type: 'text/plain' }); - return [{ field: fields[0].name, file }]; + return buildTtsUploads(fields, textInput, fieldUploads); } if (inputMode === 'upload') { - if (fields.length > 1) { - const uploads: UploadField[] = []; - for (const field of fields) { - const file = fieldUploads[field.name]; - if (!file) { - if (field.required) return null; - continue; - } - uploads.push({ field: field.name, file }); - } - return uploads; - } - - if (!selectedFile) { - return null; - } - return [{ field: fields[0].name, file: selectedFile }]; + return buildUploadModeUploads(fields, fieldUploads, selectedFile); } if (!selectedAssetId || !hasHttpInput) { @@ -1318,6 +1494,14 @@ const ConvertView: React.FC = () => { const uploadFields = httpInputFields.length > 0 ? httpInputFields : [{ name: 'media', required: true }]; const isMultiUpload = uploadFields.length > 1; + const { extraFields: ttsExtraFields } = splitTtsFields(uploadFields); + const ttsMissingRequiredUploads = ttsExtraFields.some( + (field) => field.required && !fieldUploads[field.name] + ); + const singleUploadHint = + uploadFields.length > 0 + ? formatHintForField(uploadFields[0], inputFormatSpec, inferredFormats, isTTSPipeline) + : {}; const handleDownloadAudio = () => { if (!audioUrl) return; @@ -1369,7 +1553,7 @@ const ConvertView: React.FC = () => { (isNoInputPipeline ? true // No input needed for these pipelines : isTTSPipeline - ? textInput.trim() !== '' + ? textInput.trim() !== '' && !ttsMissingRequiredUploads : (() => { if (!hasHttpInput) { // Pipelines without http_input rely solely on YAML/file_reader; allow run @@ -1488,6 +1672,35 @@ const ConvertView: React.FC = () => { aria-label="Text input for TTS conversion" /> {textInput.length} characters + {ttsExtraFields.length > 0 && ( +
+ Additional uploads: + {ttsExtraFields.map((field) => { + const hint = formatHintForField( + field, + inputFormatSpec, + inferredFormats, + isTTSPipeline + ); + return ( +
+ + {field.name} + {!field.required ? ' (optional)' : ''} + + + setFieldUploads((prev) => ({ ...prev, [field.name]: file })) + } + accept={hint.accept} + hint={hint.hint} + /> +
+ ); + })} +
+ )} ) : ( <> @@ -1516,23 +1729,38 @@ const ConvertView: React.FC = () => { This pipeline expects multiple uploads. For each field, choose an upload or pick an existing asset.

- {uploadFields.map((field) => ( -
- - {field.name} - {!field.required ? ' (optional)' : ''} - - - setFieldUploads((prev) => ({ ...prev, [field.name]: file })) - } - /> -
- ))} + {uploadFields.map((field) => { + const hint = formatHintForField( + field, + inputFormatSpec, + inferredFormats, + isTTSPipeline + ); + return ( +
+ + {field.name} + {!field.required ? ' (optional)' : ''} + + + setFieldUploads((prev) => ({ ...prev, [field.name]: file })) + } + accept={hint.accept} + hint={hint.hint} + /> +
+ ); + })} ) : ( - + ) ) : ( Date: Sun, 25 Jan 2026 12:42:30 +0100 Subject: [PATCH 2/5] chore: update workflow and docker images to include pocket-tts plugin --- .github/workflows/docker.yml | 2 ++ Dockerfile.demo | 49 +++++++++++++++++++++++++++++++++++- Dockerfile.full | 36 +++++++++++++++++++++++++- Dockerfile.full-gpu | 36 +++++++++++++++++++++++++- 4 files changed, 120 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3bdcc103..69853256 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -206,6 +206,8 @@ jobs: file: ./Dockerfile.demo push: ${{ github.event_name == 'push' || inputs.push }} tags: ${{ steps.meta.outputs.tags }} + secrets: | + hf_token=${{ secrets.HF_TOKEN }} cache-from: type=local,src=/mnt/docker-cache cache-to: type=local,dest=/mnt/docker-cache,mode=max platforms: linux/amd64 diff --git a/Dockerfile.demo b/Dockerfile.demo index ccd4f9d9..42569316 100644 --- a/Dockerfile.demo +++ b/Dockerfile.demo @@ -393,7 +393,50 @@ RUN mkdir -p /build/models && \ cd matcha-icefall-en_US-ljspeech && \ wget -O vocos-22khz-univ.onnx https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -# Stage 10: Build Helsinki Translation plugin (CPU-only) +# Stage 10: Build Pocket TTS plugin +FROM rust:1.92-slim-bookworm AS pocket-tts-builder + +WORKDIR /build + +# Install build dependencies (sentencepiece + python for model download) +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + g++ \ + cmake \ + curl \ + libclang-dev \ + clang \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Copy only what's needed to build pocket-tts plugin +COPY Cargo.toml Cargo.lock ./ +COPY crates/core ./crates/core +COPY sdks/plugin-sdk ./sdks/plugin-sdk +COPY plugins/native/pocket-tts ./plugins/native/pocket-tts + +# Build pocket-tts plugin +RUN --mount=type=cache,id=cargo-registry-pocket-tts,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git-pocket-tts,target=/usr/local/cargo/git \ + --mount=type=cache,id=pocket-tts-target,target=/build/plugins/native/pocket-tts/target \ + cd plugins/native/pocket-tts && \ + cargo build --release --target-dir target && \ + mkdir -p /build/plugins/native && \ + cp target/release/libpocket_tts.so /build/plugins/native/ + +# Download Pocket TTS models and voices (requires HF token) +RUN PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --no-cache-dir huggingface-hub && \ + rm -rf /root/.cache/pip + +RUN --mount=type=secret,id=hf_token \ + --mount=type=cache,id=hf-cache-pocket-tts,target=/build/models/hf \ + test -s /run/secrets/hf_token && \ + export HF_TOKEN="$(cat /run/secrets/hf_token)" && \ + export HF_HOME=/build/models/hf && \ + python3 plugins/native/pocket-tts/download-models.py +# Stage 11: Build Helsinki Translation plugin (CPU-only) FROM rust:1.92-slim-bookworm AS helsinki-builder WORKDIR /build @@ -487,6 +530,10 @@ COPY --from=vad-builder /build/models/ten-vad.onnx /opt/streamkit/models/ten-vad COPY --from=matcha-builder /build/plugins/native/* /opt/streamkit/plugins/native/ COPY --from=matcha-builder /build/models/matcha-icefall-en_US-ljspeech /opt/streamkit/models/matcha-icefall-en_US-ljspeech +# Copy pocket-tts plugin (models are gated; mount separately if needed) +COPY --from=pocket-tts-builder /build/plugins/native/* /opt/streamkit/plugins/native/ +COPY --from=pocket-tts-builder /build/models/pocket-tts /opt/streamkit/models/pocket-tts + # Copy sample pipelines + small bundled audio samples (Opus/Ogg only) COPY --chown=app:app samples/pipelines /opt/streamkit/samples/pipelines COPY --chown=app:app samples/audio/system/*.ogg samples/audio/system/*.ogg.license /opt/streamkit/samples/audio/system/ diff --git a/Dockerfile.full b/Dockerfile.full index 4321a5fb..8895385e 100644 --- a/Dockerfile.full +++ b/Dockerfile.full @@ -389,7 +389,38 @@ RUN mkdir -p /build/models && \ cd matcha-icefall-en_US-ljspeech && \ wget -O vocos-22khz-univ.onnx https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -# Stage 10: Build NLLB Translation plugin +# Stage 10: Build Pocket TTS plugin +FROM rust:1.92-slim-bookworm AS pocket-tts-builder + +WORKDIR /build + +# Install build dependencies (sentencepiece) +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + g++ \ + cmake \ + curl \ + libclang-dev \ + clang \ + && rm -rf /var/lib/apt/lists/* + +# Copy only what's needed to build pocket-tts plugin +COPY Cargo.toml Cargo.lock ./ +COPY crates/core ./crates/core +COPY sdks/plugin-sdk ./sdks/plugin-sdk +COPY plugins/native/pocket-tts ./plugins/native/pocket-tts + +# Build pocket-tts plugin +RUN --mount=type=cache,id=cargo-registry-pocket-tts,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git-pocket-tts,target=/usr/local/cargo/git \ + --mount=type=cache,id=pocket-tts-target,target=/build/plugins/native/pocket-tts/target \ + cd plugins/native/pocket-tts && \ + cargo build --release --target-dir target && \ + mkdir -p /build/plugins/native && \ + cp target/release/libpocket_tts.so /build/plugins/native/ + +# Stage 11: Build NLLB Translation plugin FROM rust:1.92-slim-bookworm AS nllb-builder WORKDIR /build @@ -497,6 +528,9 @@ COPY --from=vad-builder /build/models/ten-vad.onnx /opt/streamkit/models/ten-vad COPY --from=matcha-builder /build/plugins/native/* /opt/streamkit/plugins/native/ COPY --from=matcha-builder /build/models/matcha-icefall-en_US-ljspeech /opt/streamkit/models/matcha-icefall-en_US-ljspeech +# Copy pocket-tts plugin (models are gated; mount separately if needed) +COPY --from=pocket-tts-builder /build/plugins/native/* /opt/streamkit/plugins/native/ + # Copy sample pipelines COPY --chown=app:app samples /opt/streamkit/samples diff --git a/Dockerfile.full-gpu b/Dockerfile.full-gpu index 80c3ab4f..6d33d595 100644 --- a/Dockerfile.full-gpu +++ b/Dockerfile.full-gpu @@ -440,7 +440,38 @@ RUN mkdir -p /build/models && \ cd matcha-icefall-en_US-ljspeech && \ wget -O vocos-22khz-univ.onnx https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -# Stage 10: Build NLLB Translation plugin with GPU support +# Stage 10: Build Pocket TTS plugin +FROM rust:1.92-slim-bookworm AS pocket-tts-builder + +WORKDIR /build + +# Install build dependencies (sentencepiece) +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + g++ \ + cmake \ + curl \ + libclang-dev \ + clang \ + && rm -rf /var/lib/apt/lists/* + +# Copy only what's needed to build pocket-tts plugin +COPY Cargo.toml Cargo.lock ./ +COPY crates/core ./crates/core +COPY sdks/plugin-sdk ./sdks/plugin-sdk +COPY plugins/native/pocket-tts ./plugins/native/pocket-tts + +# Build pocket-tts plugin +RUN --mount=type=cache,id=cargo-registry-pocket-tts,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git-pocket-tts,target=/usr/local/cargo/git \ + --mount=type=cache,id=pocket-tts-target,target=/build/plugins/native/pocket-tts/target \ + cd plugins/native/pocket-tts && \ + cargo build --release --target-dir target && \ + mkdir -p /build/plugins/native && \ + cp target/release/libpocket_tts.so /build/plugins/native/ + +# Stage 11: Build NLLB Translation plugin with GPU support # NOTE: This stage explicitly waits for whisper-builder to complete first # to avoid running two heavy Rust+CUDA builds in parallel (disk space constraint on CI runners) ARG CUDA_VERSION @@ -640,6 +671,9 @@ COPY --from=vad-builder /build/models/ten-vad.onnx /opt/streamkit/models/ten-vad COPY --from=matcha-builder /build/plugins/native/* /opt/streamkit/plugins/native/ COPY --from=matcha-builder /build/models/matcha-icefall-en_US-ljspeech /opt/streamkit/models/matcha-icefall-en_US-ljspeech +# Copy pocket-tts plugin (models are gated; mount separately if needed) +COPY --from=pocket-tts-builder /build/plugins/native/* /opt/streamkit/plugins/native/ + # Copy sample pipelines COPY --chown=app:app samples /opt/streamkit/samples From c83cbb877fcc969f1b93ab9a4d27ba20271fa608 Mon Sep 17 00:00:00 2001 From: streamer45 Date: Sun, 25 Jan 2026 13:00:10 +0100 Subject: [PATCH 3/5] fix: license --- plugins/native/pocket-tts/README.md | 6 ++++++ .../pocket-tts/vendor/pocket-tts/benches/attention_bench.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/check_config.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/examples/wasm/index.html | 6 ++++++ plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/src/conditioners/text.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/src/modules/attention.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/tts_model.rs | 4 ++++ .../native/pocket-tts/vendor/pocket-tts/src/voice_state.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs | 4 ++++ plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/tests/integration_tests.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/tests/memory_usage.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/tests/parity_tests.rs | 4 ++++ .../pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs | 4 ++++ 32 files changed, 132 insertions(+) diff --git a/plugins/native/pocket-tts/README.md b/plugins/native/pocket-tts/README.md index cb646c18..2a257582 100644 --- a/plugins/native/pocket-tts/README.md +++ b/plugins/native/pocket-tts/README.md @@ -1,3 +1,9 @@ + + # Pocket TTS Native Plugin A native StreamKit plugin for Kyutai Pocket TTS using the Rust/Candle port. diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs index e5ef1897..82fd10db 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/attention_bench.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{DType, Device, Tensor}; use candle_nn::VarBuilder; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs index 346b18ed..b26621ee 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/full_benchmark.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use criterion::{Criterion, Throughput, criterion_group, criterion_main}; use pocket_tts::TTSModel; use std::time::Duration; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs index e6484a1e..736284f4 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/benches/streaming_bench.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use criterion::{Criterion, criterion_group, criterion_main}; use pocket_tts::TTSModel; use std::time::Instant; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs index 8d141ffd..1120843d 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/bench_sdpa.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{Device, Tensor}; use pocket_tts::modules::sdpa::sdpa; use std::time::Instant; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs index a649543d..e23c7d3a 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/check_config.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use pocket_tts::config::load_config; use std::path::PathBuf; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs index ceb33f28..d8554301 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/inspect_hound.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + fn main() { let cursor = std::io::Cursor::new(vec![]); let reader = hound::WavReader::new(cursor); diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs index b2d341ae..1aff1e8a 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/quantize_demo.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Quantization example for Pocket TTS //! //! Demonstrates the quantization module by simulating int8 quantization diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs index 0c2da390..adbb484b 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/scaling_bench.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use pocket_tts::TTSModel; use std::time::Instant; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs index d720f619..d49149c3 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/verify_sdpa.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{Device, Tensor}; use pocket_tts::modules::sdpa::sdpa; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html b/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html index 93b820b2..652da708 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html +++ b/plugins/native/pocket-tts/vendor/pocket-tts/examples/wasm/index.html @@ -1,3 +1,9 @@ + + diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs index 5c409343..f7922ef9 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/audio.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::Tensor; use hound::{Error as HoundError, WavReader}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs index 481c63ac..8ce6c9c9 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/mod.rs @@ -1 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + pub mod text; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs index 768ce7bd..5d903aa7 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/conditioners/text.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::Tensor; use candle_nn::{Embedding, Module, VarBuilder}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs index 70c199c1..179dfbb7 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/config.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Configuration types for pocket-tts, matching Python's utils/config.py use serde::Deserialize; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs index 7f750e03..e282f7e9 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/lib.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + pub mod audio; pub mod conditioners; pub mod config; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs index f4f5481d..9dff8109 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/attention.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use crate::ModelState; use crate::modules::rope::RotaryEmbedding; use candle_core::{DType, Result, Tensor}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs index 302f3936..240c5df4 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/conv.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use crate::ModelState; use candle_core::{DType, Result, Tensor}; use candle_nn::{Conv1d, Conv1dConfig, ConvTranspose1d, ConvTranspose1dConfig, Module, VarBuilder}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs index c6c65e1e..81dfd253 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mlp.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{DType, Result, Tensor}; use candle_nn::{Linear, Module, VarBuilder}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs index 718234c4..cd03870f 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/mod.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + pub mod attention; pub mod conv; pub mod mlp; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs index dd635b75..2221195c 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/rope.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{DType, Result, Tensor}; #[derive(Debug, Clone)] diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs index 605ec976..6eaa10f5 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/modules/sdpa.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::{D, Result, Tensor}; /// Memory-efficient Scaled Dot Product Attention diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs index 94ac2b01..eac3313b 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/pause.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Pause/silence handling for text-to-speech //! //! Supports: diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs index 8c663838..1e78b617 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/quantize.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Quantization support for Pocket TTS //! //! This module provides quantization utilities for reduced memory footprint diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs index 1048ddcd..66617df3 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/tts_model.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Main TTSModel struct - orchestrates the TTS pipeline //! //! This is the high-level API for text-to-speech generation, diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs index a90b7bcd..db8ed012 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/voice_state.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Voice state management for streaming generation and voice cloning use candle_core::{Result, Tensor}; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs index 0b228ed3..e8862b6e 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/wasm.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! WebAssembly bindings for Pocket TTS //! //! This module provides WASM-compatible entry points for browser usage. diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs index 9ff9246d..4e87b193 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/weights.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use anyhow::Result; use std::path::PathBuf; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs index 7b5aad67..0a19f921 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/integration_tests.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + //! Integration tests for TTSModel with real weights //! //! These tests require the HF_TOKEN environment variable to be set diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs index 43add0da..4dbb8318 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/memory_usage.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + #[cfg(test)] mod tests { use candle_core::Tensor; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs index 09999f93..a5e2e5c6 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/parity_tests.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + use candle_core::Tensor; use pocket_tts::TTSModel; use pocket_tts::audio::read_wav; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs b/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs index 66927220..db74304f 100644 --- a/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs +++ b/plugins/native/pocket-tts/vendor/pocket-tts/tests/streaming_tests.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + #[cfg(test)] mod tests { use anyhow::Result; From b79c44293f4831d592c0da7c94ca3497a337bb51 Mon Sep 17 00:00:00 2001 From: streamer45 Date: Sun, 25 Jan 2026 13:13:04 +0100 Subject: [PATCH 4/5] Add Apache license --- LICENSES/Apache-2.0.txt | 190 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 LICENSES/Apache-2.0.txt diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 00000000..4aeb478a --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Pocket TTS Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From b7ca6550cba5635dfe0bfa0c490892372186f5d1 Mon Sep 17 00:00:00 2001 From: streamer45 Date: Sun, 25 Jan 2026 14:39:15 +0100 Subject: [PATCH 5/5] fix: CI docker build --- .dockerignore | 2 + .gitignore | 2 + crates/plugin-native/src/wrapper.rs | 23 +- plugins/native/pocket-tts/README.md | 1 + .../vendor/pocket-tts/src/models/flow_lm.rs | 169 ++++++++ .../vendor/pocket-tts/src/models/mimi.rs | 266 ++++++++++++ .../vendor/pocket-tts/src/models/mod.rs | 8 + .../vendor/pocket-tts/src/models/seanet.rs | 407 ++++++++++++++++++ .../pocket-tts/src/models/transformer.rs | 262 +++++++++++ 9 files changed, 1126 insertions(+), 14 deletions(-) create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/models/flow_lm.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/models/mimi.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/models/mod.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/models/seanet.rs create mode 100644 plugins/native/pocket-tts/vendor/pocket-tts/src/models/transformer.rs diff --git a/.dockerignore b/.dockerignore index 8424d193..ead4c35a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,8 @@ target/ # Large local artifacts (not needed for image builds) models/ .plugins/ +!plugins/native/pocket-tts/vendor/pocket-tts/src/models/ +!plugins/native/pocket-tts/vendor/pocket-tts/src/models/** # Node modules node_modules/ diff --git a/.gitignore b/.gitignore index 1a75003f..2e070495 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,8 @@ samples/audio/system/*.flac samples/audio/system/*.mp3 samples/audio/system/*.m4a models +!plugins/native/pocket-tts/vendor/pocket-tts/src/models/ +!plugins/native/pocket-tts/vendor/pocket-tts/src/models/** # Example plugin build outputs /examples/plugins/*/build/ diff --git a/crates/plugin-native/src/wrapper.rs b/crates/plugin-native/src/wrapper.rs index 93e0d6e4..f2274bd1 100644 --- a/crates/plugin-native/src/wrapper.rs +++ b/crates/plugin-native/src/wrapper.rs @@ -310,9 +310,6 @@ impl ProcessorNode for NativeNodeWrapper { // Move the blocking FFI call to spawn_blocking let state = Arc::clone(&self.state); - // spawn_blocking can only fail with JoinError if the task panics. - // If that happens, it's a serious bug that should crash. - #[allow(clippy::expect_used)] let error_msg = tokio::task::spawn_blocking(move || { let handle = state.begin_call()?; @@ -338,8 +335,11 @@ impl ProcessorNode for NativeNodeWrapper { error }) .await - // spawn_blocking only panics if the task panics, which indicates a serious bug - .expect("Update params task panicked"); + .map_err(|e| { + StreamKitError::Runtime(format!( + "Update params task panicked: {e}" + )) + })?; if let Some(err) = error_msg { warn!(node = %node_name, error = %err, "Parameter update failed"); @@ -369,7 +369,6 @@ impl ProcessorNode for NativeNodeWrapper { let session_id = context.session_id.clone(); let node_id = node_name.clone(); - #[allow(clippy::expect_used)] let (outputs, error) = tokio::task::spawn_blocking(move || { let Some(handle) = state.begin_call() else { return (Vec::new(), None); @@ -418,7 +417,7 @@ impl ProcessorNode for NativeNodeWrapper { (outputs, error) }) .await - .expect("Plugin flush task panicked"); + .map_err(|e| StreamKitError::Runtime(format!("Plugin flush task panicked: {e}")))?; // Send flush outputs for (pin, pkt) in outputs { @@ -439,12 +438,7 @@ impl ProcessorNode for NativeNodeWrapper { let telemetry_tx = context.telemetry_tx.clone(); let session_id = context.session_id.clone(); let node_id = node_name.clone(); - // spawn_blocking can only fail with JoinError if the task panics. - // If that happens, it's a serious bug that should crash. let pin_cstr = Arc::clone(&input_pin_cstrs[pin_index]); - // spawn_blocking can only fail with JoinError if the task panics. - // If that happens, it's a serious bug that should crash. - #[allow(clippy::expect_used)] let (outputs, error) = tokio::task::spawn_blocking(move || { let Some(handle) = state.begin_call() else { return (Vec::new(), None); @@ -499,8 +493,9 @@ impl ProcessorNode for NativeNodeWrapper { (outputs, error) }) .await - // spawn_blocking only panics if the task panics, which indicates a serious bug - .expect("Plugin processing task panicked"); + .map_err(|e| { + StreamKitError::Runtime(format!("Plugin processing task panicked: {e}")) + })?; // Now send outputs (after dropping c_packet and result) for (pin, pkt) in outputs { diff --git a/plugins/native/pocket-tts/README.md b/plugins/native/pocket-tts/README.md index 2a257582..40a25c8f 100644 --- a/plugins/native/pocket-tts/README.md +++ b/plugins/native/pocket-tts/README.md @@ -7,6 +7,7 @@ SPDX-License-Identifier: MPL-2.0 # Pocket TTS Native Plugin A native StreamKit plugin for Kyutai Pocket TTS using the Rust/Candle port. +Upstream Rust port: https://github.com/babybirdprd/pocket-tts This plugin runs fully on CPU and streams 24kHz mono audio. ## Build diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/models/flow_lm.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/flow_lm.rs new file mode 100644 index 00000000..b6792209 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/flow_lm.rs @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::ModelState; +use crate::models::transformer::StreamingTransformer; +use crate::modules::mlp::{LayerNorm, ModulationParams, SimpleMLPAdaLN}; +use candle_core::{Result, Tensor}; +use candle_nn::{Linear, Module, VarBuilder}; + +pub fn lsd_decode( + flow_net: &SimpleMLPAdaLN, + modulations: &[Vec], + x_0: &Tensor, +) -> Result { + let mut current = x_0.clone(); + let num_steps = modulations.len(); + + let step_factor = 1.0 / num_steps as f64; + for step_mod in modulations { + // Use forward_step_cached with pre-computed modulation batch for this ODE step + let flow_dir = flow_net.forward_step_cached(¤t, step_mod)?; + current = (current + flow_dir.affine(step_factor, 0.0)?)?; + } + Ok(current) +} + +#[derive(Clone)] +pub struct FlowLMModel { + pub flow_net: SimpleMLPAdaLN, + pub transformer: StreamingTransformer, + pub input_linear: Linear, + pub out_norm: LayerNorm, + pub out_eos: Linear, + pub bos_emb: Tensor, + pub emb_mean: Tensor, + pub emb_std: Tensor, + pub ldim: usize, + pub dim: usize, + pub noise_clamp: Option, +} + +fn sample_noise( + device: &candle_core::Device, + shape: (usize, usize), + temp: f32, + clamp: Option, +) -> Result { + let std = temp.sqrt(); + match clamp { + None => Tensor::randn(0.0f32, std, shape, device), + Some(limit) => { + // Rejection sampling for truncated normal + let count = shape.0 * shape.1; + let mut data = Vec::with_capacity(count); + let mut rng = rand::thread_rng(); + let dist = rand_distr::Normal::new(0.0f32, std) + .map_err(|e| candle_core::Error::Msg(e.to_string()))?; + + while data.len() < count { + let v = rand_distr::Distribution::sample(&dist, &mut rng); + if v.abs() <= limit { + data.push(v); + } + } + Tensor::from_vec(data, shape, device) + } + } +} + +impl FlowLMModel { + pub fn new( + flow_net: SimpleMLPAdaLN, + transformer: StreamingTransformer, + ldim: usize, + dim: usize, + vb: VarBuilder, + ) -> Result { + let input_linear = candle_nn::linear_no_bias(ldim, dim, vb.pp("input_linear"))?; + let out_norm = LayerNorm::new(dim, 1e-5, true, vb.pp("out_norm"))?; + let out_eos = candle_nn::linear(dim, 1, vb.pp("out_eos"))?; + let bos_emb = vb.get(ldim, "bos_emb")?; + let emb_mean = vb.get(ldim, "emb_mean")?; + let emb_std = vb.get(ldim, "emb_std")?; + + Ok(Self { + flow_net, + transformer, + input_linear, + out_norm, + out_eos, + bos_emb, + emb_mean, + emb_std, + ldim, + dim, + noise_clamp: None, // Default to no clamp + }) + } + + #[allow(clippy::too_many_arguments)] + pub fn forward( + &self, + sequence: &Tensor, + text_embeddings: &Tensor, + model_state: &mut ModelState, + time_embeddings: &Tensor, + temp: f32, + eos_threshold: f32, + step: usize, + ) -> Result<(Tensor, bool)> { + // sequence is [B, T, ldim] + // text_embeddings is [B, S, dim] + + // Handle BOS (if NaN, use bos_emb) - simplistic check for NaN + // In Candle we can use `Tensor::where_cond` + // But for now let's assume sequence passed in doesn't have NaNs or handled upstream. + // Original: sequence = torch.where(torch.isnan(sequence), self.bos_emb, sequence) + + // Let's assume BOS is handled by caller for now or if sequence empty. + + let x = self.input_linear.forward(sequence)?; + let s_len = text_embeddings.dims()[1]; + + // Cat text embeddings and sequence embeddings only if text_embeddings is not empty + let transformer_out_pre_norm = if s_len > 0 { + let input = Tensor::cat(&[text_embeddings, &x], 1)?; + let mut out = self.transformer.forward(&input, model_state, step)?; + // Remove prefix (text embeddings length) + out = out.narrow(1, s_len, out.dims()[1] - s_len)?; + out + } else { + self.transformer.forward(&x, model_state, step)? + }; + + let transformer_out = self.out_norm.forward(&transformer_out_pre_norm)?; + + // Only use the last frame for generation + let last_frame = transformer_out + .narrow(1, transformer_out.dims()[1] - 1, 1)? + .squeeze(1)?; + + let eos_score = self + .out_eos + .forward(&last_frame)? + .squeeze(0)? + .squeeze(0)? + .to_scalar::()?; + let is_eos = eos_score > eos_threshold; + + // Generate noise with optional clamping + let noise = sample_noise( + last_frame.device(), + (last_frame.dims()[0], self.ldim), + temp, + self.noise_clamp, + )?; + + // Pre-compute all modulations for this frame's ODE steps (8 steps * N blocks) in batch + let c_emb = self.flow_net.embed_condition(&last_frame)?; + let modulations = self + .flow_net + .precompute_modulations(&c_emb, time_embeddings)?; + + let next_latent = lsd_decode(&self.flow_net, &modulations, &noise)?; + + Ok((next_latent, is_eos)) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mimi.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mimi.rs new file mode 100644 index 00000000..44d7cbd1 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mimi.rs @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::ModelState; +use crate::models::seanet::{SEANetDecoder, SEANetEncoder}; +use crate::models::transformer::ProjectedTransformer; +use crate::modules::conv::{ConvDownsample1d, ConvTrUpsample1d}; +use candle_core::{Result, Tensor}; +use candle_nn::{Conv1d, Conv1dConfig, Module, VarBuilder}; + +#[derive(Clone)] +pub struct Quantizer { + output_proj: Conv1d, +} + +impl Quantizer { + pub fn new(dimension: usize, output_dimension: usize, vb: VarBuilder) -> Result { + let config = Conv1dConfig { + groups: 1, + padding: 0, + stride: 1, + dilation: 1, + ..Default::default() + }; + let output_proj = candle_nn::conv1d_no_bias( + dimension, + output_dimension, + 1, + config, + vb.pp("output_proj"), + )?; + Ok(Self { output_proj }) + } + + pub fn forward(&self, x: &Tensor) -> Result { + // x is [B, C, T] + // Conv1d expects [B, C, T] and returns [B, C_out, T] + self.output_proj.forward(x) + } +} + +#[derive(Clone)] +pub struct MimiModel { + pub encoder: SEANetEncoder, + pub decoder: SEANetDecoder, + pub encoder_transformer: ProjectedTransformer, + pub decoder_transformer: ProjectedTransformer, + pub quantizer: Quantizer, + pub downsample: Option, + pub upsample: Option, + pub frame_rate: f64, + pub encoder_frame_rate: f64, + pub sample_rate: usize, + pub channels: usize, + pub dimension: usize, +} + +impl MimiModel { + #[allow(clippy::too_many_arguments)] + pub fn new( + encoder: SEANetEncoder, + decoder: SEANetDecoder, + encoder_transformer: ProjectedTransformer, + decoder_transformer: ProjectedTransformer, + frame_rate: f64, + encoder_frame_rate: f64, + sample_rate: usize, + channels: usize, + dimension: usize, // The quantizer input dimension (32) + output_dimension: usize, // The decoder input dimension (512) + name: &str, + vb: VarBuilder, + ) -> Result { + let quantizer = Quantizer::new(dimension, output_dimension, vb.pp("quantizer"))?; + + let (downsample, upsample) = if encoder_frame_rate != frame_rate { + let stride = (encoder_frame_rate / frame_rate) as usize; + ( + Some(ConvDownsample1d::new( + stride, + output_dimension, + &format!("{}.downsample", name), + vb.pp("downsample"), + )?), + Some(ConvTrUpsample1d::new( + stride, + output_dimension, + &format!("{}.upsample", name), + vb.pp("upsample"), + )?), + ) + } else { + (None, None) + }; + + Ok(Self { + encoder, + decoder, + encoder_transformer, + decoder_transformer, + quantizer, + downsample, + upsample, + frame_rate, + encoder_frame_rate, + sample_rate, + channels, + dimension, + }) + } + + pub fn frame_size(&self) -> usize { + (self.sample_rate as f64 / self.frame_rate) as usize + } + + pub fn encode_to_latent( + &self, + x: &Tensor, + model_state: &mut ModelState, + step: usize, + ) -> Result { + // x shape [B, C, T] + let _frame_size = self.frame_size(); + let (b, c, _t_orig) = x.dims3()?; + + let t = x.dims()[2]; + let hop = self.frame_size(); + let x = if !t.is_multiple_of(hop) { + let padding = hop - (t % hop); + let pad = Tensor::zeros((b, c, padding), x.dtype(), x.device())?; + Tensor::cat(&[x, &pad], 2)? + } else { + x.clone() + }; + + let mut emb = self.encoder.forward(&x, model_state, step)?; + let mut embs = self.encoder_transformer.forward(&emb, model_state, step)?; + emb = embs.remove(0); + + if let Some(down) = &self.downsample { + emb = down.forward(&emb, model_state, step)?; + } + Ok(emb) + } + + pub fn decode_from_latent( + &self, + latent: &Tensor, + model_state: &mut ModelState, + step: usize, + ) -> Result { + let mut emb = latent.clone(); + if let Some(up) = &self.upsample { + emb = up.forward(&emb, model_state, step)?; + } + let mut embs = self.decoder_transformer.forward(&emb, model_state, step)?; + emb = embs.remove(0); + let out = self.decoder.forward(&emb, model_state, step)?; + Ok(out) + } + pub fn quantize(&self, x: &Tensor) -> Result { + self.quantizer.forward(x) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::{DType, Device, Tensor}; + use candle_nn::VarBuilder; + use std::collections::HashMap; + + #[test] + fn test_mimi_shapes() -> Result<()> { + let device = Device::Cpu; + let vb = VarBuilder::zeros(DType::F32, &device); + + let encoder = SEANetEncoder::new( + 1, + 128, + 32, + 1, + &[2, 2], + 7, + 7, + 3, + 2, + "constant", + 2, + "encoder", + vb.pp("encoder"), + )?; + let decoder = SEANetDecoder::new( + 1, + 128, + 32, + 1, + &[2, 2], + 7, + 7, + 3, + 2, + "constant", + 2, + "decoder", + vb.pp("decoder"), + )?; + + let encoder_transformer = ProjectedTransformer::new( + 128, + vec![128], + 128, + 4, + 1, + 0.1, + 10, + 10000.0, + 512, + "enc_tr", + vb.pp("enc_tr"), + )?; + let decoder_transformer = ProjectedTransformer::new( + 128, + vec![128], + 128, + 4, + 1, + 0.1, + 10, + 10000.0, + 512, + "dec_tr", + vb.pp("dec_tr"), + )?; + + let mimi = MimiModel::new( + encoder, + decoder, + encoder_transformer, + decoder_transformer, + 12.5, + 50.0, + 16000, + 1, + 128, + 512, + "mimi", + vb.pp("mimi"), + )?; + + let _audio = Tensor::zeros((1, 1, 1280), DType::F32, &device)?; // 1280 samples = 0.08s + + // Mock state + let mut _model_state: HashMap> = HashMap::new(); + // We need to initialize state for all submodules. This is complex manually. + // For shape test, we might want a simpler way or just skip stateful forward for now if init complex. + // But our forward REQUIRES state. + + // I'll skip the actual forward test here because initializing state for ALL sub-layers is tedious. + // I'll implement a helper to init states in Phase 3. + + assert_eq!(mimi.frame_size(), 1280); + Ok(()) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mod.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mod.rs new file mode 100644 index 00000000..5ad416c6 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/mod.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub mod flow_lm; +pub mod mimi; +pub mod seanet; +pub mod transformer; diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/models/seanet.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/seanet.rs new file mode 100644 index 00000000..b7e570c2 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/seanet.rs @@ -0,0 +1,407 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::ModelState; +use crate::modules::conv::{StreamingConv1d, StreamingConvTranspose1d}; +use candle_core::{Result, Tensor}; +use candle_nn::VarBuilder; + +#[derive(Clone)] +pub struct SEANetResnetBlock { + pub layers: Vec>, + pub _name: String, +} + +pub trait StreamingLayer: Send + Sync { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result; + fn clone_box(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +impl StreamingLayer for StreamingConv1d { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +#[derive(Clone)] +pub struct EluLayer; +impl StreamingLayer for EluLayer { + fn forward(&self, x: &Tensor, _model_state: &mut ModelState, _step: usize) -> Result { + x.elu(1.0) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl SEANetResnetBlock { + pub fn new( + dim: usize, + kernel_sizes: &[usize], + dilations: &[usize], + pad_mode: &str, + compress: usize, + name: &str, + vb: VarBuilder, + ) -> Result { + let hidden = dim / compress; + let mut layers: Vec> = Vec::new(); + for i in 0..kernel_sizes.len() { + let in_chs = if i == 0 { dim } else { hidden }; + let out_chs = if i == kernel_sizes.len() - 1 { + dim + } else { + hidden + }; + layers.push(Box::new(EluLayer)); + layers.push(Box::new(StreamingConv1d::new( + in_chs, + out_chs, + kernel_sizes[i], + 1, + dilations[i], + 1, + true, + pad_mode, + &format!("{}.block.{}", name, i * 2 + 1), + vb.pp(format!("block.{}", i * 2 + 1)), + )?)); + } + Ok(Self { + layers, + _name: name.to_string(), + }) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + let mut v = x.clone(); + for layer in &self.layers { + v = layer.forward(&v, model_state, step)?; + } + x + v + } +} + +#[derive(Clone)] +pub struct SEANetEncoder { + pub layers: Vec>, + pub hop_length: usize, + pub _name: String, +} + +pub trait StreamingLayerWrapper: Send + Sync { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result; + fn clone_box(&self) -> Box; + fn weight(&self) -> Option<&Tensor> { + None + } + fn bias(&self) -> Option<&Tensor> { + None + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +impl StreamingLayerWrapper for StreamingConv1d { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + fn weight(&self) -> Option<&Tensor> { + Some(self.weight()) + } + fn bias(&self) -> Option<&Tensor> { + self.bias() + } +} + +impl StreamingLayerWrapper for SEANetResnetBlock { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl StreamingLayerWrapper for EluLayer { + fn forward(&self, x: &Tensor, _model_state: &mut ModelState, _step: usize) -> Result { + x.elu(1.0) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl SEANetEncoder { + #[allow(clippy::too_many_arguments)] + pub fn new( + channels: usize, + dimension: usize, + n_filters: usize, + n_residual_layers: usize, + ratios: &[usize], + kernel_size: usize, + last_kernel_size: usize, + residual_kernel_size: usize, + dilation_base: usize, + pad_mode: &str, + compress: usize, + name: &str, + vb: VarBuilder, + ) -> Result { + let ratios: Vec = ratios.iter().copied().rev().collect(); + let hop_length = ratios.iter().product(); + let mut layers: Vec> = Vec::new(); + + let mut mult = 1; + layers.push(Box::new(StreamingConv1d::new( + channels, + mult * n_filters, + kernel_size, + 1, + 1, + 1, + true, + pad_mode, + &format!("{}.model.0", name), + vb.pp("model.0"), + )?)); + + let mut layer_idx = 1; + for ratio in ratios { + for j in range(n_residual_layers) { + layers.push(Box::new(SEANetResnetBlock::new( + mult * n_filters, + &[residual_kernel_size, 1], + &[dilation_base.pow(j as u32), 1], + pad_mode, + compress, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + layer_idx += 1; + } + + layers.push(Box::new(EluLayer)); + layer_idx += 1; + + layers.push(Box::new(StreamingConv1d::new( + mult * n_filters, + mult * n_filters * 2, + ratio * 2, + ratio, + 1, + 1, + true, + pad_mode, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + layer_idx += 1; + mult *= 2; + } + + layers.push(Box::new(EluLayer)); + layer_idx += 1; + + layers.push(Box::new(StreamingConv1d::new( + mult * n_filters, + dimension, + last_kernel_size, + 1, + 1, + 1, + true, + pad_mode, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + + Ok(Self { + layers, + hop_length, + _name: name.to_string(), + }) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + let mut x = x.clone(); + for layer in &self.layers { + x = layer.forward(&x, model_state, step)?; + } + Ok(x) + } +} + +fn range(n: usize) -> std::ops::Range { + 0..n +} + +#[derive(Clone)] +pub struct SEANetDecoder { + pub layers: Vec>, + pub hop_length: usize, + pub _name: String, +} + +pub trait StreamingLayerDecoderWrapper: Send + Sync { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result; + fn clone_box(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +impl StreamingLayerDecoderWrapper for StreamingConv1d { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl StreamingLayerDecoderWrapper for StreamingConvTranspose1d { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl StreamingLayerDecoderWrapper for SEANetResnetBlock { + fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + self.forward(x, model_state, step) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl StreamingLayerDecoderWrapper for EluLayer { + fn forward(&self, x: &Tensor, _model_state: &mut ModelState, _step: usize) -> Result { + x.elu(1.0) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl SEANetDecoder { + #[allow(clippy::too_many_arguments)] + pub fn new( + channels: usize, + dimension: usize, + n_filters: usize, + n_residual_layers: usize, + ratios: &[usize], + kernel_size: usize, + last_kernel_size: usize, + residual_kernel_size: usize, + dilation_base: usize, + pad_mode: &str, + compress: usize, + name: &str, + vb: VarBuilder, + ) -> Result { + let hop_length = ratios.iter().product(); + let mut layers: Vec> = Vec::new(); + + let mut mult = 2usize.pow(ratios.len() as u32); + layers.push(Box::new(StreamingConv1d::new( + dimension, + mult * n_filters, + kernel_size, + 1, + 1, + 1, + true, + pad_mode, + &format!("{}.model.0", name), + vb.pp("model.0"), + )?)); + + let mut layer_idx = 1; + for ratio in ratios { + layers.push(Box::new(EluLayer)); + layer_idx += 1; + + layers.push(Box::new(StreamingConvTranspose1d::new( + mult * n_filters, + mult * n_filters / 2, + ratio * 2, + *ratio, + 1, + true, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + layer_idx += 1; + + for j in range(n_residual_layers) { + layers.push(Box::new(SEANetResnetBlock::new( + mult * n_filters / 2, + &[residual_kernel_size, 1], + &[dilation_base.pow(j as u32), 1], + pad_mode, + compress, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + layer_idx += 1; + } + mult /= 2; + } + + layers.push(Box::new(EluLayer)); + layer_idx += 1; + + layers.push(Box::new(StreamingConv1d::new( + n_filters, + channels, + last_kernel_size, + 1, + 1, + 1, + true, + pad_mode, + &format!("{}.model.{}", name, layer_idx), + vb.pp(format!("model.{}", layer_idx)), + )?)); + + Ok(Self { + layers, + hop_length, + _name: name.to_string(), + }) + } + + pub fn forward(&self, x: &Tensor, model_state: &mut ModelState, step: usize) -> Result { + let mut x = x.clone(); + for layer in &self.layers { + x = layer.forward(&x, model_state, step)?; + } + Ok(x) + } +} diff --git a/plugins/native/pocket-tts/vendor/pocket-tts/src/models/transformer.rs b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/transformer.rs new file mode 100644 index 00000000..425bf063 --- /dev/null +++ b/plugins/native/pocket-tts/vendor/pocket-tts/src/models/transformer.rs @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024 Pocket TTS Contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::ModelState; +use crate::modules::attention::StreamingMultiheadAttention; +use crate::modules::mlp::{LayerNorm, LayerScale}; +use crate::modules::rope::RotaryEmbedding; +use candle_core::{Result, Tensor}; +use candle_nn::{Linear, Module, VarBuilder}; + +#[derive(Clone)] +pub struct StreamingTransformerLayer { + self_attn: StreamingMultiheadAttention, + norm1: LayerNorm, + norm2: LayerNorm, + linear1: Linear, + linear2: Linear, + layer_scale_1: Option, + layer_scale_2: Option, +} + +impl StreamingTransformerLayer { + #[allow(clippy::too_many_arguments)] + pub fn new( + d_model: usize, + num_heads: usize, + dim_feedforward: usize, + context: Option, + rope: RotaryEmbedding, + layer_scale: Option, + _attention_kind: &str, + name: &str, + vb: VarBuilder, + ) -> Result { + let self_attn = StreamingMultiheadAttention::new( + d_model, + num_heads, + rope, + context, + &format!("{}.self_attn", name), + vb.pp("self_attn"), + )?; + let norm1 = LayerNorm::new(d_model, 1e-5, true, vb.pp("norm1"))?; + let norm2 = LayerNorm::new(d_model, 1e-5, true, vb.pp("norm2"))?; + let linear1 = candle_nn::linear_no_bias(d_model, dim_feedforward, vb.pp("linear1"))?; + let linear2 = candle_nn::linear_no_bias(dim_feedforward, d_model, vb.pp("linear2"))?; + + let (layer_scale_1, layer_scale_2) = if let Some(init) = layer_scale { + ( + Some(LayerScale::new(d_model, init, vb.pp("layer_scale_1"))?), + Some(LayerScale::new(d_model, init, vb.pp("layer_scale_2"))?), + ) + } else { + (None, None) + }; + + Ok(Self { + self_attn, + norm1, + norm2, + linear1, + linear2, + layer_scale_1, + layer_scale_2, + }) + } + + pub fn forward( + &self, + x: &Tensor, + model_state: &mut ModelState, + current_pos: usize, + current_len: usize, + ) -> Result { + let x_orig = x.clone(); + let h = self.norm1.forward(x)?; + let mut update = self + .self_attn + .forward(&h, model_state, current_pos, current_len)?; + if let Some(ls) = &self.layer_scale_1 { + update = ls.forward(&update)?; + } + let x = (x_orig + update)?; + + let x_orig = x.clone(); + let h = self.norm2.forward(&x)?; + let mut update = self.linear2.forward(&self.linear1.forward(&h)?.gelu()?)?; + if let Some(ls) = &self.layer_scale_2 { + update = ls.forward(&update)?; + } + x_orig + update + } +} + +#[derive(Clone)] +pub struct StreamingTransformer { + layers: Vec, + _rope: RotaryEmbedding, + name: String, +} + +impl StreamingTransformer { + #[allow(clippy::too_many_arguments)] + pub fn new( + d_model: usize, + num_heads: usize, + num_layers: usize, + layer_scale: Option, + dim_feedforward: usize, + context: Option, + max_period: f32, + kind: &str, + name: &str, + vb: VarBuilder, + ) -> Result { + let rope = RotaryEmbedding::new(max_period, d_model / num_heads, vb.device())?; + let mut layers = Vec::new(); + for i in 0..num_layers { + layers.push(StreamingTransformerLayer::new( + d_model, + num_heads, + dim_feedforward, + context, + rope.clone(), + layer_scale, + kind, + &format!("{}.layers.{}", name, i), + vb.pp(format!("layers.{}", i)), + )?); + } + Ok(Self { + layers, + _rope: rope, + name: name.to_string(), + }) + } + + pub fn forward( + &self, + x: &Tensor, + model_state: &mut ModelState, + _step: usize, + ) -> Result { + let mut x = x.clone(); + // Fetch current_pos once from the first attention layer's state to avoid redundant to_scalar calls. + let first_layer_name = format!("{}.layers.0.self_attn", self.name); + let current_pos = model_state + .get(&first_layer_name) + .and_then(|s| s.get("pos")) + .and_then(|t| t.to_scalar::().ok()) + .unwrap_or(0) as usize; + let current_len = model_state + .get(&first_layer_name) + .and_then(|s| s.get("l")) + .and_then(|t| t.to_scalar::().ok()) + .unwrap_or(0) as usize; + + for layer in &self.layers { + x = layer.forward(&x, model_state, current_pos, current_len)?; + } + Ok(x) + } +} + +#[derive(Clone)] +pub struct ProjectedTransformer { + transformer: StreamingTransformer, + input_proj: Option, + output_projs: Vec>, + _input_dimension: usize, + _output_dimensions: Vec, + _d_model: usize, +} + +impl ProjectedTransformer { + #[allow(clippy::too_many_arguments)] + pub fn new( + input_dimension: usize, + output_dimensions: Vec, + d_model: usize, + num_heads: usize, + num_layers: usize, + layer_scale: f32, + context: usize, + max_period: f32, + dim_feedforward: usize, + name: &str, + vb: VarBuilder, + ) -> Result { + let transformer = StreamingTransformer::new( + d_model, + num_heads, + num_layers, + Some(layer_scale), + dim_feedforward, + Some(context), + max_period, + "mimi", + &format!("{}.transformer", name), + vb.pp("transformer"), + )?; + + let input_proj = if d_model != input_dimension { + Some(candle_nn::linear_no_bias( + input_dimension, + d_model, + vb.pp("input_proj"), + )?) + } else { + None + }; + + let mut output_projs = Vec::new(); + for (i, &output_dim) in output_dimensions.iter().enumerate() { + if d_model == output_dim { + output_projs.push(None); + } else { + output_projs.push(Some(candle_nn::linear_no_bias( + d_model, + output_dim, + vb.pp(format!("output_projs.{}", i)), + )?)); + } + } + + Ok(Self { + transformer, + input_proj, + output_projs, + _input_dimension: input_dimension, + _output_dimensions: output_dimensions, + _d_model: d_model, + }) + } + + pub fn forward( + &self, + x: &Tensor, + model_state: &mut ModelState, + step: usize, + ) -> Result> { + // x is [B, C, T] + let mut x = x.transpose(1, 2)?; // [B, T, C] + if let Some(proj) = &self.input_proj { + x = proj.forward(&x)?; + } + let z = self.transformer.forward(&x, model_state, step)?; + + let mut ys = Vec::new(); + for output_proj in &self.output_projs { + let mut y = if let Some(proj) = output_proj { + proj.forward(&z)? + } else { + z.clone() + }; + y = y.transpose(1, 2)?; // [B, C_out, T] + ys.append(&mut vec![y]); + } + Ok(ys) + } +}