diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index a901d5010d..bdb64cb1bb 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -76,6 +76,13 @@ const overrides = new Map([ // across the local-archive + agent-metric-archive PR series. store_tests.rs // (~731 lines) is under 1000 so needs no override. ["src-tauri/src/archive/mod_tests.rs", 1208], + // PR #2520 port (STT model registry): STT_MODELS registry data (two model + // entries with pinned checksums + license texts) + select_stt_model and its + // unit tests. Load-bearing model-selection surface from the upstream PR; + // queued to split when #2520 lands and this port is dropped in the rebase. + // composer-dictation model picker adds per-model slots + SttModelInfo + + // resolve_dictation_model (~110 lines); split rides the same rebase. + ["src-tauri/src/huddle/models.rs", 1432], // unified-agent-model 1A.1: profile reconcile split to agents_profile.rs, // ratcheting 1443 -> 1295. Queued to split further in the A2 fold. // global-agent-config: resolve_deploy_model_provider + visibility exports @@ -517,7 +524,12 @@ const overrides = new Map([ // ownerPubkey) feeding the newly-added-mentions diff. Diff logic itself // lives in threading.ts (diffAddedMentionPubkeys); this is the minimal // composer-side wiring. Queued to split with the rest of this list. - ["src/features/messages/ui/MessageComposer.tsx", 1114], + // +15: dictation wiring — useDictation hook instantiation, ⌃Space handoff in + // handleEditorKeyDown, the Enter-to-send submit callback, and two toolbar + // props. All session/keyboard logic lives in lib/useDictation.ts; this is + // the minimal composer-side wiring. Queued to split with the rest of this + // list. + ["src/features/messages/ui/MessageComposer.tsx", 1129], // global-agent-config: model-tuning section (BuzzAgentModelTuningFields via // EditAgentAdvancedFields) + providerValid gate + effectiveProvider derivation // + globalProvider threading into getPersonaProviderOptions. All load-bearing diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..dde05a4500 --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,217 @@ +//! Standalone composer dictation — streaming STT without a huddle. +//! +//! Reuses the huddle `SttPipeline` in live-transcript mode: the in-progress +//! phrase is re-decoded every ~300 ms and emitted as a partial (`final: +//! false`), so words appear in the composer as they're spoken; each natural +//! pause commits the phrase as a final (`final: true`). The frontend captures +//! mic audio with `getUserMedia` + the existing AudioWorklet and pushes PCM +//! via `push_dictation_pcm`. Transcripts are emitted to the webview as +//! `dictation-transcript` events instead of being posted to the relay. +//! +//! Session model: `start_dictation` marks the session active; +//! `stop_dictation` marks it inactive and pushes ~1 s of silence into the +//! pipeline — the mic stops with the key release, so without it the VAD would +//! never see the silence that closes (and decodes) the trailing phrase. A +//! zero-length sentinel batch follows the silence; the worker echoes it back +//! and it is emitted as a `dictation-flushed` event, which tells the frontend +//! the stopped session's transcripts have all arrived (Enter-to-send waits +//! for it before submitting). + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, +}; + +use tauri::Emitter; + +use crate::app_state::AppState; +use crate::huddle::{ + models, + stt::{LiveEvent, SttPipeline}, + HuddlePhase, +}; + +/// Payload for `dictation-transcript` events. Partials (`final: false`) +/// replace the previous partial of the same phrase; a final commits it. +#[derive(Clone, serde::Serialize)] +struct TranscriptEvent { + text: String, + #[serde(rename = "final")] + is_final: bool, +} + +/// Same cap as the huddle audio ingest (huddle/mod.rs). +const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + +/// 1 s of 48 kHz f32-LE silence — comfortably past the worker's ~608 ms +/// live-mode silence-flush threshold (stt.rs LIVE_SILENCE_FLUSH_FRAMES) +/// after resampling. The margin matters: the threshold counts *consecutive* +/// silence frames, and the flush also unlocks the phrase's punctuation +/// (the model only commits it with ≥600 ms of trailing silence). +const FLUSH_SILENCE_BYTES: usize = 4 * 48_000; + +/// Managed Tauri state for dictation. +/// +/// ponytail: the pipeline stays alive after the first start (sherpa-onnx +/// model init takes seconds; ~110 MB resident). Add idle teardown if that +/// memory ever matters. +#[derive(Default)] +pub struct DictationState { + /// Loaded pipeline tagged with the model id it was built from, so a + /// changed model preference rebuilds it on the next start. + pipeline: Mutex>, + /// True while a session is live — blocks a second concurrent session and + /// gates `push_dictation_pcm` so late batches can't dirty the VAD buffer. + active: AtomicBool, +} + +/// Start (or resume) a dictation session. +/// +/// `model` is the user's model preference (Settings → Voice & dictation); it +/// is used when it names a registry model that is ready on disk, otherwise +/// the startup-selected model is used. +/// +/// Errors if the speech model isn't downloaded yet or a session is already +/// active (prevents two composers opening two mic streams into one pipeline). +#[tauri::command] +pub fn start_dictation( + app: tauri::AppHandle, + state: tauri::State<'_, DictationState>, + app_state: tauri::State<'_, AppState>, + model: Option, +) -> Result<(), String> { + if state.active.load(Ordering::Acquire) { + return Err("dictation already active".to_string()); + } + // The huddle owns the mic (and Ctrl+Space) while it's up. + if app_state + .huddle() + .is_ok_and(|hs| !matches!(hs.phase, HuddlePhase::Idle)) + { + return Err("dictation is unavailable during a huddle".to_string()); + } + + let mut guard = state + .pipeline + .lock() + .map_err(|_| "dictation state poisoned".to_string())?; + + let (model_id, model_dir, family) = models::resolve_dictation_model(model.as_deref()) + .ok_or("speech model still downloading — try again in a moment".to_string())?; + + // Clear a dead pipeline (init failure or crash) so retry works, and a + // pipeline built from a different model so a preference change applies. + if guard + .as_ref() + .is_some_and(|(id, p)| p.is_finished() || *id != model_id) + { + *guard = None; + } + + if guard.is_none() { + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(64); + // In live mode all transcripts arrive on live_rx; the pipeline's own + // text receiver stays silent and is dropped here. + let (pipeline, _text_rx) = SttPipeline::new( + model_dir, + family, + Arc::new(AtomicBool::new(false)), // no TTS during dictation + None, + None, // continuous VAD segments phrases at natural pauses + Some(live_tx), + )?; + *guard = Some((model_id, pipeline)); + + // Forward transcripts to the webview until the pipeline is dropped. + tauri::async_runtime::spawn(async move { + while let Some(event) = live_rx.recv().await { + let result = match event { + LiveEvent::Transcript { text, is_final } => { + app.emit("dictation-transcript", TranscriptEvent { text, is_final }) + } + // Stop-flush drained: no more transcripts for the stopped + // session. The frontend gates Enter-to-send on this. + LiveEvent::Flushed => app.emit("dictation-flushed", ()), + }; + if let Err(e) = result { + eprintln!("buzz-desktop: dictation event emit failed: {e}"); + } + } + }); + } + + state.active.store(true, Ordering::Release); + Ok(()) +} + +/// End the session. Pushes silence so the VAD worker closes and decodes the +/// phrase in flight — that final transcript arrives as an event shortly +/// after. Idempotent. +#[tauri::command] +pub fn stop_dictation(state: tauri::State<'_, DictationState>) { + if !state.active.swap(false, Ordering::AcqRel) { + return; + } + if let Ok(guard) = state.pipeline.lock() { + if let Some((_, ref pipeline)) = *guard { + let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); + // Zero-length sentinel — the worker echoes it back as a Flushed + // marker once everything before it (including the trailing + // phrase's decode) has been processed and delivered. + let _ = pipeline.push_audio(Vec::new()); + } + } +} + +/// Registry STT models with live install status, for the Settings picker. +#[tauri::command] +pub fn get_dictation_models() -> Result, String> { + Ok(models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")? + .stt_model_infos()) +} + +/// Start a background download of a specific STT model (Settings picker). +#[tauri::command] +pub fn download_dictation_model( + id: String, + app_state: tauri::State<'_, AppState>, +) -> Result<(), String> { + models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")? + .start_stt_download_for(&id, app_state.http_client.clone()) +} + +/// Receive raw PCM (f32 LE, 48 kHz mono) from the AudioWorklet. +/// Mirrors `push_audio_pcm` but feeds the dictation pipeline. +#[tauri::command] +pub fn push_dictation_pcm( + request: tauri::ipc::Request<'_>, + state: tauri::State<'_, DictationState>, +) -> Result<(), String> { + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + bytes.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + // Drop late batches after stop — the flush silence must be the + // last audio the VAD sees for a session. + if !state.active.load(Ordering::Acquire) { + return Ok(()); + } + let guard = state + .pipeline + .lock() + .map_err(|_| "dictation state poisoned".to_string())?; + if let Some((_, ref pipeline)) = *guard { + pipeline.push_audio(bytes.to_vec())?; + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..18394e2315 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -1,7 +1,11 @@ -//! Model download manager for STT (Parakeet TDT-CTC 110M) and TTS (Pocket TTS) models. +//! Model download manager for STT (Parakeet family) and TTS (Pocket TTS) models. +//! +//! The STT model is selectable via the `STT_MODELS` registry (issue #2478): +//! the English Parakeet TDT-CTC 110M default, or the multilingual Parakeet TDT +//! 0.6B v3, chosen by `BUZZ_STT_MODEL` / system locale in `selected_stt_model`. //! //! Mental model: -//! app launch → start_stt_download (background) → ~/.buzz/models/parakeet-tdt-ctc-110m-en/ +//! app launch → start_stt_download (background) → ~/.buzz/models// //! app launch → start_tts_download (background) → ~/.buzz/models/pocket-tts/ //! STT pipeline → is_stt_ready() → stt_model_dir() → run inference //! TTS pipeline → is_tts_ready() → tts_model_dir() → run synthesis @@ -26,17 +30,17 @@ use sha2::{Digest, Sha256}; // ── Integrity verification ──────────────────────────────────────────────────── // -// All model artifacts are verified against pinned SHA-256 hashes before +// Model artifacts are verified against pinned SHA-256 hashes before // installation. This is defense-in-depth: HTTPS protects the transport, // hashes protect the content. // // To recompute hashes: download each file, run `shasum -a 256 `, and // update the corresponding constant. - -/// SHA-256 hash of the STT archive -/// (sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2). -/// Computed from a known-good download. Update when upgrading model versions. -const STT_ARCHIVE_SHA256: &str = "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"; +// +// STT archive hashes are pinned per model in the `STT_MODELS` registry below +// (`SttModel::archive_sha256`). Both shipped models are pinned; the field is +// `Option` so a model may temporarily ship `None` (size cap + safe extraction +// + expected-files verification still apply) before its hash is computed. /// HuggingFace base URL for the sherpa-onnx Pocket TTS fp32 repackage. /// @@ -85,12 +89,8 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ // If the on-disk manifest doesn't match the compiled-in version, the model is // considered stale and re-downloaded. Increment when upgrading model files. -/// Model manifest version for the STT model. Increment when upgrading model files. -/// Bumped from "1" → "2" alongside the migration from Moonshine Tiny to -/// Parakeet TDT-CTC 110M — the model directory name also changed, so this -/// is technically belt-and-suspenders, but it keeps the manifest semantics -/// honest (each version tag identifies one specific set of model bytes). -const STT_MODEL_VERSION: &str = "2"; +// STT manifest versions are per model — see `SttModel::version` in the +// `STT_MODELS` registry below. /// Model manifest version for Pocket TTS. Increment when upgrading model files. /// Bumped "1" → "2" when the bundled reference voice changed from KevinAHM's @@ -107,44 +107,214 @@ const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; // ── Constants ───────────────────────────────────────────────────────────────── -/// Maximum expected STT archive size (200 MB — actual is ~100 MB). -const MAX_STT_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; - /// Maximum expected Pocket TTS file size (400 MB per file — largest is /// `lm_main.onnx` at ~303 MB fp32). const MAX_TTS_FILE_BYTES: u64 = 400 * 1024 * 1024; -/// NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx by -/// k2-fsa. Single ONNX file (CTC head) + tokens.txt. Avg WER ~7.5% across -/// the OpenASR-style benchmarks; ~half the WER of Moonshine Tiny at ~2× the -/// disk footprint. CTC blank-token decoding eliminates the silence/cut-audio -/// hallucination class that hurts encoder-decoder models on noisy huddle audio. -/// License: CC-BY-4.0 (attribution required — see About dialog). -const STT_DOWNLOAD_URL: &str = - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ - sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2"; +// ── STT model registry (multilingual — issue #2478) ─────────────────────────── +// +// Historically the huddle STT model was hard-pinned to an English-only build, +// so non-English speech transcribed as garbage (issue #2478). The registry +// makes the model selectable: the English default stays the default, and a +// multilingual model is picked automatically for non-English locales (or +// forced with the `BUZZ_STT_MODEL` env override). Adding a model is pure data +// here plus, for a new sherpa-onnx model family, one match arm in `stt.rs`. + +/// Attribution sidecar filename written next to every STT model's files. +const STT_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt"; -/// Subdirectory name produced by `tar xjf` on the archive. -const STT_ARCHIVE_SUBDIR: &str = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8"; +/// sherpa-onnx model family — decides how the offline recognizer is configured +/// (`stt.rs`) and which ONNX files must be present on disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SttFamily { + /// Single-file NeMo CTC head (e.g. Parakeet TDT-CTC 110M English). + NemoCtc, + /// NeMo transducer: encoder + decoder + joiner (e.g. Parakeet TDT 0.6B v3). + Transducer, +} -/// Final directory name under `~/.buzz/models/`. -const STT_MODEL_DIR_NAME: &str = "parakeet-tdt-ctc-110m-en"; +/// One selectable speech-to-text model. `STT_MODELS` is the single source of +/// truth; `select_stt_model` picks one at startup. +pub struct SttModel { + /// Stable id used by the `BUZZ_STT_MODEL` override and in logs. + pub id: &'static str, + /// Directory name under `~/.buzz/models/`. + pub dir_name: &'static str, + /// Download URL for the `.tar.bz2` archive. + pub download_url: &'static str, + /// Directory name produced by `tar xjf` on the archive. + pub archive_subdir: &'static str, + /// SHA-256 of the archive, or `None` if not yet pinned. `None` still + /// enforces the size cap, safe extraction, and expected-files check — it + /// only skips the content hash. The English default is always `Some`. + pub archive_sha256: Option<&'static str>, + /// Hard cap on the downloaded archive size, in bytes. + pub max_download_bytes: u64, + /// Model files (excluding the license sidecar) required for "ready". + pub model_files: &'static [&'static str], + /// sherpa-onnx model family. + pub family: SttFamily, + /// Manifest version — bump to force re-download of this model. + pub version: &'static str, + /// CC-BY-4.0 §3(a)(1) attribution written next to the model bytes. + pub license_text: &'static str, + /// Human-readable language coverage (About dialog / logs). + pub languages: &'static str, + /// `true` when multilingual — used by the non-English locale auto-select. + pub multilingual: bool, +} -/// All files that must be present for the model to be considered ready. +/// Registry of selectable STT models. Index 0 is the default (English). +static STT_MODELS: &[SttModel] = &[ + // NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx + // by k2-fsa. Single ONNX file (CTC head) + tokens.txt. Avg WER ~7.5% across + // the OpenASR-style benchmarks; CTC blank-token decoding eliminates the + // silence/cut-audio hallucination class that hurts encoder-decoder models + // on noisy huddle audio. This remains the default for English. + SttModel { + id: "parakeet-en", + dir_name: "parakeet-tdt-ctc-110m-en", + download_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2", + archive_subdir: "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8", + archive_sha256: Some("17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"), + max_download_bytes: 200 * 1024 * 1024, + model_files: &["model.int8.onnx", "tokens.txt"], + family: SttFamily::NemoCtc, + version: "2", + license_text: STT_EN_LICENSE_TEXT, + languages: "English", + multilingual: false, + }, + // NVIDIA Parakeet TDT 0.6B v3 (multilingual, int8) — packaged for + // sherpa-onnx by k2-fsa. Transducer family (encoder/decoder/joiner). Auto + // language-ID + punctuation across 25 European languages. This is the + // multilingual default for non-English locales (issue #2478). + // + // Coverage note: Parakeet v3 covers European languages only. CJK/Korean + // (the concrete case in #2478) is not covered by this model; the registry + // makes adding a CJK model (e.g. SenseVoice-Small) a follow-up — one data + // entry, no engine change beyond a family already handled here. + // + // Checksum: upstream publishes no SHA-256, so this hash was computed from + // the k2-fsa `asr-models` release archive + // (sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2, ~465 MB) with + // `shasum -a 256`. Recompute and re-pin if k2-fsa republishes the asset. + SttModel { + id: "parakeet-v3", + dir_name: "parakeet-tdt-0.6b-v3", + download_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", + archive_subdir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", + archive_sha256: Some("5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf"), + max_download_bytes: 800 * 1024 * 1024, + model_files: &[ + "encoder.int8.onnx", + "decoder.int8.onnx", + "joiner.int8.onnx", + "tokens.txt", + ], + family: SttFamily::Transducer, + version: "1", + license_text: STT_V3_LICENSE_TEXT, + languages: "25 European languages (auto-detected): Bulgarian, Croatian, \ + Czech, Danish, Dutch, English, Estonian, Finnish, French, \ + German, Greek, Hungarian, Italian, Latvian, Lithuanian, \ + Maltese, Polish, Portuguese, Romanian, Russian, Slovak, \ + Slovenian, Spanish, Swedish, Ukrainian", + multilingual: true, + }, +]; + +/// The default STT model (English). Used when no override/locale applies. +fn default_stt_model() -> &'static SttModel { + &STT_MODELS[0] +} + +/// Look up a model by id (case-insensitive). `None` if unknown. +fn stt_model_by_id(id: &str) -> Option<&'static SttModel> { + STT_MODELS.iter().find(|m| m.id.eq_ignore_ascii_case(id)) +} + +/// Files that must be present for a model to be "ready": its model files plus +/// the Buzz-written attribution sidecar (the upstream archives ship no LICENSE, +/// so readiness requires the local CC-BY-4.0 notice to travel with the bytes). +fn stt_expected_files(model: &SttModel) -> Vec<&'static str> { + let mut files = model.model_files.to_vec(); + files.push(STT_LICENSE_FILE_NAME); + files +} + +/// Pick the STT model from an explicit override id and a best-effort locale. +/// +/// Precedence (issue #2478 options 2 + 3): +/// 1. `override_id` (from `BUZZ_STT_MODEL`) when it names a known model. +/// 2. a non-English locale → the multilingual model. +/// 3. otherwise the English default. /// -/// Includes the attribution sidecar written by Buzz during install. The -/// upstream archive does not ship a license file, so readiness should require -/// the local CC-BY-4.0 attribution to travel with the cached model bytes. -const STT_EXPECTED_FILES: &[&str] = &["model.int8.onnx", "tokens.txt", STT_LICENSE_FILE_NAME]; - -/// CC-BY-4.0 §3(a)(1) attribution block written next to the STT model files -/// after install. Travels with the bytes — if a user copies the model -/// directory, the attribution comes with it. Mirrored in About/Credits. +/// Pure and dependency-free so it is unit-testable without touching disk/env. +pub fn select_stt_model(override_id: Option<&str>, locale: Option<&str>) -> &'static SttModel { + if let Some(id) = override_id { + let id = id.trim(); + if !id.is_empty() { + if let Some(model) = stt_model_by_id(id) { + return model; + } + eprintln!( + "buzz-desktop: BUZZ_STT_MODEL='{id}' is not a known STT model id — ignoring \ + (valid ids: {})", + STT_MODELS + .iter() + .map(|m| m.id) + .collect::>() + .join(", ") + ); + } + } + if let Some(locale) = locale { + // Take the primary language subtag: "de-DE"/"uk_UA" → "de"/"uk". + let lang = locale + .split(['-', '_', '.']) + .next() + .unwrap_or("") + .to_ascii_lowercase(); + if !lang.is_empty() && lang != "en" { + if let Some(model) = STT_MODELS.iter().find(|m| m.multilingual) { + return model; + } + } + } + default_stt_model() +} + +/// Best-effort system locale from the environment (dependency-free). /// +/// Reads the standard POSIX locale variables in precedence order. Returns +/// `None` when unset or set to the neutral `C`/`POSIX` locale, in which case +/// selection falls back to the English default. +fn detect_locale() -> Option { + for key in ["LC_ALL", "LC_MESSAGES", "LANG", "LANGUAGE"] { + if let Ok(value) = std::env::var(key) { + let value = value.trim(); + if !value.is_empty() && value != "C" && value != "POSIX" { + return Some(value.to_string()); + } + } + } + None +} + +/// Resolve the STT model to use for this process: `BUZZ_STT_MODEL` override, +/// else system-locale auto-select, else English default. +fn selected_stt_model() -> &'static SttModel { + let override_id = std::env::var("BUZZ_STT_MODEL").ok(); + select_stt_model(override_id.as_deref(), detect_locale().as_deref()) +} + +/// CC-BY-4.0 §3(a)(1) attribution for Parakeet TDT-CTC 110M (English). /// Covers all five §3(a)(1) bullets: creator, copyright notice, license /// notice, warranty disclaimer reference, and URI to the source material. -const STT_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt"; -const STT_LICENSE_TEXT: &str = "\ +const STT_EN_LICENSE_TEXT: &str = "\ NVIDIA Parakeet TDT-CTC 110M (English) © NVIDIA Corporation. @@ -160,6 +330,23 @@ Provided \"AS IS\", without warranty of any kind, express or implied. See the license text for full warranty disclaimer. "; +/// CC-BY-4.0 §3(a)(1) attribution for Parakeet TDT 0.6B v3 (multilingual). +const STT_V3_LICENSE_TEXT: &str = "\ +NVIDIA Parakeet TDT 0.6B v3 (multilingual) +© NVIDIA Corporation. + +Licensed under the Creative Commons Attribution 4.0 International License +(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/ + +Original model: https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 +Converted to ONNX with int8 quantization by the sherpa-onnx project +(https://github.com/k2-fsa/sherpa-onnx); Buzz ships this conversion +unmodified. + +Provided \"AS IS\", without warranty of any kind, express or implied. See the +license text for full warranty disclaimer. +"; + // ── Pocket TTS model ────────────────────────────────────────────────────────── /// Final directory name under `~/.buzz/models/`. @@ -238,6 +425,19 @@ pub struct VoiceModelStatus { pub tts: ModelStatus, } +/// One `STT_MODELS` registry entry with live status, for the dictation model +/// picker in Settings. +#[derive(Debug, Clone, Serialize)] +pub struct SttModelInfo { + pub id: &'static str, + pub languages: &'static str, + pub multilingual: bool, + /// `true` for the model the app selected at startup (override / locale / + /// default) — what dictation uses when no explicit preference is set. + pub selected: bool, + pub status: ModelStatus, +} + // ── Safe archive extraction ─────────────────────────────────────────────────── /// Extract a .tar.bz2 archive safely using Rust-native crates. @@ -401,9 +601,9 @@ where /// Per-model state + config. `ModelManager` owns two of these (stt, tts). #[derive(Clone)] struct ModelSlot { - dir_name: &'static str, // subdir under ~/.buzz/models/ - expected_files: &'static [&'static str], // files required for "ready" - version: &'static str, // manifest version; increment to force re-download + dir_name: &'static str, // subdir under ~/.buzz/models/ + expected_files: Vec<&'static str>, // files required for "ready" + version: &'static str, // manifest version; increment to force re-download status: Arc>, just_ready: Arc, // fires once when download completes } @@ -411,7 +611,7 @@ struct ModelSlot { impl ModelSlot { fn new( dir_name: &'static str, - expected_files: &'static [&'static str], + expected_files: Vec<&'static str>, version: &'static str, ) -> Self { Self { @@ -551,25 +751,65 @@ impl ModelSlot { pub struct ModelManager { /// `~/.buzz/models/` models_dir: PathBuf, + /// The STT model selected for this process (override / locale / default). + stt_model: &'static SttModel, stt: ModelSlot, + /// One slot per `STT_MODELS` entry (dictation model picker). The entry for + /// the startup-selected model shares `stt`'s Arcs so status has one truth. + stt_slots: Vec, tts: ModelSlot, } impl ModelManager { /// Create a new `ModelManager` rooted at `~/.buzz/models/`. /// + /// The STT model is resolved once here from `BUZZ_STT_MODEL`, then the + /// system locale, then the English default (issue #2478). + /// /// Returns `None` if the home directory cannot be resolved. pub fn new() -> Option { let models_dir = dirs::home_dir()?.join(".buzz").join("models"); + let stt_model = selected_stt_model(); + eprintln!( + "buzz-desktop: STT model '{}' selected ({})", + stt_model.id, stt_model.languages + ); + let stt = ModelSlot::new( + stt_model.dir_name, + stt_expected_files(stt_model), + stt_model.version, + ); + let stt_slots = STT_MODELS + .iter() + .map(|m| { + if m.id == stt_model.id { + stt.clone() + } else { + ModelSlot::new(m.dir_name, stt_expected_files(m), m.version) + } + }) + .collect(); Some(Self { models_dir, - stt: ModelSlot::new(STT_MODEL_DIR_NAME, STT_EXPECTED_FILES, STT_MODEL_VERSION), - tts: ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION), + stt_model, + stt, + stt_slots, + tts: ModelSlot::new( + TTS_MODEL_DIR_NAME, + TTS_EXPECTED_FILES.to_vec(), + TTS_MODEL_VERSION, + ), }) } // ── STT accessors ──────────────────────────────────────────────────────── + /// The sherpa-onnx model family of the selected STT model. The huddle STT + /// pipeline uses this to configure the offline recognizer. + pub fn stt_family(&self) -> SttFamily { + self.stt_model.family + } + /// Path to the STT model directory, or `None` if not ready. pub fn stt_model_dir(&self) -> Option { self.stt.dir_if_ready(&self.models_dir) @@ -587,6 +827,78 @@ impl ModelManager { self.stt.take_ready() } + // ── Per-model STT accessors (dictation model picker) ───────────────────── + + /// Registry model + its slot by id (case-insensitive). + fn stt_slot_for(&self, id: &str) -> Option<(&'static SttModel, &ModelSlot)> { + STT_MODELS + .iter() + .zip(&self.stt_slots) + .find(|(m, _)| m.id.eq_ignore_ascii_case(id)) + } + + /// Registry entries with live status, for the dictation model picker. + pub fn stt_model_infos(&self) -> Vec { + STT_MODELS + .iter() + .zip(&self.stt_slots) + .map(|(m, slot)| SttModelInfo { + id: m.id, + languages: m.languages, + multilingual: m.multilingual, + selected: m.id == self.stt_model.id, + // A slot that never started a download this run still reports + // NotDownloaded in memory — trust the disk check first. + status: if slot.is_ready(&self.models_dir) { + ModelStatus::Ready + } else { + slot.status() + }, + }) + .collect() + } + + /// Resolve the model for a dictation session: the preferred id when known + /// and ready on disk, else the startup-selected model. `None` when nothing + /// is ready. + pub fn resolve_dictation_model( + &self, + preferred: Option<&str>, + ) -> Option<(&'static str, PathBuf, SttFamily)> { + if let Some((model, slot)) = preferred.and_then(|id| self.stt_slot_for(id)) { + if let Some(dir) = slot.dir_if_ready(&self.models_dir) { + return Some((model.id, dir, model.family)); + } + } + let dir = self.stt.dir_if_ready(&self.models_dir)?; + Some((self.stt_model.id, dir, self.stt_model.family)) + } + + /// Start a background download of a specific registry model. No-op if it + /// is already ready or downloading. + pub fn start_stt_download_for( + &self, + id: &str, + http_client: reqwest::Client, + ) -> Result<(), String> { + let (model, slot) = self + .stt_slot_for(id) + .ok_or_else(|| format!("unknown STT model id: {id}"))?; + let manager = self.clone(); + let slot_for_task = slot.clone(); + slot.start_download( + &self.models_dir, + http_client, + "stt", + move |client| async move { + manager + .download_stt_model_for(model, slot_for_task, client) + .await + }, + ); + Ok(()) + } + // ── TTS accessors ───────────────────────────────────────────────────────── /// Path to the TTS model directory, or `None` if not ready. @@ -651,33 +963,45 @@ impl ModelManager { // ── Private download implementations ───────────────────────────────────── - /// Download, extract, and verify the STT model archive. + /// Download, extract, and verify the startup-selected STT model archive. async fn download_stt_model(&self, http_client: reqwest::Client) -> Result<(), String> { + self.download_stt_model_for(self.stt_model, self.stt.clone(), http_client) + .await + } + + /// Download, extract, and verify a specific STT model archive into `slot`. + async fn download_stt_model_for( + &self, + model: &'static SttModel, + slot: ModelSlot, + http_client: reqwest::Client, + ) -> Result<(), String> { tokio::fs::create_dir_all(&self.models_dir) .await .map_err(|e| format!("create models dir: {e}"))?; // Temp filenames derive from the final directory name to avoid colliding // with leftovers from any previous STT model (e.g. moonshine-tiny.*). - let archive_path = self - .models_dir - .join(format!("{STT_MODEL_DIR_NAME}.tar.bz2")); - let temp_dir = self.models_dir.join(format!("{STT_MODEL_DIR_NAME}.tmp")); + let archive_path = self.models_dir.join(format!("{}.tar.bz2", model.dir_name)); + let temp_dir = self.models_dir.join(format!("{}.tmp", model.dir_name)); - eprintln!("buzz-desktop: downloading STT model from {STT_DOWNLOAD_URL}"); - let response = fetch_url(&http_client, STT_DOWNLOAD_URL, "stt archive").await?; + eprintln!( + "buzz-desktop: downloading STT model '{}' from {}", + model.id, model.download_url + ); + let response = fetch_url(&http_client, model.download_url, "stt archive").await?; - let slot = self.stt.clone(); + let progress_slot = slot.clone(); let bytes = download_file( response, &archive_path, - MAX_STT_DOWNLOAD_BYTES, + model.max_download_bytes, "stt archive", |downloaded, content_length| { if let Some(pct) = content_length.and_then(|total| (downloaded * 89).checked_div(total)) { - slot.set_status(ModelStatus::Downloading { + progress_slot.set_status(ModelStatus::Downloading { progress_percent: pct.min(89) as u8, }); } @@ -686,16 +1010,30 @@ impl ModelManager { .await?; eprintln!("buzz-desktop: downloaded {bytes} bytes, wrote to disk"); - // Verify archive integrity before extraction. - let hash = sha256_file(&archive_path).await?; - if hash != STT_ARCHIVE_SHA256 { - let _ = tokio::fs::remove_file(&archive_path).await; - return Err(format!( - "STT archive integrity check failed: expected {STT_ARCHIVE_SHA256}, got {hash}" - )); + // Verify archive integrity before extraction. Models with a pinned + // hash are content-verified; a `None` hash relies on the size cap + // (already enforced above), safe extraction, and the expected-files + // check in `verify_and_install`. + match model.archive_sha256 { + Some(expected) => { + let hash = sha256_file(&archive_path).await?; + if hash != expected { + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(format!( + "STT archive integrity check failed: expected {expected}, got {hash}" + )); + } + } + None => { + eprintln!( + "buzz-desktop: STT model '{}' has no pinned SHA-256 — \ + skipping content hash (size cap + safe extraction still enforced)", + model.id + ); + } } - self.stt.set_status(ModelStatus::Downloading { + slot.set_status(ModelStatus::Downloading { progress_percent: 90, }); fresh_temp_dir(&temp_dir).await?; @@ -706,11 +1044,12 @@ impl ModelManager { .await .map_err(|e| format!("tar task panicked: {e}"))??; - let extracted_subdir = temp_dir.join(STT_ARCHIVE_SUBDIR); + let extracted_subdir = temp_dir.join(model.archive_subdir); if !extracted_subdir.is_dir() { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( - "expected subdir '{STT_ARCHIVE_SUBDIR}' not found after extraction" + "expected subdir '{}' not found after extraction", + model.archive_subdir )); } @@ -719,15 +1058,14 @@ impl ModelManager { // upstream tarball ships no LICENSE/NOTICE, so we provide it ourselves // per §3(a)(1) (license must travel with Shared material). let license_path = extracted_subdir.join(STT_LICENSE_FILE_NAME); - if let Err(e) = tokio::fs::write(&license_path, STT_LICENSE_TEXT).await { + if let Err(e) = tokio::fs::write(&license_path, model.license_text).await { let _ = tokio::fs::remove_dir_all(&temp_dir).await; let _ = tokio::fs::remove_file(&archive_path).await; return Err(format!("write model license sidecar: {e}")); } // verify_and_install takes the subdir (actual model files); temp_cleanup removes outer dir. - if let Err(e) = self - .stt + if let Err(e) = slot .verify_and_install(&self.models_dir, &extracted_subdir, Some(&temp_dir)) .await { @@ -746,7 +1084,7 @@ impl ModelManager { eprintln!( "buzz-desktop: STT model ready at {}", - self.stt.model_dir(&self.models_dir).display() + slot.model_dir(&self.models_dir).display() ); Ok(()) } @@ -883,6 +1221,15 @@ pub fn stt_model_dir() -> Option { global_model_manager()?.stt_model_dir() } +/// sherpa-onnx model family of the selected STT model (English default when the +/// manager is unavailable). The huddle STT pipeline uses this to configure the +/// offline recognizer for the right model family. +pub fn stt_model_family() -> SttFamily { + global_model_manager() + .map(|m| m.stt_family()) + .unwrap_or(SttFamily::NemoCtc) +} + /// `true` if all expected STT model files are present on disk. pub fn is_stt_ready() -> bool { global_model_manager() @@ -890,6 +1237,14 @@ pub fn is_stt_ready() -> bool { .unwrap_or(false) } +/// Resolve the model for a dictation session (see +/// `ModelManager::resolve_dictation_model`). +pub fn resolve_dictation_model( + preferred: Option<&str>, +) -> Option<(&'static str, PathBuf, SttFamily)> { + global_model_manager()?.resolve_dictation_model(preferred) +} + /// Best-effort cleanup of the legacy Moonshine STT model directory. /// /// Removes `~/.buzz/models/moonshine-tiny/` if present (~70 MB on disk). @@ -937,7 +1292,11 @@ mod tests { #[test] fn tts_readiness_requires_license_sidecar() { let temp = tempfile::tempdir().expect("tempdir"); - let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); + let slot = ModelSlot::new( + TTS_MODEL_DIR_NAME, + TTS_EXPECTED_FILES.to_vec(), + TTS_MODEL_VERSION, + ); let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); std::fs::create_dir_all(&model_dir).expect("create model dir"); @@ -951,4 +1310,122 @@ mod tests { std::fs::remove_file(model_dir.join(TTS_LICENSE_FILE_NAME)).expect("remove sidecar"); assert!(!slot.is_ready(temp.path())); } + + // ── STT model selection (issue #2478) ───────────────────────────────────── + + #[test] + fn defaults_to_english_without_override_or_locale() { + assert_eq!(select_stt_model(None, None).id, "parakeet-en"); + assert_eq!(select_stt_model(None, Some("en-US")).id, "parakeet-en"); + assert_eq!(select_stt_model(None, Some("en")).id, "parakeet-en"); + // Neutral C/POSIX locales are dropped by detect_locale, but pass the raw + // primary subtag here to prove "en" specifically stays on English. + assert!(!select_stt_model(None, None).multilingual); + } + + #[test] + fn non_english_locale_selects_multilingual_model() { + for locale in ["de-DE", "uk_UA", "fr", "es-ES", "pl_PL.UTF-8"] { + let model = select_stt_model(None, Some(locale)); + assert!(model.multilingual, "locale {locale} should be multilingual"); + assert_eq!(model.id, "parakeet-v3", "locale {locale}"); + } + } + + #[test] + fn explicit_override_wins_over_locale() { + // Force multilingual even on an English locale. + assert_eq!( + select_stt_model(Some("parakeet-v3"), Some("en-US")).id, + "parakeet-v3" + ); + // Force English even on a non-English locale. + assert_eq!( + select_stt_model(Some("parakeet-en"), Some("de-DE")).id, + "parakeet-en" + ); + // Override id is case-insensitive. + assert_eq!( + select_stt_model(Some("PARAKEET-V3"), None).id, + "parakeet-v3" + ); + } + + #[test] + fn unknown_or_empty_override_falls_back() { + // Unknown override id → fall through to locale, then default. + assert_eq!( + select_stt_model(Some("does-not-exist"), Some("en-US")).id, + "parakeet-en" + ); + assert_eq!( + select_stt_model(Some("does-not-exist"), Some("fr-FR")).id, + "parakeet-v3" + ); + // Empty / whitespace override is ignored. + assert_eq!(select_stt_model(Some(" "), None).id, "parakeet-en"); + } + + #[test] + fn registry_invariants_hold() { + assert!(!STT_MODELS.is_empty()); + assert_eq!(default_stt_model().id, "parakeet-en"); + // The English default must always be integrity-pinned. + assert!( + default_stt_model().archive_sha256.is_some(), + "English default must ship a pinned SHA-256" + ); + // At least one multilingual model exists (covers #2478). + assert!(STT_MODELS.iter().any(|m| m.multilingual)); + // Ids are unique (case-insensitive); every model declares files + a cap. + for (i, model) in STT_MODELS.iter().enumerate() { + assert!(!model.model_files.is_empty(), "{} has no files", model.id); + assert!(model.max_download_bytes > 0, "{} has no size cap", model.id); + for other in &STT_MODELS[i + 1..] { + assert!( + !model.id.eq_ignore_ascii_case(other.id), + "duplicate model id {}", + model.id + ); + } + } + } + + #[test] + fn expected_files_always_include_license_sidecar() { + for model in STT_MODELS { + let files = stt_expected_files(model); + assert!( + files.contains(&STT_LICENSE_FILE_NAME), + "{} missing license sidecar in expected files", + model.id + ); + for f in model.model_files { + assert!(files.contains(f), "{} missing {f}", model.id); + } + } + } + + #[test] + fn readiness_uses_per_model_expected_files() { + // A transducer model needs encoder/decoder/joiner — the single-file + // check must not pass until all three plus tokens + license exist. + let v3 = stt_model_by_id("parakeet-v3").expect("v3 registered"); + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(v3.dir_name, stt_expected_files(v3), v3.version); + let dir = temp.path().join(v3.dir_name); + std::fs::create_dir_all(&dir).expect("create dir"); + std::fs::write(dir.join(MANIFEST_FILENAME), v3.version).expect("manifest"); + + // Only some files present → not ready. + std::fs::write(dir.join("encoder.int8.onnx"), b"x").expect("write"); + std::fs::write(dir.join("tokens.txt"), b"x").expect("write"); + assert!(!slot.is_ready(temp.path())); + + // All expected files present → ready. + for f in stt_expected_files(v3) { + std::fs::write(dir.join(f), b"x").expect("write"); + } + assert!(slot.is_ready(temp.path())); + } } diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index ceccedd8b6..a5489a4acc 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -94,6 +94,9 @@ pub(crate) async fn maybe_start_stt_pipeline( return Ok(false); // Models not downloaded yet — voice-only mode. } let model_dir = models::stt_model_dir().ok_or("STT model directory not found")?; + // The selected model's family decides how the offline recognizer is + // configured (English CTC vs multilingual transducer — issue #2478). + let stt_family = models::stt_model_family(); let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; @@ -138,7 +141,14 @@ pub(crate) async fn maybe_start_stt_pipeline( drop(old_stt); let constructed = tokio::task::spawn_blocking(move || { - stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt) + stt::SttPipeline::new( + model_dir, + stt_family, + tts_active, + tts_cancel, + ptt_active_for_stt, + None, + ) }) .await; let (pipeline, text_rx) = match constructed { diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..22e5e91afc 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -31,16 +31,34 @@ use std::{ use tokio::sync::mpsc as tokio_mpsc; +use super::models::SttFamily; + // ── Public pipeline handle ──────────────────────────────────────────────────── -/// Bounded audio queue capacity. -/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. -const AUDIO_QUEUE_DEPTH: usize = 50; +/// Bounded audio queue capacity: 100 ms batches at 48 kHz ≈ 19 KB each → +/// 300 slots ≈ 30 s / ~6 MB backlog, riding out the worst live-mode decode +/// stall (~2 s flush of a 30 s phrase). Overflow silently drops mic audio, +/// splicing the phrase so later decodes delete words already shown. +const AUDIO_QUEUE_DEPTH: usize = 300; /// Maximum speech buffer size: 30 seconds at 16 kHz. /// Prevents OOM if VAD stays in speech mode (noisy environment). const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +/// Event on the live-transcript (dictation) channel. Strictly ordered — a +/// `Flushed` sent after a session's audio guarantees every `Transcript` of +/// that session was already delivered. +#[derive(Debug, PartialEq)] +pub enum LiveEvent { + /// Partial (`is_final: false`) re-decodes replace each other; a final + /// commits the phrase. An empty final means the phrase decoded to nothing. + Transcript { text: String, is_final: bool }, + /// Echo of a zero-length audio batch (the stop-flush sentinel): all audio + /// pushed before it has been processed, so the stopped session has no + /// further transcripts coming. + Flushed, +} + /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. @@ -75,6 +93,15 @@ impl SttPipeline { /// pipeline only accumulates speech while the flag is true (key held). /// When `None`, the pipeline runs in continuous VAD mode. /// + /// `live_tx` (optional) switches the pipeline into live-transcript mode + /// (used by composer dictation): the in-progress phrase is re-decoded + /// every `PARTIAL_DECODE_STEP` new samples and sent as a partial + /// `Transcript`, and finals commit it — all on this one channel so + /// ordering is preserved. A zero-length audio batch is echoed back as + /// `Flushed` (see `LiveEvent`). In this mode nothing is sent on the + /// returned text receiver. Pass `None` for huddle transcription (finals + /// only). + /// /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — /// the pipeline handle is still returned but will never produce text. @@ -85,9 +112,11 @@ impl SttPipeline { /// thread on every `recv_timeout` call). pub fn new( model_dir: PathBuf, + family: SttFamily, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + live_tx: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); @@ -101,12 +130,14 @@ impl SttPipeline { .spawn(move || { stt_worker( model_dir, + family, audio_rx, text_tx, shutdown_worker, tts_active, tts_cancel_worker, ptt_active_worker, + live_tx, ) }) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; @@ -143,7 +174,11 @@ impl SttPipeline { )); } // Drop audio if the pipeline can't keep up — better than blocking the UI. - let _ = self.audio_tx.try_send(pcm_bytes); + if self.audio_tx.try_send(pcm_bytes).is_err() { + // A drop splices the phrase buffer and corrupts later decodes of + // that phrase — rare, but worth a trail in the log when it happens. + eprintln!("buzz-desktop: STT audio queue full — batch dropped"); + } Ok(()) } } @@ -167,6 +202,17 @@ impl Drop for SttPipeline { /// Previous value (28 frames / 450 ms) felt sluggish in conversation. const SILENCE_FLUSH_FRAMES: usize = 19; +/// Silence-flush threshold for live (dictation) mode: ~608 ms. +/// +/// Parakeet only commits sentence-final punctuation and capitalization when +/// the decoded audio ends in ≥600 ms of silence (measured: 300 ms → "is there +/// anything we can do about that", 600 ms → "…about that?"). The huddle's +/// ~300 ms window cut finals just under that threshold, so dictated text lost +/// its punctuation at every phrase commit. The longer window also merges +/// mid-sentence thinking pauses into fuller phrases; perceived latency is +/// unchanged because partials keep streaming while the window runs. +const LIVE_SILENCE_FLUSH_FRAMES: usize = 38; + /// Consecutive VAD speech frames required before triggering barge-in during TTS. /// 20 frames × 256 samples / 16 kHz ≈ 320 ms — must be long enough to filter /// speaker-to-mic feedback (TTS audio bleeding through the mic) while still @@ -184,6 +230,25 @@ const VAD_THRESHOLD: f32 = 0.5; /// How long the worker waits on the audio channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(50); +/// Live-transcript mode: re-decode the in-progress phrase every this many new +/// 16 kHz samples (~300 ms) so words stream out while the user is talking. +/// The model is an offline recognizer, so "streaming" is repeated decoding of +/// the growing phrase buffer, and decode cost grows with it (~70 ms per second +/// of audio for the 0.6B model, measured in `v3_long_buffer`). So this is only +/// the MINIMUM step: after every decode (partials AND phrase flushes) the +/// worker holds off 2× that decode's duration before the next partial, capping +/// decode duty at ~50%. The hold must be WALL-CLOCK, not new-samples: a +/// post-stall backlog arrives faster than real time, so samples-based back-off +/// collapses into decode storms that overflow the audio queue. +const PARTIAL_DECODE_STEP: usize = 4800; + +/// Consecutive VAD speech frames (16 ms each, ~96 ms total) required to open a +/// phrase in live (dictation) mode. Single-frame blips (keyboard click, +/// breath, background noise) make Parakeet hallucinate filler ("yeah", "uh"); +/// real speech sustains the VAD easily, and the onset audio is buffered while +/// it proves itself, so nothing is lost. Huddle mode is untouched. +const ONSET_DEBOUNCE_FRAMES: usize = 6; + /// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. /// Previous value (200 ms) was eating the first word when the user spoke @@ -201,14 +266,25 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); /// shows it's safe on the minimum-spec target. const STT_NUM_THREADS: i32 = 1; +/// ONNX threads in live-transcript (dictation) mode. Live mode re-decodes the +/// whole in-progress phrase every ~300 ms, so per-decode latency bounds how +/// fresh the streamed words are. Measured with Parakeet v3 0.6B int8 +/// (`v3_punctuation` experiment): a 6.5 s phrase decodes in ~1.4 s at 1 +/// thread but ~460 ms at 4 — the difference between laggy and live. No +/// huddle/LiveKit stack runs during dictation, so the extra cores are free. +const LIVE_STT_NUM_THREADS: i32 = 4; + +#[allow(clippy::too_many_arguments)] fn stt_worker( model_dir: PathBuf, + family: SttFamily, audio_rx: Receiver>, text_tx: tokio_mpsc::Sender, shutdown: Arc, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + live_tx: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -228,18 +304,21 @@ fn stt_worker( // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── // - // Parakeet TDT-CTC 110M ships as a single `model.int8.onnx` (CTC head) plus - // `tokens.txt`. sherpa-onnx infers the model family from which inner config - // has a `model` path set, so we don't need to set `model_type` explicitly. - // (See rust-api-examples/parakeet_tdt_ctc_simulate_streaming_microphone.rs - // in k2-fsa/sherpa-onnx.) + // sherpa-onnx infers the model family from which inner config has model + // paths set, so we populate exactly one family sub-config (issue #2478): + // + // NemoCtc — single `model.int8.onnx` (CTC head), e.g. Parakeet 110M en. + // Transducer — `encoder/decoder/joiner.int8.onnx`, e.g. Parakeet 0.6B v3 + // (multilingual). See k2-fsa/sherpa-onnx offline-transducer + // NeMo examples. + // + // `tokens.txt` is shared by every family and lives on the parent config. use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); - if !tokens_path.exists() || !model_path.exists() { + if !tokens_path.exists() { eprintln!( - "buzz-desktop: STT model not found at {} — STT disabled", + "buzz-desktop: STT tokens.txt not found at {} — STT disabled", model_dir.display() ); drain_until_shutdown(audio_rx, &shutdown); @@ -247,9 +326,43 @@ fn stt_worker( } let mut cfg = OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); + match family { + SttFamily::NemoCtc => { + let model_path = model_dir.join("model.int8.onnx"); + if !model_path.exists() { + eprintln!( + "buzz-desktop: STT model.int8.onnx not found at {} — STT disabled", + model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); + } + SttFamily::Transducer => { + let encoder = model_dir.join("encoder.int8.onnx"); + let decoder = model_dir.join("decoder.int8.onnx"); + let joiner = model_dir.join("joiner.int8.onnx"); + if !encoder.exists() || !decoder.exists() || !joiner.exists() { + eprintln!( + "buzz-desktop: STT transducer files (encoder/decoder/joiner) missing at {} \ + — STT disabled", + model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + cfg.model_config.transducer.encoder = Some(encoder.to_string_lossy().into_owned()); + cfg.model_config.transducer.decoder = Some(decoder.to_string_lossy().into_owned()); + cfg.model_config.transducer.joiner = Some(joiner.to_string_lossy().into_owned()); + } + } cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); - cfg.model_config.num_threads = STT_NUM_THREADS; + cfg.model_config.num_threads = if live_tx.is_some() { + LIVE_STT_NUM_THREADS + } else { + STT_NUM_THREADS + }; // Explicit — defaults are not part of the API contract, and noisy debug // logging in release builds would be expensive on every VAD chunk. cfg.model_config.debug = false; @@ -276,6 +389,20 @@ fn stt_worker( let mut in_speech = false; // Consecutive speech frames seen during TTS — used for barge-in debounce. let mut barge_in_frames: usize = 0; + // Live mode: speech_buf length at the last partial decode. + let mut last_partial_len: usize = 0; + // Live mode: no partial decode before this instant (see + // PARTIAL_DECODE_STEP — the 2× wall-clock hold after every decode). + let mut decode_hold_until = std::time::Instant::now(); + // Live mode: VAD-positive frames buffered while a speech onset proves + // itself (ONSET_DEBOUNCE_FRAMES) before opening a phrase. + let mut onset_buf: Vec = Vec::new(); + // Live mode: most recent partial decode of the current phrase that ended + // with terminal punctuation — see prefer_punctuated. + let mut punct_partial: Option = None; + // Live mode: best (most words) partial decode of the current phrase — + // the collapse guard, see decode_collapsed. + let mut best_partial: Option = None; // Timestamp when TTS last stopped — used for the 200 ms cooldown. let mut tts_stopped_at: Option = None; @@ -305,10 +432,18 @@ fn stt_worker( if let Some(ref ptt) = ptt_active { let ptt_now = ptt.load(Ordering::Acquire); if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { - flush_to_stt(&speech_buf, &recognizer, &text_tx); + flush_to_stt( + &speech_buf, + &recognizer, + &text_tx, + live_tx.as_ref(), + punct_partial.take(), + best_partial.take(), + ); speech_buf.clear(); silence_frames = 0; in_speech = false; + last_partial_len = 0; } ptt_was_active = ptt_now; } @@ -327,6 +462,17 @@ fn stt_worker( } for bytes in batch { + // Zero-length batch = stop-flush sentinel (dictation.rs). All + // audio pushed before it has been processed by now, so the + // stopped session has no further transcripts — tell the frontend. + if bytes.is_empty() { + if let Some(tx) = live_tx.as_ref() { + if let Err(e) = tx.blocking_send(LiveEvent::Flushed) { + eprintln!("buzz-desktop: STT live channel closed: {e}"); + } + } + continue; + } // Convert raw bytes to f32 samples (little-endian). let samples_48k = bytes_to_f32(&bytes); input_buf_48k.extend_from_slice(&samples_48k); @@ -349,6 +495,12 @@ fn stt_worker( tts_cancel.as_deref(), &mut tts_stopped_at, ptt_active.as_ref(), + live_tx.as_ref(), + &mut last_partial_len, + &mut decode_hold_until, + &mut onset_buf, + &mut punct_partial, + &mut best_partial, ); } } @@ -411,6 +563,12 @@ fn process_16k_samples( tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, + live_tx: Option<&tokio_mpsc::Sender>, + last_partial_len: &mut usize, + decode_hold_until: &mut std::time::Instant, + onset_buf: &mut Vec, + punct_partial: &mut Option, + best_partial: &mut Option, ) { leftover.extend_from_slice(samples); @@ -490,15 +648,35 @@ fn process_16k_samples( if is_speech { *silence_frames = 0; + if !*in_speech && live_tx.is_some() { + // Live mode: debounce the onset (see ONSET_DEBOUNCE_FRAMES). + // ponytail: any non-speech frame resets it; add grace if real onsets clip. + onset_buf.extend_from_slice(&frame); + if onset_buf.len() < ONSET_DEBOUNCE_FRAMES * VAD_FRAME_SAMPLES { + continue; + } + speech_buf.append(onset_buf); + } else { + speech_buf.extend_from_slice(&frame); + } *in_speech = true; - speech_buf.extend_from_slice(&frame); // OOM guard: flush and reset if the buffer exceeds 30 s of audio. if speech_buf.len() >= MAX_SPEECH_SAMPLES { - flush_to_stt(speech_buf, recognizer, text_tx); + let t0 = std::time::Instant::now(); + flush_to_stt( + speech_buf, + recognizer, + text_tx, + live_tx, + punct_partial.take(), + best_partial.take(), + ); + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; *in_speech = false; + *last_partial_len = 0; } } else if *in_speech { // Still accumulate during brief silence gaps. @@ -509,45 +687,284 @@ fn process_16k_samples( // key-hold as one utterance. The PTT release edge in the main // loop handles the flush. In VAD mode, flush after the silence // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { - // End of utterance — transcribe. - flush_to_stt(speech_buf, recognizer, text_tx); + let flush_frames = if live_tx.is_some() { + LIVE_SILENCE_FLUSH_FRAMES + } else { + SILENCE_FLUSH_FRAMES + }; + if ptt_active.is_none() && *silence_frames >= flush_frames { + // End of utterance — transcribe. Flush decodes join the same + // wall-clock hold as partials (see PARTIAL_DECODE_STEP). + let t0 = std::time::Instant::now(); + flush_to_stt( + speech_buf, + recognizer, + text_tx, + live_tx, + punct_partial.take(), + best_partial.take(), + ); + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; *in_speech = false; + *last_partial_len = 0; + } + } else { + // Not in speech: discard the frame, and drop any pending onset — + // the blip ended before it proved itself to be speech. + onset_buf.clear(); + } + + // Live mode: re-decode the in-progress phrase every PARTIAL_DECODE_STEP + // new samples so words appear while the user is still talking. Later + // partials of the same phrase supersede earlier ones — the frontend + // replaces the partial region — so decode wobble is self-correcting. + if let Some(tx) = live_tx { + if *in_speech + && speech_buf.len() >= *last_partial_len + PARTIAL_DECODE_STEP + && std::time::Instant::now() >= *decode_hold_until + { + *last_partial_len = speech_buf.len(); + let t0 = std::time::Instant::now(); + let text = decode_speech(speech_buf, recognizer); + // 2× wall-clock hold (see PARTIAL_DECODE_STEP); 1× measured + // at ~100% duty on a 35 s ramble — no headroom. + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; + // Collapse guard (see decode_collapsed): a re-decode of the + // same growing phrase that lost words is a collapse, not a + // revision — suppress it so it neither replaces the shown + // partial nor poisons the best-partial baseline. + let collapsed = best_partial + .as_deref() + .is_some_and(|best| decode_collapsed(&text, best)); + if collapsed { + eprintln!("buzz-desktop: STT partial collapsed ({text:?}) — suppressed"); + } else if lone_filler(&text) { + // Not shown and not recorded as best/punct partial: a + // filler-only phrase must reach flush_to_stt with + // best_partial=None so its final is dropped there instead + // of resurrected by the collapse guard. + eprintln!("buzz-desktop: STT partial is a lone filler ({text:?}) — suppressed"); + } else if !text.is_empty() { + let mut text = text; + // Second collapse shape — see stitch_prefix_collapse. + if let Some(best) = best_partial.as_deref() { + if let Some(stitched) = stitch_prefix_collapse(best, &text) { + eprintln!( + "buzz-desktop: STT partial dropped its front ({text:?}) — stitched" + ); + text = stitched; + } + } + if text.ends_with(['.', '?', '!', ',']) { + *punct_partial = Some(text.clone()); + } + *best_partial = Some(text.clone()); + let event = LiveEvent::Transcript { + text, + is_final: false, + }; + if let Err(e) = tx.blocking_send(event) { + eprintln!("buzz-desktop: STT live channel closed: {e}"); + } + } } } - // If not in speech and not accumulating, just discard the frame. } } /// Run sherpa-onnx on the accumulated speech buffer and send the text. /// /// Uses `blocking_send` because this runs on a `std::thread` (not async). -/// The tokio channel's `blocking_send` is safe to call from sync contexts. +/// Lowercased words only — all punctuation and case stripped. Two decodes of +/// the same audio that differ only in the model's (unstable) punctuation and +/// capitalization normalize to the same string. +fn norm_words(s: &str) -> String { + let mapped: String = s + .chars() + .map(|c| { + if c.is_alphanumeric() || c == '\'' { + c.to_ascii_lowercase() + } else { + ' ' + } + }) + .collect(); + mapped.split_whitespace().collect::>().join(" ") +} + +/// A decode that is nothing but one filler word. Parakeet hallucinates these +/// ("Yeah.", "Mm.") on breaths/hums/clicks that outlast the VAD onset +/// debounce (see ONSET_DEBOUNCE_FRAMES), and the composer ends up with a +/// "Yeah." for every random noise. A phrase whose decodes never grew past a +/// lone filler never contained speech — suppress it end to end. +/// ponytail: word-list heuristic; a deliberately dictated bare "Yeah." is +/// also dropped — as part of any sentence it survives. +fn lone_filler(text: &str) -> bool { + matches!( + norm_words(text).as_str(), + "yeah" | "uh" | "um" | "mm" | "hmm" | "mhm" | "huh" | "uh huh" | "mm hmm" + ) +} + +/// Punctuation from the 110M model is unstable: re-decodes of the same phrase +/// flicker between "…that" and "…that?" (and internally, "question." vs +/// "question,") depending on the exact trailing-noise composition — see the +/// `silence_experiment` tests. If a partial of this phrase ended in +/// punctuation, prefer it: +/// - same words as the final (punctuation/case-insensitive) → keep the whole +/// punctuated hint; +/// - words differ but the last word agrees and the hint ends a sentence +/// (. ? !) → graft just that mark onto the final. +/// +/// Never invents words: any word-level difference and the final decode wins. +fn prefer_punctuated(final_text: String, punct_hint: Option<&str>) -> String { + let Some(hint) = punct_hint else { + return final_text; + }; + if final_text.is_empty() || hint.is_empty() { + return final_text; + } + let norm_final = norm_words(&final_text); + if norm_words(hint) == norm_final { + return hint.to_string(); + } + let term = hint.chars().last().unwrap_or(' '); + if matches!(term, '.' | '?' | '!') && !final_text.ends_with(['.', '?', '!']) { + let last_word = |s: &str| s.rsplit(' ').next().map(str::to_string); + if last_word(&norm_final) == last_word(&norm_words(hint)) { + let mut grafted = final_text.trim_end_matches([',', ' ']).to_string(); + grafted.push(term); + return grafted; + } + } + final_text +} + +/// True when a re-decode of the same phrase lost words against the best +/// decode seen so far — a collapse, not a revision. Every decode of a phrase +/// sees a superset of the audio any earlier one saw (partials decode a +/// growing buffer; the final decodes all of it), so word count can honestly +/// grow or hold but not shrink. The v3 transducer occasionally collapses a +/// re-decode to empty or a junk word ("Yeah") on audio whose earlier decodes +/// were fine; trusting it deletes the sentence the user watched appear. +/// ponytail: a rare legitimate word-merge revision ("any better"→"anybody") +/// trips this and keeps the earlier decode — words the user already saw, +/// far cheaper than losing real speech. +fn decode_collapsed(new_text: &str, best: &str) -> bool { + let words = |s: &str| norm_words(s).split_whitespace().count(); + words(new_text) < words(best) +} + +/// The v3 collapse's second shape: once a phrase contains an internal pause, +/// a re-decode sometimes returns only the words after the pause — dropping +/// the front while gaining newly spoken tail words, so decode_collapsed's +/// word-count check passes it. Detect: an established phrase (best ≥ 4 +/// words) whose re-decode agrees on neither of its first two words — early +/// decodes legitimately rewrite a 1–3 word front, but a settled front never +/// vanishes wholesale. Repair: anchor the new decode's opening words inside +/// best and splice best's preserved front onto new (new wins from the anchor +/// on, so tail revisions survive); with no anchor the drop swallowed the +/// overlap too, so append new after best. +/// ponytail: anchor window is 3 words then 2 — a 1-word anchor false-matches +/// on common words, and a missed overlap only duplicates ≤2 words. +fn stitch_prefix_collapse(best: &str, new: &str) -> Option { + // Per-token norms keep display and normalized tokens 1:1 aligned. + let b_disp: Vec<&str> = best.split_whitespace().collect(); + let b_norm: Vec = b_disp.iter().map(|t| norm_words(t)).collect(); + let n_norm: Vec = new.split_whitespace().map(norm_words).collect(); + if b_norm.len() < 4 || n_norm.len() < 2 { + return None; + } + if b_norm[0] == n_norm[0] || b_norm[1] == n_norm[1] { + return None; // front agrees — a revision, not a drop + } + for m in [3usize, 2] { + if n_norm.len() < m { + continue; + } + let window = &n_norm[..m]; + if let Some(j) = (0..=b_norm.len() - m) + .rev() + .find(|&j| &b_norm[j..j + m] == window) + { + let mut out = b_disp[..j].join(" "); + if !out.is_empty() { + out.push(' '); + } + out.push_str(new); + return Some(out); + } + } + Some(format!("{best} {new}")) +} + fn flush_to_stt( speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, text_tx: &tokio_mpsc::Sender, + live_tx: Option<&tokio_mpsc::Sender>, + punct_hint: Option, + best_partial: Option, ) { - if speech_buf.is_empty() { - return; + let mut text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); + // Noise gate — see lone_filler. best_partial=None means no real partial + // was ever shown for this phrase (filler partials are never recorded), so + // a lone-filler final is a hallucinated noise, not speech the user watched + // appear. The empty final still flows through live_tx so the frontend + // clears any state for the phrase. + if live_tx.is_some() && best_partial.is_none() && lone_filler(&text) { + eprintln!("buzz-desktop: STT final is a lone filler ({text:?}) — dropped"); + text = String::new(); + } + // Collapse guard — see decode_collapsed. A collapsed final would replace + // the sentence on screen with nothing (or junk); keep the best partial + // the user was looking at instead. + if let Some(best) = best_partial { + if decode_collapsed(&text, &best) { + eprintln!("buzz-desktop: STT final collapsed ({text:?}) — keeping best partial"); + text = best; + } else if let Some(stitched) = stitch_prefix_collapse(&best, &text) { + eprintln!("buzz-desktop: STT final dropped its front ({text:?}) — stitched"); + text = stitched; + } } + // In live mode finals ride the same channel as partials so the receiver + // never sees a stale partial after its final — and an EMPTY final is still + // sent so the frontend clears the partial shown for a phrase that decoded + // to nothing (noise); skipping it strands that partial on screen. + // (With the collapse guard above, "empty" here means no partial was ever + // shown either — a phrase the guard let through.) + let send_err = match live_tx { + Some(tx) => tx + .blocking_send(LiveEvent::Transcript { + text, + is_final: true, + }) + .err() + .map(|e| e.to_string()), + None if text.is_empty() => None, + None => text_tx.blocking_send(text).err().map(|e| e.to_string()), + }; + if let Some(e) = send_err { + eprintln!("buzz-desktop: STT text channel closed: {e}"); + } +} +/// Run sherpa-onnx on the accumulated speech and return the trimmed text +/// (empty on empty input or decode failure). +fn decode_speech(speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer) -> String { + if speech_buf.is_empty() { + return String::new(); + } let stream = recognizer.create_stream(); stream.accept_waveform(16_000, speech_buf); recognizer.decode(&stream); - - let text = stream + stream .get_result() .map(|r| r.text.trim().to_string()) - .unwrap_or_default(); - - if !text.is_empty() { - if let Err(e) = text_tx.blocking_send(text) { - eprintln!("buzz-desktop: STT text channel closed: {e}"); - } - } + .unwrap_or_default() } /// Convert raw bytes (f32 LE) to f32 samples. @@ -565,3 +982,9 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +#[path = "stt_tests.rs"] +mod tests; diff --git a/desktop/src-tauri/src/huddle/stt_tests.rs b/desktop/src-tauri/src/huddle/stt_tests.rs new file mode 100644 index 0000000000..362dc2467e --- /dev/null +++ b/desktop/src-tauri/src/huddle/stt_tests.rs @@ -0,0 +1,641 @@ +//! Tests for huddle/stt.rs — split into a sibling file to keep stt.rs +//! focused. Unit tests for the decode guards (collapse, prefix stitch, +//! lone-filler noise gate, sticky punctuation) plus the #[ignore]d decode +//! experiments that document measured model behavior. + +#[cfg(test)] +mod stitch_prefix_collapse_tests { + use super::super::stitch_prefix_collapse; + + // Strings from chl's 2026-07-24 trace (/tmp/buzz-dev-dictation.log). + #[test] + fn splices_at_the_overlap_anchor() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking.", + "I need to start asking whether it's a little bit more." + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether it's a little bit more.") + ); + } + + #[test] + fn appends_when_no_overlap_survived() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out.", + "I need to start asking whether it's" + ) + .as_deref(), + Some("Big questions are coming out. I need to start asking whether it's") + ); + } + + #[test] + fn tail_revision_wins_from_the_anchor_on() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking whether it's a little bit more.", + "I need to start asking whether or not that's the same" + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether or not that's the same") + ); + } + + #[test] + fn honest_revisions_pass_through() { + // Front agrees → not a drop. + assert!( + stitch_prefix_collapse("Big question to come.", "Big questions are coming up.") + .is_none() + ); + // Short phrases rewrite themselves legitimately. + assert!(stitch_prefix_collapse("Yeah.", "Big question.").is_none()); + assert!(stitch_prefix_collapse("Costing.", "Calls passing.").is_none()); + } +} + +#[cfg(test)] +mod lone_filler_tests { + use super::super::lone_filler; + + #[test] + fn trace_strings() { + // The noise phrases from the 2026-07-24 ramble trace. + assert!(lone_filler("Yeah.")); + assert!(lone_filler("Mm.")); + assert!(lone_filler("Uh-huh.")); + // Real speech that merely starts with or resembles filler survives. + assert!(!lone_filler("Yeah, so")); + assert!(!lone_filler("The problem.")); + assert!(!lone_filler("Testing.")); + assert!(!lone_filler("")); + } +} + +#[cfg(test)] +mod decode_collapsed_tests { + use super::super::decode_collapsed; + + #[test] + fn collapse_detection() { + let best = "caused by some other things"; + assert!(decode_collapsed("", best)); + assert!(decode_collapsed("Yeah.", best)); + assert!(decode_collapsed("Cause personal things.", best)); + // Honest revisions and growth keep the new decode. + assert!(!decode_collapsed("Caused by some other things.", best)); + assert!(!decode_collapsed("caused by some other things too", best)); + // No baseline words → nothing to protect. + assert!(!decode_collapsed("", "")); + } +} + +/// The tokio channel's `blocking_send` is safe to call from sync contexts. +#[cfg(test)] +mod prefer_punctuated_tests { + use super::super::prefer_punctuated; + + #[test] + fn keeps_dropped_terminal_punctuation() { + let hint = Some("Is there anything we can do about that?"); + assert_eq!( + prefer_punctuated("is there anything we can do about that".into(), hint), + "Is there anything we can do about that?" + ); + } + + #[test] + fn internal_punctuation_flicker_still_matches() { + // Observed in live_partial_sequence: the final decode swapped an + // internal "." for "," — same words, so the punctuated hint wins. + let hint = Some("I'm going to ask a question. Do you know why that happened?"); + assert_eq!( + prefer_punctuated( + "I'm going to ask a question, Do you know why that happened".into(), + hint + ), + "I'm going to ask a question. Do you know why that happened?" + ); + } + + #[test] + fn grafts_terminal_mark_when_only_last_word_agrees() { + assert_eq!( + prefer_punctuated( + "do you know er why that happened,".into(), + Some("Do you know why that happened?") + ), + "do you know er why that happened?" + ); + } + + #[test] + fn comma_hint_is_never_grafted() { + assert_eq!( + prefer_punctuated("hello there friend".into(), Some("hello there,")), + "hello there friend" + ); + } + + #[test] + fn ignores_hint_when_text_differs() { + assert_eq!( + prefer_punctuated("hello world again".into(), Some("hello world.")), + "hello world again" + ); + } + + #[test] + fn never_replaces_empty_final() { + assert_eq!(prefer_punctuated(String::new(), Some("hello.")), ""); + } + + #[test] + fn no_hint_passes_through() { + assert_eq!(prefer_punctuated("hello".into(), None), "hello"); + } +} + +#[cfg(test)] +mod silence_experiment { + use super::super::{decode_speech, prefer_punctuated, process_16k_samples}; + + /// Manual experiment: does Parakeet v3 (0.6B transducer) produce stable + /// punctuation under noisy pauses where the 110M CTC model flickers? + /// Also times each decode — the live mode re-decodes the whole phrase + /// every ~300 ms, so per-decode latency bounds the streaming cadence. + /// Run: cargo test v3_punctuation -- --ignored --nocapture + /// Needs ~/.buzz/models/parakeet-tdt-0.6b-v3 downloaded (see models.rs). + #[test] + #[ignore] + fn v3_punctuation() { + for threads in [1i32, 2, 4] { + let recognizer = v3_recognizer(threads); + println!("--- num_threads = {threads} ---"); + run_decodes(&recognizer); + } + } + + fn v3_recognizer(threads: i32) -> sherpa_onnx::OfflineRecognizer { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-0.6b-v3"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.transducer.encoder = Some( + model_dir + .join("encoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.decoder = Some( + model_dir + .join("decoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.joiner = Some( + model_dir + .join("joiner.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.debug = false; + cfg.model_config.num_threads = threads; + sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap() + } + + fn read_wav_16k(path: &str) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() + } + + /// Manual experiment: chl reported that during a long continuous ramble + /// "a whole bunch of text just got removed from the end". Two suspects: + /// (a) v3 transducer output degrades/truncates as the phrase buffer grows + /// toward the 30 s MAX_SPEECH_SAMPLES cap, so a later partial decode + /// yields LESS text than an earlier one and the frontend diff deletes + /// the tail; + /// (b) per-decode latency on long buffers (~70 ms/s of audio) blows past + /// the 300 ms partial cadence, the worker falls behind and the 5 s + /// audio queue overflows. + /// This decodes growing prefixes of a 35 s ramble, timing each decode and + /// flagging any shrinkage between consecutive prefixes. + /// Run: cargo test v3_long_buffer -- --ignored --nocapture + /// Setup: say -v Samantha "" -o /tmp/dict_long.wav + /// --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_long_buffer() { + let recognizer = v3_recognizer(super::super::LIVE_STT_NUM_THREADS); + let samples = read_wav_16k("/tmp/dict_long.wav"); + println!("total: {:.1}s", samples.len() as f32 / 16_000.0); + let mut prev = String::new(); + for secs in [4, 8, 12, 16, 20, 22, 24, 26, 28, 30] { + let end = (16_000 * secs).min(samples.len()); + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..end], &recognizer); + let shrink = if text.len() < prev.len() { + " <<< SHRANK" + } else { + "" + }; + println!( + "@{secs:>2}s [{:>6.0?}ms] {} chars{shrink}: {text:?}", + t0.elapsed().as_millis(), + text.len() + ); + prev = text; + if end == samples.len() { + break; + } + } + + // Simulate the worker's adaptive partial cadence over the same audio: + // next partial waits max(PARTIAL_DECODE_STEP, last decode duration). + // Real-time safe iff cumulative decode time stays under audio time. + println!("--- adaptive cadence simulation ---"); + let mut last_len = 0usize; + let mut step = super::super::PARTIAL_DECODE_STEP; + let mut decode_total = std::time::Duration::ZERO; + let mut n = 0u32; + while last_len + step <= samples.len() { + last_len += step; + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..last_len], &recognizer); + let dt = t0.elapsed(); + decode_total += dt; + step = super::super::PARTIAL_DECODE_STEP.max(32 * dt.as_millis() as usize); + n += 1; + println!( + "partial {n} @ audio {:>4.1}s: decode {:>4.0?}ms, next step {:.1}s, tail {:?}", + last_len as f32 / 16_000.0, + dt.as_millis(), + step as f32 / 16_000.0, + &text[text.len().saturating_sub(40)..] + ); + } + let audio_secs = last_len as f32 / 16_000.0; + println!( + "{n} partials over {audio_secs:.1}s audio, total decode {:.1}s → real-time safe: {}", + decode_total.as_secs_f32(), + decode_total.as_secs_f32() < audio_secs + ); + } + + fn run_decodes(recognizer: &sherpa_onnx::OfflineRecognizer) { + for path in ["/tmp/dict_multi.wav", "/tmp/dict_q.wav"] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!( + "=== {path} ({:.1}s speech) ===", + samples.len() as f32 / 16_000.0 + ); + for (label, amp, tail_ms) in [ + ("bare", 0.0f32, 0usize), + ("300ms noise", 0.01, 300), + ("608ms noise", 0.01, 608), + ("608ms loud noise", 0.03, 608), + ] { + let mut buf = samples.clone(); + let mut state = 0x2545_f491u32; + for _ in 0..(16 * tail_ms) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + buf.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + let t0 = std::time::Instant::now(); + let text = decode_speech(&buf, recognizer); + println!(" +{label}: [{:?}] {text:?}", t0.elapsed()); + } + } + } + + /// Manual experiment: simulate the live worker's exact cadence over real + /// (synthesized) speech with a noisy tail — decode the growing buffer + /// every PARTIAL_DECODE_STEP samples, track `punct_partial` like the + /// worker does, then run the final flush through `prefer_punctuated`. + /// Shows whether the sticky-punctuation hint actually matches on + /// multi-sentence phrases. + /// Run: cargo test live_partial_sequence -- --ignored --nocapture + /// Setup: say -v Samantha "It missed the full stop on the first + /// sentence. I'm now going to try and ask a question. Do you know why + /// that happened" -o /tmp/dict_multi.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn live_partial_sequence() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for (path, amp) in [ + ("/tmp/dict_multi.wav", 0.01f32), + ("/tmp/dict_q.wav", 0.01f32), + ] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let mut samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + // Noisy silence tail up to the live flush window (~608 ms). + let mut state = 0x2545_f491u32; + for _ in 0..(16 * 608) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + samples.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + println!("=== {path} (noise amp {amp}) ==="); + let mut punct_partial: Option = None; + let mut last_len = 0usize; + while last_len + super::super::PARTIAL_DECODE_STEP <= samples.len() { + last_len += super::super::PARTIAL_DECODE_STEP; + let text = decode_speech(&samples[..last_len], &recognizer); + if text.ends_with(['.', '?', '!', ',']) { + punct_partial = Some(text.clone()); + } + println!(" partial@{last_len}: {text:?}"); + } + let final_text = decode_speech(&samples, &recognizer); + println!(" FINAL: {final_text:?}"); + println!(" HINT: {punct_partial:?}"); + println!( + " KEPT: {:?}", + prefer_punctuated(final_text, punct_partial.as_deref()) + ); + // Rejected candidate fix (measured 2026-07-24): replacing the + // noisy VAD hangover with digital zeros at flush time sometimes + // REMOVES punctuation the noisy final had — tail composition is + // non-monotonic, so don't retry audio-level recipes. + let speech_end = samples.len() - 16 * 608; + let mut zeroed = samples[..speech_end].to_vec(); + zeroed.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!(" ZEROED-TAIL: {:?}", decode_speech(&zeroed, &recognizer)); + // Same but with noise overlaid on the speech itself (real mics + // pick up room noise under the voice too, not just in pauses). + let mut state = 0x1234_5678u32; + let mut noisy: Vec = samples[..speech_end] + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect(); + noisy.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " NOISY-SPEECH+ZEROED-TAIL: {:?}", + decode_speech(&noisy, &recognizer) + ); + } + } + + /// Manual experiment: reproduce chl's "captured it, then deleted it and + /// wrote Yeah" report (2026-07-24, Enhanced/v3 model) at the worker level. + /// Drives `process_16k_samples` exactly like the live worker — real VAD, + /// onset debounce, silence flush, partial cadence — over composed audio: + /// room noise, the spoken sentence (noise overlaid), a post-speech tail, + /// then the 1 s stop-flush zeros. A frontend simulator applies the events + /// the way useDictation.ts does and prints what the composer would show. + /// Run: cargo test v3_live_worker_sim -- --ignored --nocapture + /// Setup: say -v Samantha "just testing to see if this is working any + /// better" -o /tmp/dict_better.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_live_worker_sim() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + let recognizer = v3_recognizer(super::super::LIVE_STT_NUM_THREADS); + let speech = read_wav_16k("/tmp/dict_better.wav"); + + let noise = |len: usize, amp: f32, seed: u32| { + let mut state = seed; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let overlay = |samples: &[f32], amp: f32| { + let mut state = 0x1234_5678u32; + samples + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + + // (label, pre-speech, post-speech tail before the stop flush) + let scenarios: Vec<(&str, Vec, Vec)> = vec![ + ( + "release right after speaking", + noise(16 * 500, 0.01, 0xA1), + noise(16 * 400, 0.01, 0xB2), + ), + ( + "pause >608ms then breath then release", + noise(16 * 500, 0.01, 0xA1), + [ + noise(16 * 900, 0.01, 0xB2), + noise(16 * 250, 0.08, 0xC3), // breath/exhale burst + noise(16 * 300, 0.01, 0xD4), + ] + .concat(), + ), + ( + "louder room noise", + noise(16 * 500, 0.03, 0xA1), + noise(16 * 700, 0.03, 0xB2), + ), + ]; + + for (label, pre, tail) in scenarios { + println!("=== scenario: {label} ==="); + let mut audio = pre; + audio.extend(overlay(&speech, 0.01)); + audio.extend(tail); + audio.extend(std::iter::repeat_n(0.0f32, 16_000)); // stop-flush 1s zeros + + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(1024); + let (text_tx, _text_rx) = tokio::sync::mpsc::channel::(64); + let tts_active = Arc::new(AtomicBool::new(false)); + + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut leftover = Vec::new(); + let mut speech_buf = Vec::new(); + let mut silence_frames = 0usize; + let mut in_speech = false; + let mut barge_in_frames = 0usize; + let mut tts_stopped_at = None; + let mut last_partial_len = 0usize; + let mut decode_hold_until = std::time::Instant::now(); + let mut onset_buf = Vec::new(); + let mut punct_partial: Option = None; + let mut best_partial: Option = None; + + for chunk in audio.chunks(320) { + process_16k_samples( + chunk, + &mut leftover, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &tts_active, + None, + &mut tts_stopped_at, + None, + Some(&live_tx), + &mut last_partial_len, + &mut decode_hold_until, + &mut onset_buf, + &mut punct_partial, + &mut best_partial, + ); + } + drop(live_tx); + + // Frontend simulator — the transcriptDiff mechanics reduce to this. + let mut committed = String::new(); + let mut partial = String::new(); + while let Ok(event) = live_rx.try_recv() { + match event { + super::super::LiveEvent::Transcript { text, is_final } => { + let trimmed = text.trim(); + println!( + " {} {trimmed:?}", + if is_final { "FINAL " } else { "partial" } + ); + if is_final { + if !trimmed.is_empty() { + committed.push_str(trimmed); + committed.push(' '); + } else if !partial.is_empty() { + println!(" !!! empty final wiped shown partial {partial:?}"); + } + partial.clear(); + } else if !trimmed.is_empty() { + if trimmed.len() + 10 < partial.len() { + println!(" !!! partial shrank {partial:?} -> {trimmed:?}"); + } + partial = trimmed.to_string(); + } + } + super::super::LiveEvent::Flushed => println!(" FLUSHED"), + } + } + println!(" composer: {:?}", format!("{committed}{partial}")); + } + } + + /// Manual experiment: how much trailing silence does the model need before + /// it commits sentence-final punctuation? (Answer when tuned: ≥600 ms — + /// see LIVE_SILENCE_FLUSH_FRAMES.) + /// Run: cargo test silence_vs_punctuation -- --ignored --nocapture + /// Setup: say -v Samantha "is there anything we can do about that?" \ + /// -o /tmp/dict_q.wav --data-format=LEF32@16000 (same for dict_s) + #[test] + #[ignore] + fn silence_vs_punctuation() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for path in ["/tmp/dict_q.wav", "/tmp/dict_s.wav"] { + let bytes = std::fs::read(path).unwrap(); + // Naive WAV parse: find the "data" chunk, samples are f32 LE. + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!("=== {path} ({} samples) ===", samples.len()); + for silence_ms in [0usize, 100, 300, 600] { + let mut buf = samples.clone(); + buf.extend(std::iter::repeat_n(0.0f32, 16 * silence_ms)); + println!( + " +{silence_ms}ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + // Real mic "silence" is room noise, not zeros — simulate with + // low-level deterministic pseudo-noise (LCG; no rand dep) at + // amplitudes well below the VAD speech threshold. + for amp in [0.002f32, 0.01, 0.03] { + let noise = |len: usize| { + let mut state = 0x2545_f491u32; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let mut buf = samples.clone(); + buf.extend(noise(16 * 608)); + println!( + " +608ms noise(amp {amp}): {:?}", + decode_speech(&buf, &recognizer) + ); + buf.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " +608ms noise(amp {amp}) + 600ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + } + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 6a2eec75fb..5bb40ddf65 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ mod archive; mod builderlab; mod commands; mod deep_link; +mod dictation; mod event_sync; mod events; mod huddle; @@ -36,6 +37,10 @@ use deep_link::{ acknowledge_pending_community_deep_link, handle_deep_link_url, take_pending_community_deep_link, PendingCommunityDeepLinks, }; +use dictation::{ + download_dictation_model, get_dictation_models, push_dictation_pcm, start_dictation, + stop_dictation, +}; use huddle::audio_output::{ get_audio_output_device, list_audio_output_devices, set_audio_output_device, }; @@ -352,6 +357,7 @@ pub fn run() { }) .manage(build_app_state()) .manage(ClipboardState::new()) + .manage(dictation::DictationState::default()) .manage(PendingCommunityDeepLinks::default()) .manage(BuilderlabSession::default()) .manage(BuilderlabLogin::default()) @@ -852,6 +858,11 @@ pub fn run() { end_huddle, get_huddle_state, push_audio_pcm, + start_dictation, + stop_dictation, + push_dictation_pcm, + get_dictation_models, + download_dictation_model, reconnect_huddle_audio, start_stt_pipeline, set_huddle_transcription_enabled, diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 677f4ac0a0..dcb92bf4b9 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -51,6 +51,7 @@ export type AudioWorkletHandle = { export async function setupAudioWorklet( audioTrack: MediaStreamTrack, initialTransmitting = true, + command = "push_audio_pcm", ): Promise { const audioContext = new AudioContext({ sampleRate: 48000 }); @@ -89,7 +90,7 @@ export async function setupAudioWorklet( // Create a zero-copy Uint8Array view over the same underlying buffer. // Rust reinterprets the bytes as f32 on the other side. invokeRawBinary( - "push_audio_pcm", + command, new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), ).catch(() => { /* silently drop — Rust handles backpressure */ diff --git a/desktop/src/features/messages/lib/dictationModelPreference.ts b/desktop/src/features/messages/lib/dictationModelPreference.ts new file mode 100644 index 0000000000..8be6b1a9c3 --- /dev/null +++ b/desktop/src/features/messages/lib/dictationModelPreference.ts @@ -0,0 +1,75 @@ +import * as React from "react"; + +/** + * User preference for the dictation speech-to-text model, as a model id from + * the Rust `STT_MODELS` registry (e.g. "parakeet-en", "parakeet-v3"). + * + * `null` means no explicit choice: dictation uses the model the app selected + * at startup (BUZZ_STT_MODEL override / system locale / English default). + * The preference is passed to `start_dictation`, which falls back to the + * startup-selected model when the preferred one isn't downloaded yet. + * + * Persisted in localStorage — a device-level preference, like the thread + * layout in threadViewModePreference.ts. + */ +export type DictationModelPreference = string | null; + +const STORAGE_KEY = "buzz.dictation.model"; + +const listeners = new Set<() => void>(); + +let preference: DictationModelPreference = readStoredPreference(); + +function readStoredPreference(): DictationModelPreference { + try { + return globalThis.localStorage?.getItem(STORAGE_KEY) ?? null; + } catch { + return null; + } +} + +function subscribe(listener: () => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +function getSnapshot(): DictationModelPreference { + return preference; +} + +function getServerSnapshot(): DictationModelPreference { + return null; +} + +/** Read the persisted dictation model preference outside of React. */ +export function getDictationModelPreference(): DictationModelPreference { + return preference; +} + +/** Update the dictation model preference and notify subscribed components. */ +export function setDictationModelPreference( + model: DictationModelPreference, +): void { + preference = model; + + try { + if (model === null) { + globalThis.localStorage?.removeItem(STORAGE_KEY); + } else { + globalThis.localStorage?.setItem(STORAGE_KEY, model); + } + } catch { + // Persistence is best-effort; the in-memory value still applies. + } + + for (const listener of listeners) { + listener(); + } +} + +/** The dictation model preference (null = app default). */ +export function useDictationModelPreference(): DictationModelPreference { + return React.useSyncExternalStore(subscribe, getSnapshot, getServerSnapshot); +} diff --git a/desktop/src/features/messages/lib/transcriptDiff.test.mjs b/desktop/src/features/messages/lib/transcriptDiff.test.mjs new file mode 100644 index 0000000000..fb6eb510c4 --- /dev/null +++ b/desktop/src/features/messages/lib/transcriptDiff.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { transcriptDiff } from "./transcriptDiff.ts"; + +test("identical decode is a no-op", () => { + assert.deepEqual(transcriptDiff("hello world", "hello world"), { + keep: 11, + deleteLen: 0, + insert: "", + }); +}); + +test("append-only decode inserts just the tail", () => { + assert.deepEqual(transcriptDiff("hello", "hello world"), { + keep: 5, + deleteLen: 0, + insert: " world", + }); +}); + +test("mid-phrase revision rewrites from the divergence point", () => { + assert.deepEqual(transcriptDiff("space batoot", "space bar to"), { + keep: 8, + deleteLen: 4, + insert: "r to", + }); +}); + +test("shrinking decode deletes the stale tail", () => { + assert.deepEqual(transcriptDiff("hello worlds", "hello wor"), { + keep: 9, + deleteLen: 3, + insert: "", + }); +}); + +test("first partial of a phrase inserts everything", () => { + assert.deepEqual(transcriptDiff("", "hello"), { + keep: 0, + deleteLen: 0, + insert: "hello", + }); +}); + +test("final commit appends only the trailing space", () => { + assert.deepEqual(transcriptDiff("hello world", "hello world "), { + keep: 11, + deleteLen: 0, + insert: " ", + }); +}); diff --git a/desktop/src/features/messages/lib/transcriptDiff.ts b/desktop/src/features/messages/lib/transcriptDiff.ts new file mode 100644 index 0000000000..df56946f15 --- /dev/null +++ b/desktop/src/features/messages/lib/transcriptDiff.ts @@ -0,0 +1,19 @@ +/** Minimal edit turning the currently shown phrase text into the next decode: + * keep the longest common prefix, replace only the tail. Live re-decodes are + * usually append-only (or identical, between words) — diffing means a stable + * decode touches nothing, so the caret and DOM stay still instead of being + * rewritten every ~300 ms. */ +export function transcriptDiff( + shown: string, + next: string, +): { keep: number; deleteLen: number; insert: string } { + let keep = 0; + while ( + keep < shown.length && + keep < next.length && + shown.charCodeAt(keep) === next.charCodeAt(keep) + ) { + keep += 1; + } + return { keep, deleteLen: shown.length - keep, insert: next.slice(keep) }; +} diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts new file mode 100644 index 0000000000..34f7890124 --- /dev/null +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -0,0 +1,463 @@ +import * as React from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { Selection } from "@tiptap/pm/state"; +import type { Editor } from "@tiptap/react"; +import { toast } from "sonner"; + +import { + setupAudioWorklet, + type AudioWorkletHandle, +} from "@/features/huddle/lib/audioWorklet"; +import { getDictationModelPreference } from "./dictationModelPreference"; +import { transcriptDiff } from "./transcriptDiff"; + +export type DictationStatus = "idle" | "starting" | "recording"; + +/** How long after stop() this hook still claims incoming transcripts. + * Backstop only — ownership normally releases exactly when the session's + * `dictation-flushed` marker arrives; this covers a marker that never + * comes (pipeline died mid-flush). */ +const OWNERSHIP_DECAY_MS = 15_000; + +/** Hold plain Space this long to start dictation. A quick tap still types a + * space as normal — only key-repeats during the hold are suppressed. + * 800 ms: at 1.5 s people start talking well before the mic opens and the + * first words are lost (the fragment that survives decodes as filler). */ +const SPACE_HOLD_MS = 800; + +/** Cap on waiting for a session's `dictation-flushed` marker (Enter-to-send + * submits on it; a superseded session's stragglers are discarded until it). + * It normally arrives well under a second after stop(); the cap only fires + * if the pipeline died and the marker never comes. */ +const FLUSH_MARKER_TIMEOUT_MS = 3000; + +type Session = { handle: AudioWorkletHandle; stream: MediaStream }; + +/** Payload of the Rust `dictation-transcript` event. Partials re-decode the + * in-progress phrase (~every 300 ms) and replace each other; a final commits + * the phrase. */ +type TranscriptPayload = { text: string; final: boolean }; + +/** The dictation stream's place in the doc. `anchor` is where the current + * phrase starts; `partialText` is the phrase text currently shown there + * (empty right after a phrase commits). Kept to verify the region is still + * ours before replacing it. */ +type StreamState = { anchor: number; partialText: string }; + +/** + * Hold-to-talk streaming dictation for the message composer. + * + * start(): starts the Rust STT session (`start_dictation`), opens the mic via + * getUserMedia, and streams PCM through the existing AudioWorklet to + * `push_dictation_pcm`. The Rust pipeline emits `dictation-transcript` events + * while the user talks: partials replace the in-progress phrase in place (so + * words appear as they're spoken), finals commit it. stop(): releases the mic; + * Rust flushes the trailing phrase, which arrives shortly after. + * + * Triggers: hold plain Space for SPACE_HOLD_MS (quick tap still types a + * space), hold ⌃Space (instant start), or the toolbar mic button (click + * toggle). Enter while dictation is live stops it and calls `onSubmit` when + * the session's `dictation-flushed` marker confirms every transcript — + * including the trailing phrase — is in the doc (send-by-voice). + * + * Multiple composers may mount this hook — only the one that started the + * session inserts the transcript (ownsRef), and Rust rejects a second + * concurrent session. + */ +export function useDictation(editor: Editor | null, onSubmit?: () => void) { + const [status, setStatus] = React.useState("idle"); + const sessionRef = React.useRef(null); + const streamRef = React.useRef(null); + // Incremented on every start/stop — detects "released before start finished". + const genRef = React.useRef(0); + const ownsRef = React.useRef(false); + const ownershipTimerRef = React.useRef(null); + // True while dictation was started by a held key (Space or ⌃Space). + const keyHeldRef = React.useRef(false); + // Pending "Space held long enough?" timer. + const spaceTimerRef = React.useRef(null); + const editorRef = React.useRef(editor); + editorRef.current = editor; + const onSubmitRef = React.useRef(onSubmit); + onSubmitRef.current = onSubmit; + // Non-null while an Enter-triggered submit waits for the flush marker. + const submitTimerRef = React.useRef(null); + // Non-null while transcripts are dropped because they belong to a session + // this composer already replaced (restart before the old session's flush + // marker arrived). Cleared by that marker, or by the cap if it never comes. + const discardTimerRef = React.useRef(null); + + // Stream transcripts from the Rust pipeline into the editor: each partial + // replaces the previous partial of the same phrase in place, and the final + // commits it (with a trailing space) so the next phrase starts fresh. + // + // The anchor is read from the live selection ONCE (first transcript of the + // session); after that, phrases flow strictly in sequence from it. Never + // re-read the selection or call focus() per event — chasing DOM focus and + // selection every ~300 ms made the caret flick around while talking. + React.useEffect(() => { + const unlisten = listen( + "dictation-transcript", + (event) => { + // Straggler from a session this composer already replaced. + if (discardTimerRef.current !== null) return; + if (!ownsRef.current) return; + const editor = editorRef.current; + if (!editor) return; + const text = event.payload.text.trim(); + // Empty PARTIALS carry no information — skip. An empty FINAL is a + // real signal: the phrase decoded to nothing (noise), so fall through + // and let the diff delete the partial that was shown for it. + if (!text && !event.payload.final) return; + const insert = event.payload.final && text ? `${text} ` : text; + + const doc = editor.state.doc; + let stream = streamRef.current; + if (stream) { + const end = stream.anchor + stream.partialText.length; + const intact = + end <= doc.content.size && + doc.textBetween(stream.anchor, end) === stream.partialText; + if (!intact) { + // The user edited around the stream (possible between key release + // and the trailing final) — resume at the end of the doc. + stream = { anchor: Selection.atEnd(doc).from, partialText: "" }; + } + } else { + // First transcript of the session: anchor at the cursor. + stream = { anchor: editor.state.selection.from, partialText: "" }; + } + + // Only rewrite the part of the phrase that actually changed. Most + // re-decodes append or are identical — a full delete+reinsert every + // ~300 ms churned the DOM and selection even when nothing changed, + // which is what made the caret visibly bounce while talking. + const { anchor } = stream; + const diff = transcriptDiff(stream.partialText, insert); + if (diff.deleteLen > 0 || diff.insert.length > 0) { + const from = anchor + diff.keep; + const chain = editor.chain(); + if (diff.deleteLen > 0) { + chain.deleteRange({ from, to: from + diff.deleteLen }); + } + if (diff.insert.length > 0) { + // Insert as a plain text node — never parse transcript as HTML. + chain.insertContentAt(from, { type: "text", text: diff.insert }); + } + // Pin the caret to the end of the dictated text. + chain.setTextSelection(anchor + insert.length).run(); + } + streamRef.current = event.payload.final + ? { anchor: anchor + insert.length, partialText: "" } + : { anchor, partialText: insert }; + // Hide the blinking caret while words are streaming in — it reads as + // noise next to live-updating text (chl request). It comes back at + // each pause (phrase commit) and when dictation stops. + editor.view.dom.style.caretColor = event.payload.final + ? "" + : "transparent"; + }, + ); + // The flush marker: every transcript of the stopped session is in the + // doc now (the Rust live channel is strictly ordered). + const unlistenFlushed = listen("dictation-flushed", () => { + // Marker of the session this composer replaced — stop discarding, the + // current session's transcripts follow it. + if (discardTimerRef.current !== null) { + window.clearTimeout(discardTimerRef.current); + discardTimerRef.current = null; + return; + } + // A live session's own marker can only arrive after its stop(). + if (sessionRef.current) return; + if (!ownsRef.current && submitTimerRef.current === null) return; + // The stopped session is fully drained — release ownership exactly + // instead of waiting out the decay backstop. + ownsRef.current = false; + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + ownershipTimerRef.current = null; + } + // Enter-to-send was waiting for this drain. + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + submitTimerRef.current = null; + onSubmitRef.current?.(); + } + }); + return () => { + void unlisten.then((fn) => fn()); + void unlistenFlushed.then((fn) => fn()); + }; + }, []); + + const teardownSession = React.useCallback(() => { + const session = sessionRef.current; + sessionRef.current = null; + if (session) { + session.handle.stop(); + for (const track of session.stream.getTracks()) { + track.stop(); + } + } + }, []); + + const stop = React.useCallback(() => { + genRef.current += 1; + teardownSession(); + setStatus("idle"); + // Restore the caret even if no final ever arrives for the last phrase. + const dom = editorRef.current?.view.dom; + if (dom) dom.style.caretColor = ""; + void invoke("stop_dictation").catch(() => { + /* nothing to stop */ + }); + // Keep claiming transcripts briefly — the flush arrives after stop. + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + } + ownershipTimerRef.current = window.setTimeout(() => { + ownsRef.current = false; + ownershipTimerRef.current = null; + }, OWNERSHIP_DECAY_MS); + }, [teardownSession]); + + const start = React.useCallback(async (): Promise => { + if (sessionRef.current) return; + const gen = ++genRef.current; + // A stream left over from a previous session (final never arrived, or the + // doc changed) must not swallow this session's first phrase. + streamRef.current = null; + // An Enter-to-send is still waiting on the previous session's flush + // marker. Its message must not be lost to the restart — send what's in + // the doc now; the not-yet-decoded tail (if any) is dropped with the + // rest of that session's stragglers below. + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + submitTimerRef.current = null; + onSubmitRef.current?.(); + } + // The previous session hasn't drained yet (its flush marker would have + // cleared the ownership timer) — discard its stragglers until the marker + // arrives, or they'd be spliced into this session's text. + // ponytail: per-composer. Restarting in a DIFFERENT composer within the + // drain window can still catch the old session's tail — thread a session + // id through the Rust event if it ever bites. + if ( + ownershipTimerRef.current !== null && + discardTimerRef.current === null + ) { + discardTimerRef.current = window.setTimeout(() => { + discardTimerRef.current = null; + }, FLUSH_MARKER_TIMEOUT_MS); + } + setStatus("starting"); + // Tracks whether the Rust session was started, so cleanup on a later + // failure never kills a session owned by another composer. + let rustSessionStarted = false; + try { + // Rust first: fails fast if the model is missing or a session is live. + await invoke("start_dictation", { model: getDictationModelPreference() }); + rustSessionStarted = true; + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const track = stream.getAudioTracks()[0]; + if (!track) { + for (const t of stream.getTracks()) t.stop(); + throw new Error("No microphone input available"); + } + const handle = await setupAudioWorklet(track, true, "push_dictation_pcm"); + if (gen !== genRef.current) { + // Released (or restarted) while we were setting up — undo. + handle.stop(); + for (const t of stream.getTracks()) t.stop(); + return; + } + sessionRef.current = { handle, stream }; + ownsRef.current = true; + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + ownershipTimerRef.current = null; + } + setStatus("recording"); + } catch (error) { + if (gen === genRef.current) { + setStatus("idle"); + if (rustSessionStarted) { + void invoke("stop_dictation").catch(() => {}); + } + } + throw error instanceof Error ? error : new Error(String(error)); + } + }, []); + + // Release the mic if the composer unmounts mid-recording. + React.useEffect(() => { + return () => { + if (sessionRef.current) { + teardownSession(); + void invoke("stop_dictation").catch(() => {}); + } + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + } + if (spaceTimerRef.current !== null) { + window.clearTimeout(spaceTimerRef.current); + } + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + } + if (discardTimerRef.current !== null) { + window.clearTimeout(discardTimerRef.current); + } + }; + }, [teardownSession]); + + const startWithToast = React.useCallback(() => { + void start().catch((error: Error) => { + toast.error(error.message || "Could not start dictation"); + }); + }, [start]); + + /** Queue the submit to fire when the session's flush marker arrives — + * i.e. once every transcript of the stopped session, including the + * trailing phrase Enter interrupted, is in the doc. The timer only fires + * if the marker never comes. */ + const submitAfterFlush = React.useCallback(() => { + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + } + submitTimerRef.current = window.setTimeout(() => { + submitTimerRef.current = null; + ownsRef.current = false; + onSubmitRef.current?.(); + }, FLUSH_MARKER_TIMEOUT_MS); + }, []); + + /** Mic button click — start when idle, stop otherwise. */ + const statusRef = React.useRef(status); + statusRef.current = status; + const toggle = React.useCallback(() => { + if (statusRef.current === "idle") { + startWithToast(); + } else { + stop(); + } + }, [startWithToast, stop]); + + // Hold Space (or ⌃Space) = push-to-talk. Keydown comes from the composer + // (so only the focused composer starts); the release can land anywhere — or + // nowhere, on focus loss — so keyup/blur are handled on window. + React.useEffect(() => { + const release = () => { + if (spaceTimerRef.current !== null) { + window.clearTimeout(spaceTimerRef.current); + spaceTimerRef.current = null; + } + if (!keyHeldRef.current) return; + keyHeldRef.current = false; + stop(); + }; + const onKeyUp = (event: KeyboardEvent) => { + if (event.code === "Space" || event.key === "Control") release(); + }; + // Capture phase: while a hold is pending or live, swallow Space + // key-repeats before they reach ANY element. If focus drifts off the + // composer mid-hold, leaked repeats would click focused buttons, scroll + // the page, or type spaces into the stream. + const onKeyDownCapture = (event: KeyboardEvent) => { + // Enter while dictation is live: stop and send. Capture phase so it + // beats the editor's submitOnEnter, which would send immediately — + // before the trailing phrase has been flushed into the doc. + if ( + event.key === "Enter" && + !event.shiftKey && + !event.metaKey && + !event.altKey && + !event.ctrlKey && + onSubmitRef.current && + (keyHeldRef.current || + statusRef.current !== "idle" || + // A send is already queued behind the flush marker — swallow + // repeat Enters so submitOnEnter can't double-send. + submitTimerRef.current !== null) + ) { + event.preventDefault(); + event.stopPropagation(); + if (event.repeat) return; + keyHeldRef.current = false; + stop(); + submitAfterFlush(); + return; + } + if ( + event.code === "Space" && + event.repeat && + (keyHeldRef.current || spaceTimerRef.current !== null) + ) { + event.preventDefault(); + event.stopPropagation(); + } + }; + window.addEventListener("keydown", onKeyDownCapture, true); + window.addEventListener("keyup", onKeyUp); + window.addEventListener("blur", release); + return () => { + window.removeEventListener("keydown", onKeyDownCapture, true); + window.removeEventListener("keyup", onKeyUp); + window.removeEventListener("blur", release); + }; + }, [stop, submitAfterFlush]); + + /** Returns true when the event was consumed. Handles both triggers: + * ⌃Space (instant) and plain Space held for SPACE_HOLD_MS. */ + const handleComposerKeyDown = React.useCallback( + (event: React.KeyboardEvent): boolean => { + if (event.code !== "Space" || event.metaKey || event.altKey) { + return false; + } + + // ⌃Space — instant hold-to-talk. + if (event.ctrlKey) { + event.preventDefault(); + if (!event.repeat && !keyHeldRef.current) { + keyHeldRef.current = true; + startWithToast(); + } + return true; + } + if (event.shiftKey) return false; + + // Plain Space — a quick tap types a space as normal; holding for + // SPACE_HOLD_MS starts dictation. Key-repeats are swallowed while the + // hold is pending or active so a hold doesn't spray spaces. + if (event.repeat) { + if (spaceTimerRef.current !== null || keyHeldRef.current) { + event.preventDefault(); + return true; + } + return false; + } + if (spaceTimerRef.current === null && !keyHeldRef.current) { + spaceTimerRef.current = window.setTimeout(() => { + spaceTimerRef.current = null; + keyHeldRef.current = true; + // The keydown typed a space before the hold was recognised — the + // hold means "talk", not "space", so take it back. + const ed = editorRef.current; + if (ed) { + const { from } = ed.state.selection; + if (from > 0 && ed.state.doc.textBetween(from - 1, from) === " ") { + ed.commands.deleteRange({ from: from - 1, to: from }); + } + } + startWithToast(); + }, SPACE_HOLD_MS); + } + return false; // let the space type normally + }, + [startWithToast], + ); + + return { status, start, stop, toggle, handleComposerKeyDown }; +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 8c0c574049..7052ed13c5 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -1,6 +1,7 @@ import * as React from "react"; import { EditorContent } from "@tiptap/react"; +import { useDictation } from "@/features/messages/lib/useDictation"; import { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; import { handleAgentSnapshotPaste } from "@/features/messages/lib/agentSnapshotClipboard"; import { useComposerAutofocus } from "@/features/messages/lib/useComposerAutofocus"; @@ -783,8 +784,19 @@ function MessageComposerImpl({ // Plain Enter → submit is now handled inside the Tiptap `submitOnEnter` // extension (fires before ProseMirror's splitBlock). This wrapper only // handles autocomplete arrow/enter keys and Escape for edit mode. + // ── Dictation (hold ⌃Space or mic button → speech-to-text) ───────── + // Enter while dictating stops the session and sends once the trailing + // phrase has been flushed into the doc. + const dictationSubmit = React.useCallback( + () => submitMessageRef.current(), + [], + ); + const dictation = useDictation(richText.editor, dictationSubmit); + const handleEditorKeyDown = React.useCallback( (event: React.KeyboardEvent) => { + if (dictation.handleComposerKeyDown(event)) return; + // Let autocomplete handle keys first const emojiResult = emojiAutocomplete.handleEmojiKeyDown(event); if (emojiResult.handled) { @@ -835,6 +847,7 @@ function MessageComposerImpl({ linkEditor.isCardOpen, linkEditor.focusCardFirstControl, onCancelEdit, + dictation.handleComposerKeyDown, ], ); @@ -1074,6 +1087,7 @@ function MessageComposerImpl({ void; + /** When omitted, the dictation mic button is not rendered. */ + onDictationToggle?: () => void; onEmojiPickerOpenChange: (open: boolean) => void; onEmojiSelect: (emoji: string) => void; onFormattingToggle: (pressed: boolean) => void; @@ -224,6 +231,42 @@ export const MessageComposerToolbar = React.memo( Formatting + {onDictationToggle ? ( + + + + + + {dictationStatus === "recording" + ? "Stop dictation" + : "Dictate (hold Space, or ⌃Space)"} + + + ) : null} )} diff --git a/desktop/src/features/settings/ui/SettingsPanels.tsx b/desktop/src/features/settings/ui/SettingsPanels.tsx index a203269239..a292b893d3 100644 --- a/desktop/src/features/settings/ui/SettingsPanels.tsx +++ b/desktop/src/features/settings/ui/SettingsPanels.tsx @@ -13,6 +13,7 @@ import { LayoutTemplate, LockKeyhole, MessagesSquare, + Mic, MonitorCog, Moon, ShieldAlert, @@ -84,6 +85,7 @@ import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { ProfileSettingsCard } from "./ProfileSettingsCard"; import { UpdateChecker } from "../UpdateChecker"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { VoiceSettingsCard } from "./VoiceSettingsCard"; export type SettingsSection = | "profile" @@ -94,6 +96,7 @@ export type SettingsSection = | "compute" | "appearance" | "shortcuts" + | "voice" | "hosted-communities" | "community-members" | "moderation" @@ -113,6 +116,7 @@ const SETTINGS_SECTION_VALUES: readonly SettingsSection[] = [ "compute", "appearance", "shortcuts", + "voice", "hosted-communities", "community-members", "moderation", @@ -195,6 +199,11 @@ export const settingsSections: SettingsSectionDescriptor[] = [ label: "Shortcuts", icon: Keyboard, }, + { + value: "voice", + label: "Voice & dictation", + icon: Mic, + }, { value: "hosted-communities", label: "Hosted communities", @@ -827,6 +836,8 @@ export function renderSettingsSection( return ; case "shortcuts": return ; + case "voice": + return ; case "hosted-communities": return ; case "community-members": diff --git a/desktop/src/features/settings/ui/SettingsView.tsx b/desktop/src/features/settings/ui/SettingsView.tsx index 90634e01f4..d42afe269a 100644 --- a/desktop/src/features/settings/ui/SettingsView.tsx +++ b/desktop/src/features/settings/ui/SettingsView.tsx @@ -59,6 +59,7 @@ const settingsNavGroups: Array<{ "appearance", "notifications", "shortcuts", + "voice", "custom-emoji", "local-archive", ], diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx new file mode 100644 index 0000000000..c197d2fb6b --- /dev/null +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -0,0 +1,181 @@ +import * as React from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { Check } from "lucide-react"; + +import { + setDictationModelPreference, + useDictationModelPreference, +} from "@/features/messages/lib/dictationModelPreference"; +import { cn } from "@/shared/lib/cn"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; +import { SettingsSectionHeader } from "./SettingsSectionHeader"; + +/** Mirror of the Rust `ModelStatus` serde encoding (models.rs). */ +type ModelStatus = + | "not_downloaded" + | "ready" + | { downloading: { progress_percent: number } } + | { error: string }; + +/** Mirror of the Rust `SttModelInfo` (models.rs). */ +type SttModelInfo = { + id: string; + languages: string; + multilingual: boolean; + selected: boolean; + status: ModelStatus; +}; + +/** + * Display copy per registry model id. Unknown ids (a future registry + * addition) fall back to the id itself so the picker never hides a model. + */ +const MODEL_COPY: Record< + string, + { label: string; description: string; size: string } +> = { + "parakeet-en": { + label: "Standard", + description: "Fast, English only. Light on punctuation.", + size: "~120 MB", + }, + "parakeet-v3": { + label: "Enhanced", + description: + "Best accuracy and punctuation. 25 European languages, auto-detected.", + size: "~465 MB", + }, +}; + +function isDownloading( + status: ModelStatus, +): status is { downloading: { progress_percent: number } } { + return typeof status === "object" && "downloading" in status; +} + +function statusText(info: SttModelInfo): string { + if (info.status === "ready") return "Downloaded"; + if (isDownloading(info.status)) { + return `Downloading… ${info.status.downloading.progress_percent}%`; + } + const size = MODEL_COPY[info.id]?.size; + if (typeof info.status === "object" && "error" in info.status) { + return "Download failed — select to retry"; + } + return size ? `${size} download` : "Not downloaded"; +} + +export function VoiceSettingsCard() { + const preference = useDictationModelPreference(); + const [models, setModels] = React.useState(null); + const [loadError, setLoadError] = React.useState(null); + + const refresh = React.useCallback(async () => { + try { + setModels(await invoke("get_dictation_models")); + setLoadError(null); + } catch (e) { + setLoadError(e instanceof Error ? e.message : String(e)); + } + }, []); + + React.useEffect(() => { + void refresh(); + }, [refresh]); + + // Poll for download progress only while a download is running. + const downloading = models?.some((m) => isDownloading(m.status)) ?? false; + React.useEffect(() => { + if (!downloading) return; + const timer = window.setInterval(() => void refresh(), 1000); + return () => window.clearInterval(timer); + }, [downloading, refresh]); + + // With no explicit preference, dictation uses the startup-selected model. + const activeId = preference ?? models?.find((m) => m.selected)?.id ?? null; + const activeModel = models?.find((m) => m.id === activeId); + const fallbackModel = models?.find((m) => m.selected); + + const handleSelect = (info: SttModelInfo) => { + setDictationModelPreference(info.id); + if (info.status !== "ready") { + void invoke("download_dictation_model", { id: info.id }) + .then(refresh) + .catch((e) => setLoadError(e instanceof Error ? e.message : String(e))); + } + }; + + return ( +
+ + +

+ Dictation model +

+ + {(models ?? []).map((info) => { + const copy = MODEL_COPY[info.id] ?? { + label: info.id, + description: info.languages, + size: "", + }; + const active = info.id === activeId; + return ( + + + + ); + })} + + + {activeModel && activeModel.status !== "ready" && fallbackModel ? ( +

+ {MODEL_COPY[fallbackModel.id]?.label ?? fallbackModel.id} is used + until the download finishes. +

+ ) : null} + {loadError ? ( +

{loadError}

+ ) : null} +
+ ); +}