From 1a82440a03d47551cb56bd18510a6d08406dfe83 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 14:12:55 +1000 Subject: [PATCH 01/10] feat(desktop): push-to-talk dictation in message composer Hold-key dictation using the local huddle STT stack: live streaming decode with self-pacing re-decode loop, stable text commitment at pauses, sticky punctuation, Parakeet v3 (0.6B) model support via the model registry, and caret hidden while words stream in. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/scripts/check-file-sizes.mjs | 17 +- desktop/src-tauri/src/dictation.rs | 167 +++++ desktop/src-tauri/src/huddle/models.rs | 490 +++++++++++-- desktop/src-tauri/src/huddle/pipeline.rs | 12 +- desktop/src-tauri/src/huddle/stt.rs | 656 +++++++++++++++++- desktop/src-tauri/src/lib.rs | 6 + .../src/features/huddle/lib/audioWorklet.ts | 3 +- .../messages/lib/transcriptDiff.test.mjs | 52 ++ .../features/messages/lib/transcriptDiff.ts | 19 + .../src/features/messages/lib/useDictation.ts | 344 +++++++++ .../features/messages/ui/MessageComposer.tsx | 9 + .../messages/ui/MessageComposerToolbar.tsx | 45 +- 12 files changed, 1726 insertions(+), 94 deletions(-) create mode 100644 desktop/src-tauri/src/dictation.rs create mode 100644 desktop/src/features/messages/lib/transcriptDiff.test.mjs create mode 100644 desktop/src/features/messages/lib/transcriptDiff.ts create mode 100644 desktop/src/features/messages/lib/useDictation.ts diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index a901d5010d..066cd9de83 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -76,6 +76,17 @@ const overrides = new Map([ // across the local-archive + agent-metric-archive PR series. store_tests.rs // (~731 lines) is under 1000 so needs no override. ["src-tauri/src/archive/mod_tests.rs", 1208], + // PR #2520 port (STT model registry): STT_MODELS registry data (two model + // entries with pinned checksums + license texts) + select_stt_model and its + // unit tests. Load-bearing model-selection surface from the upstream PR; + // queued to split when #2520 lands and this port is dropped in the rebase. + ["src-tauri/src/huddle/models.rs", 1320], + // composer-dictation: live-transcript mode (live_tx partial re-decode loop), + // sticky-punctuation heuristic + unit tests, and two #[ignore]d decode + // experiments (silence/noise vs punctuation, v3 comparison) that document + // measured model behavior. Experiments are test-only content; queued to + // split into stt/tests.rs with the feature PR review. + ["src-tauri/src/huddle/stt.rs", 1200], // unified-agent-model 1A.1: profile reconcile split to agents_profile.rs, // ratcheting 1443 -> 1295. Queued to split further in the A2 fold. // global-agent-config: resolve_deploy_model_provider + visibility exports @@ -517,7 +528,11 @@ const overrides = new Map([ // ownerPubkey) feeding the newly-added-mentions diff. Diff logic itself // lives in threading.ts (diffAddedMentionPubkeys); this is the minimal // composer-side wiring. Queued to split with the rest of this list. - ["src/features/messages/ui/MessageComposer.tsx", 1114], + // +9: dictation wiring — useDictation hook instantiation, ⌃Space handoff in + // handleEditorKeyDown, and two toolbar props. All session/keyboard logic + // lives in lib/useDictation.ts; this is the minimal composer-side wiring. + // Queued to split with the rest of this list. + ["src/features/messages/ui/MessageComposer.tsx", 1123], // global-agent-config: model-tuning section (BuzzAgentModelTuningFields via // EditAgentAdvancedFields) + providerValid gate + effectiveProvider derivation // + globalProvider threading into getPersonaProviderOptions. All load-bearing diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..351505115b --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,167 @@ +//! Standalone composer dictation — streaming STT without a huddle. +//! +//! Reuses the huddle `SttPipeline` in live-transcript mode: the in-progress +//! phrase is re-decoded every ~300 ms and emitted as a partial (`final: +//! false`), so words appear in the composer as they're spoken; each natural +//! pause commits the phrase as a final (`final: true`). The frontend captures +//! mic audio with `getUserMedia` + the existing AudioWorklet and pushes PCM +//! via `push_dictation_pcm`. Transcripts are emitted to the webview as +//! `dictation-transcript` events instead of being posted to the relay. +//! +//! Session model: `start_dictation` marks the session active; +//! `stop_dictation` marks it inactive and pushes ~1 s of silence into the +//! pipeline — the mic stops with the key release, so without it the VAD would +//! never see the silence that closes (and decodes) the trailing phrase. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, +}; + +use tauri::Emitter; + +use crate::app_state::AppState; +use crate::huddle::{models, stt::SttPipeline, HuddlePhase}; + +/// Payload for `dictation-transcript` events. Partials (`final: false`) +/// replace the previous partial of the same phrase; a final commits it. +#[derive(Clone, serde::Serialize)] +struct TranscriptEvent { + text: String, + #[serde(rename = "final")] + is_final: bool, +} + +/// Same cap as the huddle audio ingest (huddle/mod.rs). +const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + +/// 1 s of 48 kHz f32-LE silence — comfortably past the worker's ~608 ms +/// live-mode silence-flush threshold (stt.rs LIVE_SILENCE_FLUSH_FRAMES) +/// after resampling. The margin matters: the threshold counts *consecutive* +/// silence frames, and the flush also unlocks the phrase's punctuation +/// (the model only commits it with ≥600 ms of trailing silence). +const FLUSH_SILENCE_BYTES: usize = 4 * 48_000; + +/// Managed Tauri state for dictation. +/// +/// ponytail: the pipeline stays alive after the first start (sherpa-onnx +/// model init takes seconds; ~110 MB resident). Add idle teardown if that +/// memory ever matters. +#[derive(Default)] +pub struct DictationState { + pipeline: Mutex>, + /// True while a session is live — blocks a second concurrent session and + /// gates `push_dictation_pcm` so late batches can't dirty the VAD buffer. + active: AtomicBool, +} + +/// Start (or resume) a dictation session. +/// +/// Errors if the speech model isn't downloaded yet or a session is already +/// active (prevents two composers opening two mic streams into one pipeline). +#[tauri::command] +pub fn start_dictation( + app: tauri::AppHandle, + state: tauri::State<'_, DictationState>, + app_state: tauri::State<'_, AppState>, +) -> Result<(), String> { + if state.active.load(Ordering::Acquire) { + return Err("dictation already active".to_string()); + } + // The huddle owns the mic (and Ctrl+Space) while it's up. + if app_state + .huddle() + .is_ok_and(|hs| !matches!(hs.phase, HuddlePhase::Idle)) + { + return Err("dictation is unavailable during a huddle".to_string()); + } + + let mut guard = state + .pipeline + .lock() + .map_err(|_| "dictation state poisoned".to_string())?; + + // Clear a dead pipeline (init failure or crash) so retry works. + if guard.as_ref().is_some_and(|p| p.is_finished()) { + *guard = None; + } + + if guard.is_none() { + let model_dir = models::stt_model_dir() + .ok_or("speech model still downloading — try again in a moment".to_string())?; + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::<(String, bool)>(64); + // In live mode all transcripts arrive on live_rx; the pipeline's own + // text receiver stays silent and is dropped here. + let (pipeline, _text_rx) = SttPipeline::new( + model_dir, + models::stt_model_family(), + Arc::new(AtomicBool::new(false)), // no TTS during dictation + None, + None, // continuous VAD segments phrases at natural pauses + Some(live_tx), + )?; + *guard = Some(pipeline); + + // Forward transcripts to the webview until the pipeline is dropped. + tauri::async_runtime::spawn(async move { + while let Some((text, is_final)) = live_rx.recv().await { + let event = TranscriptEvent { text, is_final }; + if let Err(e) = app.emit("dictation-transcript", event) { + eprintln!("buzz-desktop: dictation transcript emit failed: {e}"); + } + } + }); + } + + state.active.store(true, Ordering::Release); + Ok(()) +} + +/// End the session. Pushes silence so the VAD worker closes and decodes the +/// phrase in flight — that final transcript arrives as an event shortly +/// after. Idempotent. +#[tauri::command] +pub fn stop_dictation(state: tauri::State<'_, DictationState>) { + if !state.active.swap(false, Ordering::AcqRel) { + return; + } + if let Ok(guard) = state.pipeline.lock() { + if let Some(ref pipeline) = *guard { + let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); + } + } +} + +/// Receive raw PCM (f32 LE, 48 kHz mono) from the AudioWorklet. +/// Mirrors `push_audio_pcm` but feeds the dictation pipeline. +#[tauri::command] +pub fn push_dictation_pcm( + request: tauri::ipc::Request<'_>, + state: tauri::State<'_, DictationState>, +) -> Result<(), String> { + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + bytes.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + // Drop late batches after stop — the flush silence must be the + // last audio the VAD sees for a session. + if !state.active.load(Ordering::Acquire) { + return Ok(()); + } + let guard = state + .pipeline + .lock() + .map_err(|_| "dictation state poisoned".to_string())?; + if let Some(ref pipeline) = *guard { + pipeline.push_audio(bytes.to_vec())?; + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..04521c5f04 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -1,7 +1,11 @@ -//! Model download manager for STT (Parakeet TDT-CTC 110M) and TTS (Pocket TTS) models. +//! Model download manager for STT (Parakeet family) and TTS (Pocket TTS) models. +//! +//! The STT model is selectable via the `STT_MODELS` registry (issue #2478): +//! the English Parakeet TDT-CTC 110M default, or the multilingual Parakeet TDT +//! 0.6B v3, chosen by `BUZZ_STT_MODEL` / system locale in `selected_stt_model`. //! //! Mental model: -//! app launch → start_stt_download (background) → ~/.buzz/models/parakeet-tdt-ctc-110m-en/ +//! app launch → start_stt_download (background) → ~/.buzz/models// //! app launch → start_tts_download (background) → ~/.buzz/models/pocket-tts/ //! STT pipeline → is_stt_ready() → stt_model_dir() → run inference //! TTS pipeline → is_tts_ready() → tts_model_dir() → run synthesis @@ -26,17 +30,17 @@ use sha2::{Digest, Sha256}; // ── Integrity verification ──────────────────────────────────────────────────── // -// All model artifacts are verified against pinned SHA-256 hashes before +// Model artifacts are verified against pinned SHA-256 hashes before // installation. This is defense-in-depth: HTTPS protects the transport, // hashes protect the content. // // To recompute hashes: download each file, run `shasum -a 256 `, and // update the corresponding constant. - -/// SHA-256 hash of the STT archive -/// (sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2). -/// Computed from a known-good download. Update when upgrading model versions. -const STT_ARCHIVE_SHA256: &str = "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"; +// +// STT archive hashes are pinned per model in the `STT_MODELS` registry below +// (`SttModel::archive_sha256`). Both shipped models are pinned; the field is +// `Option` so a model may temporarily ship `None` (size cap + safe extraction +// + expected-files verification still apply) before its hash is computed. /// HuggingFace base URL for the sherpa-onnx Pocket TTS fp32 repackage. /// @@ -85,12 +89,8 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ // If the on-disk manifest doesn't match the compiled-in version, the model is // considered stale and re-downloaded. Increment when upgrading model files. -/// Model manifest version for the STT model. Increment when upgrading model files. -/// Bumped from "1" → "2" alongside the migration from Moonshine Tiny to -/// Parakeet TDT-CTC 110M — the model directory name also changed, so this -/// is technically belt-and-suspenders, but it keeps the manifest semantics -/// honest (each version tag identifies one specific set of model bytes). -const STT_MODEL_VERSION: &str = "2"; +// STT manifest versions are per model — see `SttModel::version` in the +// `STT_MODELS` registry below. /// Model manifest version for Pocket TTS. Increment when upgrading model files. /// Bumped "1" → "2" when the bundled reference voice changed from KevinAHM's @@ -107,44 +107,214 @@ const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; // ── Constants ───────────────────────────────────────────────────────────────── -/// Maximum expected STT archive size (200 MB — actual is ~100 MB). -const MAX_STT_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; - /// Maximum expected Pocket TTS file size (400 MB per file — largest is /// `lm_main.onnx` at ~303 MB fp32). const MAX_TTS_FILE_BYTES: u64 = 400 * 1024 * 1024; -/// NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx by -/// k2-fsa. Single ONNX file (CTC head) + tokens.txt. Avg WER ~7.5% across -/// the OpenASR-style benchmarks; ~half the WER of Moonshine Tiny at ~2× the -/// disk footprint. CTC blank-token decoding eliminates the silence/cut-audio -/// hallucination class that hurts encoder-decoder models on noisy huddle audio. -/// License: CC-BY-4.0 (attribution required — see About dialog). -const STT_DOWNLOAD_URL: &str = - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ - sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2"; +// ── STT model registry (multilingual — issue #2478) ─────────────────────────── +// +// Historically the huddle STT model was hard-pinned to an English-only build, +// so non-English speech transcribed as garbage (issue #2478). The registry +// makes the model selectable: the English default stays the default, and a +// multilingual model is picked automatically for non-English locales (or +// forced with the `BUZZ_STT_MODEL` env override). Adding a model is pure data +// here plus, for a new sherpa-onnx model family, one match arm in `stt.rs`. + +/// Attribution sidecar filename written next to every STT model's files. +const STT_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt"; -/// Subdirectory name produced by `tar xjf` on the archive. -const STT_ARCHIVE_SUBDIR: &str = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8"; +/// sherpa-onnx model family — decides how the offline recognizer is configured +/// (`stt.rs`) and which ONNX files must be present on disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SttFamily { + /// Single-file NeMo CTC head (e.g. Parakeet TDT-CTC 110M English). + NemoCtc, + /// NeMo transducer: encoder + decoder + joiner (e.g. Parakeet TDT 0.6B v3). + Transducer, +} -/// Final directory name under `~/.buzz/models/`. -const STT_MODEL_DIR_NAME: &str = "parakeet-tdt-ctc-110m-en"; +/// One selectable speech-to-text model. `STT_MODELS` is the single source of +/// truth; `select_stt_model` picks one at startup. +pub struct SttModel { + /// Stable id used by the `BUZZ_STT_MODEL` override and in logs. + pub id: &'static str, + /// Directory name under `~/.buzz/models/`. + pub dir_name: &'static str, + /// Download URL for the `.tar.bz2` archive. + pub download_url: &'static str, + /// Directory name produced by `tar xjf` on the archive. + pub archive_subdir: &'static str, + /// SHA-256 of the archive, or `None` if not yet pinned. `None` still + /// enforces the size cap, safe extraction, and expected-files check — it + /// only skips the content hash. The English default is always `Some`. + pub archive_sha256: Option<&'static str>, + /// Hard cap on the downloaded archive size, in bytes. + pub max_download_bytes: u64, + /// Model files (excluding the license sidecar) required for "ready". + pub model_files: &'static [&'static str], + /// sherpa-onnx model family. + pub family: SttFamily, + /// Manifest version — bump to force re-download of this model. + pub version: &'static str, + /// CC-BY-4.0 §3(a)(1) attribution written next to the model bytes. + pub license_text: &'static str, + /// Human-readable language coverage (About dialog / logs). + pub languages: &'static str, + /// `true` when multilingual — used by the non-English locale auto-select. + pub multilingual: bool, +} + +/// Registry of selectable STT models. Index 0 is the default (English). +static STT_MODELS: &[SttModel] = &[ + // NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx + // by k2-fsa. Single ONNX file (CTC head) + tokens.txt. Avg WER ~7.5% across + // the OpenASR-style benchmarks; CTC blank-token decoding eliminates the + // silence/cut-audio hallucination class that hurts encoder-decoder models + // on noisy huddle audio. This remains the default for English. + SttModel { + id: "parakeet-en", + dir_name: "parakeet-tdt-ctc-110m-en", + download_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2", + archive_subdir: "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8", + archive_sha256: Some("17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"), + max_download_bytes: 200 * 1024 * 1024, + model_files: &["model.int8.onnx", "tokens.txt"], + family: SttFamily::NemoCtc, + version: "2", + license_text: STT_EN_LICENSE_TEXT, + languages: "English", + multilingual: false, + }, + // NVIDIA Parakeet TDT 0.6B v3 (multilingual, int8) — packaged for + // sherpa-onnx by k2-fsa. Transducer family (encoder/decoder/joiner). Auto + // language-ID + punctuation across 25 European languages. This is the + // multilingual default for non-English locales (issue #2478). + // + // Coverage note: Parakeet v3 covers European languages only. CJK/Korean + // (the concrete case in #2478) is not covered by this model; the registry + // makes adding a CJK model (e.g. SenseVoice-Small) a follow-up — one data + // entry, no engine change beyond a family already handled here. + // + // Checksum: upstream publishes no SHA-256, so this hash was computed from + // the k2-fsa `asr-models` release archive + // (sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2, ~465 MB) with + // `shasum -a 256`. Recompute and re-pin if k2-fsa republishes the asset. + SttModel { + id: "parakeet-v3", + dir_name: "parakeet-tdt-0.6b-v3", + download_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", + archive_subdir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", + archive_sha256: Some("5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf"), + max_download_bytes: 800 * 1024 * 1024, + model_files: &[ + "encoder.int8.onnx", + "decoder.int8.onnx", + "joiner.int8.onnx", + "tokens.txt", + ], + family: SttFamily::Transducer, + version: "1", + license_text: STT_V3_LICENSE_TEXT, + languages: "25 European languages (auto-detected): Bulgarian, Croatian, \ + Czech, Danish, Dutch, English, Estonian, Finnish, French, \ + German, Greek, Hungarian, Italian, Latvian, Lithuanian, \ + Maltese, Polish, Portuguese, Romanian, Russian, Slovak, \ + Slovenian, Spanish, Swedish, Ukrainian", + multilingual: true, + }, +]; + +/// The default STT model (English). Used when no override/locale applies. +fn default_stt_model() -> &'static SttModel { + &STT_MODELS[0] +} -/// All files that must be present for the model to be considered ready. +/// Look up a model by id (case-insensitive). `None` if unknown. +fn stt_model_by_id(id: &str) -> Option<&'static SttModel> { + STT_MODELS.iter().find(|m| m.id.eq_ignore_ascii_case(id)) +} + +/// Files that must be present for a model to be "ready": its model files plus +/// the Buzz-written attribution sidecar (the upstream archives ship no LICENSE, +/// so readiness requires the local CC-BY-4.0 notice to travel with the bytes). +fn stt_expected_files(model: &SttModel) -> Vec<&'static str> { + let mut files = model.model_files.to_vec(); + files.push(STT_LICENSE_FILE_NAME); + files +} + +/// Pick the STT model from an explicit override id and a best-effort locale. /// -/// Includes the attribution sidecar written by Buzz during install. The -/// upstream archive does not ship a license file, so readiness should require -/// the local CC-BY-4.0 attribution to travel with the cached model bytes. -const STT_EXPECTED_FILES: &[&str] = &["model.int8.onnx", "tokens.txt", STT_LICENSE_FILE_NAME]; - -/// CC-BY-4.0 §3(a)(1) attribution block written next to the STT model files -/// after install. Travels with the bytes — if a user copies the model -/// directory, the attribution comes with it. Mirrored in About/Credits. +/// Precedence (issue #2478 options 2 + 3): +/// 1. `override_id` (from `BUZZ_STT_MODEL`) when it names a known model. +/// 2. a non-English locale → the multilingual model. +/// 3. otherwise the English default. +/// +/// Pure and dependency-free so it is unit-testable without touching disk/env. +pub fn select_stt_model(override_id: Option<&str>, locale: Option<&str>) -> &'static SttModel { + if let Some(id) = override_id { + let id = id.trim(); + if !id.is_empty() { + if let Some(model) = stt_model_by_id(id) { + return model; + } + eprintln!( + "buzz-desktop: BUZZ_STT_MODEL='{id}' is not a known STT model id — ignoring \ + (valid ids: {})", + STT_MODELS + .iter() + .map(|m| m.id) + .collect::>() + .join(", ") + ); + } + } + if let Some(locale) = locale { + // Take the primary language subtag: "de-DE"/"uk_UA" → "de"/"uk". + let lang = locale + .split(['-', '_', '.']) + .next() + .unwrap_or("") + .to_ascii_lowercase(); + if !lang.is_empty() && lang != "en" { + if let Some(model) = STT_MODELS.iter().find(|m| m.multilingual) { + return model; + } + } + } + default_stt_model() +} + +/// Best-effort system locale from the environment (dependency-free). /// +/// Reads the standard POSIX locale variables in precedence order. Returns +/// `None` when unset or set to the neutral `C`/`POSIX` locale, in which case +/// selection falls back to the English default. +fn detect_locale() -> Option { + for key in ["LC_ALL", "LC_MESSAGES", "LANG", "LANGUAGE"] { + if let Ok(value) = std::env::var(key) { + let value = value.trim(); + if !value.is_empty() && value != "C" && value != "POSIX" { + return Some(value.to_string()); + } + } + } + None +} + +/// Resolve the STT model to use for this process: `BUZZ_STT_MODEL` override, +/// else system-locale auto-select, else English default. +fn selected_stt_model() -> &'static SttModel { + let override_id = std::env::var("BUZZ_STT_MODEL").ok(); + select_stt_model(override_id.as_deref(), detect_locale().as_deref()) +} + +/// CC-BY-4.0 §3(a)(1) attribution for Parakeet TDT-CTC 110M (English). /// Covers all five §3(a)(1) bullets: creator, copyright notice, license /// notice, warranty disclaimer reference, and URI to the source material. -const STT_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt"; -const STT_LICENSE_TEXT: &str = "\ +const STT_EN_LICENSE_TEXT: &str = "\ NVIDIA Parakeet TDT-CTC 110M (English) © NVIDIA Corporation. @@ -160,6 +330,23 @@ Provided \"AS IS\", without warranty of any kind, express or implied. See the license text for full warranty disclaimer. "; +/// CC-BY-4.0 §3(a)(1) attribution for Parakeet TDT 0.6B v3 (multilingual). +const STT_V3_LICENSE_TEXT: &str = "\ +NVIDIA Parakeet TDT 0.6B v3 (multilingual) +© NVIDIA Corporation. + +Licensed under the Creative Commons Attribution 4.0 International License +(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/ + +Original model: https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 +Converted to ONNX with int8 quantization by the sherpa-onnx project +(https://github.com/k2-fsa/sherpa-onnx); Buzz ships this conversion +unmodified. + +Provided \"AS IS\", without warranty of any kind, express or implied. See the +license text for full warranty disclaimer. +"; + // ── Pocket TTS model ────────────────────────────────────────────────────────── /// Final directory name under `~/.buzz/models/`. @@ -401,9 +588,9 @@ where /// Per-model state + config. `ModelManager` owns two of these (stt, tts). #[derive(Clone)] struct ModelSlot { - dir_name: &'static str, // subdir under ~/.buzz/models/ - expected_files: &'static [&'static str], // files required for "ready" - version: &'static str, // manifest version; increment to force re-download + dir_name: &'static str, // subdir under ~/.buzz/models/ + expected_files: Vec<&'static str>, // files required for "ready" + version: &'static str, // manifest version; increment to force re-download status: Arc>, just_ready: Arc, // fires once when download completes } @@ -411,7 +598,7 @@ struct ModelSlot { impl ModelSlot { fn new( dir_name: &'static str, - expected_files: &'static [&'static str], + expected_files: Vec<&'static str>, version: &'static str, ) -> Self { Self { @@ -551,6 +738,8 @@ impl ModelSlot { pub struct ModelManager { /// `~/.buzz/models/` models_dir: PathBuf, + /// The STT model selected for this process (override / locale / default). + stt_model: &'static SttModel, stt: ModelSlot, tts: ModelSlot, } @@ -558,18 +747,41 @@ pub struct ModelManager { impl ModelManager { /// Create a new `ModelManager` rooted at `~/.buzz/models/`. /// + /// The STT model is resolved once here from `BUZZ_STT_MODEL`, then the + /// system locale, then the English default (issue #2478). + /// /// Returns `None` if the home directory cannot be resolved. pub fn new() -> Option { let models_dir = dirs::home_dir()?.join(".buzz").join("models"); + let stt_model = selected_stt_model(); + eprintln!( + "buzz-desktop: STT model '{}' selected ({})", + stt_model.id, stt_model.languages + ); Some(Self { models_dir, - stt: ModelSlot::new(STT_MODEL_DIR_NAME, STT_EXPECTED_FILES, STT_MODEL_VERSION), - tts: ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION), + stt_model, + stt: ModelSlot::new( + stt_model.dir_name, + stt_expected_files(stt_model), + stt_model.version, + ), + tts: ModelSlot::new( + TTS_MODEL_DIR_NAME, + TTS_EXPECTED_FILES.to_vec(), + TTS_MODEL_VERSION, + ), }) } // ── STT accessors ──────────────────────────────────────────────────────── + /// The sherpa-onnx model family of the selected STT model. The huddle STT + /// pipeline uses this to configure the offline recognizer. + pub fn stt_family(&self) -> SttFamily { + self.stt_model.family + } + /// Path to the STT model directory, or `None` if not ready. pub fn stt_model_dir(&self) -> Option { self.stt.dir_if_ready(&self.models_dir) @@ -659,19 +871,21 @@ impl ModelManager { // Temp filenames derive from the final directory name to avoid colliding // with leftovers from any previous STT model (e.g. moonshine-tiny.*). - let archive_path = self - .models_dir - .join(format!("{STT_MODEL_DIR_NAME}.tar.bz2")); - let temp_dir = self.models_dir.join(format!("{STT_MODEL_DIR_NAME}.tmp")); + let model = self.stt_model; + let archive_path = self.models_dir.join(format!("{}.tar.bz2", model.dir_name)); + let temp_dir = self.models_dir.join(format!("{}.tmp", model.dir_name)); - eprintln!("buzz-desktop: downloading STT model from {STT_DOWNLOAD_URL}"); - let response = fetch_url(&http_client, STT_DOWNLOAD_URL, "stt archive").await?; + eprintln!( + "buzz-desktop: downloading STT model '{}' from {}", + model.id, model.download_url + ); + let response = fetch_url(&http_client, model.download_url, "stt archive").await?; let slot = self.stt.clone(); let bytes = download_file( response, &archive_path, - MAX_STT_DOWNLOAD_BYTES, + model.max_download_bytes, "stt archive", |downloaded, content_length| { if let Some(pct) = @@ -686,13 +900,27 @@ impl ModelManager { .await?; eprintln!("buzz-desktop: downloaded {bytes} bytes, wrote to disk"); - // Verify archive integrity before extraction. - let hash = sha256_file(&archive_path).await?; - if hash != STT_ARCHIVE_SHA256 { - let _ = tokio::fs::remove_file(&archive_path).await; - return Err(format!( - "STT archive integrity check failed: expected {STT_ARCHIVE_SHA256}, got {hash}" - )); + // Verify archive integrity before extraction. Models with a pinned + // hash are content-verified; a `None` hash relies on the size cap + // (already enforced above), safe extraction, and the expected-files + // check in `verify_and_install`. + match model.archive_sha256 { + Some(expected) => { + let hash = sha256_file(&archive_path).await?; + if hash != expected { + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(format!( + "STT archive integrity check failed: expected {expected}, got {hash}" + )); + } + } + None => { + eprintln!( + "buzz-desktop: STT model '{}' has no pinned SHA-256 — \ + skipping content hash (size cap + safe extraction still enforced)", + model.id + ); + } } self.stt.set_status(ModelStatus::Downloading { @@ -706,11 +934,12 @@ impl ModelManager { .await .map_err(|e| format!("tar task panicked: {e}"))??; - let extracted_subdir = temp_dir.join(STT_ARCHIVE_SUBDIR); + let extracted_subdir = temp_dir.join(model.archive_subdir); if !extracted_subdir.is_dir() { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( - "expected subdir '{STT_ARCHIVE_SUBDIR}' not found after extraction" + "expected subdir '{}' not found after extraction", + model.archive_subdir )); } @@ -719,7 +948,7 @@ impl ModelManager { // upstream tarball ships no LICENSE/NOTICE, so we provide it ourselves // per §3(a)(1) (license must travel with Shared material). let license_path = extracted_subdir.join(STT_LICENSE_FILE_NAME); - if let Err(e) = tokio::fs::write(&license_path, STT_LICENSE_TEXT).await { + if let Err(e) = tokio::fs::write(&license_path, model.license_text).await { let _ = tokio::fs::remove_dir_all(&temp_dir).await; let _ = tokio::fs::remove_file(&archive_path).await; return Err(format!("write model license sidecar: {e}")); @@ -883,6 +1112,15 @@ pub fn stt_model_dir() -> Option { global_model_manager()?.stt_model_dir() } +/// sherpa-onnx model family of the selected STT model (English default when the +/// manager is unavailable). The huddle STT pipeline uses this to configure the +/// offline recognizer for the right model family. +pub fn stt_model_family() -> SttFamily { + global_model_manager() + .map(|m| m.stt_family()) + .unwrap_or(SttFamily::NemoCtc) +} + /// `true` if all expected STT model files are present on disk. pub fn is_stt_ready() -> bool { global_model_manager() @@ -937,7 +1175,11 @@ mod tests { #[test] fn tts_readiness_requires_license_sidecar() { let temp = tempfile::tempdir().expect("tempdir"); - let slot = ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION); + let slot = ModelSlot::new( + TTS_MODEL_DIR_NAME, + TTS_EXPECTED_FILES.to_vec(), + TTS_MODEL_VERSION, + ); let model_dir = temp.path().join(TTS_MODEL_DIR_NAME); std::fs::create_dir_all(&model_dir).expect("create model dir"); @@ -951,4 +1193,122 @@ mod tests { std::fs::remove_file(model_dir.join(TTS_LICENSE_FILE_NAME)).expect("remove sidecar"); assert!(!slot.is_ready(temp.path())); } + + // ── STT model selection (issue #2478) ───────────────────────────────────── + + #[test] + fn defaults_to_english_without_override_or_locale() { + assert_eq!(select_stt_model(None, None).id, "parakeet-en"); + assert_eq!(select_stt_model(None, Some("en-US")).id, "parakeet-en"); + assert_eq!(select_stt_model(None, Some("en")).id, "parakeet-en"); + // Neutral C/POSIX locales are dropped by detect_locale, but pass the raw + // primary subtag here to prove "en" specifically stays on English. + assert!(!select_stt_model(None, None).multilingual); + } + + #[test] + fn non_english_locale_selects_multilingual_model() { + for locale in ["de-DE", "uk_UA", "fr", "es-ES", "pl_PL.UTF-8"] { + let model = select_stt_model(None, Some(locale)); + assert!(model.multilingual, "locale {locale} should be multilingual"); + assert_eq!(model.id, "parakeet-v3", "locale {locale}"); + } + } + + #[test] + fn explicit_override_wins_over_locale() { + // Force multilingual even on an English locale. + assert_eq!( + select_stt_model(Some("parakeet-v3"), Some("en-US")).id, + "parakeet-v3" + ); + // Force English even on a non-English locale. + assert_eq!( + select_stt_model(Some("parakeet-en"), Some("de-DE")).id, + "parakeet-en" + ); + // Override id is case-insensitive. + assert_eq!( + select_stt_model(Some("PARAKEET-V3"), None).id, + "parakeet-v3" + ); + } + + #[test] + fn unknown_or_empty_override_falls_back() { + // Unknown override id → fall through to locale, then default. + assert_eq!( + select_stt_model(Some("does-not-exist"), Some("en-US")).id, + "parakeet-en" + ); + assert_eq!( + select_stt_model(Some("does-not-exist"), Some("fr-FR")).id, + "parakeet-v3" + ); + // Empty / whitespace override is ignored. + assert_eq!(select_stt_model(Some(" "), None).id, "parakeet-en"); + } + + #[test] + fn registry_invariants_hold() { + assert!(!STT_MODELS.is_empty()); + assert_eq!(default_stt_model().id, "parakeet-en"); + // The English default must always be integrity-pinned. + assert!( + default_stt_model().archive_sha256.is_some(), + "English default must ship a pinned SHA-256" + ); + // At least one multilingual model exists (covers #2478). + assert!(STT_MODELS.iter().any(|m| m.multilingual)); + // Ids are unique (case-insensitive); every model declares files + a cap. + for (i, model) in STT_MODELS.iter().enumerate() { + assert!(!model.model_files.is_empty(), "{} has no files", model.id); + assert!(model.max_download_bytes > 0, "{} has no size cap", model.id); + for other in &STT_MODELS[i + 1..] { + assert!( + !model.id.eq_ignore_ascii_case(other.id), + "duplicate model id {}", + model.id + ); + } + } + } + + #[test] + fn expected_files_always_include_license_sidecar() { + for model in STT_MODELS { + let files = stt_expected_files(model); + assert!( + files.contains(&STT_LICENSE_FILE_NAME), + "{} missing license sidecar in expected files", + model.id + ); + for f in model.model_files { + assert!(files.contains(f), "{} missing {f}", model.id); + } + } + } + + #[test] + fn readiness_uses_per_model_expected_files() { + // A transducer model needs encoder/decoder/joiner — the single-file + // check must not pass until all three plus tokens + license exist. + let v3 = stt_model_by_id("parakeet-v3").expect("v3 registered"); + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(v3.dir_name, stt_expected_files(v3), v3.version); + let dir = temp.path().join(v3.dir_name); + std::fs::create_dir_all(&dir).expect("create dir"); + std::fs::write(dir.join(MANIFEST_FILENAME), v3.version).expect("manifest"); + + // Only some files present → not ready. + std::fs::write(dir.join("encoder.int8.onnx"), b"x").expect("write"); + std::fs::write(dir.join("tokens.txt"), b"x").expect("write"); + assert!(!slot.is_ready(temp.path())); + + // All expected files present → ready. + for f in stt_expected_files(v3) { + std::fs::write(dir.join(f), b"x").expect("write"); + } + assert!(slot.is_ready(temp.path())); + } } diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index ceccedd8b6..a5489a4acc 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -94,6 +94,9 @@ pub(crate) async fn maybe_start_stt_pipeline( return Ok(false); // Models not downloaded yet — voice-only mode. } let model_dir = models::stt_model_dir().ok_or("STT model directory not found")?; + // The selected model's family decides how the offline recognizer is + // configured (English CTC vs multilingual transducer — issue #2478). + let stt_family = models::stt_model_family(); let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; @@ -138,7 +141,14 @@ pub(crate) async fn maybe_start_stt_pipeline( drop(old_stt); let constructed = tokio::task::spawn_blocking(move || { - stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt) + stt::SttPipeline::new( + model_dir, + stt_family, + tts_active, + tts_cancel, + ptt_active_for_stt, + None, + ) }) .await; let (pipeline, text_rx) = match constructed { diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..b5ddb3769c 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -31,6 +31,8 @@ use std::{ use tokio::sync::mpsc as tokio_mpsc; +use super::models::SttFamily; + // ── Public pipeline handle ──────────────────────────────────────────────────── /// Bounded audio queue capacity. @@ -75,6 +77,13 @@ impl SttPipeline { /// pipeline only accumulates speech while the flag is true (key held). /// When `None`, the pipeline runs in continuous VAD mode. /// + /// `live_tx` (optional) switches the pipeline into live-transcript mode + /// (used by composer dictation): the in-progress phrase is re-decoded + /// every `PARTIAL_DECODE_STEP` new samples and sent as `(text, false)`, + /// and finals are sent as `(text, true)` — all on this one channel so + /// ordering is preserved. In this mode nothing is sent on the returned + /// text receiver. Pass `None` for huddle transcription (finals only). + /// /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — /// the pipeline handle is still returned but will never produce text. @@ -85,9 +94,11 @@ impl SttPipeline { /// thread on every `recv_timeout` call). pub fn new( model_dir: PathBuf, + family: SttFamily, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + live_tx: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); @@ -101,12 +112,14 @@ impl SttPipeline { .spawn(move || { stt_worker( model_dir, + family, audio_rx, text_tx, shutdown_worker, tts_active, tts_cancel_worker, ptt_active_worker, + live_tx, ) }) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; @@ -167,6 +180,17 @@ impl Drop for SttPipeline { /// Previous value (28 frames / 450 ms) felt sluggish in conversation. const SILENCE_FLUSH_FRAMES: usize = 19; +/// Silence-flush threshold for live (dictation) mode: ~608 ms. +/// +/// Parakeet only commits sentence-final punctuation and capitalization when +/// the decoded audio ends in ≥600 ms of silence (measured: 300 ms → "is there +/// anything we can do about that", 600 ms → "…about that?"). The huddle's +/// ~300 ms window cut finals just under that threshold, so dictated text lost +/// its punctuation at every phrase commit. The longer window also merges +/// mid-sentence thinking pauses into fuller phrases; perceived latency is +/// unchanged because partials keep streaming while the window runs. +const LIVE_SILENCE_FLUSH_FRAMES: usize = 38; + /// Consecutive VAD speech frames required before triggering barge-in during TTS. /// 20 frames × 256 samples / 16 kHz ≈ 320 ms — must be long enough to filter /// speaker-to-mic feedback (TTS audio bleeding through the mic) while still @@ -184,6 +208,20 @@ const VAD_THRESHOLD: f32 = 0.5; /// How long the worker waits on the audio channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(50); +/// Live-transcript mode: re-decode the in-progress phrase every this many new +/// 16 kHz samples (~300 ms) so words stream out while the user is talking. +/// The model is an offline recognizer, so "streaming" is repeated decoding of +/// the growing phrase buffer — phrases reset at every VAD pause, but a long +/// unbroken ramble grows toward MAX_SPEECH_SAMPLES. Decode cost grows with the +/// buffer (~70 ms per second of audio for the 0.6B model, measured in +/// `v3_long_buffer`), so this is only the MINIMUM step: the worker also waits +/// at least the previous decode's duration worth of new audio before the next +/// partial. Without that back-off the worker falls behind real time past ~5 s +/// of phrase, the bounded audio queue overflows, and mic audio is silently +/// dropped — splicing the phrase and making later decodes remove words that +/// earlier partials showed. +const PARTIAL_DECODE_STEP: usize = 4800; + /// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. /// Previous value (200 ms) was eating the first word when the user spoke @@ -201,14 +239,25 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); /// shows it's safe on the minimum-spec target. const STT_NUM_THREADS: i32 = 1; +/// ONNX threads in live-transcript (dictation) mode. Live mode re-decodes the +/// whole in-progress phrase every ~300 ms, so per-decode latency bounds how +/// fresh the streamed words are. Measured with Parakeet v3 0.6B int8 +/// (`v3_punctuation` experiment): a 6.5 s phrase decodes in ~1.4 s at 1 +/// thread but ~460 ms at 4 — the difference between laggy and live. No +/// huddle/LiveKit stack runs during dictation, so the extra cores are free. +const LIVE_STT_NUM_THREADS: i32 = 4; + +#[allow(clippy::too_many_arguments)] fn stt_worker( model_dir: PathBuf, + family: SttFamily, audio_rx: Receiver>, text_tx: tokio_mpsc::Sender, shutdown: Arc, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, + live_tx: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -228,18 +277,21 @@ fn stt_worker( // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── // - // Parakeet TDT-CTC 110M ships as a single `model.int8.onnx` (CTC head) plus - // `tokens.txt`. sherpa-onnx infers the model family from which inner config - // has a `model` path set, so we don't need to set `model_type` explicitly. - // (See rust-api-examples/parakeet_tdt_ctc_simulate_streaming_microphone.rs - // in k2-fsa/sherpa-onnx.) + // sherpa-onnx infers the model family from which inner config has model + // paths set, so we populate exactly one family sub-config (issue #2478): + // + // NemoCtc — single `model.int8.onnx` (CTC head), e.g. Parakeet 110M en. + // Transducer — `encoder/decoder/joiner.int8.onnx`, e.g. Parakeet 0.6B v3 + // (multilingual). See k2-fsa/sherpa-onnx offline-transducer + // NeMo examples. + // + // `tokens.txt` is shared by every family and lives on the parent config. use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); - if !tokens_path.exists() || !model_path.exists() { + if !tokens_path.exists() { eprintln!( - "buzz-desktop: STT model not found at {} — STT disabled", + "buzz-desktop: STT tokens.txt not found at {} — STT disabled", model_dir.display() ); drain_until_shutdown(audio_rx, &shutdown); @@ -247,9 +299,43 @@ fn stt_worker( } let mut cfg = OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); + match family { + SttFamily::NemoCtc => { + let model_path = model_dir.join("model.int8.onnx"); + if !model_path.exists() { + eprintln!( + "buzz-desktop: STT model.int8.onnx not found at {} — STT disabled", + model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); + } + SttFamily::Transducer => { + let encoder = model_dir.join("encoder.int8.onnx"); + let decoder = model_dir.join("decoder.int8.onnx"); + let joiner = model_dir.join("joiner.int8.onnx"); + if !encoder.exists() || !decoder.exists() || !joiner.exists() { + eprintln!( + "buzz-desktop: STT transducer files (encoder/decoder/joiner) missing at {} \ + — STT disabled", + model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + cfg.model_config.transducer.encoder = Some(encoder.to_string_lossy().into_owned()); + cfg.model_config.transducer.decoder = Some(decoder.to_string_lossy().into_owned()); + cfg.model_config.transducer.joiner = Some(joiner.to_string_lossy().into_owned()); + } + } cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); - cfg.model_config.num_threads = STT_NUM_THREADS; + cfg.model_config.num_threads = if live_tx.is_some() { + LIVE_STT_NUM_THREADS + } else { + STT_NUM_THREADS + }; // Explicit — defaults are not part of the API contract, and noisy debug // logging in release builds would be expensive on every VAD chunk. cfg.model_config.debug = false; @@ -276,6 +362,15 @@ fn stt_worker( let mut in_speech = false; // Consecutive speech frames seen during TTS — used for barge-in debounce. let mut barge_in_frames: usize = 0; + // Live mode: speech_buf length at the last partial decode. + let mut last_partial_len: usize = 0; + // Live mode: minimum new samples before the next partial decode — + // PARTIAL_DECODE_STEP, raised to the last decode's duration so partial + // decoding never outruns real time (see PARTIAL_DECODE_STEP docs). + let mut partial_step: usize = PARTIAL_DECODE_STEP; + // Live mode: most recent partial decode of the current phrase that ended + // with terminal punctuation — see prefer_punctuated. + let mut punct_partial: Option = None; // Timestamp when TTS last stopped — used for the 200 ms cooldown. let mut tts_stopped_at: Option = None; @@ -305,10 +400,18 @@ fn stt_worker( if let Some(ref ptt) = ptt_active { let ptt_now = ptt.load(Ordering::Acquire); if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { - flush_to_stt(&speech_buf, &recognizer, &text_tx); + flush_to_stt( + &speech_buf, + &recognizer, + &text_tx, + live_tx.as_ref(), + punct_partial.take(), + ); speech_buf.clear(); silence_frames = 0; in_speech = false; + last_partial_len = 0; + partial_step = PARTIAL_DECODE_STEP; } ptt_was_active = ptt_now; } @@ -349,6 +452,10 @@ fn stt_worker( tts_cancel.as_deref(), &mut tts_stopped_at, ptt_active.as_ref(), + live_tx.as_ref(), + &mut last_partial_len, + &mut partial_step, + &mut punct_partial, ); } } @@ -411,6 +518,10 @@ fn process_16k_samples( tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, + live_tx: Option<&tokio_mpsc::Sender<(String, bool)>>, + last_partial_len: &mut usize, + partial_step: &mut usize, + punct_partial: &mut Option, ) { leftover.extend_from_slice(samples); @@ -495,10 +606,12 @@ fn process_16k_samples( // OOM guard: flush and reset if the buffer exceeds 30 s of audio. if speech_buf.len() >= MAX_SPEECH_SAMPLES { - flush_to_stt(speech_buf, recognizer, text_tx); + flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); speech_buf.clear(); *silence_frames = 0; *in_speech = false; + *last_partial_len = 0; + *partial_step = PARTIAL_DECODE_STEP; } } else if *in_speech { // Still accumulate during brief silence gaps. @@ -509,45 +622,141 @@ fn process_16k_samples( // key-hold as one utterance. The PTT release edge in the main // loop handles the flush. In VAD mode, flush after the silence // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { + let flush_frames = if live_tx.is_some() { + LIVE_SILENCE_FLUSH_FRAMES + } else { + SILENCE_FLUSH_FRAMES + }; + if ptt_active.is_none() && *silence_frames >= flush_frames { // End of utterance — transcribe. - flush_to_stt(speech_buf, recognizer, text_tx); + flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); speech_buf.clear(); *silence_frames = 0; *in_speech = false; + *last_partial_len = 0; + *partial_step = PARTIAL_DECODE_STEP; } } // If not in speech and not accumulating, just discard the frame. + + // Live mode: re-decode the in-progress phrase every PARTIAL_DECODE_STEP + // new samples so words appear while the user is still talking. Later + // partials of the same phrase supersede earlier ones — the frontend + // replaces the partial region — so decode wobble is self-correcting. + if let Some(tx) = live_tx { + if *in_speech && speech_buf.len() >= *last_partial_len + *partial_step { + *last_partial_len = speech_buf.len(); + let t0 = std::time::Instant::now(); + let text = decode_speech(speech_buf, recognizer); + // Wait at least 2× this decode's duration (in samples: 16/ms + // at 16 kHz) before the next partial, capping decode duty + // cycle at ~50% so the worker can't fall behind the mic and + // overflow the audio queue (which silently drops samples). + // 1× measured at ~100% duty on a 35 s ramble — no headroom. + *partial_step = + PARTIAL_DECODE_STEP.max(32 * t0.elapsed().as_millis() as usize); + if !text.is_empty() { + if text.ends_with(['.', '?', '!', ',']) { + *punct_partial = Some(text.clone()); + } + if let Err(e) = tx.blocking_send((text, false)) { + eprintln!("buzz-desktop: STT live channel closed: {e}"); + } + } + } + } } } /// Run sherpa-onnx on the accumulated speech buffer and send the text. /// /// Uses `blocking_send` because this runs on a `std::thread` (not async). +/// Lowercased words only — all punctuation and case stripped. Two decodes of +/// the same audio that differ only in the model's (unstable) punctuation and +/// capitalization normalize to the same string. +fn norm_words(s: &str) -> String { + let mapped: String = s + .chars() + .map(|c| { + if c.is_alphanumeric() || c == '\'' { + c.to_ascii_lowercase() + } else { + ' ' + } + }) + .collect(); + mapped.split_whitespace().collect::>().join(" ") +} + +/// Punctuation from the 110M model is unstable: re-decodes of the same phrase +/// flicker between "…that" and "…that?" (and internally, "question." vs +/// "question,") depending on the exact trailing-noise composition — see the +/// `silence_experiment` tests. If a partial of this phrase ended in +/// punctuation, prefer it: +/// - same words as the final (punctuation/case-insensitive) → keep the whole +/// punctuated hint; +/// - words differ but the last word agrees and the hint ends a sentence +/// (. ? !) → graft just that mark onto the final. +/// +/// Never invents words: any word-level difference and the final decode wins. +fn prefer_punctuated(final_text: String, punct_hint: Option<&str>) -> String { + let Some(hint) = punct_hint else { + return final_text; + }; + if final_text.is_empty() || hint.is_empty() { + return final_text; + } + let norm_final = norm_words(&final_text); + if norm_words(hint) == norm_final { + return hint.to_string(); + } + let term = hint.chars().last().unwrap_or(' '); + if matches!(term, '.' | '?' | '!') && !final_text.ends_with(['.', '?', '!']) { + let last_word = |s: &str| s.rsplit(' ').next().map(str::to_string); + if last_word(&norm_final) == last_word(&norm_words(hint)) { + let mut grafted = final_text.trim_end_matches([',', ' ']).to_string(); + grafted.push(term); + return grafted; + } + } + final_text +} + /// The tokio channel's `blocking_send` is safe to call from sync contexts. fn flush_to_stt( speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, text_tx: &tokio_mpsc::Sender, + live_tx: Option<&tokio_mpsc::Sender<(String, bool)>>, + punct_hint: Option, ) { - if speech_buf.is_empty() { - return; + let text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); + if !text.is_empty() { + // In live mode finals ride the same channel as partials so the + // receiver never sees a stale partial after its final. + let send_err = match live_tx { + Some(tx) => tx.blocking_send((text, true)).err().map(|e| e.to_string()), + None => text_tx.blocking_send(text).err().map(|e| e.to_string()), + }; + if let Some(e) = send_err { + eprintln!("buzz-desktop: STT text channel closed: {e}"); + } } +} +/// Run sherpa-onnx on the accumulated speech and return the trimmed text +/// (empty on empty input or decode failure). +fn decode_speech(speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer) -> String { + if speech_buf.is_empty() { + return String::new(); + } let stream = recognizer.create_stream(); stream.accept_waveform(16_000, speech_buf); recognizer.decode(&stream); - - let text = stream + stream .get_result() .map(|r| r.text.trim().to_string()) - .unwrap_or_default(); - - if !text.is_empty() { - if let Err(e) = text_tx.blocking_send(text) { - eprintln!("buzz-desktop: STT text channel closed: {e}"); - } - } + .unwrap_or_default() } /// Convert raw bytes (f32 LE) to f32 samples. @@ -565,3 +774,400 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; + +#[cfg(test)] +mod prefer_punctuated_tests { + use super::prefer_punctuated; + + #[test] + fn keeps_dropped_terminal_punctuation() { + let hint = Some("Is there anything we can do about that?"); + assert_eq!( + prefer_punctuated("is there anything we can do about that".into(), hint), + "Is there anything we can do about that?" + ); + } + + #[test] + fn internal_punctuation_flicker_still_matches() { + // Observed in live_partial_sequence: the final decode swapped an + // internal "." for "," — same words, so the punctuated hint wins. + let hint = Some("I'm going to ask a question. Do you know why that happened?"); + assert_eq!( + prefer_punctuated( + "I'm going to ask a question, Do you know why that happened".into(), + hint + ), + "I'm going to ask a question. Do you know why that happened?" + ); + } + + #[test] + fn grafts_terminal_mark_when_only_last_word_agrees() { + assert_eq!( + prefer_punctuated( + "do you know er why that happened,".into(), + Some("Do you know why that happened?") + ), + "do you know er why that happened?" + ); + } + + #[test] + fn comma_hint_is_never_grafted() { + assert_eq!( + prefer_punctuated("hello there friend".into(), Some("hello there,")), + "hello there friend" + ); + } + + #[test] + fn ignores_hint_when_text_differs() { + assert_eq!( + prefer_punctuated("hello world again".into(), Some("hello world.")), + "hello world again" + ); + } + + #[test] + fn never_replaces_empty_final() { + assert_eq!(prefer_punctuated(String::new(), Some("hello.")), ""); + } + + #[test] + fn no_hint_passes_through() { + assert_eq!(prefer_punctuated("hello".into(), None), "hello"); + } +} + +#[cfg(test)] +mod silence_experiment { + use super::{decode_speech, prefer_punctuated}; + + /// Manual experiment: does Parakeet v3 (0.6B transducer) produce stable + /// punctuation under noisy pauses where the 110M CTC model flickers? + /// Also times each decode — the live mode re-decodes the whole phrase + /// every ~300 ms, so per-decode latency bounds the streaming cadence. + /// Run: cargo test v3_punctuation -- --ignored --nocapture + /// Needs ~/.buzz/models/parakeet-tdt-0.6b-v3 downloaded (see models.rs). + #[test] + #[ignore] + fn v3_punctuation() { + for threads in [1i32, 2, 4] { + let recognizer = v3_recognizer(threads); + println!("--- num_threads = {threads} ---"); + run_decodes(&recognizer); + } + } + + fn v3_recognizer(threads: i32) -> sherpa_onnx::OfflineRecognizer { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-0.6b-v3"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.transducer.encoder = Some( + model_dir + .join("encoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.decoder = Some( + model_dir + .join("decoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.joiner = Some( + model_dir + .join("joiner.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = + Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.debug = false; + cfg.model_config.num_threads = threads; + sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap() + } + + fn read_wav_16k(path: &str) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() + } + + /// Manual experiment: chl reported that during a long continuous ramble + /// "a whole bunch of text just got removed from the end". Two suspects: + /// (a) v3 transducer output degrades/truncates as the phrase buffer grows + /// toward the 30 s MAX_SPEECH_SAMPLES cap, so a later partial decode + /// yields LESS text than an earlier one and the frontend diff deletes + /// the tail; + /// (b) per-decode latency on long buffers (~70 ms/s of audio) blows past + /// the 300 ms partial cadence, the worker falls behind and the 5 s + /// audio queue overflows. + /// This decodes growing prefixes of a 35 s ramble, timing each decode and + /// flagging any shrinkage between consecutive prefixes. + /// Run: cargo test v3_long_buffer -- --ignored --nocapture + /// Setup: say -v Samantha "" -o /tmp/dict_long.wav + /// --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_long_buffer() { + let recognizer = v3_recognizer(super::LIVE_STT_NUM_THREADS); + let samples = read_wav_16k("/tmp/dict_long.wav"); + println!("total: {:.1}s", samples.len() as f32 / 16_000.0); + let mut prev = String::new(); + for secs in [4, 8, 12, 16, 20, 22, 24, 26, 28, 30] { + let end = (16_000 * secs).min(samples.len()); + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..end], &recognizer); + let shrink = if text.len() < prev.len() { " <<< SHRANK" } else { "" }; + println!( + "@{secs:>2}s [{:>6.0?}ms] {} chars{shrink}: {text:?}", + t0.elapsed().as_millis(), + text.len() + ); + prev = text; + if end == samples.len() { + break; + } + } + + // Simulate the worker's adaptive partial cadence over the same audio: + // next partial waits max(PARTIAL_DECODE_STEP, last decode duration). + // Real-time safe iff cumulative decode time stays under audio time. + println!("--- adaptive cadence simulation ---"); + let mut last_len = 0usize; + let mut step = super::PARTIAL_DECODE_STEP; + let mut decode_total = std::time::Duration::ZERO; + let mut n = 0u32; + while last_len + step <= samples.len() { + last_len += step; + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..last_len], &recognizer); + let dt = t0.elapsed(); + decode_total += dt; + step = super::PARTIAL_DECODE_STEP.max(32 * dt.as_millis() as usize); + n += 1; + println!( + "partial {n} @ audio {:>4.1}s: decode {:>4.0?}ms, next step {:.1}s, tail {:?}", + last_len as f32 / 16_000.0, + dt.as_millis(), + step as f32 / 16_000.0, + &text[text.len().saturating_sub(40)..] + ); + } + let audio_secs = last_len as f32 / 16_000.0; + println!( + "{n} partials over {audio_secs:.1}s audio, total decode {:.1}s → real-time safe: {}", + decode_total.as_secs_f32(), + decode_total.as_secs_f32() < audio_secs + ); + } + + fn run_decodes(recognizer: &sherpa_onnx::OfflineRecognizer) { + for path in ["/tmp/dict_multi.wav", "/tmp/dict_q.wav"] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!( + "=== {path} ({:.1}s speech) ===", + samples.len() as f32 / 16_000.0 + ); + for (label, amp, tail_ms) in [ + ("bare", 0.0f32, 0usize), + ("300ms noise", 0.01, 300), + ("608ms noise", 0.01, 608), + ("608ms loud noise", 0.03, 608), + ] { + let mut buf = samples.clone(); + let mut state = 0x2545_f491u32; + for _ in 0..(16 * tail_ms) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + buf.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + let t0 = std::time::Instant::now(); + let text = decode_speech(&buf, recognizer); + println!(" +{label}: [{:?}] {text:?}", t0.elapsed()); + } + } + } + + /// Manual experiment: simulate the live worker's exact cadence over real + /// (synthesized) speech with a noisy tail — decode the growing buffer + /// every PARTIAL_DECODE_STEP samples, track `punct_partial` like the + /// worker does, then run the final flush through `prefer_punctuated`. + /// Shows whether the sticky-punctuation hint actually matches on + /// multi-sentence phrases. + /// Run: cargo test live_partial_sequence -- --ignored --nocapture + /// Setup: say -v Samantha "It missed the full stop on the first + /// sentence. I'm now going to try and ask a question. Do you know why + /// that happened" -o /tmp/dict_multi.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn live_partial_sequence() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = + Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for (path, amp) in [ + ("/tmp/dict_multi.wav", 0.01f32), + ("/tmp/dict_q.wav", 0.01f32), + ] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let mut samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + // Noisy silence tail up to the live flush window (~608 ms). + let mut state = 0x2545_f491u32; + for _ in 0..(16 * 608) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + samples.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + println!("=== {path} (noise amp {amp}) ==="); + let mut punct_partial: Option = None; + let mut last_len = 0usize; + while last_len + super::PARTIAL_DECODE_STEP <= samples.len() { + last_len += super::PARTIAL_DECODE_STEP; + let text = decode_speech(&samples[..last_len], &recognizer); + if text.ends_with(['.', '?', '!', ',']) { + punct_partial = Some(text.clone()); + } + println!(" partial@{last_len}: {text:?}"); + } + let final_text = decode_speech(&samples, &recognizer); + println!(" FINAL: {final_text:?}"); + println!(" HINT: {punct_partial:?}"); + println!( + " KEPT: {:?}", + prefer_punctuated(final_text, punct_partial.as_deref()) + ); + // Rejected candidate fix (measured 2026-07-24): replacing the + // noisy VAD hangover with digital zeros at flush time sometimes + // REMOVES punctuation the noisy final had — tail composition is + // non-monotonic, so don't retry audio-level recipes. + let speech_end = samples.len() - 16 * 608; + let mut zeroed = samples[..speech_end].to_vec(); + zeroed.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!(" ZEROED-TAIL: {:?}", decode_speech(&zeroed, &recognizer)); + // Same but with noise overlaid on the speech itself (real mics + // pick up room noise under the voice too, not just in pauses). + let mut state = 0x1234_5678u32; + let mut noisy: Vec = samples[..speech_end] + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect(); + noisy.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " NOISY-SPEECH+ZEROED-TAIL: {:?}", + decode_speech(&noisy, &recognizer) + ); + } + } + + /// Manual experiment: how much trailing silence does the model need before + /// it commits sentence-final punctuation? (Answer when tuned: ≥600 ms — + /// see LIVE_SILENCE_FLUSH_FRAMES.) + /// Run: cargo test silence_vs_punctuation -- --ignored --nocapture + /// Setup: say -v Samantha "is there anything we can do about that?" \ + /// -o /tmp/dict_q.wav --data-format=LEF32@16000 (same for dict_s) + #[test] + #[ignore] + fn silence_vs_punctuation() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = + Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for path in ["/tmp/dict_q.wav", "/tmp/dict_s.wav"] { + let bytes = std::fs::read(path).unwrap(); + // Naive WAV parse: find the "data" chunk, samples are f32 LE. + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!("=== {path} ({} samples) ===", samples.len()); + for silence_ms in [0usize, 100, 300, 600] { + let mut buf = samples.clone(); + buf.extend(std::iter::repeat_n(0.0f32, 16 * silence_ms)); + println!( + " +{silence_ms}ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + // Real mic "silence" is room noise, not zeros — simulate with + // low-level deterministic pseudo-noise (LCG; no rand dep) at + // amplitudes well below the VAD speech threshold. + for amp in [0.002f32, 0.01, 0.03] { + let noise = |len: usize| { + let mut state = 0x2545_f491u32; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let mut buf = samples.clone(); + buf.extend(noise(16 * 608)); + println!(" +608ms noise(amp {amp}): {:?}", decode_speech(&buf, &recognizer)); + buf.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " +608ms noise(amp {amp}) + 600ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + } + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 6a2eec75fb..d48c766a88 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ mod archive; mod builderlab; mod commands; mod deep_link; +mod dictation; mod event_sync; mod events; mod huddle; @@ -39,6 +40,7 @@ use deep_link::{ use huddle::audio_output::{ get_audio_output_device, list_audio_output_devices, set_audio_output_device, }; +use dictation::{push_dictation_pcm, start_dictation, stop_dictation}; use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, @@ -352,6 +354,7 @@ pub fn run() { }) .manage(build_app_state()) .manage(ClipboardState::new()) + .manage(dictation::DictationState::default()) .manage(PendingCommunityDeepLinks::default()) .manage(BuilderlabSession::default()) .manage(BuilderlabLogin::default()) @@ -852,6 +855,9 @@ pub fn run() { end_huddle, get_huddle_state, push_audio_pcm, + start_dictation, + stop_dictation, + push_dictation_pcm, reconnect_huddle_audio, start_stt_pipeline, set_huddle_transcription_enabled, diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 677f4ac0a0..dcb92bf4b9 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -51,6 +51,7 @@ export type AudioWorkletHandle = { export async function setupAudioWorklet( audioTrack: MediaStreamTrack, initialTransmitting = true, + command = "push_audio_pcm", ): Promise { const audioContext = new AudioContext({ sampleRate: 48000 }); @@ -89,7 +90,7 @@ export async function setupAudioWorklet( // Create a zero-copy Uint8Array view over the same underlying buffer. // Rust reinterprets the bytes as f32 on the other side. invokeRawBinary( - "push_audio_pcm", + command, new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), ).catch(() => { /* silently drop — Rust handles backpressure */ diff --git a/desktop/src/features/messages/lib/transcriptDiff.test.mjs b/desktop/src/features/messages/lib/transcriptDiff.test.mjs new file mode 100644 index 0000000000..fb6eb510c4 --- /dev/null +++ b/desktop/src/features/messages/lib/transcriptDiff.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { transcriptDiff } from "./transcriptDiff.ts"; + +test("identical decode is a no-op", () => { + assert.deepEqual(transcriptDiff("hello world", "hello world"), { + keep: 11, + deleteLen: 0, + insert: "", + }); +}); + +test("append-only decode inserts just the tail", () => { + assert.deepEqual(transcriptDiff("hello", "hello world"), { + keep: 5, + deleteLen: 0, + insert: " world", + }); +}); + +test("mid-phrase revision rewrites from the divergence point", () => { + assert.deepEqual(transcriptDiff("space batoot", "space bar to"), { + keep: 8, + deleteLen: 4, + insert: "r to", + }); +}); + +test("shrinking decode deletes the stale tail", () => { + assert.deepEqual(transcriptDiff("hello worlds", "hello wor"), { + keep: 9, + deleteLen: 3, + insert: "", + }); +}); + +test("first partial of a phrase inserts everything", () => { + assert.deepEqual(transcriptDiff("", "hello"), { + keep: 0, + deleteLen: 0, + insert: "hello", + }); +}); + +test("final commit appends only the trailing space", () => { + assert.deepEqual(transcriptDiff("hello world", "hello world "), { + keep: 11, + deleteLen: 0, + insert: " ", + }); +}); diff --git a/desktop/src/features/messages/lib/transcriptDiff.ts b/desktop/src/features/messages/lib/transcriptDiff.ts new file mode 100644 index 0000000000..df56946f15 --- /dev/null +++ b/desktop/src/features/messages/lib/transcriptDiff.ts @@ -0,0 +1,19 @@ +/** Minimal edit turning the currently shown phrase text into the next decode: + * keep the longest common prefix, replace only the tail. Live re-decodes are + * usually append-only (or identical, between words) — diffing means a stable + * decode touches nothing, so the caret and DOM stay still instead of being + * rewritten every ~300 ms. */ +export function transcriptDiff( + shown: string, + next: string, +): { keep: number; deleteLen: number; insert: string } { + let keep = 0; + while ( + keep < shown.length && + keep < next.length && + shown.charCodeAt(keep) === next.charCodeAt(keep) + ) { + keep += 1; + } + return { keep, deleteLen: shown.length - keep, insert: next.slice(keep) }; +} diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts new file mode 100644 index 0000000000..f6dd2bbc2a --- /dev/null +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -0,0 +1,344 @@ +import * as React from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { Selection } from "@tiptap/pm/state"; +import type { Editor } from "@tiptap/react"; +import { toast } from "sonner"; + +import { + setupAudioWorklet, + type AudioWorkletHandle, +} from "@/features/huddle/lib/audioWorklet"; +import { transcriptDiff } from "./transcriptDiff"; + +export type DictationStatus = "idle" | "starting" | "recording"; + +/** How long after stop() this hook still claims incoming transcripts. + * The Rust worker flushes the trailing phrase after the key is released; + * decoding can take a couple of seconds. */ +// ponytail: time-based ownership window; replace with a session id threaded +// through the Rust event if two composers ever dictate back-to-back within 15s. +const OWNERSHIP_DECAY_MS = 15_000; + +/** Hold plain Space this long to start dictation. A quick tap still types a + * space as normal — only key-repeats during the hold are suppressed. */ +const SPACE_HOLD_MS = 1500; + +type Session = { handle: AudioWorkletHandle; stream: MediaStream }; + +/** Payload of the Rust `dictation-transcript` event. Partials re-decode the + * in-progress phrase (~every 300 ms) and replace each other; a final commits + * the phrase. */ +type TranscriptPayload = { text: string; final: boolean }; + +/** The dictation stream's place in the doc. `anchor` is where the current + * phrase starts; `partialText` is the phrase text currently shown there + * (empty right after a phrase commits). Kept to verify the region is still + * ours before replacing it. */ +type StreamState = { anchor: number; partialText: string }; + +/** + * Hold-to-talk streaming dictation for the message composer. + * + * start(): starts the Rust STT session (`start_dictation`), opens the mic via + * getUserMedia, and streams PCM through the existing AudioWorklet to + * `push_dictation_pcm`. The Rust pipeline emits `dictation-transcript` events + * while the user talks: partials replace the in-progress phrase in place (so + * words appear as they're spoken), finals commit it. stop(): releases the mic; + * Rust flushes the trailing phrase, which arrives shortly after. + * + * Triggers: hold plain Space for SPACE_HOLD_MS (quick tap still types a + * space), hold ⌃Space (instant start), or the toolbar mic button (click + * toggle). + * + * Multiple composers may mount this hook — only the one that started the + * session inserts the transcript (ownsRef), and Rust rejects a second + * concurrent session. + */ +export function useDictation(editor: Editor | null) { + const [status, setStatus] = React.useState("idle"); + const sessionRef = React.useRef(null); + const streamRef = React.useRef(null); + // Incremented on every start/stop — detects "released before start finished". + const genRef = React.useRef(0); + const ownsRef = React.useRef(false); + const ownershipTimerRef = React.useRef(null); + // True while dictation was started by a held key (Space or ⌃Space). + const keyHeldRef = React.useRef(false); + // Pending "Space held long enough?" timer. + const spaceTimerRef = React.useRef(null); + const editorRef = React.useRef(editor); + editorRef.current = editor; + + // Stream transcripts from the Rust pipeline into the editor: each partial + // replaces the previous partial of the same phrase in place, and the final + // commits it (with a trailing space) so the next phrase starts fresh. + // + // The anchor is read from the live selection ONCE (first transcript of the + // session); after that, phrases flow strictly in sequence from it. Never + // re-read the selection or call focus() per event — chasing DOM focus and + // selection every ~300 ms made the caret flick around while talking. + React.useEffect(() => { + const unlisten = listen( + "dictation-transcript", + (event) => { + if (!ownsRef.current) return; + const editor = editorRef.current; + if (!editor) return; + const text = event.payload.text.trim(); + if (!text) return; + const insert = event.payload.final ? `${text} ` : text; + + const doc = editor.state.doc; + let stream = streamRef.current; + if (stream) { + const end = stream.anchor + stream.partialText.length; + const intact = + end <= doc.content.size && + doc.textBetween(stream.anchor, end) === stream.partialText; + if (!intact) { + // The user edited around the stream (possible between key release + // and the trailing final) — resume at the end of the doc. + stream = { anchor: Selection.atEnd(doc).from, partialText: "" }; + } + } else { + // First transcript of the session: anchor at the cursor. + stream = { anchor: editor.state.selection.from, partialText: "" }; + } + + // Only rewrite the part of the phrase that actually changed. Most + // re-decodes append or are identical — a full delete+reinsert every + // ~300 ms churned the DOM and selection even when nothing changed, + // which is what made the caret visibly bounce while talking. + const { anchor } = stream; + const diff = transcriptDiff(stream.partialText, insert); + if (diff.deleteLen > 0 || diff.insert.length > 0) { + const from = anchor + diff.keep; + const chain = editor.chain(); + if (diff.deleteLen > 0) { + chain.deleteRange({ from, to: from + diff.deleteLen }); + } + if (diff.insert.length > 0) { + // Insert as a plain text node — never parse transcript as HTML. + chain.insertContentAt(from, { type: "text", text: diff.insert }); + } + // Pin the caret to the end of the dictated text. + chain.setTextSelection(anchor + insert.length).run(); + } + streamRef.current = event.payload.final + ? { anchor: anchor + insert.length, partialText: "" } + : { anchor, partialText: insert }; + // Hide the blinking caret while words are streaming in — it reads as + // noise next to live-updating text (chl request). It comes back at + // each pause (phrase commit) and when dictation stops. + editor.view.dom.style.caretColor = event.payload.final + ? "" + : "transparent"; + }, + ); + return () => { + void unlisten.then((fn) => fn()); + }; + }, []); + + const teardownSession = React.useCallback(() => { + const session = sessionRef.current; + sessionRef.current = null; + if (session) { + session.handle.stop(); + for (const track of session.stream.getTracks()) { + track.stop(); + } + } + }, []); + + const stop = React.useCallback(() => { + genRef.current += 1; + teardownSession(); + setStatus("idle"); + // Restore the caret even if no final ever arrives for the last phrase. + const dom = editorRef.current?.view.dom; + if (dom) dom.style.caretColor = ""; + void invoke("stop_dictation").catch(() => { + /* nothing to stop */ + }); + // Keep claiming transcripts briefly — the flush arrives after stop. + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + } + ownershipTimerRef.current = window.setTimeout(() => { + ownsRef.current = false; + ownershipTimerRef.current = null; + }, OWNERSHIP_DECAY_MS); + }, [teardownSession]); + + const start = React.useCallback(async (): Promise => { + if (sessionRef.current) return; + const gen = ++genRef.current; + // A stream left over from a previous session (final never arrived, or the + // doc changed) must not swallow this session's first phrase. + streamRef.current = null; + setStatus("starting"); + // Tracks whether the Rust session was started, so cleanup on a later + // failure never kills a session owned by another composer. + let rustSessionStarted = false; + try { + // Rust first: fails fast if the model is missing or a session is live. + await invoke("start_dictation"); + rustSessionStarted = true; + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const track = stream.getAudioTracks()[0]; + if (!track) { + for (const t of stream.getTracks()) t.stop(); + throw new Error("No microphone input available"); + } + const handle = await setupAudioWorklet(track, true, "push_dictation_pcm"); + if (gen !== genRef.current) { + // Released (or restarted) while we were setting up — undo. + handle.stop(); + for (const t of stream.getTracks()) t.stop(); + return; + } + sessionRef.current = { handle, stream }; + ownsRef.current = true; + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + ownershipTimerRef.current = null; + } + setStatus("recording"); + } catch (error) { + if (gen === genRef.current) { + setStatus("idle"); + if (rustSessionStarted) { + void invoke("stop_dictation").catch(() => {}); + } + } + throw error instanceof Error ? error : new Error(String(error)); + } + }, []); + + // Release the mic if the composer unmounts mid-recording. + React.useEffect(() => { + return () => { + if (sessionRef.current) { + teardownSession(); + void invoke("stop_dictation").catch(() => {}); + } + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + } + if (spaceTimerRef.current !== null) { + window.clearTimeout(spaceTimerRef.current); + } + }; + }, [teardownSession]); + + const startWithToast = React.useCallback(() => { + void start().catch((error: Error) => { + toast.error(error.message || "Could not start dictation"); + }); + }, [start]); + + /** Mic button click — start when idle, stop otherwise. */ + const statusRef = React.useRef(status); + statusRef.current = status; + const toggle = React.useCallback(() => { + if (statusRef.current === "idle") { + startWithToast(); + } else { + stop(); + } + }, [startWithToast, stop]); + + // Hold Space (or ⌃Space) = push-to-talk. Keydown comes from the composer + // (so only the focused composer starts); the release can land anywhere — or + // nowhere, on focus loss — so keyup/blur are handled on window. + React.useEffect(() => { + const release = () => { + if (spaceTimerRef.current !== null) { + window.clearTimeout(spaceTimerRef.current); + spaceTimerRef.current = null; + } + if (!keyHeldRef.current) return; + keyHeldRef.current = false; + stop(); + }; + const onKeyUp = (event: KeyboardEvent) => { + if (event.code === "Space" || event.key === "Control") release(); + }; + // Capture phase: while a hold is pending or live, swallow Space + // key-repeats before they reach ANY element. If focus drifts off the + // composer mid-hold, leaked repeats would click focused buttons, scroll + // the page, or type spaces into the stream. + const onKeyDownCapture = (event: KeyboardEvent) => { + if ( + event.code === "Space" && + event.repeat && + (keyHeldRef.current || spaceTimerRef.current !== null) + ) { + event.preventDefault(); + event.stopPropagation(); + } + }; + window.addEventListener("keydown", onKeyDownCapture, true); + window.addEventListener("keyup", onKeyUp); + window.addEventListener("blur", release); + return () => { + window.removeEventListener("keydown", onKeyDownCapture, true); + window.removeEventListener("keyup", onKeyUp); + window.removeEventListener("blur", release); + }; + }, [stop]); + + /** Returns true when the event was consumed. Handles both triggers: + * ⌃Space (instant) and plain Space held for SPACE_HOLD_MS. */ + const handleComposerKeyDown = React.useCallback( + (event: React.KeyboardEvent): boolean => { + if (event.code !== "Space" || event.metaKey || event.altKey) { + return false; + } + + // ⌃Space — instant hold-to-talk. + if (event.ctrlKey) { + event.preventDefault(); + if (!event.repeat && !keyHeldRef.current) { + keyHeldRef.current = true; + startWithToast(); + } + return true; + } + if (event.shiftKey) return false; + + // Plain Space — a quick tap types a space as normal; holding for + // SPACE_HOLD_MS starts dictation. Key-repeats are swallowed while the + // hold is pending or active so a hold doesn't spray spaces. + if (event.repeat) { + if (spaceTimerRef.current !== null || keyHeldRef.current) { + event.preventDefault(); + return true; + } + return false; + } + if (spaceTimerRef.current === null && !keyHeldRef.current) { + spaceTimerRef.current = window.setTimeout(() => { + spaceTimerRef.current = null; + keyHeldRef.current = true; + // The keydown typed a space before the hold was recognised — the + // hold means "talk", not "space", so take it back. + const ed = editorRef.current; + if (ed) { + const { from } = ed.state.selection; + if (from > 0 && ed.state.doc.textBetween(from - 1, from) === " ") { + ed.commands.deleteRange({ from: from - 1, to: from }); + } + } + startWithToast(); + }, SPACE_HOLD_MS); + } + return false; // let the space type normally + }, + [startWithToast], + ); + + return { status, start, stop, toggle, handleComposerKeyDown }; +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 8c0c574049..73ee099e98 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -1,6 +1,7 @@ import * as React from "react"; import { EditorContent } from "@tiptap/react"; +import { useDictation } from "@/features/messages/lib/useDictation"; import { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; import { handleAgentSnapshotPaste } from "@/features/messages/lib/agentSnapshotClipboard"; import { useComposerAutofocus } from "@/features/messages/lib/useComposerAutofocus"; @@ -783,8 +784,13 @@ function MessageComposerImpl({ // Plain Enter → submit is now handled inside the Tiptap `submitOnEnter` // extension (fires before ProseMirror's splitBlock). This wrapper only // handles autocomplete arrow/enter keys and Escape for edit mode. + // ── Dictation (hold ⌃Space or mic button → speech-to-text) ───────── + const dictation = useDictation(richText.editor); + const handleEditorKeyDown = React.useCallback( (event: React.KeyboardEvent) => { + if (dictation.handleComposerKeyDown(event)) return; + // Let autocomplete handle keys first const emojiResult = emojiAutocomplete.handleEmojiKeyDown(event); if (emojiResult.handled) { @@ -835,6 +841,7 @@ function MessageComposerImpl({ linkEditor.isCardOpen, linkEditor.focusCardFirstControl, onCancelEdit, + dictation.handleComposerKeyDown, ], ); @@ -1074,6 +1081,7 @@ function MessageComposerImpl({ void; + /** When omitted, the dictation mic button is not rendered. */ + onDictationToggle?: () => void; onEmojiPickerOpenChange: (open: boolean) => void; onEmojiSelect: (emoji: string) => void; onFormattingToggle: (pressed: boolean) => void; @@ -224,6 +231,42 @@ export const MessageComposerToolbar = React.memo( Formatting + {onDictationToggle ? ( + + + + + + {dictationStatus === "recording" + ? "Stop dictation" + : "Dictate (hold Space, or ⌃Space)"} + + + ) : null} )} From 32a8f16e5a9c42b86d1fd53fe8f83b6dc90e0551 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 14:32:07 +1000 Subject: [PATCH 02/10] fix(desktop): dictation audio drops and noise hallucinations in live STT Three fixes for live (dictation) mode, found by measuring a 35 s ramble through the stt test harness: - Wall-clock decode hold: after every decode (partials and flushes) the worker waits 2x that decode's duration before the next partial. The previous samples-based back-off collapsed into decode storms after a long flush stall, overflowing the audio queue and silently dropping mic audio - which spliced the phrase and made later decodes delete words earlier partials had shown (pause-then-resume text loss). - Audio queue deepened 50 -> 300 slots (5 s -> 30 s) so the worst-case flush stall of a 30 s phrase cannot overflow it. - Onset debounce (~96 ms of sustained VAD speech) before opening a phrase, so keyboard clicks and background blips no longer decode to hallucinated filler ('yeah', 'uh'). Onset audio is buffered while it proves itself; huddle mode untouched. - Empty finals now propagate to the frontend so a partial shown for a phrase that decoded to nothing is cleared instead of stranded. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/src-tauri/src/huddle/stt.rs | 110 +++++++++++------- .../src/features/messages/lib/useDictation.ts | 7 +- 2 files changed, 73 insertions(+), 44 deletions(-) diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index b5ddb3769c..faf5bbf65f 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -35,9 +35,11 @@ use super::models::SttFamily; // ── Public pipeline handle ──────────────────────────────────────────────────── -/// Bounded audio queue capacity. -/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. -const AUDIO_QUEUE_DEPTH: usize = 50; +/// Bounded audio queue capacity: 100 ms batches at 48 kHz ≈ 19 KB each → +/// 300 slots ≈ 30 s / ~6 MB backlog, riding out the worst live-mode decode +/// stall (~2 s flush of a 30 s phrase). Overflow silently drops mic audio, +/// splicing the phrase so later decodes delete words already shown. +const AUDIO_QUEUE_DEPTH: usize = 300; /// Maximum speech buffer size: 30 seconds at 16 kHz. /// Prevents OOM if VAD stays in speech mode (noisy environment). @@ -211,17 +213,22 @@ const RECV_TIMEOUT: Duration = Duration::from_millis(50); /// Live-transcript mode: re-decode the in-progress phrase every this many new /// 16 kHz samples (~300 ms) so words stream out while the user is talking. /// The model is an offline recognizer, so "streaming" is repeated decoding of -/// the growing phrase buffer — phrases reset at every VAD pause, but a long -/// unbroken ramble grows toward MAX_SPEECH_SAMPLES. Decode cost grows with the -/// buffer (~70 ms per second of audio for the 0.6B model, measured in -/// `v3_long_buffer`), so this is only the MINIMUM step: the worker also waits -/// at least the previous decode's duration worth of new audio before the next -/// partial. Without that back-off the worker falls behind real time past ~5 s -/// of phrase, the bounded audio queue overflows, and mic audio is silently -/// dropped — splicing the phrase and making later decodes remove words that -/// earlier partials showed. +/// the growing phrase buffer, and decode cost grows with it (~70 ms per second +/// of audio for the 0.6B model, measured in `v3_long_buffer`). So this is only +/// the MINIMUM step: after every decode (partials AND phrase flushes) the +/// worker holds off 2× that decode's duration before the next partial, capping +/// decode duty at ~50%. The hold must be WALL-CLOCK, not new-samples: a +/// post-stall backlog arrives faster than real time, so samples-based back-off +/// collapses into decode storms that overflow the audio queue. const PARTIAL_DECODE_STEP: usize = 4800; +/// Consecutive VAD speech frames (16 ms each, ~96 ms total) required to open a +/// phrase in live (dictation) mode. Single-frame blips (keyboard click, +/// breath, background noise) make Parakeet hallucinate filler ("yeah", "uh"); +/// real speech sustains the VAD easily, and the onset audio is buffered while +/// it proves itself, so nothing is lost. Huddle mode is untouched. +const ONSET_DEBOUNCE_FRAMES: usize = 6; + /// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. /// Previous value (200 ms) was eating the first word when the user spoke @@ -364,10 +371,12 @@ fn stt_worker( let mut barge_in_frames: usize = 0; // Live mode: speech_buf length at the last partial decode. let mut last_partial_len: usize = 0; - // Live mode: minimum new samples before the next partial decode — - // PARTIAL_DECODE_STEP, raised to the last decode's duration so partial - // decoding never outruns real time (see PARTIAL_DECODE_STEP docs). - let mut partial_step: usize = PARTIAL_DECODE_STEP; + // Live mode: no partial decode before this instant (see + // PARTIAL_DECODE_STEP — the 2× wall-clock hold after every decode). + let mut decode_hold_until = std::time::Instant::now(); + // Live mode: VAD-positive frames buffered while a speech onset proves + // itself (ONSET_DEBOUNCE_FRAMES) before opening a phrase. + let mut onset_buf: Vec = Vec::new(); // Live mode: most recent partial decode of the current phrase that ended // with terminal punctuation — see prefer_punctuated. let mut punct_partial: Option = None; @@ -411,7 +420,6 @@ fn stt_worker( silence_frames = 0; in_speech = false; last_partial_len = 0; - partial_step = PARTIAL_DECODE_STEP; } ptt_was_active = ptt_now; } @@ -454,7 +462,8 @@ fn stt_worker( ptt_active.as_ref(), live_tx.as_ref(), &mut last_partial_len, - &mut partial_step, + &mut decode_hold_until, + &mut onset_buf, &mut punct_partial, ); } @@ -520,7 +529,8 @@ fn process_16k_samples( ptt_active: Option<&Arc>, live_tx: Option<&tokio_mpsc::Sender<(String, bool)>>, last_partial_len: &mut usize, - partial_step: &mut usize, + decode_hold_until: &mut std::time::Instant, + onset_buf: &mut Vec, punct_partial: &mut Option, ) { leftover.extend_from_slice(samples); @@ -601,17 +611,28 @@ fn process_16k_samples( if is_speech { *silence_frames = 0; + if !*in_speech && live_tx.is_some() { + // Live mode: debounce the onset (see ONSET_DEBOUNCE_FRAMES). + // ponytail: any non-speech frame resets it; add grace if real onsets clip. + onset_buf.extend_from_slice(&frame); + if onset_buf.len() < ONSET_DEBOUNCE_FRAMES * VAD_FRAME_SAMPLES { + continue; + } + speech_buf.append(onset_buf); + } else { + speech_buf.extend_from_slice(&frame); + } *in_speech = true; - speech_buf.extend_from_slice(&frame); // OOM guard: flush and reset if the buffer exceeds 30 s of audio. if speech_buf.len() >= MAX_SPEECH_SAMPLES { + let t0 = std::time::Instant::now(); flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; *in_speech = false; *last_partial_len = 0; - *partial_step = PARTIAL_DECODE_STEP; } } else if *in_speech { // Still accumulate during brief silence gaps. @@ -628,33 +649,37 @@ fn process_16k_samples( SILENCE_FLUSH_FRAMES }; if ptt_active.is_none() && *silence_frames >= flush_frames { - // End of utterance — transcribe. + // End of utterance — transcribe. Flush decodes join the same + // wall-clock hold as partials (see PARTIAL_DECODE_STEP). + let t0 = std::time::Instant::now(); flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; *in_speech = false; *last_partial_len = 0; - *partial_step = PARTIAL_DECODE_STEP; } + } else { + // Not in speech: discard the frame, and drop any pending onset — + // the blip ended before it proved itself to be speech. + onset_buf.clear(); } - // If not in speech and not accumulating, just discard the frame. // Live mode: re-decode the in-progress phrase every PARTIAL_DECODE_STEP // new samples so words appear while the user is still talking. Later // partials of the same phrase supersede earlier ones — the frontend // replaces the partial region — so decode wobble is self-correcting. if let Some(tx) = live_tx { - if *in_speech && speech_buf.len() >= *last_partial_len + *partial_step { + if *in_speech + && speech_buf.len() >= *last_partial_len + PARTIAL_DECODE_STEP + && std::time::Instant::now() >= *decode_hold_until + { *last_partial_len = speech_buf.len(); let t0 = std::time::Instant::now(); let text = decode_speech(speech_buf, recognizer); - // Wait at least 2× this decode's duration (in samples: 16/ms - // at 16 kHz) before the next partial, capping decode duty - // cycle at ~50% so the worker can't fall behind the mic and - // overflow the audio queue (which silently drops samples). - // 1× measured at ~100% duty on a 35 s ramble — no headroom. - *partial_step = - PARTIAL_DECODE_STEP.max(32 * t0.elapsed().as_millis() as usize); + // 2× wall-clock hold (see PARTIAL_DECODE_STEP); 1× measured + // at ~100% duty on a 35 s ramble — no headroom. + *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; if !text.is_empty() { if text.ends_with(['.', '?', '!', ',']) { *punct_partial = Some(text.clone()); @@ -731,16 +756,17 @@ fn flush_to_stt( punct_hint: Option, ) { let text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); - if !text.is_empty() { - // In live mode finals ride the same channel as partials so the - // receiver never sees a stale partial after its final. - let send_err = match live_tx { - Some(tx) => tx.blocking_send((text, true)).err().map(|e| e.to_string()), - None => text_tx.blocking_send(text).err().map(|e| e.to_string()), - }; - if let Some(e) = send_err { - eprintln!("buzz-desktop: STT text channel closed: {e}"); - } + // In live mode finals ride the same channel as partials so the receiver + // never sees a stale partial after its final — and an EMPTY final is still + // sent so the frontend clears the partial shown for a phrase that decoded + // to nothing (noise); skipping it strands that partial on screen. + let send_err = match live_tx { + Some(tx) => tx.blocking_send((text, true)).err().map(|e| e.to_string()), + None if text.is_empty() => None, + None => text_tx.blocking_send(text).err().map(|e| e.to_string()), + }; + if let Some(e) = send_err { + eprintln!("buzz-desktop: STT text channel closed: {e}"); } } diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index f6dd2bbc2a..d396ee1e64 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -86,8 +86,11 @@ export function useDictation(editor: Editor | null) { const editor = editorRef.current; if (!editor) return; const text = event.payload.text.trim(); - if (!text) return; - const insert = event.payload.final ? `${text} ` : text; + // Empty PARTIALS carry no information — skip. An empty FINAL is a + // real signal: the phrase decoded to nothing (noise), so fall through + // and let the diff delete the partial that was shown for it. + if (!text && !event.payload.final) return; + const insert = event.payload.final && text ? `${text} ` : text; const doc = editor.state.doc; let stream = streamRef.current; From 29cad8c162c7c91d4f52a8bba63a52f69dfc637a Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 14:46:12 +1000 Subject: [PATCH 03/10] feat(dictation): Voice & dictation settings with STT model picker Adds a "Voice & dictation" section under Personal settings with a Standard (parakeet-en) / Enhanced (parakeet-v3) model picker: - models.rs: per-model download slots (startup model shares its slot), SttModelInfo for the frontend, resolve_dictation_model (preferred if ready, else startup model), start_stt_download_for. - dictation.rs: start_dictation takes an optional model id and rebuilds the pipeline when the model changes; new get_dictation_models and download_dictation_model commands. - Frontend: localStorage-backed preference (dictationModelPreference), passed to start_dictation per session; VoiceSettingsCard with download-on-select, progress polling, and fallback note while the chosen model is still downloading. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/scripts/check-file-sizes.mjs | 4 +- desktop/src-tauri/src/dictation.rs | 49 ++++- desktop/src-tauri/src/huddle/models.rs | 140 ++++++++++++-- desktop/src-tauri/src/lib.rs | 7 +- .../messages/lib/dictationModelPreference.ts | 75 ++++++++ .../src/features/messages/lib/useDictation.ts | 3 +- .../features/settings/ui/SettingsPanels.tsx | 11 ++ .../src/features/settings/ui/SettingsView.tsx | 1 + .../settings/ui/VoiceSettingsCard.tsx | 181 ++++++++++++++++++ 9 files changed, 446 insertions(+), 25 deletions(-) create mode 100644 desktop/src/features/messages/lib/dictationModelPreference.ts create mode 100644 desktop/src/features/settings/ui/VoiceSettingsCard.tsx diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 066cd9de83..21674d2660 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -80,7 +80,9 @@ const overrides = new Map([ // entries with pinned checksums + license texts) + select_stt_model and its // unit tests. Load-bearing model-selection surface from the upstream PR; // queued to split when #2520 lands and this port is dropped in the rebase. - ["src-tauri/src/huddle/models.rs", 1320], + // composer-dictation model picker adds per-model slots + SttModelInfo + + // resolve_dictation_model (~110 lines); split rides the same rebase. + ["src-tauri/src/huddle/models.rs", 1430], // composer-dictation: live-transcript mode (live_tx partial re-decode loop), // sticky-punctuation heuristic + unit tests, and two #[ignore]d decode // experiments (silence/noise vs punctuation, v3 comparison) that document diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs index 351505115b..51be5808cf 100644 --- a/desktop/src-tauri/src/dictation.rs +++ b/desktop/src-tauri/src/dictation.rs @@ -49,7 +49,9 @@ const FLUSH_SILENCE_BYTES: usize = 4 * 48_000; /// memory ever matters. #[derive(Default)] pub struct DictationState { - pipeline: Mutex>, + /// Loaded pipeline tagged with the model id it was built from, so a + /// changed model preference rebuilds it on the next start. + pipeline: Mutex>, /// True while a session is live — blocks a second concurrent session and /// gates `push_dictation_pcm` so late batches can't dirty the VAD buffer. active: AtomicBool, @@ -57,6 +59,10 @@ pub struct DictationState { /// Start (or resume) a dictation session. /// +/// `model` is the user's model preference (Settings → Voice & dictation); it +/// is used when it names a registry model that is ready on disk, otherwise +/// the startup-selected model is used. +/// /// Errors if the speech model isn't downloaded yet or a session is already /// active (prevents two composers opening two mic streams into one pipeline). #[tauri::command] @@ -64,6 +70,7 @@ pub fn start_dictation( app: tauri::AppHandle, state: tauri::State<'_, DictationState>, app_state: tauri::State<'_, AppState>, + model: Option, ) -> Result<(), String> { if state.active.load(Ordering::Acquire) { return Err("dictation already active".to_string()); @@ -81,26 +88,31 @@ pub fn start_dictation( .lock() .map_err(|_| "dictation state poisoned".to_string())?; - // Clear a dead pipeline (init failure or crash) so retry works. - if guard.as_ref().is_some_and(|p| p.is_finished()) { + let (model_id, model_dir, family) = models::resolve_dictation_model(model.as_deref()) + .ok_or("speech model still downloading — try again in a moment".to_string())?; + + // Clear a dead pipeline (init failure or crash) so retry works, and a + // pipeline built from a different model so a preference change applies. + if guard + .as_ref() + .is_some_and(|(id, p)| p.is_finished() || *id != model_id) + { *guard = None; } if guard.is_none() { - let model_dir = models::stt_model_dir() - .ok_or("speech model still downloading — try again in a moment".to_string())?; let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::<(String, bool)>(64); // In live mode all transcripts arrive on live_rx; the pipeline's own // text receiver stays silent and is dropped here. let (pipeline, _text_rx) = SttPipeline::new( model_dir, - models::stt_model_family(), + family, Arc::new(AtomicBool::new(false)), // no TTS during dictation None, None, // continuous VAD segments phrases at natural pauses Some(live_tx), )?; - *guard = Some(pipeline); + *guard = Some((model_id, pipeline)); // Forward transcripts to the webview until the pipeline is dropped. tauri::async_runtime::spawn(async move { @@ -126,12 +138,31 @@ pub fn stop_dictation(state: tauri::State<'_, DictationState>) { return; } if let Ok(guard) = state.pipeline.lock() { - if let Some(ref pipeline) = *guard { + if let Some((_, ref pipeline)) = *guard { let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); } } } +/// Registry STT models with live install status, for the Settings picker. +#[tauri::command] +pub fn get_dictation_models() -> Result, String> { + Ok(models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")? + .stt_model_infos()) +} + +/// Start a background download of a specific STT model (Settings picker). +#[tauri::command] +pub fn download_dictation_model( + id: String, + app_state: tauri::State<'_, AppState>, +) -> Result<(), String> { + models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")? + .start_stt_download_for(&id, app_state.http_client.clone()) +} + /// Receive raw PCM (f32 LE, 48 kHz mono) from the AudioWorklet. /// Mirrors `push_audio_pcm` but feeds the dictation pipeline. #[tauri::command] @@ -157,7 +188,7 @@ pub fn push_dictation_pcm( .pipeline .lock() .map_err(|_| "dictation state poisoned".to_string())?; - if let Some(ref pipeline) = *guard { + if let Some((_, ref pipeline)) = *guard { pipeline.push_audio(bytes.to_vec())?; } Ok(()) diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 04521c5f04..4c2e3e1aae 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -425,6 +425,19 @@ pub struct VoiceModelStatus { pub tts: ModelStatus, } +/// One `STT_MODELS` registry entry with live status, for the dictation model +/// picker in Settings. +#[derive(Debug, Clone, Serialize)] +pub struct SttModelInfo { + pub id: &'static str, + pub languages: &'static str, + pub multilingual: bool, + /// `true` for the model the app selected at startup (override / locale / + /// default) — what dictation uses when no explicit preference is set. + pub selected: bool, + pub status: ModelStatus, +} + // ── Safe archive extraction ─────────────────────────────────────────────────── /// Extract a .tar.bz2 archive safely using Rust-native crates. @@ -741,6 +754,9 @@ pub struct ModelManager { /// The STT model selected for this process (override / locale / default). stt_model: &'static SttModel, stt: ModelSlot, + /// One slot per `STT_MODELS` entry (dictation model picker). The entry for + /// the startup-selected model shares `stt`'s Arcs so status has one truth. + stt_slots: Vec, tts: ModelSlot, } @@ -758,14 +774,26 @@ impl ModelManager { "buzz-desktop: STT model '{}' selected ({})", stt_model.id, stt_model.languages ); + let stt = ModelSlot::new( + stt_model.dir_name, + stt_expected_files(stt_model), + stt_model.version, + ); + let stt_slots = STT_MODELS + .iter() + .map(|m| { + if m.id == stt_model.id { + stt.clone() + } else { + ModelSlot::new(m.dir_name, stt_expected_files(m), m.version) + } + }) + .collect(); Some(Self { models_dir, stt_model, - stt: ModelSlot::new( - stt_model.dir_name, - stt_expected_files(stt_model), - stt_model.version, - ), + stt, + stt_slots, tts: ModelSlot::new( TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES.to_vec(), @@ -799,6 +827,75 @@ impl ModelManager { self.stt.take_ready() } + // ── Per-model STT accessors (dictation model picker) ───────────────────── + + /// Registry model + its slot by id (case-insensitive). + fn stt_slot_for(&self, id: &str) -> Option<(&'static SttModel, &ModelSlot)> { + STT_MODELS + .iter() + .zip(&self.stt_slots) + .find(|(m, _)| m.id.eq_ignore_ascii_case(id)) + } + + /// Registry entries with live status, for the dictation model picker. + pub fn stt_model_infos(&self) -> Vec { + STT_MODELS + .iter() + .zip(&self.stt_slots) + .map(|(m, slot)| SttModelInfo { + id: m.id, + languages: m.languages, + multilingual: m.multilingual, + selected: m.id == self.stt_model.id, + // A slot that never started a download this run still reports + // NotDownloaded in memory — trust the disk check first. + status: if slot.is_ready(&self.models_dir) { + ModelStatus::Ready + } else { + slot.status() + }, + }) + .collect() + } + + /// Resolve the model for a dictation session: the preferred id when known + /// and ready on disk, else the startup-selected model. `None` when nothing + /// is ready. + pub fn resolve_dictation_model( + &self, + preferred: Option<&str>, + ) -> Option<(&'static str, PathBuf, SttFamily)> { + if let Some((model, slot)) = preferred.and_then(|id| self.stt_slot_for(id)) { + if let Some(dir) = slot.dir_if_ready(&self.models_dir) { + return Some((model.id, dir, model.family)); + } + } + let dir = self.stt.dir_if_ready(&self.models_dir)?; + Some((self.stt_model.id, dir, self.stt_model.family)) + } + + /// Start a background download of a specific registry model. No-op if it + /// is already ready or downloading. + pub fn start_stt_download_for( + &self, + id: &str, + http_client: reqwest::Client, + ) -> Result<(), String> { + let (model, slot) = self + .stt_slot_for(id) + .ok_or_else(|| format!("unknown STT model id: {id}"))?; + let manager = self.clone(); + let slot_for_task = slot.clone(); + slot.start_download(&self.models_dir, http_client, "stt", move |client| { + async move { + manager + .download_stt_model_for(model, slot_for_task, client) + .await + } + }); + Ok(()) + } + // ── TTS accessors ───────────────────────────────────────────────────────── /// Path to the TTS model directory, or `None` if not ready. @@ -863,15 +960,25 @@ impl ModelManager { // ── Private download implementations ───────────────────────────────────── - /// Download, extract, and verify the STT model archive. + /// Download, extract, and verify the startup-selected STT model archive. async fn download_stt_model(&self, http_client: reqwest::Client) -> Result<(), String> { + self.download_stt_model_for(self.stt_model, self.stt.clone(), http_client) + .await + } + + /// Download, extract, and verify a specific STT model archive into `slot`. + async fn download_stt_model_for( + &self, + model: &'static SttModel, + slot: ModelSlot, + http_client: reqwest::Client, + ) -> Result<(), String> { tokio::fs::create_dir_all(&self.models_dir) .await .map_err(|e| format!("create models dir: {e}"))?; // Temp filenames derive from the final directory name to avoid colliding // with leftovers from any previous STT model (e.g. moonshine-tiny.*). - let model = self.stt_model; let archive_path = self.models_dir.join(format!("{}.tar.bz2", model.dir_name)); let temp_dir = self.models_dir.join(format!("{}.tmp", model.dir_name)); @@ -881,7 +988,7 @@ impl ModelManager { ); let response = fetch_url(&http_client, model.download_url, "stt archive").await?; - let slot = self.stt.clone(); + let progress_slot = slot.clone(); let bytes = download_file( response, &archive_path, @@ -891,7 +998,7 @@ impl ModelManager { if let Some(pct) = content_length.and_then(|total| (downloaded * 89).checked_div(total)) { - slot.set_status(ModelStatus::Downloading { + progress_slot.set_status(ModelStatus::Downloading { progress_percent: pct.min(89) as u8, }); } @@ -923,7 +1030,7 @@ impl ModelManager { } } - self.stt.set_status(ModelStatus::Downloading { + slot.set_status(ModelStatus::Downloading { progress_percent: 90, }); fresh_temp_dir(&temp_dir).await?; @@ -955,8 +1062,7 @@ impl ModelManager { } // verify_and_install takes the subdir (actual model files); temp_cleanup removes outer dir. - if let Err(e) = self - .stt + if let Err(e) = slot .verify_and_install(&self.models_dir, &extracted_subdir, Some(&temp_dir)) .await { @@ -975,7 +1081,7 @@ impl ModelManager { eprintln!( "buzz-desktop: STT model ready at {}", - self.stt.model_dir(&self.models_dir).display() + slot.model_dir(&self.models_dir).display() ); Ok(()) } @@ -1128,6 +1234,14 @@ pub fn is_stt_ready() -> bool { .unwrap_or(false) } +/// Resolve the model for a dictation session (see +/// `ModelManager::resolve_dictation_model`). +pub fn resolve_dictation_model( + preferred: Option<&str>, +) -> Option<(&'static str, PathBuf, SttFamily)> { + global_model_manager()?.resolve_dictation_model(preferred) +} + /// Best-effort cleanup of the legacy Moonshine STT model directory. /// /// Removes `~/.buzz/models/moonshine-tiny/` if present (~70 MB on disk). diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index d48c766a88..10b6b8f24b 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -40,7 +40,10 @@ use deep_link::{ use huddle::audio_output::{ get_audio_output_device, list_audio_output_devices, set_audio_output_device, }; -use dictation::{push_dictation_pcm, start_dictation, stop_dictation}; +use dictation::{ + download_dictation_model, get_dictation_models, push_dictation_pcm, start_dictation, + stop_dictation, +}; use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, @@ -858,6 +861,8 @@ pub fn run() { start_dictation, stop_dictation, push_dictation_pcm, + get_dictation_models, + download_dictation_model, reconnect_huddle_audio, start_stt_pipeline, set_huddle_transcription_enabled, diff --git a/desktop/src/features/messages/lib/dictationModelPreference.ts b/desktop/src/features/messages/lib/dictationModelPreference.ts new file mode 100644 index 0000000000..8be6b1a9c3 --- /dev/null +++ b/desktop/src/features/messages/lib/dictationModelPreference.ts @@ -0,0 +1,75 @@ +import * as React from "react"; + +/** + * User preference for the dictation speech-to-text model, as a model id from + * the Rust `STT_MODELS` registry (e.g. "parakeet-en", "parakeet-v3"). + * + * `null` means no explicit choice: dictation uses the model the app selected + * at startup (BUZZ_STT_MODEL override / system locale / English default). + * The preference is passed to `start_dictation`, which falls back to the + * startup-selected model when the preferred one isn't downloaded yet. + * + * Persisted in localStorage — a device-level preference, like the thread + * layout in threadViewModePreference.ts. + */ +export type DictationModelPreference = string | null; + +const STORAGE_KEY = "buzz.dictation.model"; + +const listeners = new Set<() => void>(); + +let preference: DictationModelPreference = readStoredPreference(); + +function readStoredPreference(): DictationModelPreference { + try { + return globalThis.localStorage?.getItem(STORAGE_KEY) ?? null; + } catch { + return null; + } +} + +function subscribe(listener: () => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +function getSnapshot(): DictationModelPreference { + return preference; +} + +function getServerSnapshot(): DictationModelPreference { + return null; +} + +/** Read the persisted dictation model preference outside of React. */ +export function getDictationModelPreference(): DictationModelPreference { + return preference; +} + +/** Update the dictation model preference and notify subscribed components. */ +export function setDictationModelPreference( + model: DictationModelPreference, +): void { + preference = model; + + try { + if (model === null) { + globalThis.localStorage?.removeItem(STORAGE_KEY); + } else { + globalThis.localStorage?.setItem(STORAGE_KEY, model); + } + } catch { + // Persistence is best-effort; the in-memory value still applies. + } + + for (const listener of listeners) { + listener(); + } +} + +/** The dictation model preference (null = app default). */ +export function useDictationModelPreference(): DictationModelPreference { + return React.useSyncExternalStore(subscribe, getSnapshot, getServerSnapshot); +} diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index d396ee1e64..09cbf2e73c 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -9,6 +9,7 @@ import { setupAudioWorklet, type AudioWorkletHandle, } from "@/features/huddle/lib/audioWorklet"; +import { getDictationModelPreference } from "./dictationModelPreference"; import { transcriptDiff } from "./transcriptDiff"; export type DictationStatus = "idle" | "starting" | "recording"; @@ -187,7 +188,7 @@ export function useDictation(editor: Editor | null) { let rustSessionStarted = false; try { // Rust first: fails fast if the model is missing or a session is live. - await invoke("start_dictation"); + await invoke("start_dictation", { model: getDictationModelPreference() }); rustSessionStarted = true; const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const track = stream.getAudioTracks()[0]; diff --git a/desktop/src/features/settings/ui/SettingsPanels.tsx b/desktop/src/features/settings/ui/SettingsPanels.tsx index a203269239..a292b893d3 100644 --- a/desktop/src/features/settings/ui/SettingsPanels.tsx +++ b/desktop/src/features/settings/ui/SettingsPanels.tsx @@ -13,6 +13,7 @@ import { LayoutTemplate, LockKeyhole, MessagesSquare, + Mic, MonitorCog, Moon, ShieldAlert, @@ -84,6 +85,7 @@ import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { ProfileSettingsCard } from "./ProfileSettingsCard"; import { UpdateChecker } from "../UpdateChecker"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; +import { VoiceSettingsCard } from "./VoiceSettingsCard"; export type SettingsSection = | "profile" @@ -94,6 +96,7 @@ export type SettingsSection = | "compute" | "appearance" | "shortcuts" + | "voice" | "hosted-communities" | "community-members" | "moderation" @@ -113,6 +116,7 @@ const SETTINGS_SECTION_VALUES: readonly SettingsSection[] = [ "compute", "appearance", "shortcuts", + "voice", "hosted-communities", "community-members", "moderation", @@ -195,6 +199,11 @@ export const settingsSections: SettingsSectionDescriptor[] = [ label: "Shortcuts", icon: Keyboard, }, + { + value: "voice", + label: "Voice & dictation", + icon: Mic, + }, { value: "hosted-communities", label: "Hosted communities", @@ -827,6 +836,8 @@ export function renderSettingsSection( return ; case "shortcuts": return ; + case "voice": + return ; case "hosted-communities": return ; case "community-members": diff --git a/desktop/src/features/settings/ui/SettingsView.tsx b/desktop/src/features/settings/ui/SettingsView.tsx index 90634e01f4..d42afe269a 100644 --- a/desktop/src/features/settings/ui/SettingsView.tsx +++ b/desktop/src/features/settings/ui/SettingsView.tsx @@ -59,6 +59,7 @@ const settingsNavGroups: Array<{ "appearance", "notifications", "shortcuts", + "voice", "custom-emoji", "local-archive", ], diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx new file mode 100644 index 0000000000..b0675cb200 --- /dev/null +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -0,0 +1,181 @@ +import * as React from "react"; +import { invoke } from "@tauri-apps/api/core"; +import { Check } from "lucide-react"; + +import { + setDictationModelPreference, + useDictationModelPreference, +} from "@/features/messages/lib/dictationModelPreference"; +import { cn } from "@/shared/lib/cn"; +import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; +import { SettingsSectionHeader } from "./SettingsSectionHeader"; + +/** Mirror of the Rust `ModelStatus` serde encoding (models.rs). */ +type ModelStatus = + | "not_downloaded" + | "ready" + | { downloading: { progress_percent: number } } + | { error: string }; + +/** Mirror of the Rust `SttModelInfo` (models.rs). */ +type SttModelInfo = { + id: string; + languages: string; + multilingual: boolean; + selected: boolean; + status: ModelStatus; +}; + +/** + * Display copy per registry model id. Unknown ids (a future registry + * addition) fall back to the id itself so the picker never hides a model. + */ +const MODEL_COPY: Record< + string, + { label: string; description: string; size: string } +> = { + "parakeet-en": { + label: "Standard", + description: "Fast, English only. Light on punctuation.", + size: "~120 MB", + }, + "parakeet-v3": { + label: "Enhanced", + description: + "Best accuracy and punctuation. 25 European languages, auto-detected.", + size: "~465 MB", + }, +}; + +function isDownloading( + status: ModelStatus, +): status is { downloading: { progress_percent: number } } { + return typeof status === "object" && "downloading" in status; +} + +function statusText(info: SttModelInfo): string { + if (info.status === "ready") return "Downloaded"; + if (isDownloading(info.status)) { + return `Downloading… ${info.status.downloading.progress_percent}%`; + } + const size = MODEL_COPY[info.id]?.size; + if (typeof info.status === "object" && "error" in info.status) { + return "Download failed — select to retry"; + } + return size ? `${size} download` : "Not downloaded"; +} + +export function VoiceSettingsCard() { + const preference = useDictationModelPreference(); + const [models, setModels] = React.useState(null); + const [loadError, setLoadError] = React.useState(null); + + const refresh = React.useCallback(async () => { + try { + setModels(await invoke("get_dictation_models")); + setLoadError(null); + } catch (e) { + setLoadError(e instanceof Error ? e.message : String(e)); + } + }, []); + + React.useEffect(() => { + void refresh(); + }, [refresh]); + + // Poll for download progress only while a download is running. + const downloading = models?.some((m) => isDownloading(m.status)) ?? false; + React.useEffect(() => { + if (!downloading) return; + const timer = window.setInterval(() => void refresh(), 1000); + return () => window.clearInterval(timer); + }, [downloading, refresh]); + + // With no explicit preference, dictation uses the startup-selected model. + const activeId = preference ?? models?.find((m) => m.selected)?.id ?? null; + const activeModel = models?.find((m) => m.id === activeId); + const fallbackModel = models?.find((m) => m.selected); + + const handleSelect = (info: SttModelInfo) => { + setDictationModelPreference(info.id); + if (info.status !== "ready") { + void invoke("download_dictation_model", { id: info.id }) + .then(refresh) + .catch((e) => setLoadError(e instanceof Error ? e.message : String(e))); + } + }; + + return ( +
+ + +

+ Dictation model +

+ + {(models ?? []).map((info) => { + const copy = MODEL_COPY[info.id] ?? { + label: info.id, + description: info.languages, + size: "", + }; + const active = info.id === activeId; + return ( + + + + ); + })} + + + {activeModel && activeModel.status !== "ready" && fallbackModel ? ( +

+ {MODEL_COPY[fallbackModel.id]?.label ?? fallbackModel.id} is used + until the download finishes. +

+ ) : null} + {loadError ? ( +

{loadError}

+ ) : null} +
+ ); +} From ae21f9acd743d6291c2d443b739840ce5e31033a Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 15:13:08 +1000 Subject: [PATCH 04/10] feat(dictation): Enter while dictating sends the message Enter during a live dictation session is intercepted at window capture (ahead of the editor's submitOnEnter, which would send before the trailing phrase decoded): dictation stops, and the submit fires once the trailing final lands in the doc (3 s cap if it never arrives; immediate when no partial is showing). Repeat Enters while the send is queued are swallowed so it can't double-send. Also de-slops the Voice & dictation settings description (em dash out, Enter-to-send mentioned). Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/scripts/check-file-sizes.mjs | 11 +-- .../src/features/messages/lib/useDictation.ts | 79 ++++++++++++++++++- .../features/messages/ui/MessageComposer.tsx | 8 +- .../settings/ui/VoiceSettingsCard.tsx | 2 +- 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 21674d2660..caf79dc4ec 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -530,11 +530,12 @@ const overrides = new Map([ // ownerPubkey) feeding the newly-added-mentions diff. Diff logic itself // lives in threading.ts (diffAddedMentionPubkeys); this is the minimal // composer-side wiring. Queued to split with the rest of this list. - // +9: dictation wiring — useDictation hook instantiation, ⌃Space handoff in - // handleEditorKeyDown, and two toolbar props. All session/keyboard logic - // lives in lib/useDictation.ts; this is the minimal composer-side wiring. - // Queued to split with the rest of this list. - ["src/features/messages/ui/MessageComposer.tsx", 1123], + // +15: dictation wiring — useDictation hook instantiation, ⌃Space handoff in + // handleEditorKeyDown, the Enter-to-send submit callback, and two toolbar + // props. All session/keyboard logic lives in lib/useDictation.ts; this is + // the minimal composer-side wiring. Queued to split with the rest of this + // list. + ["src/features/messages/ui/MessageComposer.tsx", 1129], // global-agent-config: model-tuning section (BuzzAgentModelTuningFields via // EditAgentAdvancedFields) + providerValid gate + effectiveProvider derivation // + globalProvider threading into getPersonaProviderOptions. All load-bearing diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index 09cbf2e73c..85347e3857 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -25,6 +25,11 @@ const OWNERSHIP_DECAY_MS = 15_000; * space as normal — only key-repeats during the hold are suppressed. */ const SPACE_HOLD_MS = 1500; +/** Enter during dictation sends the message — but the trailing phrase is + * still being flushed/decoded when the key lands, so the submit waits for + * that final. This caps the wait if the final never arrives. */ +const SUBMIT_FLUSH_TIMEOUT_MS = 3000; + type Session = { handle: AudioWorkletHandle; stream: MediaStream }; /** Payload of the Rust `dictation-transcript` event. Partials re-decode the @@ -50,13 +55,14 @@ type StreamState = { anchor: number; partialText: string }; * * Triggers: hold plain Space for SPACE_HOLD_MS (quick tap still types a * space), hold ⌃Space (instant start), or the toolbar mic button (click - * toggle). + * toggle). Enter while dictation is live stops it and calls `onSubmit` once + * the trailing phrase has been flushed into the doc (send-by-voice). * * Multiple composers may mount this hook — only the one that started the * session inserts the transcript (ownsRef), and Rust rejects a second * concurrent session. */ -export function useDictation(editor: Editor | null) { +export function useDictation(editor: Editor | null, onSubmit?: () => void) { const [status, setStatus] = React.useState("idle"); const sessionRef = React.useRef(null); const streamRef = React.useRef(null); @@ -70,6 +76,10 @@ export function useDictation(editor: Editor | null) { const spaceTimerRef = React.useRef(null); const editorRef = React.useRef(editor); editorRef.current = editor; + const onSubmitRef = React.useRef(onSubmit); + onSubmitRef.current = onSubmit; + // Non-null while an Enter-triggered submit waits for the trailing final. + const submitTimerRef = React.useRef(null); // Stream transcripts from the Rust pipeline into the editor: each partial // replaces the previous partial of the same phrase in place, and the final @@ -138,6 +148,13 @@ export function useDictation(editor: Editor | null) { editor.view.dom.style.caretColor = event.payload.final ? "" : "transparent"; + // Enter-to-send was waiting for this final — it's in the doc now. + if (event.payload.final && submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + submitTimerRef.current = null; + ownsRef.current = false; // a stray later final must not land in the sent composer + onSubmitRef.current?.(); + } }, ); return () => { @@ -182,6 +199,12 @@ export function useDictation(editor: Editor | null) { // A stream left over from a previous session (final never arrived, or the // doc changed) must not swallow this session's first phrase. streamRef.current = null; + // Restarting dictation supersedes an Enter-to-send still waiting on its + // trailing final — don't fire a send under the new session. + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + submitTimerRef.current = null; + } setStatus("starting"); // Tracks whether the Rust session was started, so cleanup on a later // failure never kills a session owned by another composer. @@ -234,6 +257,9 @@ export function useDictation(editor: Editor | null) { if (spaceTimerRef.current !== null) { window.clearTimeout(spaceTimerRef.current); } + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + } }; }, [teardownSession]); @@ -243,6 +269,29 @@ export function useDictation(editor: Editor | null) { }); }, [start]); + /** Submit now if the doc is settled, else once the trailing final lands. + * A visible partial means a phrase is in flight and its final is coming + * (stop() pushes flush silence); with nothing showing there is usually + * nothing to wait for. */ + // ponytail: "no partial showing" can also mean speech too fresh to have + // decoded yet (<~600 ms) — that tail is dropped. Thread a session-scoped + // flush ack through the Rust event if it ever bites. + const submitAfterFlush = React.useCallback(() => { + if (!streamRef.current?.partialText) { + ownsRef.current = false; + onSubmitRef.current?.(); + return; + } + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + } + submitTimerRef.current = window.setTimeout(() => { + submitTimerRef.current = null; + ownsRef.current = false; + onSubmitRef.current?.(); + }, SUBMIT_FLUSH_TIMEOUT_MS); + }, []); + /** Mic button click — start when idle, stop otherwise. */ const statusRef = React.useRef(status); statusRef.current = status; @@ -275,6 +324,30 @@ export function useDictation(editor: Editor | null) { // composer mid-hold, leaked repeats would click focused buttons, scroll // the page, or type spaces into the stream. const onKeyDownCapture = (event: KeyboardEvent) => { + // Enter while dictation is live: stop and send. Capture phase so it + // beats the editor's submitOnEnter, which would send immediately — + // before the trailing phrase has been flushed into the doc. + if ( + event.key === "Enter" && + !event.shiftKey && + !event.metaKey && + !event.altKey && + !event.ctrlKey && + onSubmitRef.current && + (keyHeldRef.current || + statusRef.current !== "idle" || + // A send is already queued behind the trailing final — swallow + // repeat Enters so submitOnEnter can't double-send. + submitTimerRef.current !== null) + ) { + event.preventDefault(); + event.stopPropagation(); + if (event.repeat) return; + keyHeldRef.current = false; + stop(); + submitAfterFlush(); + return; + } if ( event.code === "Space" && event.repeat && @@ -292,7 +365,7 @@ export function useDictation(editor: Editor | null) { window.removeEventListener("keyup", onKeyUp); window.removeEventListener("blur", release); }; - }, [stop]); + }, [stop, submitAfterFlush]); /** Returns true when the event was consumed. Handles both triggers: * ⌃Space (instant) and plain Space held for SPACE_HOLD_MS. */ diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 73ee099e98..7052ed13c5 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -785,7 +785,13 @@ function MessageComposerImpl({ // extension (fires before ProseMirror's splitBlock). This wrapper only // handles autocomplete arrow/enter keys and Escape for edit mode. // ── Dictation (hold ⌃Space or mic button → speech-to-text) ───────── - const dictation = useDictation(richText.editor); + // Enter while dictating stops the session and sends once the trailing + // phrase has been flushed into the doc. + const dictationSubmit = React.useCallback( + () => submitMessageRef.current(), + [], + ); + const dictation = useDictation(richText.editor, dictationSubmit); const handleEditorKeyDown = React.useCallback( (event: React.KeyboardEvent) => { diff --git a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx index b0675cb200..c197d2fb6b 100644 --- a/desktop/src/features/settings/ui/VoiceSettingsCard.tsx +++ b/desktop/src/features/settings/ui/VoiceSettingsCard.tsx @@ -109,7 +109,7 @@ export function VoiceSettingsCard() {

From 6962eeddc39b2cf7034e2b00b9e42cc4f0ff0f30 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 15:53:02 +1000 Subject: [PATCH 05/10] fix(dictation): flush marker ends the send-on-Enter guesswork Rapid dictate -> Enter -> dictate cycles lost messages two ways: a restart cancelled a send still waiting on its trailing phrase (the message never went out), and the "nothing on screen, send now" fast path fired before the trailing phrase had decoded (the last words were cut off). Stragglers from the stopped session could also splice into the next session's text through the 15 s ownership window. Replace the heuristics with a deterministic marker: stop_dictation pushes a zero-length sentinel batch after the flush silence, the STT worker echoes it back on the (strictly ordered) live channel as LiveEvent::Flushed, and it reaches the webview as a dictation-flushed event -- a hard guarantee that every transcript of the stopped session has been delivered. Enter-to-send now submits on the marker; restarting with a send pending fires that send immediately instead of dropping it, and discards the old session's stragglers until its marker arrives. Ownership also releases on the marker instead of decaying for 15 s. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/scripts/check-file-sizes.mjs | 5 +- desktop/src-tauri/src/dictation.rs | 33 ++++-- desktop/src-tauri/src/huddle/stt.rs | 57 +++++++-- .../src/features/messages/lib/useDictation.ts | 108 ++++++++++++------ 4 files changed, 150 insertions(+), 53 deletions(-) diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index caf79dc4ec..a8b124a696 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -87,8 +87,9 @@ const overrides = new Map([ // sticky-punctuation heuristic + unit tests, and two #[ignore]d decode // experiments (silence/noise vs punctuation, v3 comparison) that document // measured model behavior. Experiments are test-only content; queued to - // split into stt/tests.rs with the feature PR review. - ["src-tauri/src/huddle/stt.rs", 1200], + // split into stt/tests.rs with the feature PR review. +37: LiveEvent enum + // and the stop-flush sentinel echo (Enter-to-send flush marker). + ["src-tauri/src/huddle/stt.rs", 1240], // unified-agent-model 1A.1: profile reconcile split to agents_profile.rs, // ratcheting 1443 -> 1295. Queued to split further in the A2 fold. // global-agent-config: resolve_deploy_model_provider + visibility exports diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs index 51be5808cf..dde05a4500 100644 --- a/desktop/src-tauri/src/dictation.rs +++ b/desktop/src-tauri/src/dictation.rs @@ -11,7 +11,11 @@ //! Session model: `start_dictation` marks the session active; //! `stop_dictation` marks it inactive and pushes ~1 s of silence into the //! pipeline — the mic stops with the key release, so without it the VAD would -//! never see the silence that closes (and decodes) the trailing phrase. +//! never see the silence that closes (and decodes) the trailing phrase. A +//! zero-length sentinel batch follows the silence; the worker echoes it back +//! and it is emitted as a `dictation-flushed` event, which tells the frontend +//! the stopped session's transcripts have all arrived (Enter-to-send waits +//! for it before submitting). use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -21,7 +25,11 @@ use std::sync::{ use tauri::Emitter; use crate::app_state::AppState; -use crate::huddle::{models, stt::SttPipeline, HuddlePhase}; +use crate::huddle::{ + models, + stt::{LiveEvent, SttPipeline}, + HuddlePhase, +}; /// Payload for `dictation-transcript` events. Partials (`final: false`) /// replace the previous partial of the same phrase; a final commits it. @@ -101,7 +109,7 @@ pub fn start_dictation( } if guard.is_none() { - let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::<(String, bool)>(64); + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(64); // In live mode all transcripts arrive on live_rx; the pipeline's own // text receiver stays silent and is dropped here. let (pipeline, _text_rx) = SttPipeline::new( @@ -116,10 +124,17 @@ pub fn start_dictation( // Forward transcripts to the webview until the pipeline is dropped. tauri::async_runtime::spawn(async move { - while let Some((text, is_final)) = live_rx.recv().await { - let event = TranscriptEvent { text, is_final }; - if let Err(e) = app.emit("dictation-transcript", event) { - eprintln!("buzz-desktop: dictation transcript emit failed: {e}"); + while let Some(event) = live_rx.recv().await { + let result = match event { + LiveEvent::Transcript { text, is_final } => { + app.emit("dictation-transcript", TranscriptEvent { text, is_final }) + } + // Stop-flush drained: no more transcripts for the stopped + // session. The frontend gates Enter-to-send on this. + LiveEvent::Flushed => app.emit("dictation-flushed", ()), + }; + if let Err(e) = result { + eprintln!("buzz-desktop: dictation event emit failed: {e}"); } } }); @@ -140,6 +155,10 @@ pub fn stop_dictation(state: tauri::State<'_, DictationState>) { if let Ok(guard) = state.pipeline.lock() { if let Some((_, ref pipeline)) = *guard { let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); + // Zero-length sentinel — the worker echoes it back as a Flushed + // marker once everything before it (including the trailing + // phrase's decode) has been processed and delivered. + let _ = pipeline.push_audio(Vec::new()); } } } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index faf5bbf65f..110e2d5592 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -45,6 +45,20 @@ const AUDIO_QUEUE_DEPTH: usize = 300; /// Prevents OOM if VAD stays in speech mode (noisy environment). const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +/// Event on the live-transcript (dictation) channel. Strictly ordered — a +/// `Flushed` sent after a session's audio guarantees every `Transcript` of +/// that session was already delivered. +#[derive(Debug, PartialEq)] +pub enum LiveEvent { + /// Partial (`is_final: false`) re-decodes replace each other; a final + /// commits the phrase. An empty final means the phrase decoded to nothing. + Transcript { text: String, is_final: bool }, + /// Echo of a zero-length audio batch (the stop-flush sentinel): all audio + /// pushed before it has been processed, so the stopped session has no + /// further transcripts coming. + Flushed, +} + /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. @@ -81,10 +95,12 @@ impl SttPipeline { /// /// `live_tx` (optional) switches the pipeline into live-transcript mode /// (used by composer dictation): the in-progress phrase is re-decoded - /// every `PARTIAL_DECODE_STEP` new samples and sent as `(text, false)`, - /// and finals are sent as `(text, true)` — all on this one channel so - /// ordering is preserved. In this mode nothing is sent on the returned - /// text receiver. Pass `None` for huddle transcription (finals only). + /// every `PARTIAL_DECODE_STEP` new samples and sent as a partial + /// `Transcript`, and finals commit it — all on this one channel so + /// ordering is preserved. A zero-length audio batch is echoed back as + /// `Flushed` (see `LiveEvent`). In this mode nothing is sent on the + /// returned text receiver. Pass `None` for huddle transcription (finals + /// only). /// /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — @@ -100,7 +116,7 @@ impl SttPipeline { tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, - live_tx: Option>, + live_tx: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); @@ -264,7 +280,7 @@ fn stt_worker( tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, - live_tx: Option>, + live_tx: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -438,6 +454,17 @@ fn stt_worker( } for bytes in batch { + // Zero-length batch = stop-flush sentinel (dictation.rs). All + // audio pushed before it has been processed by now, so the + // stopped session has no further transcripts — tell the frontend. + if bytes.is_empty() { + if let Some(tx) = live_tx.as_ref() { + if let Err(e) = tx.blocking_send(LiveEvent::Flushed) { + eprintln!("buzz-desktop: STT live channel closed: {e}"); + } + } + continue; + } // Convert raw bytes to f32 samples (little-endian). let samples_48k = bytes_to_f32(&bytes); input_buf_48k.extend_from_slice(&samples_48k); @@ -527,7 +554,7 @@ fn process_16k_samples( tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, - live_tx: Option<&tokio_mpsc::Sender<(String, bool)>>, + live_tx: Option<&tokio_mpsc::Sender>, last_partial_len: &mut usize, decode_hold_until: &mut std::time::Instant, onset_buf: &mut Vec, @@ -684,7 +711,11 @@ fn process_16k_samples( if text.ends_with(['.', '?', '!', ',']) { *punct_partial = Some(text.clone()); } - if let Err(e) = tx.blocking_send((text, false)) { + let event = LiveEvent::Transcript { + text, + is_final: false, + }; + if let Err(e) = tx.blocking_send(event) { eprintln!("buzz-desktop: STT live channel closed: {e}"); } } @@ -752,7 +783,7 @@ fn flush_to_stt( speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, text_tx: &tokio_mpsc::Sender, - live_tx: Option<&tokio_mpsc::Sender<(String, bool)>>, + live_tx: Option<&tokio_mpsc::Sender>, punct_hint: Option, ) { let text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); @@ -761,7 +792,13 @@ fn flush_to_stt( // sent so the frontend clears the partial shown for a phrase that decoded // to nothing (noise); skipping it strands that partial on screen. let send_err = match live_tx { - Some(tx) => tx.blocking_send((text, true)).err().map(|e| e.to_string()), + Some(tx) => tx + .blocking_send(LiveEvent::Transcript { + text, + is_final: true, + }) + .err() + .map(|e| e.to_string()), None if text.is_empty() => None, None => text_tx.blocking_send(text).err().map(|e| e.to_string()), }; diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index 85347e3857..60c9c45056 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -15,20 +15,20 @@ import { transcriptDiff } from "./transcriptDiff"; export type DictationStatus = "idle" | "starting" | "recording"; /** How long after stop() this hook still claims incoming transcripts. - * The Rust worker flushes the trailing phrase after the key is released; - * decoding can take a couple of seconds. */ -// ponytail: time-based ownership window; replace with a session id threaded -// through the Rust event if two composers ever dictate back-to-back within 15s. + * Backstop only — ownership normally releases exactly when the session's + * `dictation-flushed` marker arrives; this covers a marker that never + * comes (pipeline died mid-flush). */ const OWNERSHIP_DECAY_MS = 15_000; /** Hold plain Space this long to start dictation. A quick tap still types a * space as normal — only key-repeats during the hold are suppressed. */ const SPACE_HOLD_MS = 1500; -/** Enter during dictation sends the message — but the trailing phrase is - * still being flushed/decoded when the key lands, so the submit waits for - * that final. This caps the wait if the final never arrives. */ -const SUBMIT_FLUSH_TIMEOUT_MS = 3000; +/** Cap on waiting for a session's `dictation-flushed` marker (Enter-to-send + * submits on it; a superseded session's stragglers are discarded until it). + * It normally arrives well under a second after stop(); the cap only fires + * if the pipeline died and the marker never comes. */ +const FLUSH_MARKER_TIMEOUT_MS = 3000; type Session = { handle: AudioWorkletHandle; stream: MediaStream }; @@ -55,8 +55,9 @@ type StreamState = { anchor: number; partialText: string }; * * Triggers: hold plain Space for SPACE_HOLD_MS (quick tap still types a * space), hold ⌃Space (instant start), or the toolbar mic button (click - * toggle). Enter while dictation is live stops it and calls `onSubmit` once - * the trailing phrase has been flushed into the doc (send-by-voice). + * toggle). Enter while dictation is live stops it and calls `onSubmit` when + * the session's `dictation-flushed` marker confirms every transcript — + * including the trailing phrase — is in the doc (send-by-voice). * * Multiple composers may mount this hook — only the one that started the * session inserts the transcript (ownsRef), and Rust rejects a second @@ -78,8 +79,12 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { editorRef.current = editor; const onSubmitRef = React.useRef(onSubmit); onSubmitRef.current = onSubmit; - // Non-null while an Enter-triggered submit waits for the trailing final. + // Non-null while an Enter-triggered submit waits for the flush marker. const submitTimerRef = React.useRef(null); + // Non-null while transcripts are dropped because they belong to a session + // this composer already replaced (restart before the old session's flush + // marker arrived). Cleared by that marker, or by the cap if it never comes. + const discardTimerRef = React.useRef(null); // Stream transcripts from the Rust pipeline into the editor: each partial // replaces the previous partial of the same phrase in place, and the final @@ -93,6 +98,8 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { const unlisten = listen( "dictation-transcript", (event) => { + // Straggler from a session this composer already replaced. + if (discardTimerRef.current !== null) return; if (!ownsRef.current) return; const editor = editorRef.current; if (!editor) return; @@ -148,17 +155,38 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { editor.view.dom.style.caretColor = event.payload.final ? "" : "transparent"; - // Enter-to-send was waiting for this final — it's in the doc now. - if (event.payload.final && submitTimerRef.current !== null) { - window.clearTimeout(submitTimerRef.current); - submitTimerRef.current = null; - ownsRef.current = false; // a stray later final must not land in the sent composer - onSubmitRef.current?.(); - } }, ); + // The flush marker: every transcript of the stopped session is in the + // doc now (the Rust live channel is strictly ordered). + const unlistenFlushed = listen("dictation-flushed", () => { + // Marker of the session this composer replaced — stop discarding, the + // current session's transcripts follow it. + if (discardTimerRef.current !== null) { + window.clearTimeout(discardTimerRef.current); + discardTimerRef.current = null; + return; + } + // A live session's own marker can only arrive after its stop(). + if (sessionRef.current) return; + if (!ownsRef.current && submitTimerRef.current === null) return; + // The stopped session is fully drained — release ownership exactly + // instead of waiting out the decay backstop. + ownsRef.current = false; + if (ownershipTimerRef.current !== null) { + window.clearTimeout(ownershipTimerRef.current); + ownershipTimerRef.current = null; + } + // Enter-to-send was waiting for this drain. + if (submitTimerRef.current !== null) { + window.clearTimeout(submitTimerRef.current); + submitTimerRef.current = null; + onSubmitRef.current?.(); + } + }); return () => { void unlisten.then((fn) => fn()); + void unlistenFlushed.then((fn) => fn()); }; }, []); @@ -199,11 +227,28 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { // A stream left over from a previous session (final never arrived, or the // doc changed) must not swallow this session's first phrase. streamRef.current = null; - // Restarting dictation supersedes an Enter-to-send still waiting on its - // trailing final — don't fire a send under the new session. + // An Enter-to-send is still waiting on the previous session's flush + // marker. Its message must not be lost to the restart — send what's in + // the doc now; the not-yet-decoded tail (if any) is dropped with the + // rest of that session's stragglers below. if (submitTimerRef.current !== null) { window.clearTimeout(submitTimerRef.current); submitTimerRef.current = null; + onSubmitRef.current?.(); + } + // The previous session hasn't drained yet (its flush marker would have + // cleared the ownership timer) — discard its stragglers until the marker + // arrives, or they'd be spliced into this session's text. + // ponytail: per-composer. Restarting in a DIFFERENT composer within the + // drain window can still catch the old session's tail — thread a session + // id through the Rust event if it ever bites. + if ( + ownershipTimerRef.current !== null && + discardTimerRef.current === null + ) { + discardTimerRef.current = window.setTimeout(() => { + discardTimerRef.current = null; + }, FLUSH_MARKER_TIMEOUT_MS); } setStatus("starting"); // Tracks whether the Rust session was started, so cleanup on a later @@ -260,6 +305,9 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { if (submitTimerRef.current !== null) { window.clearTimeout(submitTimerRef.current); } + if (discardTimerRef.current !== null) { + window.clearTimeout(discardTimerRef.current); + } }; }, [teardownSession]); @@ -269,19 +317,11 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { }); }, [start]); - /** Submit now if the doc is settled, else once the trailing final lands. - * A visible partial means a phrase is in flight and its final is coming - * (stop() pushes flush silence); with nothing showing there is usually - * nothing to wait for. */ - // ponytail: "no partial showing" can also mean speech too fresh to have - // decoded yet (<~600 ms) — that tail is dropped. Thread a session-scoped - // flush ack through the Rust event if it ever bites. + /** Queue the submit to fire when the session's flush marker arrives — + * i.e. once every transcript of the stopped session, including the + * trailing phrase Enter interrupted, is in the doc. The timer only fires + * if the marker never comes. */ const submitAfterFlush = React.useCallback(() => { - if (!streamRef.current?.partialText) { - ownsRef.current = false; - onSubmitRef.current?.(); - return; - } if (submitTimerRef.current !== null) { window.clearTimeout(submitTimerRef.current); } @@ -289,7 +329,7 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { submitTimerRef.current = null; ownsRef.current = false; onSubmitRef.current?.(); - }, SUBMIT_FLUSH_TIMEOUT_MS); + }, FLUSH_MARKER_TIMEOUT_MS); }, []); /** Mic button click — start when idle, stop otherwise. */ @@ -336,7 +376,7 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { onSubmitRef.current && (keyHeldRef.current || statusRef.current !== "idle" || - // A send is already queued behind the trailing final — swallow + // A send is already queued behind the flush marker — swallow // repeat Enters so submitOnEnter can't double-send. submitTimerRef.current !== null) ) { From 039c70189a253737d679ccfb0e66e784f4ee8efd Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 15:57:53 +1000 Subject: [PATCH 06/10] fix(dictation): shorten the Space hold to 800 ms At 1.5 s people start talking well before the mic opens, so the first words never reach the pipeline and the surviving fragment decodes as filler ("Yeah"). Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/src/features/messages/lib/useDictation.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index 60c9c45056..34f7890124 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -21,8 +21,10 @@ export type DictationStatus = "idle" | "starting" | "recording"; const OWNERSHIP_DECAY_MS = 15_000; /** Hold plain Space this long to start dictation. A quick tap still types a - * space as normal — only key-repeats during the hold are suppressed. */ -const SPACE_HOLD_MS = 1500; + * space as normal — only key-repeats during the hold are suppressed. + * 800 ms: at 1.5 s people start talking well before the mic opens and the + * first words are lost (the fragment that survives decodes as filler). */ +const SPACE_HOLD_MS = 800; /** Cap on waiting for a session's `dictation-flushed` marker (Enter-to-send * submits on it; a superseded session's stragglers are discarded until it). From f0e093d2b74490d2a0a97a88a484165dc0cb2b02 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 20:01:09 +1000 Subject: [PATCH 07/10] fix(dictation): guard against v3 decode collapse deleting shown text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Enhanced (parakeet-v3) transducer occasionally collapses a re-decode of a phrase to empty or one junk word ("Yeah") even though earlier decodes of the same growing buffer were fine. The frontend trusted finals unconditionally, so a collapsed final replaced the sentence on screen with nothing. Every decode of a phrase sees a superset of the audio any earlier one saw, so word count can honestly grow or hold but not shrink. Track the best partial per phrase; suppress partials that lose words against it, and when the final loses words, commit the best partial instead. Also includes temporary DEBUG(dictation-yeah) tracing on both sides of the event stream — to be stripped before the PR. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/src-tauri/src/dictation.rs | 18 +- desktop/src-tauri/src/huddle/stt.rs | 236 +++++++++++++++++- .../src/features/messages/lib/useDictation.ts | 23 ++ 3 files changed, 270 insertions(+), 7 deletions(-) diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs index dde05a4500..9e535a1e12 100644 --- a/desktop/src-tauri/src/dictation.rs +++ b/desktop/src-tauri/src/dictation.rs @@ -108,6 +108,11 @@ pub fn start_dictation( *guard = None; } + // DEBUG(dictation-yeah): temporary event trace. + eprintln!( + "buzz-desktop: dictation start — model={model_id} rebuild={}", + guard.is_none() + ); if guard.is_none() { let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(64); // In live mode all transcripts arrive on live_rx; the pipeline's own @@ -127,11 +132,20 @@ pub fn start_dictation( while let Some(event) = live_rx.recv().await { let result = match event { LiveEvent::Transcript { text, is_final } => { + // DEBUG(dictation-yeah): temporary event trace. + eprintln!( + "buzz-desktop: dictation {} {text:?}", + if is_final { "FINAL " } else { "partial" } + ); app.emit("dictation-transcript", TranscriptEvent { text, is_final }) } // Stop-flush drained: no more transcripts for the stopped // session. The frontend gates Enter-to-send on this. - LiveEvent::Flushed => app.emit("dictation-flushed", ()), + LiveEvent::Flushed => { + // DEBUG(dictation-yeah): temporary event trace. + eprintln!("buzz-desktop: dictation FLUSHED"); + app.emit("dictation-flushed", ()) + } }; if let Err(e) = result { eprintln!("buzz-desktop: dictation event emit failed: {e}"); @@ -152,6 +166,8 @@ pub fn stop_dictation(state: tauri::State<'_, DictationState>) { if !state.active.swap(false, Ordering::AcqRel) { return; } + // DEBUG(dictation-yeah): temporary event trace. + eprintln!("buzz-desktop: dictation stop — pushing flush silence + sentinel"); if let Ok(guard) = state.pipeline.lock() { if let Some((_, ref pipeline)) = *guard { let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 110e2d5592..4c4e32afcf 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -174,7 +174,11 @@ impl SttPipeline { )); } // Drop audio if the pipeline can't keep up — better than blocking the UI. - let _ = self.audio_tx.try_send(pcm_bytes); + if self.audio_tx.try_send(pcm_bytes).is_err() { + // DEBUG(dictation-yeah): temporary — audio drops splice the phrase + // buffer and corrupt later decodes; log to confirm/rule out. + eprintln!("buzz-desktop: STT audio queue full — batch dropped"); + } Ok(()) } } @@ -396,6 +400,9 @@ fn stt_worker( // Live mode: most recent partial decode of the current phrase that ended // with terminal punctuation — see prefer_punctuated. let mut punct_partial: Option = None; + // Live mode: best (most words) partial decode of the current phrase — + // the collapse guard, see decode_collapsed. + let mut best_partial: Option = None; // Timestamp when TTS last stopped — used for the 200 ms cooldown. let mut tts_stopped_at: Option = None; @@ -431,6 +438,7 @@ fn stt_worker( &text_tx, live_tx.as_ref(), punct_partial.take(), + best_partial.take(), ); speech_buf.clear(); silence_frames = 0; @@ -492,6 +500,7 @@ fn stt_worker( &mut decode_hold_until, &mut onset_buf, &mut punct_partial, + &mut best_partial, ); } } @@ -559,6 +568,7 @@ fn process_16k_samples( decode_hold_until: &mut std::time::Instant, onset_buf: &mut Vec, punct_partial: &mut Option, + best_partial: &mut Option, ) { leftover.extend_from_slice(samples); @@ -654,7 +664,14 @@ fn process_16k_samples( // OOM guard: flush and reset if the buffer exceeds 30 s of audio. if speech_buf.len() >= MAX_SPEECH_SAMPLES { let t0 = std::time::Instant::now(); - flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); + flush_to_stt( + speech_buf, + recognizer, + text_tx, + live_tx, + punct_partial.take(), + best_partial.take(), + ); *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; @@ -679,7 +696,14 @@ fn process_16k_samples( // End of utterance — transcribe. Flush decodes join the same // wall-clock hold as partials (see PARTIAL_DECODE_STEP). let t0 = std::time::Instant::now(); - flush_to_stt(speech_buf, recognizer, text_tx, live_tx, punct_partial.take()); + flush_to_stt( + speech_buf, + recognizer, + text_tx, + live_tx, + punct_partial.take(), + best_partial.take(), + ); *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; speech_buf.clear(); *silence_frames = 0; @@ -707,10 +731,20 @@ fn process_16k_samples( // 2× wall-clock hold (see PARTIAL_DECODE_STEP); 1× measured // at ~100% duty on a 35 s ramble — no headroom. *decode_hold_until = std::time::Instant::now() + t0.elapsed() * 2; - if !text.is_empty() { + // Collapse guard (see decode_collapsed): a re-decode of the + // same growing phrase that lost words is a collapse, not a + // revision — suppress it so it neither replaces the shown + // partial nor poisons the best-partial baseline. + let collapsed = best_partial + .as_deref() + .is_some_and(|best| decode_collapsed(&text, best)); + if collapsed { + eprintln!("buzz-desktop: STT partial collapsed ({text:?}) — suppressed"); + } else if !text.is_empty() { if text.ends_with(['.', '?', '!', ',']) { *punct_partial = Some(text.clone()); } + *best_partial = Some(text.clone()); let event = LiveEvent::Transcript { text, is_final: false, @@ -778,6 +812,39 @@ fn prefer_punctuated(final_text: String, punct_hint: Option<&str>) -> String { final_text } +/// True when a re-decode of the same phrase lost words against the best +/// decode seen so far — a collapse, not a revision. Every decode of a phrase +/// sees a superset of the audio any earlier one saw (partials decode a +/// growing buffer; the final decodes all of it), so word count can honestly +/// grow or hold but not shrink. The v3 transducer occasionally collapses a +/// re-decode to empty or a junk word ("Yeah") on audio whose earlier decodes +/// were fine; trusting it deletes the sentence the user watched appear. +/// ponytail: a rare legitimate word-merge revision ("any better"→"anybody") +/// trips this and keeps the earlier decode — words the user already saw, +/// far cheaper than losing real speech. +fn decode_collapsed(new_text: &str, best: &str) -> bool { + let words = |s: &str| norm_words(s).split_whitespace().count(); + words(new_text) < words(best) +} + +#[cfg(test)] +mod decode_collapsed_tests { + use super::decode_collapsed; + + #[test] + fn collapse_detection() { + let best = "caused by some other things"; + assert!(decode_collapsed("", best)); + assert!(decode_collapsed("Yeah.", best)); + assert!(decode_collapsed("Cause personal things.", best)); + // Honest revisions and growth keep the new decode. + assert!(!decode_collapsed("Caused by some other things.", best)); + assert!(!decode_collapsed("caused by some other things too", best)); + // No baseline words → nothing to protect. + assert!(!decode_collapsed("", "")); + } +} + /// The tokio channel's `blocking_send` is safe to call from sync contexts. fn flush_to_stt( speech_buf: &[f32], @@ -785,12 +852,24 @@ fn flush_to_stt( text_tx: &tokio_mpsc::Sender, live_tx: Option<&tokio_mpsc::Sender>, punct_hint: Option, + best_partial: Option, ) { - let text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); + let mut text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); + // Collapse guard — see decode_collapsed. A collapsed final would replace + // the sentence on screen with nothing (or junk); keep the best partial + // the user was looking at instead. + if let Some(best) = best_partial { + if decode_collapsed(&text, &best) { + eprintln!("buzz-desktop: STT final collapsed ({text:?}) — keeping best partial"); + text = best; + } + } // In live mode finals ride the same channel as partials so the receiver // never sees a stale partial after its final — and an EMPTY final is still // sent so the frontend clears the partial shown for a phrase that decoded // to nothing (noise); skipping it strands that partial on screen. + // (With the collapse guard above, "empty" here means no partial was ever + // shown either — a phrase the guard let through.) let send_err = match live_tx { Some(tx) => tx .blocking_send(LiveEvent::Transcript { @@ -905,7 +984,7 @@ mod prefer_punctuated_tests { #[cfg(test)] mod silence_experiment { - use super::{decode_speech, prefer_punctuated}; + use super::{decode_speech, prefer_punctuated, process_16k_samples}; /// Manual experiment: does Parakeet v3 (0.6B transducer) produce stable /// punctuation under noisy pauses where the 110M CTC model flickers? @@ -1163,6 +1242,151 @@ mod silence_experiment { } } + /// Manual experiment: reproduce chl's "captured it, then deleted it and + /// wrote Yeah" report (2026-07-24, Enhanced/v3 model) at the worker level. + /// Drives `process_16k_samples` exactly like the live worker — real VAD, + /// onset debounce, silence flush, partial cadence — over composed audio: + /// room noise, the spoken sentence (noise overlaid), a post-speech tail, + /// then the 1 s stop-flush zeros. A frontend simulator applies the events + /// the way useDictation.ts does and prints what the composer would show. + /// Run: cargo test v3_live_worker_sim -- --ignored --nocapture + /// Setup: say -v Samantha "just testing to see if this is working any + /// better" -o /tmp/dict_better.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_live_worker_sim() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + let recognizer = v3_recognizer(super::LIVE_STT_NUM_THREADS); + let speech = read_wav_16k("/tmp/dict_better.wav"); + + let noise = |len: usize, amp: f32, seed: u32| { + let mut state = seed; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let overlay = |samples: &[f32], amp: f32| { + let mut state = 0x1234_5678u32; + samples + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + + // (label, pre-speech, post-speech tail before the stop flush) + let scenarios: Vec<(&str, Vec, Vec)> = vec![ + ( + "release right after speaking", + noise(16 * 500, 0.01, 0xA1), + noise(16 * 400, 0.01, 0xB2), + ), + ( + "pause >608ms then breath then release", + noise(16 * 500, 0.01, 0xA1), + [ + noise(16 * 900, 0.01, 0xB2), + noise(16 * 250, 0.08, 0xC3), // breath/exhale burst + noise(16 * 300, 0.01, 0xD4), + ] + .concat(), + ), + ( + "louder room noise", + noise(16 * 500, 0.03, 0xA1), + noise(16 * 700, 0.03, 0xB2), + ), + ]; + + for (label, pre, tail) in scenarios { + println!("=== scenario: {label} ==="); + let mut audio = pre; + audio.extend(overlay(&speech, 0.01)); + audio.extend(tail); + audio.extend(std::iter::repeat_n(0.0f32, 16_000)); // stop-flush 1s zeros + + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(1024); + let (text_tx, _text_rx) = tokio::sync::mpsc::channel::(64); + let tts_active = Arc::new(AtomicBool::new(false)); + + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut leftover = Vec::new(); + let mut speech_buf = Vec::new(); + let mut silence_frames = 0usize; + let mut in_speech = false; + let mut barge_in_frames = 0usize; + let mut tts_stopped_at = None; + let mut last_partial_len = 0usize; + let mut decode_hold_until = std::time::Instant::now(); + let mut onset_buf = Vec::new(); + let mut punct_partial: Option = None; + let mut best_partial: Option = None; + + for chunk in audio.chunks(320) { + process_16k_samples( + chunk, + &mut leftover, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &tts_active, + None, + &mut tts_stopped_at, + None, + Some(&live_tx), + &mut last_partial_len, + &mut decode_hold_until, + &mut onset_buf, + &mut punct_partial, + &mut best_partial, + ); + } + drop(live_tx); + + // Frontend simulator — the transcriptDiff mechanics reduce to this. + let mut committed = String::new(); + let mut partial = String::new(); + while let Ok(event) = live_rx.try_recv() { + match event { + super::LiveEvent::Transcript { text, is_final } => { + let trimmed = text.trim(); + println!( + " {} {trimmed:?}", + if is_final { "FINAL " } else { "partial" } + ); + if is_final { + if !trimmed.is_empty() { + committed.push_str(trimmed); + committed.push(' '); + } else if !partial.is_empty() { + println!(" !!! empty final wiped shown partial {partial:?}"); + } + partial.clear(); + } else if !trimmed.is_empty() { + if trimmed.len() + 10 < partial.len() { + println!(" !!! partial shrank {partial:?} -> {trimmed:?}"); + } + partial = trimmed.to_string(); + } + } + super::LiveEvent::Flushed => println!(" FLUSHED"), + } + } + println!(" composer: {:?}", format!("{committed}{partial}")); + } + } + /// Manual experiment: how much trailing silence does the model need before /// it commits sentence-final punctuation? (Answer when tuned: ≥600 ms — /// see LIVE_SILENCE_FLUSH_FRAMES.) diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index 34f7890124..62e7640da3 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -100,6 +100,11 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { const unlisten = listen( "dictation-transcript", (event) => { + // DEBUG(dictation-yeah): temporary event trace (console.warn lands in + // the vite dev log). + console.warn( + `[dictation] ${event.payload.final ? "FINAL" : "partial"} ${JSON.stringify(event.payload.text)} owns=${ownsRef.current} discard=${discardTimerRef.current !== null} stream=${JSON.stringify(streamRef.current)}`, + ); // Straggler from a session this composer already replaced. if (discardTimerRef.current !== null) return; if (!ownsRef.current) return; @@ -120,6 +125,10 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { end <= doc.content.size && doc.textBetween(stream.anchor, end) === stream.partialText; if (!intact) { + // DEBUG(dictation-yeah): temporary event trace. + console.warn( + `[dictation] stream NOT intact: expected ${JSON.stringify(stream.partialText)} at ${stream.anchor}, doc has ${JSON.stringify(end <= doc.content.size ? doc.textBetween(stream.anchor, end) : "")}`, + ); // The user edited around the stream (possible between key release // and the trailing final) — resume at the end of the doc. stream = { anchor: Selection.atEnd(doc).from, partialText: "" }; @@ -162,6 +171,10 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { // The flush marker: every transcript of the stopped session is in the // doc now (the Rust live channel is strictly ordered). const unlistenFlushed = listen("dictation-flushed", () => { + // DEBUG(dictation-yeah): temporary event trace. + console.warn( + `[dictation] FLUSHED owns=${ownsRef.current} discard=${discardTimerRef.current !== null} session=${sessionRef.current !== null} submitPending=${submitTimerRef.current !== null}`, + ); // Marker of the session this composer replaced — stop discarding, the // current session's transcripts follow it. if (discardTimerRef.current !== null) { @@ -183,6 +196,8 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { if (submitTimerRef.current !== null) { window.clearTimeout(submitTimerRef.current); submitTimerRef.current = null; + // DEBUG(dictation-yeah): temporary event trace. + console.warn("[dictation] submit (marker)"); onSubmitRef.current?.(); } }); @@ -204,6 +219,8 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { }, []); const stop = React.useCallback(() => { + // DEBUG(dictation-yeah): temporary event trace. + console.warn("[dictation] stop()"); genRef.current += 1; teardownSession(); setStatus("idle"); @@ -225,6 +242,10 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { const start = React.useCallback(async (): Promise => { if (sessionRef.current) return; + // DEBUG(dictation-yeah): temporary event trace. + console.warn( + `[dictation] start() pendingSubmit=${submitTimerRef.current !== null} prevUndrained=${ownershipTimerRef.current !== null}`, + ); const gen = ++genRef.current; // A stream left over from a previous session (final never arrived, or the // doc changed) must not swallow this session's first phrase. @@ -330,6 +351,8 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { submitTimerRef.current = window.setTimeout(() => { submitTimerRef.current = null; ownsRef.current = false; + // DEBUG(dictation-yeah): temporary event trace. + console.warn("[dictation] submit (3s cap — marker never arrived)"); onSubmitRef.current?.(); }, FLUSH_MARKER_TIMEOUT_MS); }, []); From da48aa4f67c18f6a53e47b6042f4f4e7883d5297 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 20:14:59 +1000 Subject: [PATCH 08/10] fix(dictation): stitch v3 prefix collapse back together MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The word-count guard (dff5f8d4) catches decodes that collapse outright, but chl's next trace showed a second shape: once a phrase contains an internal pause, a re-decode sometimes returns only the words after the pause — dropping the front while gaining new tail words, so the word count still grows and the guard passes it ("Big questions are coming out, I need to start asking." -> "I need to start asking whether it's..."). A settled front (>= 4 words) never legitimately vanishes wholesale — early decodes only rewrite a 1-3 word start. When a re-decode disagrees on both of the phrase's first two words, anchor its opening words inside the best partial and splice the preserved front back on; with no anchor, append. Applied to partials and finals. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/src-tauri/src/huddle/stt.rs | 111 ++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 4c4e32afcf..e7dd631b14 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -741,6 +741,16 @@ fn process_16k_samples( if collapsed { eprintln!("buzz-desktop: STT partial collapsed ({text:?}) — suppressed"); } else if !text.is_empty() { + let mut text = text; + // Second collapse shape — see stitch_prefix_collapse. + if let Some(best) = best_partial.as_deref() { + if let Some(stitched) = stitch_prefix_collapse(best, &text) { + eprintln!( + "buzz-desktop: STT partial dropped its front ({text:?}) — stitched" + ); + text = stitched; + } + } if text.ends_with(['.', '?', '!', ',']) { *punct_partial = Some(text.clone()); } @@ -827,6 +837,104 @@ fn decode_collapsed(new_text: &str, best: &str) -> bool { words(new_text) < words(best) } +/// The v3 collapse's second shape: once a phrase contains an internal pause, +/// a re-decode sometimes returns only the words after the pause — dropping +/// the front while gaining newly spoken tail words, so decode_collapsed's +/// word-count check passes it. Detect: an established phrase (best ≥ 4 +/// words) whose re-decode agrees on neither of its first two words — early +/// decodes legitimately rewrite a 1–3 word front, but a settled front never +/// vanishes wholesale. Repair: anchor the new decode's opening words inside +/// best and splice best's preserved front onto new (new wins from the anchor +/// on, so tail revisions survive); with no anchor the drop swallowed the +/// overlap too, so append new after best. +/// ponytail: anchor window is 3 words then 2 — a 1-word anchor false-matches +/// on common words, and a missed overlap only duplicates ≤2 words. +fn stitch_prefix_collapse(best: &str, new: &str) -> Option { + // Per-token norms keep display and normalized tokens 1:1 aligned. + let b_disp: Vec<&str> = best.split_whitespace().collect(); + let b_norm: Vec = b_disp.iter().map(|t| norm_words(t)).collect(); + let n_norm: Vec = new.split_whitespace().map(norm_words).collect(); + if b_norm.len() < 4 || n_norm.len() < 2 { + return None; + } + if b_norm[0] == n_norm[0] || b_norm[1] == n_norm[1] { + return None; // front agrees — a revision, not a drop + } + for m in [3usize, 2] { + if n_norm.len() < m { + continue; + } + let window = &n_norm[..m]; + if let Some(j) = (0..=b_norm.len() - m) + .rev() + .find(|&j| &b_norm[j..j + m] == window) + { + let mut out = b_disp[..j].join(" "); + if !out.is_empty() { + out.push(' '); + } + out.push_str(new); + return Some(out); + } + } + Some(format!("{best} {new}")) +} + +#[cfg(test)] +mod stitch_prefix_collapse_tests { + use super::stitch_prefix_collapse; + + // Strings from chl's 2026-07-24 trace (/tmp/buzz-dev-dictation.log). + #[test] + fn splices_at_the_overlap_anchor() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking.", + "I need to start asking whether it's a little bit more." + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether it's a little bit more.") + ); + } + + #[test] + fn appends_when_no_overlap_survived() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out.", + "I need to start asking whether it's" + ) + .as_deref(), + Some("Big questions are coming out. I need to start asking whether it's") + ); + } + + #[test] + fn tail_revision_wins_from_the_anchor_on() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking whether it's a little bit more.", + "I need to start asking whether or not that's the same" + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether or not that's the same") + ); + } + + #[test] + fn honest_revisions_pass_through() { + // Front agrees → not a drop. + assert!(stitch_prefix_collapse( + "Big question to come.", + "Big questions are coming up." + ) + .is_none()); + // Short phrases rewrite themselves legitimately. + assert!(stitch_prefix_collapse("Yeah.", "Big question.").is_none()); + assert!(stitch_prefix_collapse("Costing.", "Calls passing.").is_none()); + } +} + #[cfg(test)] mod decode_collapsed_tests { use super::decode_collapsed; @@ -862,6 +970,9 @@ fn flush_to_stt( if decode_collapsed(&text, &best) { eprintln!("buzz-desktop: STT final collapsed ({text:?}) — keeping best partial"); text = best; + } else if let Some(stitched) = stitch_prefix_collapse(&best, &text) { + eprintln!("buzz-desktop: STT final dropped its front ({text:?}) — stitched"); + text = stitched; } } // In live mode finals ride the same channel as partials so the receiver From db5d550f007d783f53ba5fe064995fda3fbb2b29 Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 20:42:54 +1000 Subject: [PATCH 09/10] fix(dictation): drop lone-filler hallucinations on noise-only phrases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parakeet decodes breaths/hums/clicks that outlast the VAD onset debounce as a lone filler word ("Yeah.", "Mm."), so every random noise inserted "Yeah." into the composer. A phrase whose decodes never grow past a lone filler never contained speech: its partials are suppressed (and never recorded as best/punct partial), and its final is dropped — the empty final still flows to the frontend to clear phrase state. Trace strings from the 2026-07-24 ramble session are unit-tested. Deliberately dictating a bare "Yeah." is dropped too — known tradeoff. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/src-tauri/src/huddle/stt.rs | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index e7dd631b14..ce9fdb8c51 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -740,6 +740,12 @@ fn process_16k_samples( .is_some_and(|best| decode_collapsed(&text, best)); if collapsed { eprintln!("buzz-desktop: STT partial collapsed ({text:?}) — suppressed"); + } else if lone_filler(&text) { + // Not shown and not recorded as best/punct partial: a + // filler-only phrase must reach flush_to_stt with + // best_partial=None so its final is dropped there instead + // of resurrected by the collapse guard. + eprintln!("buzz-desktop: STT partial is a lone filler ({text:?}) — suppressed"); } else if !text.is_empty() { let mut text = text; // Second collapse shape — see stitch_prefix_collapse. @@ -788,6 +794,20 @@ fn norm_words(s: &str) -> String { mapped.split_whitespace().collect::>().join(" ") } +/// A decode that is nothing but one filler word. Parakeet hallucinates these +/// ("Yeah.", "Mm.") on breaths/hums/clicks that outlast the VAD onset +/// debounce (see ONSET_DEBOUNCE_FRAMES), and the composer ends up with a +/// "Yeah." for every random noise. A phrase whose decodes never grew past a +/// lone filler never contained speech — suppress it end to end. +/// ponytail: word-list heuristic; a deliberately dictated bare "Yeah." is +/// also dropped — as part of any sentence it survives. +fn lone_filler(text: &str) -> bool { + matches!( + norm_words(text).as_str(), + "yeah" | "uh" | "um" | "mm" | "hmm" | "mhm" | "huh" | "uh huh" | "mm hmm" + ) +} + /// Punctuation from the 110M model is unstable: re-decodes of the same phrase /// flicker between "…that" and "…that?" (and internally, "question." vs /// "question,") depending on the exact trailing-noise composition — see the @@ -935,6 +955,24 @@ mod stitch_prefix_collapse_tests { } } +#[cfg(test)] +mod lone_filler_tests { + use super::lone_filler; + + #[test] + fn trace_strings() { + // The noise phrases from the 2026-07-24 ramble trace. + assert!(lone_filler("Yeah.")); + assert!(lone_filler("Mm.")); + assert!(lone_filler("Uh-huh.")); + // Real speech that merely starts with or resembles filler survives. + assert!(!lone_filler("Yeah, so")); + assert!(!lone_filler("The problem.")); + assert!(!lone_filler("Testing.")); + assert!(!lone_filler("")); + } +} + #[cfg(test)] mod decode_collapsed_tests { use super::decode_collapsed; @@ -963,6 +1001,15 @@ fn flush_to_stt( best_partial: Option, ) { let mut text = prefer_punctuated(decode_speech(speech_buf, recognizer), punct_hint.as_deref()); + // Noise gate — see lone_filler. best_partial=None means no real partial + // was ever shown for this phrase (filler partials are never recorded), so + // a lone-filler final is a hallucinated noise, not speech the user watched + // appear. The empty final still flows through live_tx so the frontend + // clears any state for the phrase. + if live_tx.is_some() && best_partial.is_none() && lone_filler(&text) { + eprintln!("buzz-desktop: STT final is a lone filler ({text:?}) — dropped"); + text = String::new(); + } // Collapse guard — see decode_collapsed. A collapsed final would replace // the sentence on screen with nothing (or junk); keep the best partial // the user was looking at instead. From 29e83551eb06295675c389409b6bffb335e95f7a Mon Sep 17 00:00:00 2001 From: chillerno1 Date: Fri, 24 Jul 2026 21:18:29 +1000 Subject: [PATCH 10/10] chore(dictation): strip debug tracing, split stt tests to stt_tests.rs Removes the temporary DEBUG(dictation-yeah) event traces from the Rust dictation commands and the composer hook now that the decode-collapse, prefix-stitch, and lone-filler guards are verified against live repros. Moves the stt.rs unit tests and #[ignore]d decode experiments into a sibling stt_tests.rs, dropping the file-size override for stt.rs. Co-Authored-By: Claude Fable 5 Signed-off-by: chillerno1 --- desktop/scripts/check-file-sizes.mjs | 9 +- desktop/src-tauri/src/dictation.rs | 18 +- desktop/src-tauri/src/huddle/models.rs | 11 +- desktop/src-tauri/src/huddle/stt.rs | 638 +---------------- desktop/src-tauri/src/huddle/stt_tests.rs | 641 ++++++++++++++++++ desktop/src-tauri/src/lib.rs | 6 +- .../src/features/messages/lib/useDictation.ts | 23 - 7 files changed, 658 insertions(+), 688 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/stt_tests.rs diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index a8b124a696..bdb64cb1bb 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -82,14 +82,7 @@ const overrides = new Map([ // queued to split when #2520 lands and this port is dropped in the rebase. // composer-dictation model picker adds per-model slots + SttModelInfo + // resolve_dictation_model (~110 lines); split rides the same rebase. - ["src-tauri/src/huddle/models.rs", 1430], - // composer-dictation: live-transcript mode (live_tx partial re-decode loop), - // sticky-punctuation heuristic + unit tests, and two #[ignore]d decode - // experiments (silence/noise vs punctuation, v3 comparison) that document - // measured model behavior. Experiments are test-only content; queued to - // split into stt/tests.rs with the feature PR review. +37: LiveEvent enum - // and the stop-flush sentinel echo (Enter-to-send flush marker). - ["src-tauri/src/huddle/stt.rs", 1240], + ["src-tauri/src/huddle/models.rs", 1432], // unified-agent-model 1A.1: profile reconcile split to agents_profile.rs, // ratcheting 1443 -> 1295. Queued to split further in the A2 fold. // global-agent-config: resolve_deploy_model_provider + visibility exports diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs index 9e535a1e12..dde05a4500 100644 --- a/desktop/src-tauri/src/dictation.rs +++ b/desktop/src-tauri/src/dictation.rs @@ -108,11 +108,6 @@ pub fn start_dictation( *guard = None; } - // DEBUG(dictation-yeah): temporary event trace. - eprintln!( - "buzz-desktop: dictation start — model={model_id} rebuild={}", - guard.is_none() - ); if guard.is_none() { let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(64); // In live mode all transcripts arrive on live_rx; the pipeline's own @@ -132,20 +127,11 @@ pub fn start_dictation( while let Some(event) = live_rx.recv().await { let result = match event { LiveEvent::Transcript { text, is_final } => { - // DEBUG(dictation-yeah): temporary event trace. - eprintln!( - "buzz-desktop: dictation {} {text:?}", - if is_final { "FINAL " } else { "partial" } - ); app.emit("dictation-transcript", TranscriptEvent { text, is_final }) } // Stop-flush drained: no more transcripts for the stopped // session. The frontend gates Enter-to-send on this. - LiveEvent::Flushed => { - // DEBUG(dictation-yeah): temporary event trace. - eprintln!("buzz-desktop: dictation FLUSHED"); - app.emit("dictation-flushed", ()) - } + LiveEvent::Flushed => app.emit("dictation-flushed", ()), }; if let Err(e) = result { eprintln!("buzz-desktop: dictation event emit failed: {e}"); @@ -166,8 +152,6 @@ pub fn stop_dictation(state: tauri::State<'_, DictationState>) { if !state.active.swap(false, Ordering::AcqRel) { return; } - // DEBUG(dictation-yeah): temporary event trace. - eprintln!("buzz-desktop: dictation stop — pushing flush silence + sentinel"); if let Ok(guard) = state.pipeline.lock() { if let Some((_, ref pipeline)) = *guard { let _ = pipeline.push_audio(vec![0u8; FLUSH_SILENCE_BYTES]); diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 4c2e3e1aae..18394e2315 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -886,13 +886,16 @@ impl ModelManager { .ok_or_else(|| format!("unknown STT model id: {id}"))?; let manager = self.clone(); let slot_for_task = slot.clone(); - slot.start_download(&self.models_dir, http_client, "stt", move |client| { - async move { + slot.start_download( + &self.models_dir, + http_client, + "stt", + move |client| async move { manager .download_stt_model_for(model, slot_for_task, client) .await - } - }); + }, + ); Ok(()) } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index ce9fdb8c51..22e5e91afc 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -175,8 +175,8 @@ impl SttPipeline { } // Drop audio if the pipeline can't keep up — better than blocking the UI. if self.audio_tx.try_send(pcm_bytes).is_err() { - // DEBUG(dictation-yeah): temporary — audio drops splice the phrase - // buffer and corrupt later decodes; log to confirm/rule out. + // A drop splices the phrase buffer and corrupts later decodes of + // that phrase — rare, but worth a trail in the log when it happens. eprintln!("buzz-desktop: STT audio queue full — batch dropped"); } Ok(()) @@ -900,98 +900,6 @@ fn stitch_prefix_collapse(best: &str, new: &str) -> Option { Some(format!("{best} {new}")) } -#[cfg(test)] -mod stitch_prefix_collapse_tests { - use super::stitch_prefix_collapse; - - // Strings from chl's 2026-07-24 trace (/tmp/buzz-dev-dictation.log). - #[test] - fn splices_at_the_overlap_anchor() { - assert_eq!( - stitch_prefix_collapse( - "Big questions are coming out, I need to start asking.", - "I need to start asking whether it's a little bit more." - ) - .as_deref(), - Some("Big questions are coming out, I need to start asking whether it's a little bit more.") - ); - } - - #[test] - fn appends_when_no_overlap_survived() { - assert_eq!( - stitch_prefix_collapse( - "Big questions are coming out.", - "I need to start asking whether it's" - ) - .as_deref(), - Some("Big questions are coming out. I need to start asking whether it's") - ); - } - - #[test] - fn tail_revision_wins_from_the_anchor_on() { - assert_eq!( - stitch_prefix_collapse( - "Big questions are coming out, I need to start asking whether it's a little bit more.", - "I need to start asking whether or not that's the same" - ) - .as_deref(), - Some("Big questions are coming out, I need to start asking whether or not that's the same") - ); - } - - #[test] - fn honest_revisions_pass_through() { - // Front agrees → not a drop. - assert!(stitch_prefix_collapse( - "Big question to come.", - "Big questions are coming up." - ) - .is_none()); - // Short phrases rewrite themselves legitimately. - assert!(stitch_prefix_collapse("Yeah.", "Big question.").is_none()); - assert!(stitch_prefix_collapse("Costing.", "Calls passing.").is_none()); - } -} - -#[cfg(test)] -mod lone_filler_tests { - use super::lone_filler; - - #[test] - fn trace_strings() { - // The noise phrases from the 2026-07-24 ramble trace. - assert!(lone_filler("Yeah.")); - assert!(lone_filler("Mm.")); - assert!(lone_filler("Uh-huh.")); - // Real speech that merely starts with or resembles filler survives. - assert!(!lone_filler("Yeah, so")); - assert!(!lone_filler("The problem.")); - assert!(!lone_filler("Testing.")); - assert!(!lone_filler("")); - } -} - -#[cfg(test)] -mod decode_collapsed_tests { - use super::decode_collapsed; - - #[test] - fn collapse_detection() { - let best = "caused by some other things"; - assert!(decode_collapsed("", best)); - assert!(decode_collapsed("Yeah.", best)); - assert!(decode_collapsed("Cause personal things.", best)); - // Honest revisions and growth keep the new decode. - assert!(!decode_collapsed("Caused by some other things.", best)); - assert!(!decode_collapsed("caused by some other things too", best)); - // No baseline words → nothing to protect. - assert!(!decode_collapsed("", "")); - } -} - -/// The tokio channel's `blocking_send` is safe to call from sync contexts. fn flush_to_stt( speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, @@ -1075,544 +983,8 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; -#[cfg(test)] -mod prefer_punctuated_tests { - use super::prefer_punctuated; - - #[test] - fn keeps_dropped_terminal_punctuation() { - let hint = Some("Is there anything we can do about that?"); - assert_eq!( - prefer_punctuated("is there anything we can do about that".into(), hint), - "Is there anything we can do about that?" - ); - } - - #[test] - fn internal_punctuation_flicker_still_matches() { - // Observed in live_partial_sequence: the final decode swapped an - // internal "." for "," — same words, so the punctuated hint wins. - let hint = Some("I'm going to ask a question. Do you know why that happened?"); - assert_eq!( - prefer_punctuated( - "I'm going to ask a question, Do you know why that happened".into(), - hint - ), - "I'm going to ask a question. Do you know why that happened?" - ); - } - - #[test] - fn grafts_terminal_mark_when_only_last_word_agrees() { - assert_eq!( - prefer_punctuated( - "do you know er why that happened,".into(), - Some("Do you know why that happened?") - ), - "do you know er why that happened?" - ); - } - - #[test] - fn comma_hint_is_never_grafted() { - assert_eq!( - prefer_punctuated("hello there friend".into(), Some("hello there,")), - "hello there friend" - ); - } - - #[test] - fn ignores_hint_when_text_differs() { - assert_eq!( - prefer_punctuated("hello world again".into(), Some("hello world.")), - "hello world again" - ); - } - - #[test] - fn never_replaces_empty_final() { - assert_eq!(prefer_punctuated(String::new(), Some("hello.")), ""); - } - - #[test] - fn no_hint_passes_through() { - assert_eq!(prefer_punctuated("hello".into(), None), "hello"); - } -} +// ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] -mod silence_experiment { - use super::{decode_speech, prefer_punctuated, process_16k_samples}; - - /// Manual experiment: does Parakeet v3 (0.6B transducer) produce stable - /// punctuation under noisy pauses where the 110M CTC model flickers? - /// Also times each decode — the live mode re-decodes the whole phrase - /// every ~300 ms, so per-decode latency bounds the streaming cadence. - /// Run: cargo test v3_punctuation -- --ignored --nocapture - /// Needs ~/.buzz/models/parakeet-tdt-0.6b-v3 downloaded (see models.rs). - #[test] - #[ignore] - fn v3_punctuation() { - for threads in [1i32, 2, 4] { - let recognizer = v3_recognizer(threads); - println!("--- num_threads = {threads} ---"); - run_decodes(&recognizer); - } - } - - fn v3_recognizer(threads: i32) -> sherpa_onnx::OfflineRecognizer { - let model_dir = dirs::home_dir() - .unwrap() - .join(".buzz/models/parakeet-tdt-0.6b-v3"); - let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); - cfg.model_config.transducer.encoder = Some( - model_dir - .join("encoder.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - cfg.model_config.transducer.decoder = Some( - model_dir - .join("decoder.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - cfg.model_config.transducer.joiner = Some( - model_dir - .join("joiner.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - cfg.model_config.tokens = - Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); - cfg.model_config.debug = false; - cfg.model_config.num_threads = threads; - sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap() - } - - fn read_wav_16k(path: &str) -> Vec { - let bytes = std::fs::read(path).unwrap(); - let data_pos = bytes - .windows(4) - .position(|w| w == b"data") - .expect("no data chunk") - + 8; - bytes[data_pos..] - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect() - } - - /// Manual experiment: chl reported that during a long continuous ramble - /// "a whole bunch of text just got removed from the end". Two suspects: - /// (a) v3 transducer output degrades/truncates as the phrase buffer grows - /// toward the 30 s MAX_SPEECH_SAMPLES cap, so a later partial decode - /// yields LESS text than an earlier one and the frontend diff deletes - /// the tail; - /// (b) per-decode latency on long buffers (~70 ms/s of audio) blows past - /// the 300 ms partial cadence, the worker falls behind and the 5 s - /// audio queue overflows. - /// This decodes growing prefixes of a 35 s ramble, timing each decode and - /// flagging any shrinkage between consecutive prefixes. - /// Run: cargo test v3_long_buffer -- --ignored --nocapture - /// Setup: say -v Samantha "" -o /tmp/dict_long.wav - /// --data-format=LEF32@16000 - #[test] - #[ignore] - fn v3_long_buffer() { - let recognizer = v3_recognizer(super::LIVE_STT_NUM_THREADS); - let samples = read_wav_16k("/tmp/dict_long.wav"); - println!("total: {:.1}s", samples.len() as f32 / 16_000.0); - let mut prev = String::new(); - for secs in [4, 8, 12, 16, 20, 22, 24, 26, 28, 30] { - let end = (16_000 * secs).min(samples.len()); - let t0 = std::time::Instant::now(); - let text = decode_speech(&samples[..end], &recognizer); - let shrink = if text.len() < prev.len() { " <<< SHRANK" } else { "" }; - println!( - "@{secs:>2}s [{:>6.0?}ms] {} chars{shrink}: {text:?}", - t0.elapsed().as_millis(), - text.len() - ); - prev = text; - if end == samples.len() { - break; - } - } - - // Simulate the worker's adaptive partial cadence over the same audio: - // next partial waits max(PARTIAL_DECODE_STEP, last decode duration). - // Real-time safe iff cumulative decode time stays under audio time. - println!("--- adaptive cadence simulation ---"); - let mut last_len = 0usize; - let mut step = super::PARTIAL_DECODE_STEP; - let mut decode_total = std::time::Duration::ZERO; - let mut n = 0u32; - while last_len + step <= samples.len() { - last_len += step; - let t0 = std::time::Instant::now(); - let text = decode_speech(&samples[..last_len], &recognizer); - let dt = t0.elapsed(); - decode_total += dt; - step = super::PARTIAL_DECODE_STEP.max(32 * dt.as_millis() as usize); - n += 1; - println!( - "partial {n} @ audio {:>4.1}s: decode {:>4.0?}ms, next step {:.1}s, tail {:?}", - last_len as f32 / 16_000.0, - dt.as_millis(), - step as f32 / 16_000.0, - &text[text.len().saturating_sub(40)..] - ); - } - let audio_secs = last_len as f32 / 16_000.0; - println!( - "{n} partials over {audio_secs:.1}s audio, total decode {:.1}s → real-time safe: {}", - decode_total.as_secs_f32(), - decode_total.as_secs_f32() < audio_secs - ); - } - - fn run_decodes(recognizer: &sherpa_onnx::OfflineRecognizer) { - for path in ["/tmp/dict_multi.wav", "/tmp/dict_q.wav"] { - let bytes = std::fs::read(path).unwrap(); - let data_pos = bytes - .windows(4) - .position(|w| w == b"data") - .expect("no data chunk") - + 8; - let samples: Vec = bytes[data_pos..] - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect(); - println!( - "=== {path} ({:.1}s speech) ===", - samples.len() as f32 / 16_000.0 - ); - for (label, amp, tail_ms) in [ - ("bare", 0.0f32, 0usize), - ("300ms noise", 0.01, 300), - ("608ms noise", 0.01, 608), - ("608ms loud noise", 0.03, 608), - ] { - let mut buf = samples.clone(); - let mut state = 0x2545_f491u32; - for _ in 0..(16 * tail_ms) { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - buf.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); - } - let t0 = std::time::Instant::now(); - let text = decode_speech(&buf, recognizer); - println!(" +{label}: [{:?}] {text:?}", t0.elapsed()); - } - } - } - - /// Manual experiment: simulate the live worker's exact cadence over real - /// (synthesized) speech with a noisy tail — decode the growing buffer - /// every PARTIAL_DECODE_STEP samples, track `punct_partial` like the - /// worker does, then run the final flush through `prefer_punctuated`. - /// Shows whether the sticky-punctuation hint actually matches on - /// multi-sentence phrases. - /// Run: cargo test live_partial_sequence -- --ignored --nocapture - /// Setup: say -v Samantha "It missed the full stop on the first - /// sentence. I'm now going to try and ask a question. Do you know why - /// that happened" -o /tmp/dict_multi.wav --data-format=LEF32@16000 - #[test] - #[ignore] - fn live_partial_sequence() { - let model_dir = dirs::home_dir() - .unwrap() - .join(".buzz/models/parakeet-tdt-ctc-110m-en"); - let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some( - model_dir - .join("model.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - cfg.model_config.tokens = - Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); - cfg.model_config.num_threads = 1; - cfg.model_config.debug = false; - let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); - - for (path, amp) in [ - ("/tmp/dict_multi.wav", 0.01f32), - ("/tmp/dict_q.wav", 0.01f32), - ] { - let bytes = std::fs::read(path).unwrap(); - let data_pos = bytes - .windows(4) - .position(|w| w == b"data") - .expect("no data chunk") - + 8; - let mut samples: Vec = bytes[data_pos..] - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect(); - // Noisy silence tail up to the live flush window (~608 ms). - let mut state = 0x2545_f491u32; - for _ in 0..(16 * 608) { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - samples.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); - } - println!("=== {path} (noise amp {amp}) ==="); - let mut punct_partial: Option = None; - let mut last_len = 0usize; - while last_len + super::PARTIAL_DECODE_STEP <= samples.len() { - last_len += super::PARTIAL_DECODE_STEP; - let text = decode_speech(&samples[..last_len], &recognizer); - if text.ends_with(['.', '?', '!', ',']) { - punct_partial = Some(text.clone()); - } - println!(" partial@{last_len}: {text:?}"); - } - let final_text = decode_speech(&samples, &recognizer); - println!(" FINAL: {final_text:?}"); - println!(" HINT: {punct_partial:?}"); - println!( - " KEPT: {:?}", - prefer_punctuated(final_text, punct_partial.as_deref()) - ); - // Rejected candidate fix (measured 2026-07-24): replacing the - // noisy VAD hangover with digital zeros at flush time sometimes - // REMOVES punctuation the noisy final had — tail composition is - // non-monotonic, so don't retry audio-level recipes. - let speech_end = samples.len() - 16 * 608; - let mut zeroed = samples[..speech_end].to_vec(); - zeroed.extend(std::iter::repeat_n(0.0f32, 16 * 600)); - println!(" ZEROED-TAIL: {:?}", decode_speech(&zeroed, &recognizer)); - // Same but with noise overlaid on the speech itself (real mics - // pick up room noise under the voice too, not just in pauses). - let mut state = 0x1234_5678u32; - let mut noisy: Vec = samples[..speech_end] - .iter() - .map(|s| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp - }) - .collect(); - noisy.extend(std::iter::repeat_n(0.0f32, 16 * 600)); - println!( - " NOISY-SPEECH+ZEROED-TAIL: {:?}", - decode_speech(&noisy, &recognizer) - ); - } - } - - /// Manual experiment: reproduce chl's "captured it, then deleted it and - /// wrote Yeah" report (2026-07-24, Enhanced/v3 model) at the worker level. - /// Drives `process_16k_samples` exactly like the live worker — real VAD, - /// onset debounce, silence flush, partial cadence — over composed audio: - /// room noise, the spoken sentence (noise overlaid), a post-speech tail, - /// then the 1 s stop-flush zeros. A frontend simulator applies the events - /// the way useDictation.ts does and prints what the composer would show. - /// Run: cargo test v3_live_worker_sim -- --ignored --nocapture - /// Setup: say -v Samantha "just testing to see if this is working any - /// better" -o /tmp/dict_better.wav --data-format=LEF32@16000 - #[test] - #[ignore] - fn v3_live_worker_sim() { - use std::sync::atomic::AtomicBool; - use std::sync::Arc; - - let recognizer = v3_recognizer(super::LIVE_STT_NUM_THREADS); - let speech = read_wav_16k("/tmp/dict_better.wav"); - - let noise = |len: usize, amp: f32, seed: u32| { - let mut state = seed; - (0..len) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp - }) - .collect::>() - }; - let overlay = |samples: &[f32], amp: f32| { - let mut state = 0x1234_5678u32; - samples - .iter() - .map(|s| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp - }) - .collect::>() - }; - - // (label, pre-speech, post-speech tail before the stop flush) - let scenarios: Vec<(&str, Vec, Vec)> = vec![ - ( - "release right after speaking", - noise(16 * 500, 0.01, 0xA1), - noise(16 * 400, 0.01, 0xB2), - ), - ( - "pause >608ms then breath then release", - noise(16 * 500, 0.01, 0xA1), - [ - noise(16 * 900, 0.01, 0xB2), - noise(16 * 250, 0.08, 0xC3), // breath/exhale burst - noise(16 * 300, 0.01, 0xD4), - ] - .concat(), - ), - ( - "louder room noise", - noise(16 * 500, 0.03, 0xA1), - noise(16 * 700, 0.03, 0xB2), - ), - ]; - - for (label, pre, tail) in scenarios { - println!("=== scenario: {label} ==="); - let mut audio = pre; - audio.extend(overlay(&speech, 0.01)); - audio.extend(tail); - audio.extend(std::iter::repeat_n(0.0f32, 16_000)); // stop-flush 1s zeros - - let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(1024); - let (text_tx, _text_rx) = tokio::sync::mpsc::channel::(64); - let tts_active = Arc::new(AtomicBool::new(false)); - - let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); - let mut leftover = Vec::new(); - let mut speech_buf = Vec::new(); - let mut silence_frames = 0usize; - let mut in_speech = false; - let mut barge_in_frames = 0usize; - let mut tts_stopped_at = None; - let mut last_partial_len = 0usize; - let mut decode_hold_until = std::time::Instant::now(); - let mut onset_buf = Vec::new(); - let mut punct_partial: Option = None; - let mut best_partial: Option = None; - - for chunk in audio.chunks(320) { - process_16k_samples( - chunk, - &mut leftover, - &mut vad, - &mut speech_buf, - &mut silence_frames, - &mut in_speech, - &mut barge_in_frames, - &recognizer, - &text_tx, - &tts_active, - None, - &mut tts_stopped_at, - None, - Some(&live_tx), - &mut last_partial_len, - &mut decode_hold_until, - &mut onset_buf, - &mut punct_partial, - &mut best_partial, - ); - } - drop(live_tx); - - // Frontend simulator — the transcriptDiff mechanics reduce to this. - let mut committed = String::new(); - let mut partial = String::new(); - while let Ok(event) = live_rx.try_recv() { - match event { - super::LiveEvent::Transcript { text, is_final } => { - let trimmed = text.trim(); - println!( - " {} {trimmed:?}", - if is_final { "FINAL " } else { "partial" } - ); - if is_final { - if !trimmed.is_empty() { - committed.push_str(trimmed); - committed.push(' '); - } else if !partial.is_empty() { - println!(" !!! empty final wiped shown partial {partial:?}"); - } - partial.clear(); - } else if !trimmed.is_empty() { - if trimmed.len() + 10 < partial.len() { - println!(" !!! partial shrank {partial:?} -> {trimmed:?}"); - } - partial = trimmed.to_string(); - } - } - super::LiveEvent::Flushed => println!(" FLUSHED"), - } - } - println!(" composer: {:?}", format!("{committed}{partial}")); - } - } - - /// Manual experiment: how much trailing silence does the model need before - /// it commits sentence-final punctuation? (Answer when tuned: ≥600 ms — - /// see LIVE_SILENCE_FLUSH_FRAMES.) - /// Run: cargo test silence_vs_punctuation -- --ignored --nocapture - /// Setup: say -v Samantha "is there anything we can do about that?" \ - /// -o /tmp/dict_q.wav --data-format=LEF32@16000 (same for dict_s) - #[test] - #[ignore] - fn silence_vs_punctuation() { - let model_dir = dirs::home_dir() - .unwrap() - .join(".buzz/models/parakeet-tdt-ctc-110m-en"); - let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some( - model_dir - .join("model.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - cfg.model_config.tokens = - Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); - cfg.model_config.num_threads = 1; - cfg.model_config.debug = false; - let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); - - for path in ["/tmp/dict_q.wav", "/tmp/dict_s.wav"] { - let bytes = std::fs::read(path).unwrap(); - // Naive WAV parse: find the "data" chunk, samples are f32 LE. - let data_pos = bytes - .windows(4) - .position(|w| w == b"data") - .expect("no data chunk") - + 8; - let samples: Vec = bytes[data_pos..] - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect(); - println!("=== {path} ({} samples) ===", samples.len()); - for silence_ms in [0usize, 100, 300, 600] { - let mut buf = samples.clone(); - buf.extend(std::iter::repeat_n(0.0f32, 16 * silence_ms)); - println!( - " +{silence_ms}ms zeros: {:?}", - decode_speech(&buf, &recognizer) - ); - } - // Real mic "silence" is room noise, not zeros — simulate with - // low-level deterministic pseudo-noise (LCG; no rand dep) at - // amplitudes well below the VAD speech threshold. - for amp in [0.002f32, 0.01, 0.03] { - let noise = |len: usize| { - let mut state = 0x2545_f491u32; - (0..len) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp - }) - .collect::>() - }; - let mut buf = samples.clone(); - buf.extend(noise(16 * 608)); - println!(" +608ms noise(amp {amp}): {:?}", decode_speech(&buf, &recognizer)); - buf.extend(std::iter::repeat_n(0.0f32, 16 * 600)); - println!( - " +608ms noise(amp {amp}) + 600ms zeros: {:?}", - decode_speech(&buf, &recognizer) - ); - } - } - } -} +#[path = "stt_tests.rs"] +mod tests; diff --git a/desktop/src-tauri/src/huddle/stt_tests.rs b/desktop/src-tauri/src/huddle/stt_tests.rs new file mode 100644 index 0000000000..362dc2467e --- /dev/null +++ b/desktop/src-tauri/src/huddle/stt_tests.rs @@ -0,0 +1,641 @@ +//! Tests for huddle/stt.rs — split into a sibling file to keep stt.rs +//! focused. Unit tests for the decode guards (collapse, prefix stitch, +//! lone-filler noise gate, sticky punctuation) plus the #[ignore]d decode +//! experiments that document measured model behavior. + +#[cfg(test)] +mod stitch_prefix_collapse_tests { + use super::super::stitch_prefix_collapse; + + // Strings from chl's 2026-07-24 trace (/tmp/buzz-dev-dictation.log). + #[test] + fn splices_at_the_overlap_anchor() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking.", + "I need to start asking whether it's a little bit more." + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether it's a little bit more.") + ); + } + + #[test] + fn appends_when_no_overlap_survived() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out.", + "I need to start asking whether it's" + ) + .as_deref(), + Some("Big questions are coming out. I need to start asking whether it's") + ); + } + + #[test] + fn tail_revision_wins_from_the_anchor_on() { + assert_eq!( + stitch_prefix_collapse( + "Big questions are coming out, I need to start asking whether it's a little bit more.", + "I need to start asking whether or not that's the same" + ) + .as_deref(), + Some("Big questions are coming out, I need to start asking whether or not that's the same") + ); + } + + #[test] + fn honest_revisions_pass_through() { + // Front agrees → not a drop. + assert!( + stitch_prefix_collapse("Big question to come.", "Big questions are coming up.") + .is_none() + ); + // Short phrases rewrite themselves legitimately. + assert!(stitch_prefix_collapse("Yeah.", "Big question.").is_none()); + assert!(stitch_prefix_collapse("Costing.", "Calls passing.").is_none()); + } +} + +#[cfg(test)] +mod lone_filler_tests { + use super::super::lone_filler; + + #[test] + fn trace_strings() { + // The noise phrases from the 2026-07-24 ramble trace. + assert!(lone_filler("Yeah.")); + assert!(lone_filler("Mm.")); + assert!(lone_filler("Uh-huh.")); + // Real speech that merely starts with or resembles filler survives. + assert!(!lone_filler("Yeah, so")); + assert!(!lone_filler("The problem.")); + assert!(!lone_filler("Testing.")); + assert!(!lone_filler("")); + } +} + +#[cfg(test)] +mod decode_collapsed_tests { + use super::super::decode_collapsed; + + #[test] + fn collapse_detection() { + let best = "caused by some other things"; + assert!(decode_collapsed("", best)); + assert!(decode_collapsed("Yeah.", best)); + assert!(decode_collapsed("Cause personal things.", best)); + // Honest revisions and growth keep the new decode. + assert!(!decode_collapsed("Caused by some other things.", best)); + assert!(!decode_collapsed("caused by some other things too", best)); + // No baseline words → nothing to protect. + assert!(!decode_collapsed("", "")); + } +} + +/// The tokio channel's `blocking_send` is safe to call from sync contexts. +#[cfg(test)] +mod prefer_punctuated_tests { + use super::super::prefer_punctuated; + + #[test] + fn keeps_dropped_terminal_punctuation() { + let hint = Some("Is there anything we can do about that?"); + assert_eq!( + prefer_punctuated("is there anything we can do about that".into(), hint), + "Is there anything we can do about that?" + ); + } + + #[test] + fn internal_punctuation_flicker_still_matches() { + // Observed in live_partial_sequence: the final decode swapped an + // internal "." for "," — same words, so the punctuated hint wins. + let hint = Some("I'm going to ask a question. Do you know why that happened?"); + assert_eq!( + prefer_punctuated( + "I'm going to ask a question, Do you know why that happened".into(), + hint + ), + "I'm going to ask a question. Do you know why that happened?" + ); + } + + #[test] + fn grafts_terminal_mark_when_only_last_word_agrees() { + assert_eq!( + prefer_punctuated( + "do you know er why that happened,".into(), + Some("Do you know why that happened?") + ), + "do you know er why that happened?" + ); + } + + #[test] + fn comma_hint_is_never_grafted() { + assert_eq!( + prefer_punctuated("hello there friend".into(), Some("hello there,")), + "hello there friend" + ); + } + + #[test] + fn ignores_hint_when_text_differs() { + assert_eq!( + prefer_punctuated("hello world again".into(), Some("hello world.")), + "hello world again" + ); + } + + #[test] + fn never_replaces_empty_final() { + assert_eq!(prefer_punctuated(String::new(), Some("hello.")), ""); + } + + #[test] + fn no_hint_passes_through() { + assert_eq!(prefer_punctuated("hello".into(), None), "hello"); + } +} + +#[cfg(test)] +mod silence_experiment { + use super::super::{decode_speech, prefer_punctuated, process_16k_samples}; + + /// Manual experiment: does Parakeet v3 (0.6B transducer) produce stable + /// punctuation under noisy pauses where the 110M CTC model flickers? + /// Also times each decode — the live mode re-decodes the whole phrase + /// every ~300 ms, so per-decode latency bounds the streaming cadence. + /// Run: cargo test v3_punctuation -- --ignored --nocapture + /// Needs ~/.buzz/models/parakeet-tdt-0.6b-v3 downloaded (see models.rs). + #[test] + #[ignore] + fn v3_punctuation() { + for threads in [1i32, 2, 4] { + let recognizer = v3_recognizer(threads); + println!("--- num_threads = {threads} ---"); + run_decodes(&recognizer); + } + } + + fn v3_recognizer(threads: i32) -> sherpa_onnx::OfflineRecognizer { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-0.6b-v3"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.transducer.encoder = Some( + model_dir + .join("encoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.decoder = Some( + model_dir + .join("decoder.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.transducer.joiner = Some( + model_dir + .join("joiner.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.debug = false; + cfg.model_config.num_threads = threads; + sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap() + } + + fn read_wav_16k(path: &str) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() + } + + /// Manual experiment: chl reported that during a long continuous ramble + /// "a whole bunch of text just got removed from the end". Two suspects: + /// (a) v3 transducer output degrades/truncates as the phrase buffer grows + /// toward the 30 s MAX_SPEECH_SAMPLES cap, so a later partial decode + /// yields LESS text than an earlier one and the frontend diff deletes + /// the tail; + /// (b) per-decode latency on long buffers (~70 ms/s of audio) blows past + /// the 300 ms partial cadence, the worker falls behind and the 5 s + /// audio queue overflows. + /// This decodes growing prefixes of a 35 s ramble, timing each decode and + /// flagging any shrinkage between consecutive prefixes. + /// Run: cargo test v3_long_buffer -- --ignored --nocapture + /// Setup: say -v Samantha "" -o /tmp/dict_long.wav + /// --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_long_buffer() { + let recognizer = v3_recognizer(super::super::LIVE_STT_NUM_THREADS); + let samples = read_wav_16k("/tmp/dict_long.wav"); + println!("total: {:.1}s", samples.len() as f32 / 16_000.0); + let mut prev = String::new(); + for secs in [4, 8, 12, 16, 20, 22, 24, 26, 28, 30] { + let end = (16_000 * secs).min(samples.len()); + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..end], &recognizer); + let shrink = if text.len() < prev.len() { + " <<< SHRANK" + } else { + "" + }; + println!( + "@{secs:>2}s [{:>6.0?}ms] {} chars{shrink}: {text:?}", + t0.elapsed().as_millis(), + text.len() + ); + prev = text; + if end == samples.len() { + break; + } + } + + // Simulate the worker's adaptive partial cadence over the same audio: + // next partial waits max(PARTIAL_DECODE_STEP, last decode duration). + // Real-time safe iff cumulative decode time stays under audio time. + println!("--- adaptive cadence simulation ---"); + let mut last_len = 0usize; + let mut step = super::super::PARTIAL_DECODE_STEP; + let mut decode_total = std::time::Duration::ZERO; + let mut n = 0u32; + while last_len + step <= samples.len() { + last_len += step; + let t0 = std::time::Instant::now(); + let text = decode_speech(&samples[..last_len], &recognizer); + let dt = t0.elapsed(); + decode_total += dt; + step = super::super::PARTIAL_DECODE_STEP.max(32 * dt.as_millis() as usize); + n += 1; + println!( + "partial {n} @ audio {:>4.1}s: decode {:>4.0?}ms, next step {:.1}s, tail {:?}", + last_len as f32 / 16_000.0, + dt.as_millis(), + step as f32 / 16_000.0, + &text[text.len().saturating_sub(40)..] + ); + } + let audio_secs = last_len as f32 / 16_000.0; + println!( + "{n} partials over {audio_secs:.1}s audio, total decode {:.1}s → real-time safe: {}", + decode_total.as_secs_f32(), + decode_total.as_secs_f32() < audio_secs + ); + } + + fn run_decodes(recognizer: &sherpa_onnx::OfflineRecognizer) { + for path in ["/tmp/dict_multi.wav", "/tmp/dict_q.wav"] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!( + "=== {path} ({:.1}s speech) ===", + samples.len() as f32 / 16_000.0 + ); + for (label, amp, tail_ms) in [ + ("bare", 0.0f32, 0usize), + ("300ms noise", 0.01, 300), + ("608ms noise", 0.01, 608), + ("608ms loud noise", 0.03, 608), + ] { + let mut buf = samples.clone(); + let mut state = 0x2545_f491u32; + for _ in 0..(16 * tail_ms) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + buf.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + let t0 = std::time::Instant::now(); + let text = decode_speech(&buf, recognizer); + println!(" +{label}: [{:?}] {text:?}", t0.elapsed()); + } + } + } + + /// Manual experiment: simulate the live worker's exact cadence over real + /// (synthesized) speech with a noisy tail — decode the growing buffer + /// every PARTIAL_DECODE_STEP samples, track `punct_partial` like the + /// worker does, then run the final flush through `prefer_punctuated`. + /// Shows whether the sticky-punctuation hint actually matches on + /// multi-sentence phrases. + /// Run: cargo test live_partial_sequence -- --ignored --nocapture + /// Setup: say -v Samantha "It missed the full stop on the first + /// sentence. I'm now going to try and ask a question. Do you know why + /// that happened" -o /tmp/dict_multi.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn live_partial_sequence() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for (path, amp) in [ + ("/tmp/dict_multi.wav", 0.01f32), + ("/tmp/dict_q.wav", 0.01f32), + ] { + let bytes = std::fs::read(path).unwrap(); + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let mut samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + // Noisy silence tail up to the live flush window (~608 ms). + let mut state = 0x2545_f491u32; + for _ in 0..(16 * 608) { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + samples.push((state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp); + } + println!("=== {path} (noise amp {amp}) ==="); + let mut punct_partial: Option = None; + let mut last_len = 0usize; + while last_len + super::super::PARTIAL_DECODE_STEP <= samples.len() { + last_len += super::super::PARTIAL_DECODE_STEP; + let text = decode_speech(&samples[..last_len], &recognizer); + if text.ends_with(['.', '?', '!', ',']) { + punct_partial = Some(text.clone()); + } + println!(" partial@{last_len}: {text:?}"); + } + let final_text = decode_speech(&samples, &recognizer); + println!(" FINAL: {final_text:?}"); + println!(" HINT: {punct_partial:?}"); + println!( + " KEPT: {:?}", + prefer_punctuated(final_text, punct_partial.as_deref()) + ); + // Rejected candidate fix (measured 2026-07-24): replacing the + // noisy VAD hangover with digital zeros at flush time sometimes + // REMOVES punctuation the noisy final had — tail composition is + // non-monotonic, so don't retry audio-level recipes. + let speech_end = samples.len() - 16 * 608; + let mut zeroed = samples[..speech_end].to_vec(); + zeroed.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!(" ZEROED-TAIL: {:?}", decode_speech(&zeroed, &recognizer)); + // Same but with noise overlaid on the speech itself (real mics + // pick up room noise under the voice too, not just in pauses). + let mut state = 0x1234_5678u32; + let mut noisy: Vec = samples[..speech_end] + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect(); + noisy.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " NOISY-SPEECH+ZEROED-TAIL: {:?}", + decode_speech(&noisy, &recognizer) + ); + } + } + + /// Manual experiment: reproduce chl's "captured it, then deleted it and + /// wrote Yeah" report (2026-07-24, Enhanced/v3 model) at the worker level. + /// Drives `process_16k_samples` exactly like the live worker — real VAD, + /// onset debounce, silence flush, partial cadence — over composed audio: + /// room noise, the spoken sentence (noise overlaid), a post-speech tail, + /// then the 1 s stop-flush zeros. A frontend simulator applies the events + /// the way useDictation.ts does and prints what the composer would show. + /// Run: cargo test v3_live_worker_sim -- --ignored --nocapture + /// Setup: say -v Samantha "just testing to see if this is working any + /// better" -o /tmp/dict_better.wav --data-format=LEF32@16000 + #[test] + #[ignore] + fn v3_live_worker_sim() { + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + let recognizer = v3_recognizer(super::super::LIVE_STT_NUM_THREADS); + let speech = read_wav_16k("/tmp/dict_better.wav"); + + let noise = |len: usize, amp: f32, seed: u32| { + let mut state = seed; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let overlay = |samples: &[f32], amp: f32| { + let mut state = 0x1234_5678u32; + samples + .iter() + .map(|s| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + s + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + + // (label, pre-speech, post-speech tail before the stop flush) + let scenarios: Vec<(&str, Vec, Vec)> = vec![ + ( + "release right after speaking", + noise(16 * 500, 0.01, 0xA1), + noise(16 * 400, 0.01, 0xB2), + ), + ( + "pause >608ms then breath then release", + noise(16 * 500, 0.01, 0xA1), + [ + noise(16 * 900, 0.01, 0xB2), + noise(16 * 250, 0.08, 0xC3), // breath/exhale burst + noise(16 * 300, 0.01, 0xD4), + ] + .concat(), + ), + ( + "louder room noise", + noise(16 * 500, 0.03, 0xA1), + noise(16 * 700, 0.03, 0xB2), + ), + ]; + + for (label, pre, tail) in scenarios { + println!("=== scenario: {label} ==="); + let mut audio = pre; + audio.extend(overlay(&speech, 0.01)); + audio.extend(tail); + audio.extend(std::iter::repeat_n(0.0f32, 16_000)); // stop-flush 1s zeros + + let (live_tx, mut live_rx) = tokio::sync::mpsc::channel::(1024); + let (text_tx, _text_rx) = tokio::sync::mpsc::channel::(64); + let tts_active = Arc::new(AtomicBool::new(false)); + + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut leftover = Vec::new(); + let mut speech_buf = Vec::new(); + let mut silence_frames = 0usize; + let mut in_speech = false; + let mut barge_in_frames = 0usize; + let mut tts_stopped_at = None; + let mut last_partial_len = 0usize; + let mut decode_hold_until = std::time::Instant::now(); + let mut onset_buf = Vec::new(); + let mut punct_partial: Option = None; + let mut best_partial: Option = None; + + for chunk in audio.chunks(320) { + process_16k_samples( + chunk, + &mut leftover, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &tts_active, + None, + &mut tts_stopped_at, + None, + Some(&live_tx), + &mut last_partial_len, + &mut decode_hold_until, + &mut onset_buf, + &mut punct_partial, + &mut best_partial, + ); + } + drop(live_tx); + + // Frontend simulator — the transcriptDiff mechanics reduce to this. + let mut committed = String::new(); + let mut partial = String::new(); + while let Ok(event) = live_rx.try_recv() { + match event { + super::super::LiveEvent::Transcript { text, is_final } => { + let trimmed = text.trim(); + println!( + " {} {trimmed:?}", + if is_final { "FINAL " } else { "partial" } + ); + if is_final { + if !trimmed.is_empty() { + committed.push_str(trimmed); + committed.push(' '); + } else if !partial.is_empty() { + println!(" !!! empty final wiped shown partial {partial:?}"); + } + partial.clear(); + } else if !trimmed.is_empty() { + if trimmed.len() + 10 < partial.len() { + println!(" !!! partial shrank {partial:?} -> {trimmed:?}"); + } + partial = trimmed.to_string(); + } + } + super::super::LiveEvent::Flushed => println!(" FLUSHED"), + } + } + println!(" composer: {:?}", format!("{committed}{partial}")); + } + } + + /// Manual experiment: how much trailing silence does the model need before + /// it commits sentence-final punctuation? (Answer when tuned: ≥600 ms — + /// see LIVE_SILENCE_FLUSH_FRAMES.) + /// Run: cargo test silence_vs_punctuation -- --ignored --nocapture + /// Setup: say -v Samantha "is there anything we can do about that?" \ + /// -o /tmp/dict_q.wav --data-format=LEF32@16000 (same for dict_s) + #[test] + #[ignore] + fn silence_vs_punctuation() { + let model_dir = dirs::home_dir() + .unwrap() + .join(".buzz/models/parakeet-tdt-ctc-110m-en"); + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.debug = false; + let recognizer = sherpa_onnx::OfflineRecognizer::create(&cfg).unwrap(); + + for path in ["/tmp/dict_q.wav", "/tmp/dict_s.wav"] { + let bytes = std::fs::read(path).unwrap(); + // Naive WAV parse: find the "data" chunk, samples are f32 LE. + let data_pos = bytes + .windows(4) + .position(|w| w == b"data") + .expect("no data chunk") + + 8; + let samples: Vec = bytes[data_pos..] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + println!("=== {path} ({} samples) ===", samples.len()); + for silence_ms in [0usize, 100, 300, 600] { + let mut buf = samples.clone(); + buf.extend(std::iter::repeat_n(0.0f32, 16 * silence_ms)); + println!( + " +{silence_ms}ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + // Real mic "silence" is room noise, not zeros — simulate with + // low-level deterministic pseudo-noise (LCG; no rand dep) at + // amplitudes well below the VAD speech threshold. + for amp in [0.002f32, 0.01, 0.03] { + let noise = |len: usize| { + let mut state = 0x2545_f491u32; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state as f32 / u32::MAX as f32 - 0.5) * 2.0 * amp + }) + .collect::>() + }; + let mut buf = samples.clone(); + buf.extend(noise(16 * 608)); + println!( + " +608ms noise(amp {amp}): {:?}", + decode_speech(&buf, &recognizer) + ); + buf.extend(std::iter::repeat_n(0.0f32, 16 * 600)); + println!( + " +608ms noise(amp {amp}) + 600ms zeros: {:?}", + decode_speech(&buf, &recognizer) + ); + } + } + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 10b6b8f24b..5bb40ddf65 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -37,13 +37,13 @@ use deep_link::{ acknowledge_pending_community_deep_link, handle_deep_link_url, take_pending_community_deep_link, PendingCommunityDeepLinks, }; -use huddle::audio_output::{ - get_audio_output_device, list_audio_output_devices, set_audio_output_device, -}; use dictation::{ download_dictation_model, get_dictation_models, push_dictation_pcm, start_dictation, stop_dictation, }; +use huddle::audio_output::{ + get_audio_output_device, list_audio_output_devices, set_audio_output_device, +}; use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, diff --git a/desktop/src/features/messages/lib/useDictation.ts b/desktop/src/features/messages/lib/useDictation.ts index 62e7640da3..34f7890124 100644 --- a/desktop/src/features/messages/lib/useDictation.ts +++ b/desktop/src/features/messages/lib/useDictation.ts @@ -100,11 +100,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { const unlisten = listen( "dictation-transcript", (event) => { - // DEBUG(dictation-yeah): temporary event trace (console.warn lands in - // the vite dev log). - console.warn( - `[dictation] ${event.payload.final ? "FINAL" : "partial"} ${JSON.stringify(event.payload.text)} owns=${ownsRef.current} discard=${discardTimerRef.current !== null} stream=${JSON.stringify(streamRef.current)}`, - ); // Straggler from a session this composer already replaced. if (discardTimerRef.current !== null) return; if (!ownsRef.current) return; @@ -125,10 +120,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { end <= doc.content.size && doc.textBetween(stream.anchor, end) === stream.partialText; if (!intact) { - // DEBUG(dictation-yeah): temporary event trace. - console.warn( - `[dictation] stream NOT intact: expected ${JSON.stringify(stream.partialText)} at ${stream.anchor}, doc has ${JSON.stringify(end <= doc.content.size ? doc.textBetween(stream.anchor, end) : "")}`, - ); // The user edited around the stream (possible between key release // and the trailing final) — resume at the end of the doc. stream = { anchor: Selection.atEnd(doc).from, partialText: "" }; @@ -171,10 +162,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { // The flush marker: every transcript of the stopped session is in the // doc now (the Rust live channel is strictly ordered). const unlistenFlushed = listen("dictation-flushed", () => { - // DEBUG(dictation-yeah): temporary event trace. - console.warn( - `[dictation] FLUSHED owns=${ownsRef.current} discard=${discardTimerRef.current !== null} session=${sessionRef.current !== null} submitPending=${submitTimerRef.current !== null}`, - ); // Marker of the session this composer replaced — stop discarding, the // current session's transcripts follow it. if (discardTimerRef.current !== null) { @@ -196,8 +183,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { if (submitTimerRef.current !== null) { window.clearTimeout(submitTimerRef.current); submitTimerRef.current = null; - // DEBUG(dictation-yeah): temporary event trace. - console.warn("[dictation] submit (marker)"); onSubmitRef.current?.(); } }); @@ -219,8 +204,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { }, []); const stop = React.useCallback(() => { - // DEBUG(dictation-yeah): temporary event trace. - console.warn("[dictation] stop()"); genRef.current += 1; teardownSession(); setStatus("idle"); @@ -242,10 +225,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { const start = React.useCallback(async (): Promise => { if (sessionRef.current) return; - // DEBUG(dictation-yeah): temporary event trace. - console.warn( - `[dictation] start() pendingSubmit=${submitTimerRef.current !== null} prevUndrained=${ownershipTimerRef.current !== null}`, - ); const gen = ++genRef.current; // A stream left over from a previous session (final never arrived, or the // doc changed) must not swallow this session's first phrase. @@ -351,8 +330,6 @@ export function useDictation(editor: Editor | null, onSubmit?: () => void) { submitTimerRef.current = window.setTimeout(() => { submitTimerRef.current = null; ownsRef.current = false; - // DEBUG(dictation-yeah): temporary event trace. - console.warn("[dictation] submit (3s cap — marker never arrived)"); onSubmitRef.current?.(); }, FLUSH_MARKER_TIMEOUT_MS); }, []);