From 1f13d62f9c6ea24b4cb596c137c6e7f096a2c1aa Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:11:28 -0500 Subject: [PATCH 01/12] feat(commands): add 'Hey Vox' trigger, real-time command detection, and overlay exclusivity - Add "Hey Vox" trigger keyword alongside "Vox Control" with fuzzy matching and documentation updates across the UI and routing docs. - Implement live interim transcription in the audio coordinator to detect voice commands early during active speech. - Prewarm and preload TTS models concurrently upon early command detection, significantly reducing latency for 'Say' commands. - Skip unnecessary S1-mini post-processing on spoken speech targets to preserve verbatim output and avoid latency. - Hide normal target visualizers while the system response overlay is active so overlays never overlap. - Enforce strict wake-word matching and transcript verification on voice commands to ensure normal dictation is never dropped or misclassified as a command. --- crates/voxctrl-config/src/lib.rs | 2 +- crates/voxctrl-inference/src/lib.rs | 37 ++++- crates/voxctrl-routing/src/targets.rs | 156 +++++++++++++++++++-- crates/voxctrl-routing/src/tests.rs | 138 ++++++++++++++++++ crates/voxctrl-tts/src/engine.rs | 52 ++++--- docs/routing.md | 12 +- src-tauri/src/lib.rs | 35 +++-- src-tauri/src/pipeline.rs | 162 +++++++++++++++++++++- src-tauri/src/services.rs | 8 +- src-tauri/src/state.rs | 33 ++++- src-tauri/src/tests.rs | 1 + src/lib/Overlay/Overlay.svelte | 153 ++++++++++++-------- src/lib/Settings/AudioTab.svelte | 2 +- src/lib/Settings/CommandsTab.svelte | 6 +- src/lib/Settings/TargetEditorModal.svelte | 10 +- src/stores/config.ts | 2 +- tests/svelte/Overlay.test.ts | 45 ++++++ 17 files changed, 714 insertions(+), 140 deletions(-) diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index 87c54a0a..40e37e30 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -292,7 +292,7 @@ impl Default for FeaturesConfig { fn default() -> Self { Self { remove_fillers: true, - custom_vocabulary: vec!["VoxCtrl".into()], + custom_vocabulary: vec!["VoxCtrl".into(), "Hey Vox".into()], spoken_punctuation: true, auto_format_lists: true, show_notification: None, diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 74e75b98..9b337d48 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -225,6 +225,10 @@ pub struct InferenceRequest { pub target_id: String, /// Hotkey binding ID (if triggered by a hotkey) pub binding_id: Option, + /// Whether this is an intermediate/live transcription during active speech + pub is_interim: bool, + /// Unique session identifier for the recording turn + pub session_id: u64, } /// Final output after transcription + post-processing. @@ -239,6 +243,8 @@ pub struct InferenceOutput { /// Set when transcription failed (model missing, backend error, ...). The /// UI layer surfaces this to the user; `text` is empty in that case. pub error: Option, + pub is_interim: bool, + pub session_id: u64, } // ── Engine ──────────────────────────────────────────────────────────────────── @@ -293,6 +299,9 @@ impl InferenceEngine { /// Transcribe and post-process. Returns the final text. pub fn process(&self, req: InferenceRequest) -> Result { + let is_interim = req.is_interim; + let session_id = req.session_id; + if req.audio.is_empty() { return Ok(InferenceOutput { text: String::new(), @@ -302,6 +311,8 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } @@ -345,13 +356,15 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } let dir = voxctrl_routing::config_dir(); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. "); + let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. "); // Custom vocabulary words from features config if !app_config.features.custom_vocabulary.is_empty() { @@ -477,6 +490,8 @@ impl InferenceEngine { inference_ms: result.inference_ms, language: result.language, error: None, + is_interim, + session_id, }) } @@ -616,11 +631,25 @@ pub fn run_worker_with_config( loop { crossbeam_channel::select! { recv(rx) -> req_res => { - let req = match req_res { + let mut req = match req_res { Ok(r) => r, Err(_) => break, }; + // If an interim request arrived, drain any newer requests already in the queue + // to prioritize fresher audio or the final release request. + if req.is_interim { + while let Ok(newer) = rx.try_recv() { + req = newer; + if !req.is_interim { + break; + } + } + } + + let is_interim = req.is_interim; + let session_id = req.session_id; + if !loaded { match engine.load() { Ok(()) => { @@ -637,6 +666,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); continue; } @@ -657,6 +688,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: "".to_string(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); } } diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 892f28c4..1b2352b8 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -1312,9 +1312,9 @@ fn levenshtein_distance(s1: &str, s2: &str) -> usize { /// Parse text for keyword "VoxCtrl" and target name/label matching. /// Supports both direct commands (e.g. "VoxCtrl notes Hi there") and natural, -/// conversational phrasing (e.g. "VoxCtrl add this to my notes. What are you doing here?"). -/// Returns `Some(VoiceCommandParseResult)` if the trigger keyword was found AND a target matched; -/// otherwise returns `None`. +/// Parse text for keyword "VoxCtrl" / "Hey Vox" and target name/label matching. +/// Requires an explicit wake-word trigger ("VoxCtrl", "Vox Control", "Hey Vox", etc.) +/// to protect normal dictation from being misclassified as a command. pub fn parse_voice_command( text: &str, targets: &[OutputTarget], @@ -1324,13 +1324,31 @@ pub fn parse_voice_command( let mut trigger_len = 0; // 1. Exact & standard triggers - let exact_triggers = ["voxctrl", "vox ctrl", "vox-ctrl", "vox control"]; + let exact_triggers = [ + "voxctrl", "vox ctrl", "vox-ctrl", "vox control", + "hey vox", "hey, vox", "hey-vox", "hey_vox", + ]; for trigger in &exact_triggers { - if let Some(pos) = lower_text.find(trigger) { - if found_pos.map_or(true, |p| pos < p) { - found_pos = Some(pos); - trigger_len = trigger.len(); + let mut search_start = 0; + while let Some(match_idx) = lower_text[search_start..].find(trigger) { + let pos = search_start + match_idx; + let end_idx = pos + trigger.len(); + let is_boundary_start = pos == 0 || { + let prev = lower_text[..pos].chars().last().unwrap(); + prev.is_whitespace() || prev.is_ascii_punctuation() + }; + let is_boundary_end = end_idx == lower_text.len() || { + let next = lower_text[end_idx..].chars().next().unwrap(); + next.is_whitespace() || next.is_ascii_punctuation() + }; + if is_boundary_start && is_boundary_end { + if found_pos.map_or(true, |p| pos < p) { + found_pos = Some(pos); + trigger_len = trigger.len(); + } + break; } + search_start = pos + 1; } } @@ -1361,12 +1379,32 @@ pub fn parse_voice_command( let clean_cand = candidate.trim_matches(|c: char| c.is_ascii_punctuation()); let dist1 = levenshtein_distance(clean_cand, "voxctrl"); let dist2 = levenshtein_distance(clean_cand, "vox control"); - if dist1 <= 2 || dist2 <= 3 { - if let Some(pos) = lower_text.find(clean_cand) { - found_pos = Some(pos); - trigger_len = clean_cand.len(); - break; - } + + // Check "hey vox" fuzzy match (distance <= 1) on normalized tokens + let clean_tokens: Vec<&str> = words[..len] + .iter() + .map(|w| w.trim_matches(|c: char| c.is_ascii_punctuation())) + .filter(|w| !w.is_empty()) + .collect(); + let hey_vox_cand = clean_tokens.join(" "); + let dist_hey_vox = if !hey_vox_cand.is_empty() { + levenshtein_distance(&hey_vox_cand, "hey vox") + } else { + usize::MAX + }; + + if dist1 <= 2 || dist2 <= 3 || dist_hey_vox <= 1 { + let first_word = words[0]; + let last_word = words[len - 1]; + let start_idx = lower_text.find(first_word).unwrap_or(0); + let last_pos = lower_text[start_idx..] + .find(last_word) + .map(|p| start_idx + p) + .unwrap_or(start_idx); + let end_idx = last_pos + last_word.len(); + found_pos = Some(start_idx); + trigger_len = end_idx - start_idx; + break; } } } @@ -1431,3 +1469,93 @@ pub fn parse_voice_command( None } + +/// Backwards compatibility alias +pub fn parse_voice_command_or_direct( + text: &str, + targets: &[OutputTarget], + _allow_direct: bool, +) -> Option { + parse_voice_command(text, targets) +} + +/// Check if text contains a target's label or ID on word boundaries. +pub fn text_contains_target_name(text: &str, target: &OutputTarget) -> bool { + let lower_text = text.to_lowercase(); + let mut candidates: Vec<&str> = Vec::new(); + if !target.label.is_empty() { + candidates.push(&target.label); + } + if !target.id.is_empty() { + candidates.push(&target.id); + } + candidates.sort_by(|a, b| b.len().cmp(&a.len())); + + for cand in candidates { + let cand_lower = cand.to_lowercase(); + let mut search_start = 0; + while let Some(match_idx) = lower_text[search_start..].find(&cand_lower) { + let abs_match_start = search_start + match_idx; + let abs_match_end = abs_match_start + cand_lower.len(); + + let is_boundary_start = abs_match_start == 0 || { + let prev_char = text[..abs_match_start].chars().last().unwrap(); + prev_char.is_whitespace() || prev_char.is_ascii_punctuation() + }; + let is_boundary_end = abs_match_end == text.len() || { + let next_char = text[abs_match_end..].chars().next().unwrap(); + next_char.is_whitespace() || next_char.is_ascii_punctuation() + }; + + if is_boundary_start && is_boundary_end { + return true; + } + search_start = abs_match_start + 1; + } + } + false +} + +/// Extract payload for a known matched target from transcription text. +/// Used when a session was already recognized as matching `target` (e.g. during interim speech), +/// but the final transcription varied slightly in punctuation or phrasing. +pub fn extract_payload_for_target( + text: &str, + target: &OutputTarget, +) -> String { + let lower_text = text.to_lowercase(); + let mut candidates: Vec<&str> = Vec::new(); + if !target.label.is_empty() { + candidates.push(&target.label); + } + if !target.id.is_empty() { + candidates.push(&target.id); + } + candidates.sort_by(|a, b| b.len().cmp(&a.len())); + + for cand in candidates { + let cand_lower = cand.to_lowercase(); + let mut search_start = 0; + while let Some(match_idx) = lower_text[search_start..].find(&cand_lower) { + let abs_match_start = search_start + match_idx; + let abs_match_end = abs_match_start + cand_lower.len(); + + let is_boundary_start = abs_match_start == 0 || { + let prev_char = text[..abs_match_start].chars().last().unwrap(); + prev_char.is_whitespace() || prev_char.is_ascii_punctuation() + }; + let is_boundary_end = abs_match_end == text.len() || { + let next_char = text[abs_match_end..].chars().next().unwrap(); + next_char.is_whitespace() || next_char.is_ascii_punctuation() + }; + + if is_boundary_start && is_boundary_end { + let post_target = &text[abs_match_end..]; + return clean_payload(post_target); + } + search_start = abs_match_start + 1; + } + } + + clean_payload(text) +} diff --git a/crates/voxctrl-routing/src/tests.rs b/crates/voxctrl-routing/src/tests.rs index cd68ab76..bc776b42 100644 --- a/crates/voxctrl-routing/src/tests.rs +++ b/crates/voxctrl-routing/src/tests.rs @@ -1768,6 +1768,12 @@ fn test_parse_voice_command_no_keyword() { ]; assert!(parse_voice_command("hi there", &targets).is_none()); + assert!(parse_voice_command("Hey folks, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey boss, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey guys, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("They voxelize notes and models", &targets).is_none()); + assert!(parse_voice_command("heavy box notes something", &targets).is_none()); + assert!(parse_voice_command("the box notes something", &targets).is_none()); } #[test] @@ -1854,6 +1860,60 @@ fn test_parse_voice_command_matched_target() { let res8 = parse_voice_command("Walks control speak how are you?", &speak_targets).expect("should match Walks control dynamic trigger pattern"); assert_eq!(res8.matched_target_id, "speak"); assert_eq!(res8.payload, "how are you?"); + + // Hey Vox commands + let res9 = parse_voice_command("Hey Vox, notes Hi there", &targets).expect("should match Hey Vox with comma"); + assert_eq!(res9.matched_target_id, "notes"); + assert_eq!(res9.payload, "Hi there"); + + let res10 = parse_voice_command("Hey Vox notes Hi there", &targets).expect("should match Hey Vox without comma"); + assert_eq!(res10.matched_target_id, "notes"); + assert_eq!(res10.payload, "Hi there"); + + let res11 = parse_voice_command("Hey, Vox, notes Hi there", &targets).expect("should match Hey, Vox, with internal comma"); + assert_eq!(res11.matched_target_id, "notes"); + assert_eq!(res11.payload, "Hi there"); + + let res12 = parse_voice_command("Hey Vox: Notes, please write this down", &targets).expect("should match Hey Vox with colon"); + assert_eq!(res12.matched_target_id, "notes"); + assert_eq!(res12.payload, "please write this down"); + + let res13 = parse_voice_command("Hey, Vox: Notes, please write this down", &targets).expect("should match Hey, Vox: with colon"); + assert_eq!(res13.matched_target_id, "notes"); + assert_eq!(res13.payload, "please write this down"); + + let res14 = parse_voice_command("hey vox add this to my notes. What are you doing here?", &targets).expect("should match conversational hey vox notes"); + assert_eq!(res14.matched_target_id, "notes"); + assert_eq!(res14.payload, "What are you doing here?"); + + let res15 = parse_voice_command("Hey Vox put this into my Notes: What are you doing here?", &targets).expect("should match conversational put into notes"); + assert_eq!(res15.matched_target_id, "notes"); + assert_eq!(res15.payload, "What are you doing here?"); + + let res16 = parse_voice_command("Hey, Vox, send us to my notes. I love you.", &targets).expect("should match conversational send us to my notes"); + assert_eq!(res16.matched_target_id, "notes"); + assert_eq!(res16.payload, "I love you."); + + // Hey Vox homophones (Acoustic mis-recognitions within distance <= 1) + let res17 = parse_voice_command("Hey Box speak text", &speak_targets).expect("should match Hey Box homophone"); + assert_eq!(res17.matched_target_id, "speak"); + assert_eq!(res17.payload, "text"); + + let res18 = parse_voice_command("hay vox speak text", &speak_targets).expect("should match hay vox homophone"); + assert_eq!(res18.matched_target_id, "speak"); + assert_eq!(res18.payload, "text"); + + let res19 = parse_voice_command("hey fox speak text", &speak_targets).expect("should match hey fox homophone"); + assert_eq!(res19.matched_target_id, "speak"); + assert_eq!(res19.payload, "text"); + + let res20 = parse_voice_command("hey vax speak text", &speak_targets).expect("should match hey vax homophone"); + assert_eq!(res20.matched_target_id, "speak"); + assert_eq!(res20.payload, "text"); + + let res21 = parse_voice_command("Hey, Box, speak, eat my ass.", &speak_targets).expect("should match Hey, Box, homophone with punctuation"); + assert_eq!(res21.matched_target_id, "speak"); + assert_eq!(res21.payload, "eat my ass."); } #[test] @@ -2218,3 +2278,81 @@ async fn command_targets_keep_newlines_when_not_stripping() { let text = delivered.expect("delivery produced no text"); assert_eq!(text, "first line\nsecond line "); } + +#[test] +fn test_parse_voice_command_requires_wake_word() { + use crate::targets::parse_voice_command; + + let targets = vec![ + OutputTarget { + id: "say_target".into(), + label: "Say".into(), + delivery: DeliveryType::Speak, + ..OutputTarget::default_inject() + }, + OutputTarget { + id: "notes".into(), + label: "Notes".into(), + delivery: DeliveryType::File, + ..OutputTarget::default_inject() + }, + ]; + + // Normal speech must NEVER trigger a command without wake words + assert!(parse_voice_command("say hello world", &targets).is_none()); + assert!(parse_voice_command("notes: buy groceries", &targets).is_none()); + assert!(parse_voice_command("what the hell is going on. And why is it happening?", &targets).is_none()); + assert!(parse_voice_command("There have been several changes made in this branch.", &targets).is_none()); + assert!(parse_voice_command("Do you hear me? Do you know what's happening?", &targets).is_none()); + + // Valid commands with Hey Vox or Vox Control trigger properly + let res1 = parse_voice_command("Hey Vox, say hello", &targets).expect("should match with hey vox trigger"); + assert_eq!(res1.matched_target_id, "say_target"); + assert_eq!(res1.payload, "hello"); + + let res2 = parse_voice_command("Vox Control, say hello world", &targets).expect("should match with vox control trigger"); + assert_eq!(res2.matched_target_id, "say_target"); + assert_eq!(res2.payload, "hello world"); + + let res3 = parse_voice_command("Hey Vox, add this to my notes: buy groceries", &targets).expect("should match notes with filler"); + assert_eq!(res3.matched_target_id, "notes"); + assert_eq!(res3.payload, "buy groceries"); +} + +#[test] +fn test_text_contains_target_name() { + use crate::targets::text_contains_target_name; + + let target = OutputTarget { + id: "say_target".into(), + label: "Say".into(), + delivery: DeliveryType::Speak, + ..OutputTarget::default_inject() + }; + + assert!(text_contains_target_name("Hey box, say hello", &target)); + assert!(text_contains_target_name("Say, how are you", &target)); + assert!(text_contains_target_name("please say testing", &target)); + // Normal sentences without "say" do not match + assert!(!text_contains_target_name("what the hell is going on", &target)); + assert!(!text_contains_target_name("There have been several changes made", &target)); + assert!(!text_contains_target_name("essay writing", &target)); // substring inside another word +} + +#[test] +fn test_extract_payload_for_target() { + use crate::targets::extract_payload_for_target; + + let target = OutputTarget { + id: "say_target".into(), + label: "Say".into(), + delivery: DeliveryType::Speak, + ..OutputTarget::default_inject() + }; + + assert_eq!(extract_payload_for_target("Say hello world", &target), "hello world"); + assert_eq!(extract_payload_for_target("say, what is the weather?", &target), "what is the weather?"); + assert_eq!(extract_payload_for_target("Hey folks, say how are you", &target), "how are you"); + assert_eq!(extract_payload_for_target("hello there", &target), "hello there"); +} + diff --git a/crates/voxctrl-tts/src/engine.rs b/crates/voxctrl-tts/src/engine.rs index 7b928876..572541e3 100644 --- a/crates/voxctrl-tts/src/engine.rs +++ b/crates/voxctrl-tts/src/engine.rs @@ -158,26 +158,10 @@ impl TtsEngineWorker { model_loaded: model_loaded.clone(), }; - let prewarm = match config.engine { - TtsEngine::PocketTts => config.pocket_tts.prewarm, - TtsEngine::InflectMicro => config.inflect_micro.prewarm, - TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, - TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, - _ => false, - }; - // Pre-warming loads the model at startup and keeps it there, which is - // exactly what the on-demand memory mode exists to avoid — so the two - // settings do not fight: on-demand wins and the model waits for its - // first real use (or a `preload()`). - if prewarm && !config.unloads_when_idle() { - let _ = handle.tx.send(TtsCommand::Play { - utterance: Utterance { - text: " ".into(), - voice: None, - source_label: Some("prewarm".into()), - }, - generation: 0, - }); + // In always-loaded mode, ensure the model is preloaded on startup so the + // user never experiences cold-start latency on their first utterance. + if !config.unloads_when_idle() { + let _ = handle.tx.send(TtsCommand::Preload); } let worker = Self { @@ -283,10 +267,34 @@ impl TtsEngineWorker { "TTS worker config dynamically updated (engine={:?}, memory_mode={:?})", new_cfg.engine, new_cfg.memory_mode ); + let was_unloading = current_config.unloads_when_idle(); + let now_unloading = new_cfg.unloads_when_idle(); current_config = new_cfg; - // Switching to on-demand starts the clock now rather than - // dropping a model that may be about to be used again. last_used = Instant::now(); + + // If switched from on_demand to always_loaded (e.g. toggled off in the tray), + // preload/warm the model immediately so it stays ready in memory. + if was_unloading && !now_unloading { + let _ = match current_config.engine { + TtsEngine::InflectMicro if inflect_model.is_none() => { + ensure_inflect_micro_loaded(¤t_config, &mut inflect_model).ok() + } + TtsEngine::PocketTts if audiocpp_session.is_none() => { + ensure_pocket_tts_loaded(¤t_config, &mut audiocpp_session).ok() + } + TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { + ensure_breeze_tts_2_loaded(¤t_config, &mut audiocpp_session).ok() + } + TtsEngine::VoxCpm2 if audiocpp_session.is_none() => { + ensure_vox_cpm_2_loaded(¤t_config, &mut audiocpp_session).ok() + } + _ => None, + }; + self.model_loaded.store( + inflect_model.is_some() || audiocpp_session.is_some(), + Ordering::SeqCst, + ); + } } TtsCommand::Play { mut utterance, generation } => { // Synthesis is about to touch the model; hold off the idle diff --git a/docs/routing.md b/docs/routing.md index 64945727..02e55f33 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -23,8 +23,8 @@ Both files are hot-reloaded when changed on disk. Defined in `~/.config/voxctrl/targets.toml`. Each `[[target]]` block describes one destination. -**Saying one by name.** Start dictation and say *"VoxCtrl"*, then the command's -name, then the text: *"VoxCtrl notes, remember to call the plumber"* delivers +**Saying one by name.** Start dictation and say *"Hey Vox"* (or *"Vox Control"*), then the command's +name, then the text: *"Hey Vox, notes, remember to call the plumber"* (or *"Vox Control notes, remember to call the plumber"*) delivers *remember to call the plumber* to the command named **notes**, whatever the active hotkey was pointed at. Everything after the name is the payload. The full matching rules — conversational lead-ins, fuzzy name matching, the overlay — @@ -262,13 +262,13 @@ typing into the focused window — so voice command routing works the day anothe target is added, with no re-binding. **How Voice Command Routing Works:** -- **Trigger Keyword**: Listens for the `"VoxCtrl"` keyword (case-insensitive, supporting `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, and optional punctuation like `VoxCtrl:`). +- **Trigger Keyword**: Listens for the `"Hey Vox"` or `"Vox Control"` keywords (case-insensitive, supporting `Hey Vox`, `Hey, Vox`, `Hey-Vox`, `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, `vox control`, and trailing punctuation like `Hey Vox:`). - **Target Resolution**: Matches spoken target names against all configured target IDs and Labels (case-insensitively). Longest candidate target names take precedence (e.g. `"Personal Notes"` is matched before `"Notes"`). -- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"VoxCtrl send this to my notes. I love you."*, *"VoxCtrl add this to my personal notes, help"*, *"VoxCtrl put this into my Notes: hello"*, or *"VoxCtrl send us to my notes. I love you."*. +- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"Hey Vox, send this to my notes. I love you."*, *"Hey Vox, add this to my personal notes, help"*, *"Vox Control put this into my Notes: hello"*, or *"Hey, Vox, send us to my notes. I love you."*. - **Payload Extraction**: Strips transition punctuation (`.`, `:`, `,`, `;`) and connector words (`saying`, `that`, `with text`) and routes the remaining text payload to the matched target. - **Command UI Overlay**: When a voice command trigger is matched and executed, VoxCtrl automatically displays a temporary purple/indigo HUD overlay pill displaying the command name and text payload summary for a configurable duration (default: 3 seconds). -- **Auto-Trigger Detection**: Dictating with the `"VoxCtrl"` keyword (e.g. *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. -- **Fallback**: If no `"VoxCtrl"` keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). +- **Auto-Trigger Detection**: Dictating with the `"Hey Vox"` or `"Vox Control"` keyword (e.g. *"Hey Vox, notes Help me!"* or *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. +- **Fallback**: If no trigger keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). --- diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 22b20399..43d11c86 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -345,11 +345,25 @@ pub fn run() { let hotkey_health = Arc::new(voxctrl_hotkeys::ListenerHealth::default()); + // TTS initial worker + let initial_tts_handle = if cfg_data.tts.enabled { + Some(voxctrl_tts::TtsEngineWorker::start( + cfg_data.tts.clone(), + cfg_data.features.custom_vocabulary.clone(), + None, + None, + None, + )) + } else { + None + }; + let app_state = Arc::new(AppState { config: config.clone(), router: router.clone(), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(cfg_data.ui.show_overlay)), mcp_recording: Arc::new(AtomicBool::new(false)), @@ -373,7 +387,7 @@ pub fn run() { audio_tx: audio_tx.clone(), audio_wake: audio_wake_tx, inference_config_tx: inference_cfg_tx, - tts_handle: Arc::new(Mutex::new(None)), + tts_handle: Arc::new(Mutex::new(initial_tts_handle.clone())), active_fifos: Arc::new(Mutex::new(std::collections::HashSet::new())), stop_key_held: Arc::new(AtomicBool::new(false)), speaking_tx: Arc::new(std::sync::OnceLock::new()), @@ -424,26 +438,9 @@ pub fn run() { inference_cfg_rx, ); - // TTS initial worker - let _tts_handle = if cfg_data.tts.enabled { - Some(voxctrl_tts::TtsEngineWorker::start( - cfg_data.tts.clone(), - cfg_data.features.custom_vocabulary.clone(), - None, - None, - None, - )) - } else { - None - }; - let state_for_tts = app_state.clone(); - let tts_handle_clone = _tts_handle.clone(); + let tts_handle_clone = initial_tts_handle.clone(); tokio::spawn(async move { - { - let mut handle = state_for_tts.tts_handle.lock().await; - *handle = tts_handle_clone.clone(); - } if let Some(tts) = tts_handle_clone { state_for_tts.spawn_fifo_responders(tts).await; } diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 37486331..4d2b3535 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -36,6 +36,8 @@ async fn process_remote_transcription( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim: false, + session_id: 0, }); return; } @@ -69,6 +71,8 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); return; } @@ -188,6 +192,8 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); } @@ -205,6 +211,10 @@ pub fn spawn_audio_coordinator( let mut binding_id = String::new(); let mut remote_session: Option = None; let mut is_remote_backend = false; + let mut is_heavy_model = false; + let mut session_id: u64 = 0; + let mut last_interim_sample_count: usize = 0; + let mut last_interim_instant = std::time::Instant::now(); while let Ok(chunk) = audio_rx.recv() { let is_recording = state_for_audio.is_recording(); @@ -215,12 +225,21 @@ pub fn spawn_audio_coordinator( target_id = state_for_audio.active_target.blocking_lock().clone(); binding_id = state_for_audio.active_binding_id.blocking_lock().clone(); was_recording = true; + session_id = session_id.wrapping_add(1); + last_interim_sample_count = 0; + last_interim_instant = std::time::Instant::now(); + state_for_audio.set_interim_in_flight(false); let cfg = state_for_audio.config.blocking_lock().data.clone(); + is_heavy_model = cfg.engine.backend == voxctrl_config::BackendChoice::WhisperCpp + && voxctrl_inference::whisper_gpu_backend().is_none() + && (cfg.engine.whisper_cpp.model_size.starts_with("medium") + || cfg.engine.whisper_cpp.model_size.starts_with("large")); + if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { is_remote_backend = true; let mut merged_prompt = String::from( - "VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. ", + "VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. ", ); if !cfg.features.custom_vocabulary.is_empty() { merged_prompt.push_str("Vocabulary: "); @@ -248,6 +267,30 @@ pub fn spawn_audio_coordinator( session.send_chunk(chunk.clone()); } accumulated_audio.extend(chunk); + + // Live interim sliding-window transcription for fast local STT backends + if !is_remote_backend && !is_heavy_model { + let total_samples = accumulated_audio.len(); + let new_samples = total_samples.saturating_sub(last_interim_sample_count); + if total_samples >= 8000 + && new_samples >= 4800 + && last_interim_instant.elapsed() >= std::time::Duration::from_millis(400) + && !state_for_audio.is_interim_in_flight() + { + state_for_audio.set_interim_in_flight(true); + last_interim_sample_count = total_samples; + last_interim_instant = std::time::Instant::now(); + + let req = voxctrl_inference::InferenceRequest { + audio: accumulated_audio.clone(), + target_id: target_id.clone(), + binding_id: Some(binding_id.clone()), + is_interim: true, + session_id, + }; + let _ = inference_tx.send(req); + } + } } else { if was_recording { if is_remote_backend { @@ -280,6 +323,8 @@ pub fn spawn_audio_coordinator( audio: std::mem::take(&mut accumulated_audio), target_id: target_id.clone(), binding_id: Some(binding_id.clone()), + is_interim: false, + session_id, }; state_for_audio.set_processing(true); let _ = inference_tx.send(req); @@ -298,8 +343,63 @@ pub fn spawn_text_delivery_worker( rt_handle: tokio::runtime::Handle, ) { std::thread::spawn(move || { + let mut last_detected_session: Option = None; + let mut session_command_detected: Option<(u64, String)> = None; + while let Ok(output) = text_rx.recv() { + if output.is_interim { + state.set_interim_in_flight(false); + if output.text.trim().is_empty() { + continue; + } + + // Live early voice command detection: detect command word as soon as uttered + let dir = voxctrl_routing::config_dir(); + let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); + if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { + let matched_id = parsed.matched_target_id.clone(); + let matched_target = targets.iter().find(|t| t.id == matched_id); + let matched_label = matched_target + .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) + .unwrap_or_else(|| matched_id.clone()); + + let already_notified = last_detected_session == Some(output.session_id) + && session_command_detected.as_ref().map(|(_, id)| id) == Some(&matched_id); + + if !already_notified { + last_detected_session = Some(output.session_id); + session_command_detected = Some((output.session_id, matched_id.clone())); + tracing::info!( + "Early command detected during speech: '{}' (target: {})", + matched_label, + matched_id + ); + voxctrl_routing::targets::notify_command_trigger(&matched_label, &parsed.payload); + + let is_speak = matched_target + .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak || t.response_pipe.is_some()) + .unwrap_or(false); + if is_speak { + let state_c = state.clone(); + rt_handle.spawn(async move { + state_c.preload_tts().await; + }); + } + } + } + continue; + } + + state.set_interim_in_flight(false); state.set_processing(false); + + // Retrieve any command target identified earlier during speech for this session + let early_matched_id = session_command_detected + .take() + .filter(|(sess_id, _)| *sess_id == output.session_id) + .map(|(_, id)| id); + last_detected_session = None; + if let Some(ref err) = output.error { // Always surface transcription failures — without this a // fresh install with no Whisper model records audio and @@ -335,21 +435,71 @@ pub fn spawn_text_delivery_worker( let (target_id, text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { let matched_id = parsed.matched_target_id.clone(); let payload = parsed.payload; - let matched_label = targets - .iter() - .find(|t| t.id == matched_id) + let matched_target = targets.iter().find(|t| t.id == matched_id); + let matched_label = matched_target .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) .unwrap_or_else(|| matched_id.clone()); voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); - let cleaned_payload = if s1_mini_enabled && !payload.trim().is_empty() { + // For Speak targets (e.g. "Say" voice commands), do not run S1-mini LLM cleanup. + // Running S1-mini on spoken speech output adds hundreds of milliseconds + // to seconds of latency and can rewrite words the user intended to be spoken verbatim. + let is_speak_target = matched_target + .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) + .unwrap_or(false); + + let cleaned_payload = if !is_speak_target && s1_mini_enabled && !payload.trim().is_empty() { voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) } else { payload }; (matched_id, cleaned_payload) + } else if let Some(early_id) = early_matched_id { + let matched_target = targets.iter().find(|t| t.id == early_id); + let has_target_in_text = matched_target.map_or(false, |tgt| { + voxctrl_routing::targets::text_contains_target_name(&output.text, tgt) + }); + + if has_target_in_text { + let matched_tgt = matched_target.unwrap(); + let matched_label = if matched_tgt.label.is_empty() { + matched_tgt.id.clone() + } else { + matched_tgt.label.clone() + }; + let payload = voxctrl_routing::targets::extract_payload_for_target(&output.text, matched_tgt); + voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); + + let is_speak_target = matched_tgt.delivery == voxctrl_routing::DeliveryType::Speak; + + let cleaned_payload = if !is_speak_target && s1_mini_enabled && !payload.trim().is_empty() { + voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) + } else { + payload + }; + (early_id, cleaned_payload) + } else { + let is_speak_target = targets + .iter() + .find(|t| t.id == output.target_id) + .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) + .unwrap_or(false); + + let cleaned_text = if !is_speak_target && s1_mini_enabled && !output.text.trim().is_empty() { + voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) + } else { + output.text.clone() + }; + (output.target_id.clone(), cleaned_text) + } } else { - let cleaned_text = if s1_mini_enabled && !output.text.trim().is_empty() { + let is_speak_target = targets + .iter() + .find(|t| t.id == output.target_id) + .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) + .unwrap_or(false); + + let cleaned_text = if !is_speak_target && s1_mini_enabled && !output.text.trim().is_empty() { voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) } else { output.text.clone() diff --git a/src-tauri/src/services.rs b/src-tauri/src/services.rs index 1fe74eda..86effb56 100644 --- a/src-tauri/src/services.rs +++ b/src-tauri/src/services.rs @@ -224,8 +224,14 @@ pub fn setup_tts_and_fifos(app_handle: &tauri::AppHandle, state: Arc) pub fn register_speak_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); voxctrl_routing::targets::set_speak_callback(std::sync::Arc::new(move |text| { - let state = state.clone(); let text_str = text.to_string(); + if let Ok(handle) = state.tts_handle.try_lock() { + if let Some(ref tts) = *handle { + tts.speak(text_str); + return; + } + } + let state = state.clone(); tauri::async_runtime::spawn(async move { let handle = state.tts_handle.lock().await; if let Some(ref tts) = *handle { diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index 7e8f1637..c59cea28 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -14,6 +14,8 @@ pub struct AppState { pub recording: Arc, /// True while speech transcription/OpenAI post-processing is running pub processing: Arc, + /// True while an interim sliding-window transcription pass is actively running + pub interim_in_flight: Arc, /// True while TTS is playing back pub speaking: Arc, /// Live mirror of `ui.show_overlay` so the hot status-forwarding loops can @@ -150,6 +152,14 @@ impl AppState { self.processing.store(v, Ordering::SeqCst); } + pub fn is_interim_in_flight(&self) -> bool { + self.interim_in_flight.load(Ordering::SeqCst) + } + + pub fn set_interim_in_flight(&self, v: bool) { + self.interim_in_flight.store(v, Ordering::SeqCst); + } + pub fn is_audio_ready(&self) -> bool { self.audio_ready.load(Ordering::SeqCst) } @@ -291,11 +301,11 @@ impl AppState { /// hides most of that behind the time they spend talking. In always-loaded /// mode the model is already there and this is a no-op. pub async fn preload_tts(&self) { - let unloads_when_idle = { + let tts_enabled = { let cfg = self.config.lock().await; - cfg.data.tts.enabled && cfg.data.tts.unloads_when_idle() + cfg.data.tts.enabled }; - if !unloads_when_idle { + if !tts_enabled { return; } if let Some(tts) = self.tts_handle.lock().await.as_ref() { @@ -305,11 +315,24 @@ impl AppState { /// True when dictating through `target_id` is likely to end in speech — /// either the target speaks the transcript itself, or it has a response - /// pipe whose reply VoxCtrl reads back. Used to decide whether a recording - /// is worth pre-loading the TTS model for; dictating into an editor is not. + /// pipe whose reply VoxCtrl reads back, or it routes through the voice + /// command router when speech targets exist. pub async fn target_leads_to_speech(&self, target_id: &str) -> bool { use voxctrl_routing::DeliveryType; let targets = self.targets.lock().await; + + // If target is the voice command router, check if any target in the configuration uses speech + if target_id == "command" + || targets + .iter() + .any(|t| t.id == target_id && t.delivery == DeliveryType::Command) + { + return targets.iter().any(|t| { + t.delivery == DeliveryType::Speak + || t.response_pipe.as_deref().is_some_and(|p| !p.trim().is_empty()) + }); + } + targets.iter().any(|t| { t.id == target_id && (t.delivery == DeliveryType::Speak diff --git a/src-tauri/src/tests.rs b/src-tauri/src/tests.rs index c5657570..340ad21c 100644 --- a/src-tauri/src/tests.rs +++ b/src-tauri/src/tests.rs @@ -66,6 +66,7 @@ fn make_test_state() -> AppState { router: Arc::new(OutputTargetRouter::new(Vec::new())), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(true)), mcp_recording: Arc::new(AtomicBool::new(false)), diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index 1dbaeb7e..1daf0051 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -35,6 +35,7 @@ let commandOverlayActive = $state(false); let commandOverlayName = $state(""); let commandOverlayText = $state(""); + let commandTriggeredDuringRecording = $state(false); let commandTimerId: any = null; let unlistenCommandExecuted: (() => void) | null = null; let unlistenOverlayStyleSelected: (() => void) | null = null; @@ -46,6 +47,16 @@ ($mcpRecording && $config.mcp.visual_feedback) || (commandOverlayActive && $config.ui.show_command_overlay) ); + + let isSystemResponding = $derived($speaking && $config.tts.enabled && $config.tts.response_overlay); + + let showPrimaryVisualizer = $derived( + $config.ui.overlay_style !== "none" && + $config.ui.show_overlay && + !isSystemResponding && + !$speaking && + ($recording || $status.processing) + ); let renderOverlay = $state(false); let animateActive = $state(false); let timeoutId: any; @@ -267,10 +278,14 @@ commandOverlayName = event.payload.command; commandOverlayText = event.payload.summary; commandOverlayActive = true; + if ($recording) { + commandTriggeredDuringRecording = true; + } if (commandTimerId) clearTimeout(commandTimerId); const durationMs = (event.payload.duration_secs || $config.ui.command_overlay_duration_secs || 3) * 1000; commandTimerId = setTimeout(() => { commandOverlayActive = false; + commandTriggeredDuringRecording = false; }, durationMs); }).then((unlisten) => { unlistenCommandExecuted = unlisten; @@ -311,58 +326,68 @@
{#if renderOverlay && visible} - {#if $config.ui.overlay_style === "waveform"} - - {:else if $config.ui.overlay_style === "pulse"} - - {:else if $config.ui.overlay_style === "blue_wave"} - - {:else if $config.ui.overlay_style === "mono_bars"} - - {:else if $config.ui.overlay_style === "spectrum"} - - {:else if $config.ui.overlay_style === "terminal"} - - {:else if $config.ui.overlay_style === "vinyl"} - - {:else if activeCustomOverlay} - {@html ``} -
- {@html processedHtml} -
- {:else if $config.ui.overlay_style !== "none"} - - {/if} - - {#if $speaking} -
- - {#each [0, 1, 2, 3, 4] as i} - - {/each} - - - SYSTEM RESPONDING - ▸ {targetLabel} - -
- {:else if commandOverlayActive} -
- ⚡ - - {commandOverlayName.toUpperCase()} - ▸ {commandOverlayText} - -
- {:else if $mcpRecording} -
- - - RECORDING - ▸ {targetLabel} - -
- {/if} +
+ {#if showPrimaryVisualizer} +
+ {#if $config.ui.overlay_style === "waveform"} + + {:else if $config.ui.overlay_style === "pulse"} + + {:else if $config.ui.overlay_style === "blue_wave"} + + {:else if $config.ui.overlay_style === "mono_bars"} + + {:else if $config.ui.overlay_style === "spectrum"} + + {:else if $config.ui.overlay_style === "terminal"} + + {:else if $config.ui.overlay_style === "vinyl"} + + {:else if activeCustomOverlay} + {@html ``} +
+ {@html processedHtml} +
+ {:else if $config.ui.overlay_style !== "none"} + + {/if} +
+ {/if} + + {#if commandOverlayActive && $config.ui.show_command_overlay && !isSystemResponding} +
+ ⚡ + + {commandOverlayName.toUpperCase()} + ▸ {commandOverlayText} + +
+ {/if} + + {#if $speaking && $config.tts.enabled && $config.tts.response_overlay} +
+ + {#each [0, 1, 2, 3, 4] as i} + + {/each} + + + SYSTEM RESPONDING + ▸ {targetLabel} + +
+ {/if} + + {#if $mcpRecording && $config.mcp.visual_feedback} +
+ + + RECORDING + ▸ {targetLabel} + +
+ {/if} +
{/if}
@@ -395,8 +420,26 @@ overflow: hidden !important; } + .overlay-stack { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + max-width: 100%; + max-height: 100%; + pointer-events: none; + } + + .visualizer-container { + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + } + .system-response-box { - position: absolute; + position: relative; z-index: 10; display: flex; align-items: center; @@ -406,10 +449,12 @@ border-radius: 23px; font-family: 'Outfit', 'Inter', system-ui, sans-serif; opacity: 0; - transform: translateY(20px); + transform: translateY(10px); transition: transform 0.34s cubic-bezier(0.175, 0.885, 0.32, 1.25), opacity 0.28s ease; + pointer-events: none; + flex-shrink: 0; } .system-response-box.on { diff --git a/src/lib/Settings/AudioTab.svelte b/src/lib/Settings/AudioTab.svelte index e7c943a5..cbec8441 100644 --- a/src/lib/Settings/AudioTab.svelte +++ b/src/lib/Settings/AudioTab.svelte @@ -139,7 +139,7 @@

- Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "VoxCtrl" — is not lost. + Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "Hey Vox" or "Vox Control" — is not lost.

diff --git a/src/lib/Settings/CommandsTab.svelte b/src/lib/Settings/CommandsTab.svelte index 6540f745..26135d16 100644 --- a/src/lib/Settings/CommandsTab.svelte +++ b/src/lib/Settings/CommandsTab.svelte @@ -139,11 +139,11 @@

Saying a command by name. Start dictation and say - “VoxCtrl”, then the command's name, then what you want to send — for - example “VoxCtrl notes, remember to call the plumber” routes + “Hey Vox,” (or “VoxCtrl” / “Vox Control”), then the command's name, then what you want to send — for + example “Hey Vox, notes, remember to call the plumber” or “VoxCtrl notes, remember to call the plumber” routes remember to call the plumber to the command named notes. Everything after the name is the text. Natural phrasing works too - (“VoxCtrl, add this to my notes: …”). Say nothing of the sort and + (“Hey Vox, add this to my notes: …”). Say nothing of the sort and dictation goes wherever your hotkey already points.

diff --git a/src/lib/Settings/TargetEditorModal.svelte b/src/lib/Settings/TargetEditorModal.svelte index 21fe2b7a..30ba9572 100644 --- a/src/lib/Settings/TargetEditorModal.svelte +++ b/src/lib/Settings/TargetEditorModal.svelte @@ -320,8 +320,8 @@

Names this target in the app, and is what you say to send dictation here through a - Voice Command Router target: “VoxCtrl, {editingTarget.label || "Obsidian Notes"}, - buy milk tomorrow”. The Target ID works as a spoken name too, so pick something + Voice Command Router target: “Hey Vox, {editingTarget.label || "Obsidian Notes"}, + buy milk tomorrow” (or “VoxCtrl, {editingTarget.label || "Obsidian Notes"}...”). The Target ID works as a spoken name too, so pick something easy to say and easy for speech recognition to catch.

@@ -330,7 +330,7 @@ @@ -352,7 +352,7 @@
Voice Command Router Settings

- Types dictated text into your active application by default. If your dictation contains VoxCtrl <target_name> <text> (for example, "VoxCtrl Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead. + Types dictated text into your active application by default. If your dictation contains Hey Vox <target_name> <text> or Vox Control <target_name> <text> (for example, "Hey Vox, Notes Hello world" or "Vox Control Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead.

{/if} diff --git a/src/stores/config.ts b/src/stores/config.ts index 5346f8de..f048c32e 100644 --- a/src/stores/config.ts +++ b/src/stores/config.ts @@ -199,7 +199,7 @@ const defaultConfig: AppConfig = { }, features: { remove_fillers: true, - custom_vocabulary: ["VoxCtrl"], + custom_vocabulary: ["VoxCtrl", "Hey Vox"], spoken_punctuation: true, auto_format_lists: true, snippets: {}, diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index b1a935b8..ceb3eb74 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -29,6 +29,8 @@ vi.stubGlobal( vi.stubGlobal("cancelAnimationFrame", (id: number) => clearTimeout(id)); import { status, type AppStatus } from "../../src/stores/status"; +import { config } from "../../src/stores/config"; +import Overlay from "../../src/lib/Overlay/Overlay.svelte"; import Waveform from "../../src/lib/Overlay/Waveform.svelte"; import Pulse from "../../src/lib/Overlay/Pulse.svelte"; import BlueWave from "../../src/lib/Overlay/BlueWave.svelte"; @@ -189,3 +191,46 @@ describe("VoiceCard.svelte (membership card)", () => { expect(procContainer.querySelector(".field-value")?.textContent).toContain("Reading the card…"); }); }); + +describe("Overlay.svelte (root layout)", () => { + test("hides target visualizer and displays SYSTEM RESPONDING when speaking", async () => { + config.update((c) => ({ + ...c, + ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, + tts: { ...c.tts, enabled: true, response_overlay: true }, + })); + + setStatus({ speaking: true, recording: false, active_target_label: "Kitty Terminal" }); + + const { container } = render(Overlay); + await new Promise((r) => setTimeout(r, 50)); + + // SYSTEM RESPONDING pill should be present + expect(container.textContent).toContain("SYSTEM RESPONDING"); + expect(container.textContent).toContain("Kitty Terminal"); + + // Target visualizer (e.g. Waveform) must NOT be present + expect(container.querySelector(".scope")).toBeNull(); + expect(container.textContent).not.toContain("WAVEFORM // OSC-01"); + }); + + test("shows target visualizer when recording", async () => { + config.update((c) => ({ + ...c, + ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, + tts: { ...c.tts, enabled: true, response_overlay: true }, + })); + + setStatus({ speaking: false, recording: true, active_target_label: "Code Editor" }); + + const { container } = render(Overlay); + await new Promise((r) => setTimeout(r, 50)); + + // Target visualizer should be present + expect(container.querySelector(".scope")).not.toBeNull(); + expect(container.textContent).toContain("WAVEFORM // OSC-01"); + + // SYSTEM RESPONDING should NOT be present + expect(container.textContent).not.toContain("SYSTEM RESPONDING"); + }); +}); From 8c97721448ebd21de0884dd059f42369d35a4fb3 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:46:11 -0500 Subject: [PATCH 02/12] feat(commands): add "Hey Vox" trigger keyword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Accept "Hey Vox" (and "Hey, Vox", "Hey-Vox", fuzzy "hey box"/"hay vox") alongside "VoxCtrl"/"Vox Control"; exact triggers now require word boundaries so "voxelize" and similar never trigger. - Only treat " control" as a trigger when it opens the dictation and sounds like "vox" ("box control", "walks control"). Previously any "... control" anywhere matched, so "Take control, say hello" was routed to a Say target. - Match target names on an offset-preserving lowercase copy so text or labels with characters like "İ" can no longer panic the delivery thread. - Add "Hey Vox" to the default custom vocabulary and the Whisper prompt, and update Settings copy and docs/routing.md. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-config/src/lib.rs | 2 +- crates/voxctrl-inference/src/lib.rs | 2 +- crates/voxctrl-routing/src/targets.rs | 130 ++++++++++++++++------ crates/voxctrl-routing/src/tests.rs | 126 +++++++++++++++++++++ docs/routing.md | 12 +- src-tauri/src/pipeline.rs | 2 +- src/lib/Settings/AudioTab.svelte | 2 +- src/lib/Settings/CommandsTab.svelte | 6 +- src/lib/Settings/TargetEditorModal.svelte | 10 +- src/stores/config.ts | 2 +- 10 files changed, 243 insertions(+), 51 deletions(-) diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index 87c54a0a..40e37e30 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -292,7 +292,7 @@ impl Default for FeaturesConfig { fn default() -> Self { Self { remove_fillers: true, - custom_vocabulary: vec!["VoxCtrl".into()], + custom_vocabulary: vec!["VoxCtrl".into(), "Hey Vox".into()], spoken_punctuation: true, auto_format_lists: true, show_notification: None, diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 74e75b98..0a670b31 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -351,7 +351,7 @@ impl InferenceEngine { let dir = voxctrl_routing::config_dir(); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. "); + let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. "); // Custom vocabulary words from features config if !app_config.features.custom_vocabulary.is_empty() { diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 892f28c4..4d3f9307 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -1310,44 +1310,87 @@ fn levenshtein_distance(s1: &str, s2: &str) -> usize { dp[len1][len2] } -/// Parse text for keyword "VoxCtrl" and target name/label matching. -/// Supports both direct commands (e.g. "VoxCtrl notes Hi there") and natural, -/// conversational phrasing (e.g. "VoxCtrl add this to my notes. What are you doing here?"). -/// Returns `Some(VoiceCommandParseResult)` if the trigger keyword was found AND a target matched; -/// otherwise returns `None`. +/// Lowercase `s` without changing any character's byte length, so byte offsets +/// found in the result are valid in `s` too. The few characters whose lowercase +/// form is longer or shorter in UTF-8 (e.g. `İ`, `ẞ`) are left as they are, +/// which only costs them case-insensitive matching. +fn fold_case(s: &str) -> String { + s.chars() + .map(|c| { + let mut lower = c.to_lowercase(); + match (lower.next(), lower.next()) { + (Some(l), None) if l.len_utf8() == c.len_utf8() => l, + _ => c, + } + }) + .collect() +} + +/// True when `word` sounds like "vox" as speech recognition tends to mishear it +/// ("box", "fox", "walks", "locks"): short and ending in an x/ks sound. +fn sounds_like_vox(word: &str) -> bool { + word.len() <= 6 && (word.ends_with('x') || word.ends_with("ks") || word.ends_with("cs")) +} + +/// Parse text for a trigger keyword ("VoxCtrl", "Vox Control", "Hey Vox", ...) +/// followed by a target name/label. Supports both direct commands +/// (e.g. "Hey Vox notes Hi there") and natural, conversational phrasing +/// (e.g. "Hey Vox, add this to my notes. What are you doing here?"). +/// Without an explicit trigger nothing matches, so normal dictation is never +/// misclassified as a command. Returns `Some(VoiceCommandParseResult)` if the +/// trigger was found AND a target matched; otherwise `None`. pub fn parse_voice_command( text: &str, targets: &[OutputTarget], ) -> Option { - let lower_text = text.to_lowercase(); + let lower_text = fold_case(text); let mut found_pos = None; let mut trigger_len = 0; // 1. Exact & standard triggers - let exact_triggers = ["voxctrl", "vox ctrl", "vox-ctrl", "vox control"]; + let exact_triggers = [ + "voxctrl", "vox ctrl", "vox-ctrl", "vox control", + "hey vox", "hey, vox", "hey-vox", "hey_vox", + ]; for trigger in &exact_triggers { - if let Some(pos) = lower_text.find(trigger) { - if found_pos.map_or(true, |p| pos < p) { - found_pos = Some(pos); - trigger_len = trigger.len(); + let mut search_start = 0; + while let Some(match_idx) = lower_text[search_start..].find(trigger) { + let pos = search_start + match_idx; + let end_idx = pos + trigger.len(); + let is_boundary_start = pos == 0 || { + let prev = lower_text[..pos].chars().last().unwrap(); + prev.is_whitespace() || prev.is_ascii_punctuation() + }; + let is_boundary_end = end_idx == lower_text.len() || { + let next = lower_text[end_idx..].chars().next().unwrap(); + next.is_whitespace() || next.is_ascii_punctuation() + }; + if is_boundary_start && is_boundary_end { + if found_pos.map_or(true, |p| pos < p) { + found_pos = Some(pos); + trigger_len = trigger.len(); + } + break; } + search_start = pos + 1; } } - // 2. Dynamic pattern trigger for any " control" or " ctrl" phrase + // 2. Misheard "vox control" at the very start ("box control", "walks control"). + // Only the leading two words count, and the first must sound like "vox": + // "take control, say hello" or "I lost control" is ordinary dictation. if found_pos.is_none() { let words: Vec<&str> = lower_text.split_whitespace().collect(); - for (i, word) in words.iter().enumerate() { - let clean_w = word.trim_matches(|c: char| c.is_ascii_punctuation()); - if clean_w == "control" || clean_w == "ctrl" || clean_w == "ctl" || clean_w == "kontrol" { - if i > 0 { - let start_idx = lower_text.find(words[0]).unwrap_or(0); - let ctrl_pos = lower_text.find(word).unwrap_or(0); - let end_pos = ctrl_pos + word.len(); - found_pos = Some(start_idx); - trigger_len = end_pos - start_idx; - break; - } + if words.len() >= 2 { + let first = words[0].trim_matches(|c: char| c.is_ascii_punctuation()); + let second = words[1].trim_matches(|c: char| c.is_ascii_punctuation()); + if sounds_like_vox(first) && matches!(second, "control" | "ctrl" | "ctl" | "kontrol") { + let start_idx = lower_text.find(words[0]).unwrap_or(0); + let first_end = start_idx + words[0].len(); + let second_end = + first_end + lower_text[first_end..].find(words[1]).unwrap_or(0) + words[1].len(); + found_pos = Some(start_idx); + trigger_len = second_end - start_idx; } } } @@ -1361,12 +1404,32 @@ pub fn parse_voice_command( let clean_cand = candidate.trim_matches(|c: char| c.is_ascii_punctuation()); let dist1 = levenshtein_distance(clean_cand, "voxctrl"); let dist2 = levenshtein_distance(clean_cand, "vox control"); - if dist1 <= 2 || dist2 <= 3 { - if let Some(pos) = lower_text.find(clean_cand) { - found_pos = Some(pos); - trigger_len = clean_cand.len(); - break; - } + + // Check "hey vox" fuzzy match (distance <= 1) on normalized tokens + let clean_tokens: Vec<&str> = words[..len] + .iter() + .map(|w| w.trim_matches(|c: char| c.is_ascii_punctuation())) + .filter(|w| !w.is_empty()) + .collect(); + let hey_vox_cand = clean_tokens.join(" "); + let dist_hey_vox = if !hey_vox_cand.is_empty() { + levenshtein_distance(&hey_vox_cand, "hey vox") + } else { + usize::MAX + }; + + if dist1 <= 2 || dist2 <= 3 || dist_hey_vox <= 1 { + let first_word = words[0]; + let last_word = words[len - 1]; + let start_idx = lower_text.find(first_word).unwrap_or(0); + let last_pos = lower_text[start_idx..] + .find(last_word) + .map(|p| start_idx + p) + .unwrap_or(start_idx); + let end_idx = last_pos + last_word.len(); + found_pos = Some(start_idx); + trigger_len = end_idx - start_idx; + break; } } } @@ -1392,10 +1455,10 @@ pub fn parse_voice_command( candidate_entries.sort_by(|a, b| b.1.len().cmp(&a.1.len())); - let after_trigger_lower = after_trigger.to_lowercase(); + let after_trigger_lower = fold_case(after_trigger); for (target_id, candidate) in candidate_entries { - let cand_lower = candidate.to_lowercase(); + let cand_lower = fold_case(candidate); let mut search_start = 0; while let Some(match_idx) = after_trigger_lower[search_start..].find(&cand_lower) { @@ -1425,9 +1488,12 @@ pub fn parse_voice_command( } } - search_start = abs_match_start + 1; + // Step past the whole first character: target names need not be ASCII. + search_start = abs_match_start + + after_trigger_lower[abs_match_start..].chars().next().map_or(1, char::len_utf8); } } None } + diff --git a/crates/voxctrl-routing/src/tests.rs b/crates/voxctrl-routing/src/tests.rs index cd68ab76..38794034 100644 --- a/crates/voxctrl-routing/src/tests.rs +++ b/crates/voxctrl-routing/src/tests.rs @@ -1768,6 +1768,12 @@ fn test_parse_voice_command_no_keyword() { ]; assert!(parse_voice_command("hi there", &targets).is_none()); + assert!(parse_voice_command("Hey folks, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey boss, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey guys, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("They voxelize notes and models", &targets).is_none()); + assert!(parse_voice_command("heavy box notes something", &targets).is_none()); + assert!(parse_voice_command("the box notes something", &targets).is_none()); } #[test] @@ -1854,6 +1860,60 @@ fn test_parse_voice_command_matched_target() { let res8 = parse_voice_command("Walks control speak how are you?", &speak_targets).expect("should match Walks control dynamic trigger pattern"); assert_eq!(res8.matched_target_id, "speak"); assert_eq!(res8.payload, "how are you?"); + + // Hey Vox commands + let res9 = parse_voice_command("Hey Vox, notes Hi there", &targets).expect("should match Hey Vox with comma"); + assert_eq!(res9.matched_target_id, "notes"); + assert_eq!(res9.payload, "Hi there"); + + let res10 = parse_voice_command("Hey Vox notes Hi there", &targets).expect("should match Hey Vox without comma"); + assert_eq!(res10.matched_target_id, "notes"); + assert_eq!(res10.payload, "Hi there"); + + let res11 = parse_voice_command("Hey, Vox, notes Hi there", &targets).expect("should match Hey, Vox, with internal comma"); + assert_eq!(res11.matched_target_id, "notes"); + assert_eq!(res11.payload, "Hi there"); + + let res12 = parse_voice_command("Hey Vox: Notes, please write this down", &targets).expect("should match Hey Vox with colon"); + assert_eq!(res12.matched_target_id, "notes"); + assert_eq!(res12.payload, "please write this down"); + + let res13 = parse_voice_command("Hey, Vox: Notes, please write this down", &targets).expect("should match Hey, Vox: with colon"); + assert_eq!(res13.matched_target_id, "notes"); + assert_eq!(res13.payload, "please write this down"); + + let res14 = parse_voice_command("hey vox add this to my notes. What are you doing here?", &targets).expect("should match conversational hey vox notes"); + assert_eq!(res14.matched_target_id, "notes"); + assert_eq!(res14.payload, "What are you doing here?"); + + let res15 = parse_voice_command("Hey Vox put this into my Notes: What are you doing here?", &targets).expect("should match conversational put into notes"); + assert_eq!(res15.matched_target_id, "notes"); + assert_eq!(res15.payload, "What are you doing here?"); + + let res16 = parse_voice_command("Hey, Vox, send us to my notes. I love you.", &targets).expect("should match conversational send us to my notes"); + assert_eq!(res16.matched_target_id, "notes"); + assert_eq!(res16.payload, "I love you."); + + // Hey Vox homophones (Acoustic mis-recognitions within distance <= 1) + let res17 = parse_voice_command("Hey Box speak text", &speak_targets).expect("should match Hey Box homophone"); + assert_eq!(res17.matched_target_id, "speak"); + assert_eq!(res17.payload, "text"); + + let res18 = parse_voice_command("hay vox speak text", &speak_targets).expect("should match hay vox homophone"); + assert_eq!(res18.matched_target_id, "speak"); + assert_eq!(res18.payload, "text"); + + let res19 = parse_voice_command("hey fox speak text", &speak_targets).expect("should match hey fox homophone"); + assert_eq!(res19.matched_target_id, "speak"); + assert_eq!(res19.payload, "text"); + + let res20 = parse_voice_command("hey vax speak text", &speak_targets).expect("should match hey vax homophone"); + assert_eq!(res20.matched_target_id, "speak"); + assert_eq!(res20.payload, "text"); + + let res21 = parse_voice_command("Hey, Box, speak, eat my ass.", &speak_targets).expect("should match Hey, Box, homophone with punctuation"); + assert_eq!(res21.matched_target_id, "speak"); + assert_eq!(res21.payload, "eat my ass."); } #[test] @@ -2218,3 +2278,69 @@ async fn command_targets_keep_newlines_when_not_stripping() { let text = delivered.expect("delivery produced no text"); assert_eq!(text, "first line\nsecond line "); } + +#[test] +fn test_parse_voice_command_requires_wake_word() { + use crate::targets::parse_voice_command; + + let targets = vec![ + OutputTarget { + id: "say_target".into(), + label: "Say".into(), + delivery: DeliveryType::Speak, + ..OutputTarget::default_inject() + }, + OutputTarget { + id: "notes".into(), + label: "Notes".into(), + delivery: DeliveryType::File, + ..OutputTarget::default_inject() + }, + ]; + + // Normal speech must NEVER trigger a command without wake words + assert!(parse_voice_command("say hello world", &targets).is_none()); + assert!(parse_voice_command("notes: buy groceries", &targets).is_none()); + assert!(parse_voice_command("what the hell is going on. And why is it happening?", &targets).is_none()); + assert!(parse_voice_command("There have been several changes made in this branch.", &targets).is_none()); + assert!(parse_voice_command("Do you hear me? Do you know what's happening?", &targets).is_none()); + // " control" only triggers at the start, and only when sounds like "vox" + assert!(parse_voice_command("Take control, say hello", &targets).is_none()); + assert!(parse_voice_command("I lost control notes about the meeting", &targets).is_none()); + assert!(parse_voice_command("The remote control, say what you will", &targets).is_none()); + + // Valid commands with Hey Vox or Vox Control trigger properly + let res1 = parse_voice_command("Hey Vox, say hello", &targets).expect("should match with hey vox trigger"); + assert_eq!(res1.matched_target_id, "say_target"); + assert_eq!(res1.payload, "hello"); + + let res2 = parse_voice_command("Vox Control, say hello world", &targets).expect("should match with vox control trigger"); + assert_eq!(res2.matched_target_id, "say_target"); + assert_eq!(res2.payload, "hello world"); + + let res3 = parse_voice_command("Hey Vox, add this to my notes: buy groceries", &targets).expect("should match notes with filler"); + assert_eq!(res3.matched_target_id, "notes"); + assert_eq!(res3.payload, "buy groceries"); +} + +#[test] +fn test_parse_voice_command_non_ascii_text_and_targets() { + use crate::targets::parse_voice_command; + + let targets = vec![OutputTarget { + id: "uber".into(), + label: "Überblick".into(), + delivery: DeliveryType::File, + ..OutputTarget::default_inject() + }]; + + // `İ` lowercases to a longer UTF-8 sequence; offsets must still line up. + let res = parse_voice_command("İİ Hey Vox, Überblick grüße aus İstanbul", &targets) + .expect("should match non-ASCII label after non-ASCII prefix"); + assert_eq!(res.matched_target_id, "uber"); + assert_eq!(res.payload, "grüße aus İstanbul"); + + // A non-boundary occurrence of a label starting with a multibyte char + // must be skipped without slicing mid-character. + assert!(parse_voice_command("Hey Vox, xÜberblick hallo", &targets).is_none()); +} diff --git a/docs/routing.md b/docs/routing.md index 64945727..02e55f33 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -23,8 +23,8 @@ Both files are hot-reloaded when changed on disk. Defined in `~/.config/voxctrl/targets.toml`. Each `[[target]]` block describes one destination. -**Saying one by name.** Start dictation and say *"VoxCtrl"*, then the command's -name, then the text: *"VoxCtrl notes, remember to call the plumber"* delivers +**Saying one by name.** Start dictation and say *"Hey Vox"* (or *"Vox Control"*), then the command's +name, then the text: *"Hey Vox, notes, remember to call the plumber"* (or *"Vox Control notes, remember to call the plumber"*) delivers *remember to call the plumber* to the command named **notes**, whatever the active hotkey was pointed at. Everything after the name is the payload. The full matching rules — conversational lead-ins, fuzzy name matching, the overlay — @@ -262,13 +262,13 @@ typing into the focused window — so voice command routing works the day anothe target is added, with no re-binding. **How Voice Command Routing Works:** -- **Trigger Keyword**: Listens for the `"VoxCtrl"` keyword (case-insensitive, supporting `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, and optional punctuation like `VoxCtrl:`). +- **Trigger Keyword**: Listens for the `"Hey Vox"` or `"Vox Control"` keywords (case-insensitive, supporting `Hey Vox`, `Hey, Vox`, `Hey-Vox`, `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, `vox control`, and trailing punctuation like `Hey Vox:`). - **Target Resolution**: Matches spoken target names against all configured target IDs and Labels (case-insensitively). Longest candidate target names take precedence (e.g. `"Personal Notes"` is matched before `"Notes"`). -- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"VoxCtrl send this to my notes. I love you."*, *"VoxCtrl add this to my personal notes, help"*, *"VoxCtrl put this into my Notes: hello"*, or *"VoxCtrl send us to my notes. I love you."*. +- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"Hey Vox, send this to my notes. I love you."*, *"Hey Vox, add this to my personal notes, help"*, *"Vox Control put this into my Notes: hello"*, or *"Hey, Vox, send us to my notes. I love you."*. - **Payload Extraction**: Strips transition punctuation (`.`, `:`, `,`, `;`) and connector words (`saying`, `that`, `with text`) and routes the remaining text payload to the matched target. - **Command UI Overlay**: When a voice command trigger is matched and executed, VoxCtrl automatically displays a temporary purple/indigo HUD overlay pill displaying the command name and text payload summary for a configurable duration (default: 3 seconds). -- **Auto-Trigger Detection**: Dictating with the `"VoxCtrl"` keyword (e.g. *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. -- **Fallback**: If no `"VoxCtrl"` keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). +- **Auto-Trigger Detection**: Dictating with the `"Hey Vox"` or `"Vox Control"` keyword (e.g. *"Hey Vox, notes Help me!"* or *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. +- **Fallback**: If no trigger keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). --- diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 37486331..78460600 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -220,7 +220,7 @@ pub fn spawn_audio_coordinator( if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { is_remote_backend = true; let mut merged_prompt = String::from( - "VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. ", + "VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. ", ); if !cfg.features.custom_vocabulary.is_empty() { merged_prompt.push_str("Vocabulary: "); diff --git a/src/lib/Settings/AudioTab.svelte b/src/lib/Settings/AudioTab.svelte index e7c943a5..cbec8441 100644 --- a/src/lib/Settings/AudioTab.svelte +++ b/src/lib/Settings/AudioTab.svelte @@ -139,7 +139,7 @@

- Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "VoxCtrl" — is not lost. + Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "Hey Vox" or "Vox Control" — is not lost.

diff --git a/src/lib/Settings/CommandsTab.svelte b/src/lib/Settings/CommandsTab.svelte index 6540f745..26135d16 100644 --- a/src/lib/Settings/CommandsTab.svelte +++ b/src/lib/Settings/CommandsTab.svelte @@ -139,11 +139,11 @@

Saying a command by name. Start dictation and say - “VoxCtrl”, then the command's name, then what you want to send — for - example “VoxCtrl notes, remember to call the plumber” routes + “Hey Vox,” (or “VoxCtrl” / “Vox Control”), then the command's name, then what you want to send — for + example “Hey Vox, notes, remember to call the plumber” or “VoxCtrl notes, remember to call the plumber” routes remember to call the plumber to the command named notes. Everything after the name is the text. Natural phrasing works too - (“VoxCtrl, add this to my notes: …”). Say nothing of the sort and + (“Hey Vox, add this to my notes: …”). Say nothing of the sort and dictation goes wherever your hotkey already points.

diff --git a/src/lib/Settings/TargetEditorModal.svelte b/src/lib/Settings/TargetEditorModal.svelte index 21fe2b7a..30ba9572 100644 --- a/src/lib/Settings/TargetEditorModal.svelte +++ b/src/lib/Settings/TargetEditorModal.svelte @@ -320,8 +320,8 @@

Names this target in the app, and is what you say to send dictation here through a - Voice Command Router target: “VoxCtrl, {editingTarget.label || "Obsidian Notes"}, - buy milk tomorrow”. The Target ID works as a spoken name too, so pick something + Voice Command Router target: “Hey Vox, {editingTarget.label || "Obsidian Notes"}, + buy milk tomorrow” (or “VoxCtrl, {editingTarget.label || "Obsidian Notes"}...”). The Target ID works as a spoken name too, so pick something easy to say and easy for speech recognition to catch.

@@ -330,7 +330,7 @@ @@ -352,7 +352,7 @@
Voice Command Router Settings

- Types dictated text into your active application by default. If your dictation contains VoxCtrl <target_name> <text> (for example, "VoxCtrl Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead. + Types dictated text into your active application by default. If your dictation contains Hey Vox <target_name> <text> or Vox Control <target_name> <text> (for example, "Hey Vox, Notes Hello world" or "Vox Control Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead.

{/if} diff --git a/src/stores/config.ts b/src/stores/config.ts index 5346f8de..f048c32e 100644 --- a/src/stores/config.ts +++ b/src/stores/config.ts @@ -199,7 +199,7 @@ const defaultConfig: AppConfig = { }, features: { remove_fillers: true, - custom_vocabulary: ["VoxCtrl"], + custom_vocabulary: ["VoxCtrl", "Hey Vox"], spoken_punctuation: true, auto_format_lists: true, snippets: {}, From 187eec53d61fd9f7323d7f27ee3e01d6f5d671b6 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:50:06 -0500 Subject: [PATCH 03/12] feat(tts): start TTS model loads earlier without loading at every startup - preload_tts() now also works in always-loaded mode, where a model without prewarm is loaded lazily; recording against a speech target (or an early spoken command) starts that first load while the user is still talking. The worker ignores the request when the model is already resident. - Switching from on-demand to always-loaded loads the model immediately when the engine's prewarm setting is on, and logs a load failure. - The per-engine prewarm settings still decide whether the model is loaded at startup (an earlier draft loaded it unconditionally, including Breeze-TTS-2's ~5 GB, and made the prewarm toggles do nothing). - Voice Command Router hotkeys no longer count as speech targets, so on-demand mode doesn't load the model for every ordinary dictation. - Start the TTS worker before AppState is built so tts_handle is set from the start, and let the speak callback use it without a runtime hop when it's uncontended. - Share the "load the selected engine" match between Preload and the mode switch. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-tts/src/engine.rs | 73 ++++++++++++++++++++++---------- docs/tts.md | 2 +- src-tauri/src/lib.rs | 34 +++++++-------- src-tauri/src/services.rs | 8 +++- src-tauri/src/state.rs | 17 +++----- 5 files changed, 80 insertions(+), 54 deletions(-) diff --git a/crates/voxctrl-tts/src/engine.rs b/crates/voxctrl-tts/src/engine.rs index 7b928876..995ba0c8 100644 --- a/crates/voxctrl-tts/src/engine.rs +++ b/crates/voxctrl-tts/src/engine.rs @@ -123,6 +123,38 @@ impl TtsEngineHandle { } } +/// Whether the selected engine's `prewarm` setting asks for its model to be +/// loaded up front. +fn prewarm_enabled(config: &TtsConfig) -> bool { + match config.engine { + TtsEngine::PocketTts => config.pocket_tts.prewarm, + TtsEngine::InflectMicro => config.inflect_micro.prewarm, + TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, + TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, + _ => false, + } +} + +/// Load the selected engine's model if it is not resident yet. `None` when +/// there was nothing to do: already loaded, or an engine that holds no model. +fn load_selected_engine( + config: &TtsConfig, + inflect_model: &mut InflectModelSlot, + audiocpp_session: &mut Option, +) -> Option> { + match config.engine { + TtsEngine::InflectMicro if inflect_model.is_none() => { + Some(ensure_inflect_micro_loaded(config, inflect_model)) + } + TtsEngine::PocketTts if audiocpp_session.is_none() => Some(ensure_pocket_tts_loaded(config, audiocpp_session)), + TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { + Some(ensure_breeze_tts_2_loaded(config, audiocpp_session)) + } + TtsEngine::VoxCpm2 if audiocpp_session.is_none() => Some(ensure_vox_cpm_2_loaded(config, audiocpp_session)), + _ => None, + } +} + // ── TTS engine worker ───────────────────────────────────────────────────────── pub type PlaybackCallback = Arc; @@ -158,13 +190,7 @@ impl TtsEngineWorker { model_loaded: model_loaded.clone(), }; - let prewarm = match config.engine { - TtsEngine::PocketTts => config.pocket_tts.prewarm, - TtsEngine::InflectMicro => config.inflect_micro.prewarm, - TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, - TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, - _ => false, - }; + let prewarm = prewarm_enabled(&config); // Pre-warming loads the model at startup and keeps it there, which is // exactly what the on-demand memory mode exists to avoid — so the two // settings do not fight: on-demand wins and the model waits for its @@ -283,10 +309,25 @@ impl TtsEngineWorker { "TTS worker config dynamically updated (engine={:?}, memory_mode={:?})", new_cfg.engine, new_cfg.memory_mode ); + let was_unloading = current_config.unloads_when_idle(); current_config = new_cfg; // Switching to on-demand starts the clock now rather than // dropping a model that may be about to be used again. last_used = Instant::now(); + + // Switching to always-loaded (e.g. from the tray) with + // pre-warming on: load now, as a startup in this mode would. + if was_unloading && !current_config.unloads_when_idle() && prewarm_enabled(¤t_config) { + if let Some(Err(e)) = + load_selected_engine(¤t_config, &mut inflect_model, &mut audiocpp_session) + { + warn!("TTS pre-warm after switching to always-loaded failed: {e:#}"); + } + self.model_loaded.store( + inflect_model.is_some() || audiocpp_session.is_some(), + Ordering::SeqCst, + ); + } } TtsCommand::Play { mut utterance, generation } => { // Synthesis is about to touch the model; hold off the idle @@ -427,22 +468,8 @@ impl TtsEngineWorker { // Speculative: failures are logged, not surfaced. The same // error is reported properly (with a toast) if an utterance // actually arrives. - let outcome = match current_config.engine { - TtsEngine::InflectMicro if inflect_model.is_none() => { - Some(ensure_inflect_micro_loaded(¤t_config, &mut inflect_model)) - } - TtsEngine::PocketTts if audiocpp_session.is_none() => { - Some(ensure_pocket_tts_loaded(¤t_config, &mut audiocpp_session)) - } - TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { - Some(ensure_breeze_tts_2_loaded(¤t_config, &mut audiocpp_session)) - } - TtsEngine::VoxCpm2 if audiocpp_session.is_none() => { - Some(ensure_vox_cpm_2_loaded(¤t_config, &mut audiocpp_session)) - } - // Already resident, or an engine that holds no model. - _ => None, - }; + let outcome = + load_selected_engine(¤t_config, &mut inflect_model, &mut audiocpp_session); match outcome { Some(Err(e)) => debug!("TTS preload skipped: {e:#}"), Some(Ok(())) => debug!("TTS model pre-loaded and primed"), diff --git a/docs/tts.md b/docs/tts.md index e87d3167..a9bec782 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -333,7 +333,7 @@ survives being idle: | Mode | Behaviour | |---|---| -| `"always_loaded"` (default) | The model is loaded on first use and stays resident for the life of the TTS worker. Fastest, highest memory. | +| `"always_loaded"` (default) | The model is loaded on first use (or at startup with `prewarm`) and stays resident for the life of the TTS worker. Fastest, highest memory. Recording against a speech target, or a spoken command to one, starts that first load early, as in `"on_demand"` mode. Switching into this mode with `prewarm` on loads the model right away. | | `"on_demand"` | The model is loaded when VoxCtrl knows it is needed, kept primed while it keeps being used, and dropped after `idle_unload_secs` (default 900 = 15 minutes) of inactivity. | In `"on_demand"` mode TTS itself stays enabled the whole time — only the weights come and go: diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 22b20399..9a1dd8e4 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -345,6 +345,19 @@ pub fn run() { let hotkey_health = Arc::new(voxctrl_hotkeys::ListenerHealth::default()); + // TTS initial worker + let initial_tts_handle = if cfg_data.tts.enabled { + Some(voxctrl_tts::TtsEngineWorker::start( + cfg_data.tts.clone(), + cfg_data.features.custom_vocabulary.clone(), + None, + None, + None, + )) + } else { + None + }; + let app_state = Arc::new(AppState { config: config.clone(), router: router.clone(), @@ -373,7 +386,7 @@ pub fn run() { audio_tx: audio_tx.clone(), audio_wake: audio_wake_tx, inference_config_tx: inference_cfg_tx, - tts_handle: Arc::new(Mutex::new(None)), + tts_handle: Arc::new(Mutex::new(initial_tts_handle.clone())), active_fifos: Arc::new(Mutex::new(std::collections::HashSet::new())), stop_key_held: Arc::new(AtomicBool::new(false)), speaking_tx: Arc::new(std::sync::OnceLock::new()), @@ -424,26 +437,9 @@ pub fn run() { inference_cfg_rx, ); - // TTS initial worker - let _tts_handle = if cfg_data.tts.enabled { - Some(voxctrl_tts::TtsEngineWorker::start( - cfg_data.tts.clone(), - cfg_data.features.custom_vocabulary.clone(), - None, - None, - None, - )) - } else { - None - }; - let state_for_tts = app_state.clone(); - let tts_handle_clone = _tts_handle.clone(); + let tts_handle_clone = initial_tts_handle.clone(); tokio::spawn(async move { - { - let mut handle = state_for_tts.tts_handle.lock().await; - *handle = tts_handle_clone.clone(); - } if let Some(tts) = tts_handle_clone { state_for_tts.spawn_fifo_responders(tts).await; } diff --git a/src-tauri/src/services.rs b/src-tauri/src/services.rs index 1fe74eda..86effb56 100644 --- a/src-tauri/src/services.rs +++ b/src-tauri/src/services.rs @@ -224,8 +224,14 @@ pub fn setup_tts_and_fifos(app_handle: &tauri::AppHandle, state: Arc) pub fn register_speak_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); voxctrl_routing::targets::set_speak_callback(std::sync::Arc::new(move |text| { - let state = state.clone(); let text_str = text.to_string(); + if let Ok(handle) = state.tts_handle.try_lock() { + if let Some(ref tts) = *handle { + tts.speak(text_str); + return; + } + } + let state = state.clone(); tauri::async_runtime::spawn(async move { let handle = state.tts_handle.lock().await; if let Some(ref tts) = *handle { diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index 7e8f1637..fc23b1a4 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -285,17 +285,14 @@ impl AppState { /// Start loading the TTS model now, before anything asks it to speak. /// - /// Only meaningful in the on-demand memory mode, where the model is not - /// resident between uses: the load takes seconds, so kicking it off the - /// moment we know speech is coming (the user has just started dictating) - /// hides most of that behind the time they spend talking. In always-loaded - /// mode the model is already there and this is a no-op. + /// The load takes seconds, so kicking it off the moment we know speech is + /// coming (the user has just started dictating, or a spoken command was + /// recognised mid-speech) hides most of that behind the time they spend + /// talking. That holds in both memory modes: on-demand drops the model + /// when idle, and always-loaded without pre-warming loads it lazily. When + /// the model is already resident the worker treats this as a no-op. pub async fn preload_tts(&self) { - let unloads_when_idle = { - let cfg = self.config.lock().await; - cfg.data.tts.enabled && cfg.data.tts.unloads_when_idle() - }; - if !unloads_when_idle { + if !self.config.lock().await.data.tts.enabled { return; } if let Some(tts) = self.tts_handle.lock().await.as_ref() { From 973b5c3356873151b05d31d19ca876c4b6392414 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:53:07 -0500 Subject: [PATCH 04/12] feat(overlay): never show the target visualizer over SYSTEM RESPONDING Stack the overlay's parts in a column and hide the recording visualizer while the SYSTEM RESPONDING pill is up, so the two never overlap. Command and MCP pills are shown alongside the visualizer during recording. - The visualizer stays mounted while the whole overlay fades out, so its outro transition still plays (an earlier draft unmounted it the instant recording stopped). It is dropped once recording ends only when the overlay stays up for a command/MCP pill. - Overlay tests restore the config store after themselves, wait out the overlay's 25ms repaint delay explicitly, and fail the mocked get_config so the store keeps its defaults instead of a status-shaped object. Co-Authored-By: Claude Opus 5.5 --- src/lib/Overlay/Overlay.svelte | 158 ++++++++++++++++++++++----------- tests/svelte/Overlay.test.ts | 76 +++++++++++++++- 2 files changed, 177 insertions(+), 57 deletions(-) diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index 1dbaeb7e..c50022a7 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -46,7 +46,27 @@ ($mcpRecording && $config.mcp.visual_feedback) || (commandOverlayActive && $config.ui.show_command_overlay) ); + + let isSystemResponding = $derived($speaking && $config.tts.enabled && $config.tts.response_overlay); let renderOverlay = $state(false); + + // The target visualizer belongs to recording. It gives way to the SYSTEM + // RESPONDING pill so the two never overlap, and is dropped once recording + // ends if the overlay stays up only for a command/MCP pill. When the whole + // overlay is fading out it stays mounted, so its outro transition can play. + let primaryVisualizerOn = $state(false); + $effect(() => { + if (!renderOverlay || isSystemResponding) { + primaryVisualizerOn = false; + } else if ($recording || $status.processing) { + primaryVisualizerOn = true; + } else if (isRecordingOrSpeaking) { + primaryVisualizerOn = false; + } + }); + let showPrimaryVisualizer = $derived( + $config.ui.overlay_style !== "none" && $config.ui.show_overlay && primaryVisualizerOn + ); let animateActive = $state(false); let timeoutId: any; let animateTimeoutId: any; @@ -311,58 +331,68 @@
{#if renderOverlay && visible} - {#if $config.ui.overlay_style === "waveform"} - - {:else if $config.ui.overlay_style === "pulse"} - - {:else if $config.ui.overlay_style === "blue_wave"} - - {:else if $config.ui.overlay_style === "mono_bars"} - - {:else if $config.ui.overlay_style === "spectrum"} - - {:else if $config.ui.overlay_style === "terminal"} - - {:else if $config.ui.overlay_style === "vinyl"} - - {:else if activeCustomOverlay} - {@html ``} -
- {@html processedHtml} -
- {:else if $config.ui.overlay_style !== "none"} - - {/if} - - {#if $speaking} -
- - {#each [0, 1, 2, 3, 4] as i} - - {/each} - - - SYSTEM RESPONDING - ▸ {targetLabel} - -
- {:else if commandOverlayActive} -
- ⚡ - - {commandOverlayName.toUpperCase()} - ▸ {commandOverlayText} - -
- {:else if $mcpRecording} -
- - - RECORDING - ▸ {targetLabel} - -
- {/if} +
+ {#if showPrimaryVisualizer} +
+ {#if $config.ui.overlay_style === "waveform"} + + {:else if $config.ui.overlay_style === "pulse"} + + {:else if $config.ui.overlay_style === "blue_wave"} + + {:else if $config.ui.overlay_style === "mono_bars"} + + {:else if $config.ui.overlay_style === "spectrum"} + + {:else if $config.ui.overlay_style === "terminal"} + + {:else if $config.ui.overlay_style === "vinyl"} + + {:else if activeCustomOverlay} + {@html ``} +
+ {@html processedHtml} +
+ {:else if $config.ui.overlay_style !== "none"} + + {/if} +
+ {/if} + + {#if commandOverlayActive && $config.ui.show_command_overlay && !isSystemResponding} +
+ ⚡ + + {commandOverlayName.toUpperCase()} + ▸ {commandOverlayText} + +
+ {/if} + + {#if isSystemResponding} +
+ + {#each [0, 1, 2, 3, 4] as i} + + {/each} + + + SYSTEM RESPONDING + ▸ {targetLabel} + +
+ {/if} + + {#if $mcpRecording && $config.mcp.visual_feedback} +
+ + + RECORDING + ▸ {targetLabel} + +
+ {/if} +
{/if}
@@ -395,8 +425,26 @@ overflow: hidden !important; } + .overlay-stack { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + max-width: 100%; + max-height: 100%; + pointer-events: none; + } + + .visualizer-container { + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + } + .system-response-box { - position: absolute; + position: relative; z-index: 10; display: flex; align-items: center; @@ -406,10 +454,12 @@ border-radius: 23px; font-family: 'Outfit', 'Inter', system-ui, sans-serif; opacity: 0; - transform: translateY(20px); + transform: translateY(10px); transition: transform 0.34s cubic-bezier(0.175, 0.885, 0.32, 1.25), opacity 0.28s ease; + pointer-events: none; + flex-shrink: 0; } .system-response-box.on { diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index b1a935b8..21cff49f 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -1,9 +1,14 @@ -import { describe, test, expect, vi, beforeEach } from "vitest"; +import { describe, test, expect, vi, beforeEach, afterEach } from "vitest"; +import { get } from "svelte/store"; import { render } from "@testing-library/svelte"; // Mock tauri IPC used by the overlay components and the status store vi.mock("@tauri-apps/api/core", () => ({ - invoke: vi.fn(async () => ({ + // `get_config` has no backend here: failing it leaves the config store on + // its defaults instead of overwriting it with this status-shaped reply. + invoke: vi.fn(async (cmd: string) => { + if (cmd === "get_config") throw new Error("no backend in tests"); + return { recording: false, processing: false, speaking: false, @@ -12,7 +17,8 @@ vi.mock("@tauri-apps/api/core", () => ({ word_count: 0, active_target_id: "default", active_target_label: "Focused Window", - })), + }; + }), })); vi.mock("@tauri-apps/api/event", () => ({ @@ -29,6 +35,8 @@ vi.stubGlobal( vi.stubGlobal("cancelAnimationFrame", (id: number) => clearTimeout(id)); import { status, type AppStatus } from "../../src/stores/status"; +import { config } from "../../src/stores/config"; +import Overlay from "../../src/lib/Overlay/Overlay.svelte"; import Waveform from "../../src/lib/Overlay/Waveform.svelte"; import Pulse from "../../src/lib/Overlay/Pulse.svelte"; import BlueWave from "../../src/lib/Overlay/BlueWave.svelte"; @@ -189,3 +197,65 @@ describe("VoiceCard.svelte (membership card)", () => { expect(procContainer.querySelector(".field-value")?.textContent).toContain("Reading the card…"); }); }); + +describe("Overlay.svelte (root layout)", () => { + // These tests reconfigure the shared config store; put it back afterwards + // so later suites see the defaults. + let savedConfig: ReturnType>; + beforeEach(() => { + savedConfig = get(config); + config.update((c) => ({ + ...c, + ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, + tts: { ...c.tts, enabled: true, response_overlay: true }, + })); + }); + afterEach(() => { + config.set(savedConfig); + }); + + // The overlay mounts its content after a 25ms repaint delay (see the + // overlay_style effect), so wait past that rather than a single tick. + const settle = (ms = 50) => new Promise((r) => setTimeout(r, ms)); + + test("hides target visualizer and displays SYSTEM RESPONDING when speaking", async () => { + setStatus({ speaking: true, recording: false, active_target_label: "Kitty Terminal" }); + + const { container } = render(Overlay); + await settle(); + + expect(container.textContent).toContain("SYSTEM RESPONDING"); + expect(container.textContent).toContain("Kitty Terminal"); + // The target visualizer (Waveform) must not be rendered alongside it + expect(container.querySelector(".scope")).toBeNull(); + expect(container.textContent).not.toContain("WAVEFORM // OSC-01"); + }); + + test("shows target visualizer when recording", async () => { + setStatus({ speaking: false, recording: true, active_target_label: "Code Editor" }); + + const { container } = render(Overlay); + await settle(); + + expect(container.querySelector(".scope")).not.toBeNull(); + expect(container.textContent).toContain("WAVEFORM // OSC-01"); + expect(container.textContent).not.toContain("SYSTEM RESPONDING"); + }); + + test("keeps the visualizer mounted while the overlay fades out after recording", async () => { + setStatus({ recording: true }); + const { container } = render(Overlay); + await settle(); + expect(container.querySelector(".scope")).not.toBeNull(); + + setStatus({ recording: false }); + await settle(); + // Still inside the 450ms outro window: the visualizer animates out rather + // than vanishing the instant recording stops... + expect(container.querySelector(".scope")).not.toBeNull(); + + // ...and is gone once the outro has finished. + await settle(500); + expect(container.querySelector(".scope")).toBeNull(); + }); +}); From 29ab1f82faf452274079c9ca5d7a091af539b684 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:58:05 -0500 Subject: [PATCH 05/12] Revert "feat(commands): add 'Hey Vox' trigger, real-time command detection, and overlay exclusivity" This reverts commit 1f13d62f9c6ea24b4cb596c137c6e7f096a2c1aa. The commit bundled six independent changes, several of which changed runtime behaviour for users who never say a trigger word (a billed OpenAI call per interim pass, TTS models loaded at every startup, on-demand TTS memory mode defeated). It is re-landed as separate, fixed PRs. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-config/src/lib.rs | 2 +- crates/voxctrl-inference/src/lib.rs | 37 +---- crates/voxctrl-routing/src/targets.rs | 156 ++------------------- crates/voxctrl-routing/src/tests.rs | 138 ------------------ crates/voxctrl-tts/src/engine.rs | 52 +++---- docs/routing.md | 12 +- src-tauri/src/lib.rs | 35 ++--- src-tauri/src/pipeline.rs | 162 +--------------------- src-tauri/src/services.rs | 8 +- src-tauri/src/state.rs | 33 +---- src-tauri/src/tests.rs | 1 - src/lib/Overlay/Overlay.svelte | 153 ++++++++------------ src/lib/Settings/AudioTab.svelte | 2 +- src/lib/Settings/CommandsTab.svelte | 6 +- src/lib/Settings/TargetEditorModal.svelte | 10 +- src/stores/config.ts | 2 +- tests/svelte/Overlay.test.ts | 45 ------ 17 files changed, 140 insertions(+), 714 deletions(-) diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index 40e37e30..87c54a0a 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -292,7 +292,7 @@ impl Default for FeaturesConfig { fn default() -> Self { Self { remove_fillers: true, - custom_vocabulary: vec!["VoxCtrl".into(), "Hey Vox".into()], + custom_vocabulary: vec!["VoxCtrl".into()], spoken_punctuation: true, auto_format_lists: true, show_notification: None, diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 9b337d48..74e75b98 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -225,10 +225,6 @@ pub struct InferenceRequest { pub target_id: String, /// Hotkey binding ID (if triggered by a hotkey) pub binding_id: Option, - /// Whether this is an intermediate/live transcription during active speech - pub is_interim: bool, - /// Unique session identifier for the recording turn - pub session_id: u64, } /// Final output after transcription + post-processing. @@ -243,8 +239,6 @@ pub struct InferenceOutput { /// Set when transcription failed (model missing, backend error, ...). The /// UI layer surfaces this to the user; `text` is empty in that case. pub error: Option, - pub is_interim: bool, - pub session_id: u64, } // ── Engine ──────────────────────────────────────────────────────────────────── @@ -299,9 +293,6 @@ impl InferenceEngine { /// Transcribe and post-process. Returns the final text. pub fn process(&self, req: InferenceRequest) -> Result { - let is_interim = req.is_interim; - let session_id = req.session_id; - if req.audio.is_empty() { return Ok(InferenceOutput { text: String::new(), @@ -311,8 +302,6 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, - is_interim, - session_id, }); } @@ -356,15 +345,13 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, - is_interim, - session_id, }); } let dir = voxctrl_routing::config_dir(); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. "); + let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. "); // Custom vocabulary words from features config if !app_config.features.custom_vocabulary.is_empty() { @@ -490,8 +477,6 @@ impl InferenceEngine { inference_ms: result.inference_ms, language: result.language, error: None, - is_interim, - session_id, }) } @@ -631,25 +616,11 @@ pub fn run_worker_with_config( loop { crossbeam_channel::select! { recv(rx) -> req_res => { - let mut req = match req_res { + let req = match req_res { Ok(r) => r, Err(_) => break, }; - // If an interim request arrived, drain any newer requests already in the queue - // to prioritize fresher audio or the final release request. - if req.is_interim { - while let Ok(newer) = rx.try_recv() { - req = newer; - if !req.is_interim { - break; - } - } - } - - let is_interim = req.is_interim; - let session_id = req.session_id; - if !loaded { match engine.load() { Ok(()) => { @@ -666,8 +637,6 @@ pub fn run_worker_with_config( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), - is_interim, - session_id, }); continue; } @@ -688,8 +657,6 @@ pub fn run_worker_with_config( inference_ms: 0, language: "".to_string(), error: Some(format!("{e:#}")), - is_interim, - session_id, }); } } diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 1b2352b8..892f28c4 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -1312,9 +1312,9 @@ fn levenshtein_distance(s1: &str, s2: &str) -> usize { /// Parse text for keyword "VoxCtrl" and target name/label matching. /// Supports both direct commands (e.g. "VoxCtrl notes Hi there") and natural, -/// Parse text for keyword "VoxCtrl" / "Hey Vox" and target name/label matching. -/// Requires an explicit wake-word trigger ("VoxCtrl", "Vox Control", "Hey Vox", etc.) -/// to protect normal dictation from being misclassified as a command. +/// conversational phrasing (e.g. "VoxCtrl add this to my notes. What are you doing here?"). +/// Returns `Some(VoiceCommandParseResult)` if the trigger keyword was found AND a target matched; +/// otherwise returns `None`. pub fn parse_voice_command( text: &str, targets: &[OutputTarget], @@ -1324,31 +1324,13 @@ pub fn parse_voice_command( let mut trigger_len = 0; // 1. Exact & standard triggers - let exact_triggers = [ - "voxctrl", "vox ctrl", "vox-ctrl", "vox control", - "hey vox", "hey, vox", "hey-vox", "hey_vox", - ]; + let exact_triggers = ["voxctrl", "vox ctrl", "vox-ctrl", "vox control"]; for trigger in &exact_triggers { - let mut search_start = 0; - while let Some(match_idx) = lower_text[search_start..].find(trigger) { - let pos = search_start + match_idx; - let end_idx = pos + trigger.len(); - let is_boundary_start = pos == 0 || { - let prev = lower_text[..pos].chars().last().unwrap(); - prev.is_whitespace() || prev.is_ascii_punctuation() - }; - let is_boundary_end = end_idx == lower_text.len() || { - let next = lower_text[end_idx..].chars().next().unwrap(); - next.is_whitespace() || next.is_ascii_punctuation() - }; - if is_boundary_start && is_boundary_end { - if found_pos.map_or(true, |p| pos < p) { - found_pos = Some(pos); - trigger_len = trigger.len(); - } - break; + if let Some(pos) = lower_text.find(trigger) { + if found_pos.map_or(true, |p| pos < p) { + found_pos = Some(pos); + trigger_len = trigger.len(); } - search_start = pos + 1; } } @@ -1379,32 +1361,12 @@ pub fn parse_voice_command( let clean_cand = candidate.trim_matches(|c: char| c.is_ascii_punctuation()); let dist1 = levenshtein_distance(clean_cand, "voxctrl"); let dist2 = levenshtein_distance(clean_cand, "vox control"); - - // Check "hey vox" fuzzy match (distance <= 1) on normalized tokens - let clean_tokens: Vec<&str> = words[..len] - .iter() - .map(|w| w.trim_matches(|c: char| c.is_ascii_punctuation())) - .filter(|w| !w.is_empty()) - .collect(); - let hey_vox_cand = clean_tokens.join(" "); - let dist_hey_vox = if !hey_vox_cand.is_empty() { - levenshtein_distance(&hey_vox_cand, "hey vox") - } else { - usize::MAX - }; - - if dist1 <= 2 || dist2 <= 3 || dist_hey_vox <= 1 { - let first_word = words[0]; - let last_word = words[len - 1]; - let start_idx = lower_text.find(first_word).unwrap_or(0); - let last_pos = lower_text[start_idx..] - .find(last_word) - .map(|p| start_idx + p) - .unwrap_or(start_idx); - let end_idx = last_pos + last_word.len(); - found_pos = Some(start_idx); - trigger_len = end_idx - start_idx; - break; + if dist1 <= 2 || dist2 <= 3 { + if let Some(pos) = lower_text.find(clean_cand) { + found_pos = Some(pos); + trigger_len = clean_cand.len(); + break; + } } } } @@ -1469,93 +1431,3 @@ pub fn parse_voice_command( None } - -/// Backwards compatibility alias -pub fn parse_voice_command_or_direct( - text: &str, - targets: &[OutputTarget], - _allow_direct: bool, -) -> Option { - parse_voice_command(text, targets) -} - -/// Check if text contains a target's label or ID on word boundaries. -pub fn text_contains_target_name(text: &str, target: &OutputTarget) -> bool { - let lower_text = text.to_lowercase(); - let mut candidates: Vec<&str> = Vec::new(); - if !target.label.is_empty() { - candidates.push(&target.label); - } - if !target.id.is_empty() { - candidates.push(&target.id); - } - candidates.sort_by(|a, b| b.len().cmp(&a.len())); - - for cand in candidates { - let cand_lower = cand.to_lowercase(); - let mut search_start = 0; - while let Some(match_idx) = lower_text[search_start..].find(&cand_lower) { - let abs_match_start = search_start + match_idx; - let abs_match_end = abs_match_start + cand_lower.len(); - - let is_boundary_start = abs_match_start == 0 || { - let prev_char = text[..abs_match_start].chars().last().unwrap(); - prev_char.is_whitespace() || prev_char.is_ascii_punctuation() - }; - let is_boundary_end = abs_match_end == text.len() || { - let next_char = text[abs_match_end..].chars().next().unwrap(); - next_char.is_whitespace() || next_char.is_ascii_punctuation() - }; - - if is_boundary_start && is_boundary_end { - return true; - } - search_start = abs_match_start + 1; - } - } - false -} - -/// Extract payload for a known matched target from transcription text. -/// Used when a session was already recognized as matching `target` (e.g. during interim speech), -/// but the final transcription varied slightly in punctuation or phrasing. -pub fn extract_payload_for_target( - text: &str, - target: &OutputTarget, -) -> String { - let lower_text = text.to_lowercase(); - let mut candidates: Vec<&str> = Vec::new(); - if !target.label.is_empty() { - candidates.push(&target.label); - } - if !target.id.is_empty() { - candidates.push(&target.id); - } - candidates.sort_by(|a, b| b.len().cmp(&a.len())); - - for cand in candidates { - let cand_lower = cand.to_lowercase(); - let mut search_start = 0; - while let Some(match_idx) = lower_text[search_start..].find(&cand_lower) { - let abs_match_start = search_start + match_idx; - let abs_match_end = abs_match_start + cand_lower.len(); - - let is_boundary_start = abs_match_start == 0 || { - let prev_char = text[..abs_match_start].chars().last().unwrap(); - prev_char.is_whitespace() || prev_char.is_ascii_punctuation() - }; - let is_boundary_end = abs_match_end == text.len() || { - let next_char = text[abs_match_end..].chars().next().unwrap(); - next_char.is_whitespace() || next_char.is_ascii_punctuation() - }; - - if is_boundary_start && is_boundary_end { - let post_target = &text[abs_match_end..]; - return clean_payload(post_target); - } - search_start = abs_match_start + 1; - } - } - - clean_payload(text) -} diff --git a/crates/voxctrl-routing/src/tests.rs b/crates/voxctrl-routing/src/tests.rs index bc776b42..cd68ab76 100644 --- a/crates/voxctrl-routing/src/tests.rs +++ b/crates/voxctrl-routing/src/tests.rs @@ -1768,12 +1768,6 @@ fn test_parse_voice_command_no_keyword() { ]; assert!(parse_voice_command("hi there", &targets).is_none()); - assert!(parse_voice_command("Hey folks, notes about meeting", &targets).is_none()); - assert!(parse_voice_command("Hey boss, notes about meeting", &targets).is_none()); - assert!(parse_voice_command("Hey guys, notes about meeting", &targets).is_none()); - assert!(parse_voice_command("They voxelize notes and models", &targets).is_none()); - assert!(parse_voice_command("heavy box notes something", &targets).is_none()); - assert!(parse_voice_command("the box notes something", &targets).is_none()); } #[test] @@ -1860,60 +1854,6 @@ fn test_parse_voice_command_matched_target() { let res8 = parse_voice_command("Walks control speak how are you?", &speak_targets).expect("should match Walks control dynamic trigger pattern"); assert_eq!(res8.matched_target_id, "speak"); assert_eq!(res8.payload, "how are you?"); - - // Hey Vox commands - let res9 = parse_voice_command("Hey Vox, notes Hi there", &targets).expect("should match Hey Vox with comma"); - assert_eq!(res9.matched_target_id, "notes"); - assert_eq!(res9.payload, "Hi there"); - - let res10 = parse_voice_command("Hey Vox notes Hi there", &targets).expect("should match Hey Vox without comma"); - assert_eq!(res10.matched_target_id, "notes"); - assert_eq!(res10.payload, "Hi there"); - - let res11 = parse_voice_command("Hey, Vox, notes Hi there", &targets).expect("should match Hey, Vox, with internal comma"); - assert_eq!(res11.matched_target_id, "notes"); - assert_eq!(res11.payload, "Hi there"); - - let res12 = parse_voice_command("Hey Vox: Notes, please write this down", &targets).expect("should match Hey Vox with colon"); - assert_eq!(res12.matched_target_id, "notes"); - assert_eq!(res12.payload, "please write this down"); - - let res13 = parse_voice_command("Hey, Vox: Notes, please write this down", &targets).expect("should match Hey, Vox: with colon"); - assert_eq!(res13.matched_target_id, "notes"); - assert_eq!(res13.payload, "please write this down"); - - let res14 = parse_voice_command("hey vox add this to my notes. What are you doing here?", &targets).expect("should match conversational hey vox notes"); - assert_eq!(res14.matched_target_id, "notes"); - assert_eq!(res14.payload, "What are you doing here?"); - - let res15 = parse_voice_command("Hey Vox put this into my Notes: What are you doing here?", &targets).expect("should match conversational put into notes"); - assert_eq!(res15.matched_target_id, "notes"); - assert_eq!(res15.payload, "What are you doing here?"); - - let res16 = parse_voice_command("Hey, Vox, send us to my notes. I love you.", &targets).expect("should match conversational send us to my notes"); - assert_eq!(res16.matched_target_id, "notes"); - assert_eq!(res16.payload, "I love you."); - - // Hey Vox homophones (Acoustic mis-recognitions within distance <= 1) - let res17 = parse_voice_command("Hey Box speak text", &speak_targets).expect("should match Hey Box homophone"); - assert_eq!(res17.matched_target_id, "speak"); - assert_eq!(res17.payload, "text"); - - let res18 = parse_voice_command("hay vox speak text", &speak_targets).expect("should match hay vox homophone"); - assert_eq!(res18.matched_target_id, "speak"); - assert_eq!(res18.payload, "text"); - - let res19 = parse_voice_command("hey fox speak text", &speak_targets).expect("should match hey fox homophone"); - assert_eq!(res19.matched_target_id, "speak"); - assert_eq!(res19.payload, "text"); - - let res20 = parse_voice_command("hey vax speak text", &speak_targets).expect("should match hey vax homophone"); - assert_eq!(res20.matched_target_id, "speak"); - assert_eq!(res20.payload, "text"); - - let res21 = parse_voice_command("Hey, Box, speak, eat my ass.", &speak_targets).expect("should match Hey, Box, homophone with punctuation"); - assert_eq!(res21.matched_target_id, "speak"); - assert_eq!(res21.payload, "eat my ass."); } #[test] @@ -2278,81 +2218,3 @@ async fn command_targets_keep_newlines_when_not_stripping() { let text = delivered.expect("delivery produced no text"); assert_eq!(text, "first line\nsecond line "); } - -#[test] -fn test_parse_voice_command_requires_wake_word() { - use crate::targets::parse_voice_command; - - let targets = vec![ - OutputTarget { - id: "say_target".into(), - label: "Say".into(), - delivery: DeliveryType::Speak, - ..OutputTarget::default_inject() - }, - OutputTarget { - id: "notes".into(), - label: "Notes".into(), - delivery: DeliveryType::File, - ..OutputTarget::default_inject() - }, - ]; - - // Normal speech must NEVER trigger a command without wake words - assert!(parse_voice_command("say hello world", &targets).is_none()); - assert!(parse_voice_command("notes: buy groceries", &targets).is_none()); - assert!(parse_voice_command("what the hell is going on. And why is it happening?", &targets).is_none()); - assert!(parse_voice_command("There have been several changes made in this branch.", &targets).is_none()); - assert!(parse_voice_command("Do you hear me? Do you know what's happening?", &targets).is_none()); - - // Valid commands with Hey Vox or Vox Control trigger properly - let res1 = parse_voice_command("Hey Vox, say hello", &targets).expect("should match with hey vox trigger"); - assert_eq!(res1.matched_target_id, "say_target"); - assert_eq!(res1.payload, "hello"); - - let res2 = parse_voice_command("Vox Control, say hello world", &targets).expect("should match with vox control trigger"); - assert_eq!(res2.matched_target_id, "say_target"); - assert_eq!(res2.payload, "hello world"); - - let res3 = parse_voice_command("Hey Vox, add this to my notes: buy groceries", &targets).expect("should match notes with filler"); - assert_eq!(res3.matched_target_id, "notes"); - assert_eq!(res3.payload, "buy groceries"); -} - -#[test] -fn test_text_contains_target_name() { - use crate::targets::text_contains_target_name; - - let target = OutputTarget { - id: "say_target".into(), - label: "Say".into(), - delivery: DeliveryType::Speak, - ..OutputTarget::default_inject() - }; - - assert!(text_contains_target_name("Hey box, say hello", &target)); - assert!(text_contains_target_name("Say, how are you", &target)); - assert!(text_contains_target_name("please say testing", &target)); - // Normal sentences without "say" do not match - assert!(!text_contains_target_name("what the hell is going on", &target)); - assert!(!text_contains_target_name("There have been several changes made", &target)); - assert!(!text_contains_target_name("essay writing", &target)); // substring inside another word -} - -#[test] -fn test_extract_payload_for_target() { - use crate::targets::extract_payload_for_target; - - let target = OutputTarget { - id: "say_target".into(), - label: "Say".into(), - delivery: DeliveryType::Speak, - ..OutputTarget::default_inject() - }; - - assert_eq!(extract_payload_for_target("Say hello world", &target), "hello world"); - assert_eq!(extract_payload_for_target("say, what is the weather?", &target), "what is the weather?"); - assert_eq!(extract_payload_for_target("Hey folks, say how are you", &target), "how are you"); - assert_eq!(extract_payload_for_target("hello there", &target), "hello there"); -} - diff --git a/crates/voxctrl-tts/src/engine.rs b/crates/voxctrl-tts/src/engine.rs index 572541e3..7b928876 100644 --- a/crates/voxctrl-tts/src/engine.rs +++ b/crates/voxctrl-tts/src/engine.rs @@ -158,10 +158,26 @@ impl TtsEngineWorker { model_loaded: model_loaded.clone(), }; - // In always-loaded mode, ensure the model is preloaded on startup so the - // user never experiences cold-start latency on their first utterance. - if !config.unloads_when_idle() { - let _ = handle.tx.send(TtsCommand::Preload); + let prewarm = match config.engine { + TtsEngine::PocketTts => config.pocket_tts.prewarm, + TtsEngine::InflectMicro => config.inflect_micro.prewarm, + TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, + TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, + _ => false, + }; + // Pre-warming loads the model at startup and keeps it there, which is + // exactly what the on-demand memory mode exists to avoid — so the two + // settings do not fight: on-demand wins and the model waits for its + // first real use (or a `preload()`). + if prewarm && !config.unloads_when_idle() { + let _ = handle.tx.send(TtsCommand::Play { + utterance: Utterance { + text: " ".into(), + voice: None, + source_label: Some("prewarm".into()), + }, + generation: 0, + }); } let worker = Self { @@ -267,34 +283,10 @@ impl TtsEngineWorker { "TTS worker config dynamically updated (engine={:?}, memory_mode={:?})", new_cfg.engine, new_cfg.memory_mode ); - let was_unloading = current_config.unloads_when_idle(); - let now_unloading = new_cfg.unloads_when_idle(); current_config = new_cfg; + // Switching to on-demand starts the clock now rather than + // dropping a model that may be about to be used again. last_used = Instant::now(); - - // If switched from on_demand to always_loaded (e.g. toggled off in the tray), - // preload/warm the model immediately so it stays ready in memory. - if was_unloading && !now_unloading { - let _ = match current_config.engine { - TtsEngine::InflectMicro if inflect_model.is_none() => { - ensure_inflect_micro_loaded(¤t_config, &mut inflect_model).ok() - } - TtsEngine::PocketTts if audiocpp_session.is_none() => { - ensure_pocket_tts_loaded(¤t_config, &mut audiocpp_session).ok() - } - TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { - ensure_breeze_tts_2_loaded(¤t_config, &mut audiocpp_session).ok() - } - TtsEngine::VoxCpm2 if audiocpp_session.is_none() => { - ensure_vox_cpm_2_loaded(¤t_config, &mut audiocpp_session).ok() - } - _ => None, - }; - self.model_loaded.store( - inflect_model.is_some() || audiocpp_session.is_some(), - Ordering::SeqCst, - ); - } } TtsCommand::Play { mut utterance, generation } => { // Synthesis is about to touch the model; hold off the idle diff --git a/docs/routing.md b/docs/routing.md index 02e55f33..64945727 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -23,8 +23,8 @@ Both files are hot-reloaded when changed on disk. Defined in `~/.config/voxctrl/targets.toml`. Each `[[target]]` block describes one destination. -**Saying one by name.** Start dictation and say *"Hey Vox"* (or *"Vox Control"*), then the command's -name, then the text: *"Hey Vox, notes, remember to call the plumber"* (or *"Vox Control notes, remember to call the plumber"*) delivers +**Saying one by name.** Start dictation and say *"VoxCtrl"*, then the command's +name, then the text: *"VoxCtrl notes, remember to call the plumber"* delivers *remember to call the plumber* to the command named **notes**, whatever the active hotkey was pointed at. Everything after the name is the payload. The full matching rules — conversational lead-ins, fuzzy name matching, the overlay — @@ -262,13 +262,13 @@ typing into the focused window — so voice command routing works the day anothe target is added, with no re-binding. **How Voice Command Routing Works:** -- **Trigger Keyword**: Listens for the `"Hey Vox"` or `"Vox Control"` keywords (case-insensitive, supporting `Hey Vox`, `Hey, Vox`, `Hey-Vox`, `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, `vox control`, and trailing punctuation like `Hey Vox:`). +- **Trigger Keyword**: Listens for the `"VoxCtrl"` keyword (case-insensitive, supporting `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, and optional punctuation like `VoxCtrl:`). - **Target Resolution**: Matches spoken target names against all configured target IDs and Labels (case-insensitively). Longest candidate target names take precedence (e.g. `"Personal Notes"` is matched before `"Notes"`). -- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"Hey Vox, send this to my notes. I love you."*, *"Hey Vox, add this to my personal notes, help"*, *"Vox Control put this into my Notes: hello"*, or *"Hey, Vox, send us to my notes. I love you."*. +- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"VoxCtrl send this to my notes. I love you."*, *"VoxCtrl add this to my personal notes, help"*, *"VoxCtrl put this into my Notes: hello"*, or *"VoxCtrl send us to my notes. I love you."*. - **Payload Extraction**: Strips transition punctuation (`.`, `:`, `,`, `;`) and connector words (`saying`, `that`, `with text`) and routes the remaining text payload to the matched target. - **Command UI Overlay**: When a voice command trigger is matched and executed, VoxCtrl automatically displays a temporary purple/indigo HUD overlay pill displaying the command name and text payload summary for a configurable duration (default: 3 seconds). -- **Auto-Trigger Detection**: Dictating with the `"Hey Vox"` or `"Vox Control"` keyword (e.g. *"Hey Vox, notes Help me!"* or *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. -- **Fallback**: If no trigger keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). +- **Auto-Trigger Detection**: Dictating with the `"VoxCtrl"` keyword (e.g. *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. +- **Fallback**: If no `"VoxCtrl"` keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). --- diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 43d11c86..22b20399 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -345,25 +345,11 @@ pub fn run() { let hotkey_health = Arc::new(voxctrl_hotkeys::ListenerHealth::default()); - // TTS initial worker - let initial_tts_handle = if cfg_data.tts.enabled { - Some(voxctrl_tts::TtsEngineWorker::start( - cfg_data.tts.clone(), - cfg_data.features.custom_vocabulary.clone(), - None, - None, - None, - )) - } else { - None - }; - let app_state = Arc::new(AppState { config: config.clone(), router: router.clone(), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), - interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(cfg_data.ui.show_overlay)), mcp_recording: Arc::new(AtomicBool::new(false)), @@ -387,7 +373,7 @@ pub fn run() { audio_tx: audio_tx.clone(), audio_wake: audio_wake_tx, inference_config_tx: inference_cfg_tx, - tts_handle: Arc::new(Mutex::new(initial_tts_handle.clone())), + tts_handle: Arc::new(Mutex::new(None)), active_fifos: Arc::new(Mutex::new(std::collections::HashSet::new())), stop_key_held: Arc::new(AtomicBool::new(false)), speaking_tx: Arc::new(std::sync::OnceLock::new()), @@ -438,9 +424,26 @@ pub fn run() { inference_cfg_rx, ); + // TTS initial worker + let _tts_handle = if cfg_data.tts.enabled { + Some(voxctrl_tts::TtsEngineWorker::start( + cfg_data.tts.clone(), + cfg_data.features.custom_vocabulary.clone(), + None, + None, + None, + )) + } else { + None + }; + let state_for_tts = app_state.clone(); - let tts_handle_clone = initial_tts_handle.clone(); + let tts_handle_clone = _tts_handle.clone(); tokio::spawn(async move { + { + let mut handle = state_for_tts.tts_handle.lock().await; + *handle = tts_handle_clone.clone(); + } if let Some(tts) = tts_handle_clone { state_for_tts.spawn_fifo_responders(tts).await; } diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 4d2b3535..37486331 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -36,8 +36,6 @@ async fn process_remote_transcription( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), - is_interim: false, - session_id: 0, }); return; } @@ -71,8 +69,6 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, - is_interim: false, - session_id: 0, }); return; } @@ -192,8 +188,6 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, - is_interim: false, - session_id: 0, }); } @@ -211,10 +205,6 @@ pub fn spawn_audio_coordinator( let mut binding_id = String::new(); let mut remote_session: Option = None; let mut is_remote_backend = false; - let mut is_heavy_model = false; - let mut session_id: u64 = 0; - let mut last_interim_sample_count: usize = 0; - let mut last_interim_instant = std::time::Instant::now(); while let Ok(chunk) = audio_rx.recv() { let is_recording = state_for_audio.is_recording(); @@ -225,21 +215,12 @@ pub fn spawn_audio_coordinator( target_id = state_for_audio.active_target.blocking_lock().clone(); binding_id = state_for_audio.active_binding_id.blocking_lock().clone(); was_recording = true; - session_id = session_id.wrapping_add(1); - last_interim_sample_count = 0; - last_interim_instant = std::time::Instant::now(); - state_for_audio.set_interim_in_flight(false); let cfg = state_for_audio.config.blocking_lock().data.clone(); - is_heavy_model = cfg.engine.backend == voxctrl_config::BackendChoice::WhisperCpp - && voxctrl_inference::whisper_gpu_backend().is_none() - && (cfg.engine.whisper_cpp.model_size.starts_with("medium") - || cfg.engine.whisper_cpp.model_size.starts_with("large")); - if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { is_remote_backend = true; let mut merged_prompt = String::from( - "VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. ", + "VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. ", ); if !cfg.features.custom_vocabulary.is_empty() { merged_prompt.push_str("Vocabulary: "); @@ -267,30 +248,6 @@ pub fn spawn_audio_coordinator( session.send_chunk(chunk.clone()); } accumulated_audio.extend(chunk); - - // Live interim sliding-window transcription for fast local STT backends - if !is_remote_backend && !is_heavy_model { - let total_samples = accumulated_audio.len(); - let new_samples = total_samples.saturating_sub(last_interim_sample_count); - if total_samples >= 8000 - && new_samples >= 4800 - && last_interim_instant.elapsed() >= std::time::Duration::from_millis(400) - && !state_for_audio.is_interim_in_flight() - { - state_for_audio.set_interim_in_flight(true); - last_interim_sample_count = total_samples; - last_interim_instant = std::time::Instant::now(); - - let req = voxctrl_inference::InferenceRequest { - audio: accumulated_audio.clone(), - target_id: target_id.clone(), - binding_id: Some(binding_id.clone()), - is_interim: true, - session_id, - }; - let _ = inference_tx.send(req); - } - } } else { if was_recording { if is_remote_backend { @@ -323,8 +280,6 @@ pub fn spawn_audio_coordinator( audio: std::mem::take(&mut accumulated_audio), target_id: target_id.clone(), binding_id: Some(binding_id.clone()), - is_interim: false, - session_id, }; state_for_audio.set_processing(true); let _ = inference_tx.send(req); @@ -343,63 +298,8 @@ pub fn spawn_text_delivery_worker( rt_handle: tokio::runtime::Handle, ) { std::thread::spawn(move || { - let mut last_detected_session: Option = None; - let mut session_command_detected: Option<(u64, String)> = None; - while let Ok(output) = text_rx.recv() { - if output.is_interim { - state.set_interim_in_flight(false); - if output.text.trim().is_empty() { - continue; - } - - // Live early voice command detection: detect command word as soon as uttered - let dir = voxctrl_routing::config_dir(); - let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { - let matched_id = parsed.matched_target_id.clone(); - let matched_target = targets.iter().find(|t| t.id == matched_id); - let matched_label = matched_target - .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) - .unwrap_or_else(|| matched_id.clone()); - - let already_notified = last_detected_session == Some(output.session_id) - && session_command_detected.as_ref().map(|(_, id)| id) == Some(&matched_id); - - if !already_notified { - last_detected_session = Some(output.session_id); - session_command_detected = Some((output.session_id, matched_id.clone())); - tracing::info!( - "Early command detected during speech: '{}' (target: {})", - matched_label, - matched_id - ); - voxctrl_routing::targets::notify_command_trigger(&matched_label, &parsed.payload); - - let is_speak = matched_target - .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak || t.response_pipe.is_some()) - .unwrap_or(false); - if is_speak { - let state_c = state.clone(); - rt_handle.spawn(async move { - state_c.preload_tts().await; - }); - } - } - } - continue; - } - - state.set_interim_in_flight(false); state.set_processing(false); - - // Retrieve any command target identified earlier during speech for this session - let early_matched_id = session_command_detected - .take() - .filter(|(sess_id, _)| *sess_id == output.session_id) - .map(|(_, id)| id); - last_detected_session = None; - if let Some(ref err) = output.error { // Always surface transcription failures — without this a // fresh install with no Whisper model records audio and @@ -435,71 +335,21 @@ pub fn spawn_text_delivery_worker( let (target_id, text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { let matched_id = parsed.matched_target_id.clone(); let payload = parsed.payload; - let matched_target = targets.iter().find(|t| t.id == matched_id); - let matched_label = matched_target + let matched_label = targets + .iter() + .find(|t| t.id == matched_id) .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) .unwrap_or_else(|| matched_id.clone()); voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); - // For Speak targets (e.g. "Say" voice commands), do not run S1-mini LLM cleanup. - // Running S1-mini on spoken speech output adds hundreds of milliseconds - // to seconds of latency and can rewrite words the user intended to be spoken verbatim. - let is_speak_target = matched_target - .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) - .unwrap_or(false); - - let cleaned_payload = if !is_speak_target && s1_mini_enabled && !payload.trim().is_empty() { + let cleaned_payload = if s1_mini_enabled && !payload.trim().is_empty() { voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) } else { payload }; (matched_id, cleaned_payload) - } else if let Some(early_id) = early_matched_id { - let matched_target = targets.iter().find(|t| t.id == early_id); - let has_target_in_text = matched_target.map_or(false, |tgt| { - voxctrl_routing::targets::text_contains_target_name(&output.text, tgt) - }); - - if has_target_in_text { - let matched_tgt = matched_target.unwrap(); - let matched_label = if matched_tgt.label.is_empty() { - matched_tgt.id.clone() - } else { - matched_tgt.label.clone() - }; - let payload = voxctrl_routing::targets::extract_payload_for_target(&output.text, matched_tgt); - voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); - - let is_speak_target = matched_tgt.delivery == voxctrl_routing::DeliveryType::Speak; - - let cleaned_payload = if !is_speak_target && s1_mini_enabled && !payload.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) - } else { - payload - }; - (early_id, cleaned_payload) - } else { - let is_speak_target = targets - .iter() - .find(|t| t.id == output.target_id) - .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) - .unwrap_or(false); - - let cleaned_text = if !is_speak_target && s1_mini_enabled && !output.text.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) - } else { - output.text.clone() - }; - (output.target_id.clone(), cleaned_text) - } } else { - let is_speak_target = targets - .iter() - .find(|t| t.id == output.target_id) - .map(|t| t.delivery == voxctrl_routing::DeliveryType::Speak) - .unwrap_or(false); - - let cleaned_text = if !is_speak_target && s1_mini_enabled && !output.text.trim().is_empty() { + let cleaned_text = if s1_mini_enabled && !output.text.trim().is_empty() { voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) } else { output.text.clone() diff --git a/src-tauri/src/services.rs b/src-tauri/src/services.rs index 86effb56..1fe74eda 100644 --- a/src-tauri/src/services.rs +++ b/src-tauri/src/services.rs @@ -224,14 +224,8 @@ pub fn setup_tts_and_fifos(app_handle: &tauri::AppHandle, state: Arc) pub fn register_speak_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); voxctrl_routing::targets::set_speak_callback(std::sync::Arc::new(move |text| { - let text_str = text.to_string(); - if let Ok(handle) = state.tts_handle.try_lock() { - if let Some(ref tts) = *handle { - tts.speak(text_str); - return; - } - } let state = state.clone(); + let text_str = text.to_string(); tauri::async_runtime::spawn(async move { let handle = state.tts_handle.lock().await; if let Some(ref tts) = *handle { diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index c59cea28..7e8f1637 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -14,8 +14,6 @@ pub struct AppState { pub recording: Arc, /// True while speech transcription/OpenAI post-processing is running pub processing: Arc, - /// True while an interim sliding-window transcription pass is actively running - pub interim_in_flight: Arc, /// True while TTS is playing back pub speaking: Arc, /// Live mirror of `ui.show_overlay` so the hot status-forwarding loops can @@ -152,14 +150,6 @@ impl AppState { self.processing.store(v, Ordering::SeqCst); } - pub fn is_interim_in_flight(&self) -> bool { - self.interim_in_flight.load(Ordering::SeqCst) - } - - pub fn set_interim_in_flight(&self, v: bool) { - self.interim_in_flight.store(v, Ordering::SeqCst); - } - pub fn is_audio_ready(&self) -> bool { self.audio_ready.load(Ordering::SeqCst) } @@ -301,11 +291,11 @@ impl AppState { /// hides most of that behind the time they spend talking. In always-loaded /// mode the model is already there and this is a no-op. pub async fn preload_tts(&self) { - let tts_enabled = { + let unloads_when_idle = { let cfg = self.config.lock().await; - cfg.data.tts.enabled + cfg.data.tts.enabled && cfg.data.tts.unloads_when_idle() }; - if !tts_enabled { + if !unloads_when_idle { return; } if let Some(tts) = self.tts_handle.lock().await.as_ref() { @@ -315,24 +305,11 @@ impl AppState { /// True when dictating through `target_id` is likely to end in speech — /// either the target speaks the transcript itself, or it has a response - /// pipe whose reply VoxCtrl reads back, or it routes through the voice - /// command router when speech targets exist. + /// pipe whose reply VoxCtrl reads back. Used to decide whether a recording + /// is worth pre-loading the TTS model for; dictating into an editor is not. pub async fn target_leads_to_speech(&self, target_id: &str) -> bool { use voxctrl_routing::DeliveryType; let targets = self.targets.lock().await; - - // If target is the voice command router, check if any target in the configuration uses speech - if target_id == "command" - || targets - .iter() - .any(|t| t.id == target_id && t.delivery == DeliveryType::Command) - { - return targets.iter().any(|t| { - t.delivery == DeliveryType::Speak - || t.response_pipe.as_deref().is_some_and(|p| !p.trim().is_empty()) - }); - } - targets.iter().any(|t| { t.id == target_id && (t.delivery == DeliveryType::Speak diff --git a/src-tauri/src/tests.rs b/src-tauri/src/tests.rs index 340ad21c..c5657570 100644 --- a/src-tauri/src/tests.rs +++ b/src-tauri/src/tests.rs @@ -66,7 +66,6 @@ fn make_test_state() -> AppState { router: Arc::new(OutputTargetRouter::new(Vec::new())), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), - interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(true)), mcp_recording: Arc::new(AtomicBool::new(false)), diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index 1daf0051..1dbaeb7e 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -35,7 +35,6 @@ let commandOverlayActive = $state(false); let commandOverlayName = $state(""); let commandOverlayText = $state(""); - let commandTriggeredDuringRecording = $state(false); let commandTimerId: any = null; let unlistenCommandExecuted: (() => void) | null = null; let unlistenOverlayStyleSelected: (() => void) | null = null; @@ -47,16 +46,6 @@ ($mcpRecording && $config.mcp.visual_feedback) || (commandOverlayActive && $config.ui.show_command_overlay) ); - - let isSystemResponding = $derived($speaking && $config.tts.enabled && $config.tts.response_overlay); - - let showPrimaryVisualizer = $derived( - $config.ui.overlay_style !== "none" && - $config.ui.show_overlay && - !isSystemResponding && - !$speaking && - ($recording || $status.processing) - ); let renderOverlay = $state(false); let animateActive = $state(false); let timeoutId: any; @@ -278,14 +267,10 @@ commandOverlayName = event.payload.command; commandOverlayText = event.payload.summary; commandOverlayActive = true; - if ($recording) { - commandTriggeredDuringRecording = true; - } if (commandTimerId) clearTimeout(commandTimerId); const durationMs = (event.payload.duration_secs || $config.ui.command_overlay_duration_secs || 3) * 1000; commandTimerId = setTimeout(() => { commandOverlayActive = false; - commandTriggeredDuringRecording = false; }, durationMs); }).then((unlisten) => { unlistenCommandExecuted = unlisten; @@ -326,68 +311,58 @@
{#if renderOverlay && visible} -
- {#if showPrimaryVisualizer} -
- {#if $config.ui.overlay_style === "waveform"} - - {:else if $config.ui.overlay_style === "pulse"} - - {:else if $config.ui.overlay_style === "blue_wave"} - - {:else if $config.ui.overlay_style === "mono_bars"} - - {:else if $config.ui.overlay_style === "spectrum"} - - {:else if $config.ui.overlay_style === "terminal"} - - {:else if $config.ui.overlay_style === "vinyl"} - - {:else if activeCustomOverlay} - {@html ``} -
- {@html processedHtml} -
- {:else if $config.ui.overlay_style !== "none"} - - {/if} -
- {/if} - - {#if commandOverlayActive && $config.ui.show_command_overlay && !isSystemResponding} -
- ⚡ - - {commandOverlayName.toUpperCase()} - ▸ {commandOverlayText} - -
- {/if} - - {#if $speaking && $config.tts.enabled && $config.tts.response_overlay} -
- - {#each [0, 1, 2, 3, 4] as i} - - {/each} - - - SYSTEM RESPONDING - ▸ {targetLabel} - -
- {/if} - - {#if $mcpRecording && $config.mcp.visual_feedback} -
- - - RECORDING - ▸ {targetLabel} - -
- {/if} -
+ {#if $config.ui.overlay_style === "waveform"} + + {:else if $config.ui.overlay_style === "pulse"} + + {:else if $config.ui.overlay_style === "blue_wave"} + + {:else if $config.ui.overlay_style === "mono_bars"} + + {:else if $config.ui.overlay_style === "spectrum"} + + {:else if $config.ui.overlay_style === "terminal"} + + {:else if $config.ui.overlay_style === "vinyl"} + + {:else if activeCustomOverlay} + {@html ``} +
+ {@html processedHtml} +
+ {:else if $config.ui.overlay_style !== "none"} + + {/if} + + {#if $speaking} +
+ + {#each [0, 1, 2, 3, 4] as i} + + {/each} + + + SYSTEM RESPONDING + ▸ {targetLabel} + +
+ {:else if commandOverlayActive} +
+ ⚡ + + {commandOverlayName.toUpperCase()} + ▸ {commandOverlayText} + +
+ {:else if $mcpRecording} +
+ + + RECORDING + ▸ {targetLabel} + +
+ {/if} {/if}
@@ -420,26 +395,8 @@ overflow: hidden !important; } - .overlay-stack { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 12px; - max-width: 100%; - max-height: 100%; - pointer-events: none; - } - - .visualizer-container { - display: flex; - align-items: center; - justify-content: center; - pointer-events: none; - } - .system-response-box { - position: relative; + position: absolute; z-index: 10; display: flex; align-items: center; @@ -449,12 +406,10 @@ border-radius: 23px; font-family: 'Outfit', 'Inter', system-ui, sans-serif; opacity: 0; - transform: translateY(10px); + transform: translateY(20px); transition: transform 0.34s cubic-bezier(0.175, 0.885, 0.32, 1.25), opacity 0.28s ease; - pointer-events: none; - flex-shrink: 0; } .system-response-box.on { diff --git a/src/lib/Settings/AudioTab.svelte b/src/lib/Settings/AudioTab.svelte index cbec8441..e7c943a5 100644 --- a/src/lib/Settings/AudioTab.svelte +++ b/src/lib/Settings/AudioTab.svelte @@ -139,7 +139,7 @@

- Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "Hey Vox" or "Vox Control" — is not lost. + Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "VoxCtrl" — is not lost.

diff --git a/src/lib/Settings/CommandsTab.svelte b/src/lib/Settings/CommandsTab.svelte index 26135d16..6540f745 100644 --- a/src/lib/Settings/CommandsTab.svelte +++ b/src/lib/Settings/CommandsTab.svelte @@ -139,11 +139,11 @@

Saying a command by name. Start dictation and say - “Hey Vox,” (or “VoxCtrl” / “Vox Control”), then the command's name, then what you want to send — for - example “Hey Vox, notes, remember to call the plumber” or “VoxCtrl notes, remember to call the plumber” routes + “VoxCtrl”, then the command's name, then what you want to send — for + example “VoxCtrl notes, remember to call the plumber” routes remember to call the plumber to the command named notes. Everything after the name is the text. Natural phrasing works too - (“Hey Vox, add this to my notes: …”). Say nothing of the sort and + (“VoxCtrl, add this to my notes: …”). Say nothing of the sort and dictation goes wherever your hotkey already points.

diff --git a/src/lib/Settings/TargetEditorModal.svelte b/src/lib/Settings/TargetEditorModal.svelte index 30ba9572..21fe2b7a 100644 --- a/src/lib/Settings/TargetEditorModal.svelte +++ b/src/lib/Settings/TargetEditorModal.svelte @@ -320,8 +320,8 @@

Names this target in the app, and is what you say to send dictation here through a - Voice Command Router target: “Hey Vox, {editingTarget.label || "Obsidian Notes"}, - buy milk tomorrow” (or “VoxCtrl, {editingTarget.label || "Obsidian Notes"}...”). The Target ID works as a spoken name too, so pick something + Voice Command Router target: “VoxCtrl, {editingTarget.label || "Obsidian Notes"}, + buy milk tomorrow”. The Target ID works as a spoken name too, so pick something easy to say and easy for speech recognition to catch.

@@ -330,7 +330,7 @@ @@ -352,7 +352,7 @@
Voice Command Router Settings

- Types dictated text into your active application by default. If your dictation contains Hey Vox <target_name> <text> or Vox Control <target_name> <text> (for example, "Hey Vox, Notes Hello world" or "Vox Control Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead. + Types dictated text into your active application by default. If your dictation contains VoxCtrl <target_name> <text> (for example, "VoxCtrl Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead.

{/if} diff --git a/src/stores/config.ts b/src/stores/config.ts index f048c32e..5346f8de 100644 --- a/src/stores/config.ts +++ b/src/stores/config.ts @@ -199,7 +199,7 @@ const defaultConfig: AppConfig = { }, features: { remove_fillers: true, - custom_vocabulary: ["VoxCtrl", "Hey Vox"], + custom_vocabulary: ["VoxCtrl"], spoken_punctuation: true, auto_format_lists: true, snippets: {}, diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index ceb3eb74..b1a935b8 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -29,8 +29,6 @@ vi.stubGlobal( vi.stubGlobal("cancelAnimationFrame", (id: number) => clearTimeout(id)); import { status, type AppStatus } from "../../src/stores/status"; -import { config } from "../../src/stores/config"; -import Overlay from "../../src/lib/Overlay/Overlay.svelte"; import Waveform from "../../src/lib/Overlay/Waveform.svelte"; import Pulse from "../../src/lib/Overlay/Pulse.svelte"; import BlueWave from "../../src/lib/Overlay/BlueWave.svelte"; @@ -191,46 +189,3 @@ describe("VoiceCard.svelte (membership card)", () => { expect(procContainer.querySelector(".field-value")?.textContent).toContain("Reading the card…"); }); }); - -describe("Overlay.svelte (root layout)", () => { - test("hides target visualizer and displays SYSTEM RESPONDING when speaking", async () => { - config.update((c) => ({ - ...c, - ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, - tts: { ...c.tts, enabled: true, response_overlay: true }, - })); - - setStatus({ speaking: true, recording: false, active_target_label: "Kitty Terminal" }); - - const { container } = render(Overlay); - await new Promise((r) => setTimeout(r, 50)); - - // SYSTEM RESPONDING pill should be present - expect(container.textContent).toContain("SYSTEM RESPONDING"); - expect(container.textContent).toContain("Kitty Terminal"); - - // Target visualizer (e.g. Waveform) must NOT be present - expect(container.querySelector(".scope")).toBeNull(); - expect(container.textContent).not.toContain("WAVEFORM // OSC-01"); - }); - - test("shows target visualizer when recording", async () => { - config.update((c) => ({ - ...c, - ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, - tts: { ...c.tts, enabled: true, response_overlay: true }, - })); - - setStatus({ speaking: false, recording: true, active_target_label: "Code Editor" }); - - const { container } = render(Overlay); - await new Promise((r) => setTimeout(r, 50)); - - // Target visualizer should be present - expect(container.querySelector(".scope")).not.toBeNull(); - expect(container.textContent).toContain("WAVEFORM // OSC-01"); - - // SYSTEM RESPONDING should NOT be present - expect(container.textContent).not.toContain("SYSTEM RESPONDING"); - }); -}); From ebcc500d16eb8dd1c651df3f8e46d95a9055903f Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 20:00:22 -0500 Subject: [PATCH 06/12] test(routing): mock wl-copy in the empty-text clipboard test It was the one clipboard test without MockClipboardTool, so on a Wayland desktop it ran the real wl-copy: the developer's clipboard was replaced, and wl-copy's forked background server inherited the test binary's stdout, so `cargo test -p voxctrl-routing | tail` hung after every test passed. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-routing/src/tests.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/voxctrl-routing/src/tests.rs b/crates/voxctrl-routing/src/tests.rs index cd68ab76..d0a36175 100644 --- a/crates/voxctrl-routing/src/tests.rs +++ b/crates/voxctrl-routing/src/tests.rs @@ -457,6 +457,11 @@ async fn test_clipboard_target_linux_cli() { #[tokio::test] async fn test_clipboard_target_failure_empty_text() { let _env = env_lock().lock().await; + // Without the mock this ran the machine's real `wl-copy`, which replaced + // the developer's clipboard and forked a background server that kept the + // test binary's stdout open, so a piped `cargo test` never exited. + #[cfg(target_os = "linux")] + let _mock = MockClipboardTool::install(); let mut config = OutputTarget::default_inject(); config.delivery = DeliveryType::Clipboard; let target = build_target(config); From 4e47af37bd85105263ea2dae23796160663375df Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 20:04:01 -0500 Subject: [PATCH 07/12] feat(overlay): recording takes priority over SYSTEM RESPONDING; keep pills in view - While recording, show the target visualizer and hide the SYSTEM RESPONDING pill. Starting to dictate interrupts playback, but the speaking flag can lag behind it, which briefly hid all recording feedback. - Let the visualizer box shrink (trimmed, not scaled) so a tall custom overlay can't push the command/MCP pills out of the 444px window. Built-in visualizers (at most ~152px) plus two pills fit with room left. Co-Authored-By: Claude Opus 5.5 --- src/lib/Overlay/Overlay.svelte | 14 +++++++++++++- tests/svelte/Overlay.test.ts | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index c50022a7..ae8e5026 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -47,7 +47,12 @@ (commandOverlayActive && $config.ui.show_command_overlay) ); - let isSystemResponding = $derived($speaking && $config.tts.enabled && $config.tts.response_overlay); + // Recording wins: starting to dictate interrupts playback (begin_recording + // stops TTS), and until the speaking flag catches up the user must still + // see that they are being recorded rather than a stale SYSTEM RESPONDING. + let isSystemResponding = $derived( + $speaking && $config.tts.enabled && $config.tts.response_overlay && !$recording + ); let renderOverlay = $state(false); // The target visualizer belongs to recording. It gives way to the SYSTEM @@ -441,6 +446,13 @@ align-items: center; justify-content: center; pointer-events: none; + /* Built-in visualizers are at most ~152px, leaving room in the 444px + window for two pills below. A custom overlay may be taller: let this + box give way (trimmed, not scaled) so the pills are never pushed out + of the window. */ + min-height: 0; + flex-shrink: 1; + overflow: hidden; } .system-response-box { diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index 21cff49f..9f284a64 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -242,6 +242,17 @@ describe("Overlay.svelte (root layout)", () => { expect(container.textContent).not.toContain("SYSTEM RESPONDING"); }); + test("recording over a spoken reply shows the visualizer, not SYSTEM RESPONDING", async () => { + // begin_recording stops playback, but the speaking flag can lag behind it. + setStatus({ speaking: true, recording: true }); + + const { container } = render(Overlay); + await settle(); + + expect(container.querySelector(".scope")).not.toBeNull(); + expect(container.textContent).not.toContain("SYSTEM RESPONDING"); + }); + test("keeps the visualizer mounted while the overlay fades out after recording", async () => { setStatus({ recording: true }); const { container } = render(Overlay); From 3956b8a90f8af2b44321ad1f098c13500de372c3 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 20:07:11 -0500 Subject: [PATCH 08/12] Bump version to 0.7.0 Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 2 +- package-lock.json | 4 ++-- package.json | 2 +- src-tauri/tauri.conf.json | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b459709..40051222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6689,7 +6689,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "voxctrl-app" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -6728,7 +6728,7 @@ dependencies = [ [[package]] name = "voxctrl-audio" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "cpal", @@ -6743,7 +6743,7 @@ dependencies = [ [[package]] name = "voxctrl-bugreport" -version = "0.6.5" +version = "0.7.0" dependencies = [ "chrono", "dirs 5.0.1", @@ -6761,7 +6761,7 @@ dependencies = [ [[package]] name = "voxctrl-config" -version = "0.6.5" +version = "0.7.0" dependencies = [ "dirs 5.0.1", "serde", @@ -6774,7 +6774,7 @@ dependencies = [ [[package]] name = "voxctrl-dbus" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "serde", @@ -6786,7 +6786,7 @@ dependencies = [ [[package]] name = "voxctrl-hotkeys" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "ashpd", @@ -6808,7 +6808,7 @@ dependencies = [ [[package]] name = "voxctrl-inference" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "bytes", @@ -6839,7 +6839,7 @@ dependencies = [ [[package]] name = "voxctrl-inject" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", @@ -6853,7 +6853,7 @@ dependencies = [ [[package]] name = "voxctrl-llm" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "reqwest 0.12.28", @@ -6867,7 +6867,7 @@ dependencies = [ [[package]] name = "voxctrl-llm-sidecar" -version = "0.6.5" +version = "0.7.0" dependencies = [ "encoding_rs", "llama-cpp-2", @@ -6879,7 +6879,7 @@ dependencies = [ [[package]] name = "voxctrl-mcp" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "serde", @@ -6891,7 +6891,7 @@ dependencies = [ [[package]] name = "voxctrl-routing" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", @@ -6916,14 +6916,14 @@ dependencies = [ [[package]] name = "voxctrl-text" -version = "0.6.5" +version = "0.7.0" dependencies = [ "regex", ] [[package]] name = "voxctrl-tts" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "cc", @@ -6947,7 +6947,7 @@ dependencies = [ [[package]] name = "voxctrl-update" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "futures-util", @@ -6964,7 +6964,7 @@ dependencies = [ [[package]] name = "voxctrl-winput" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", diff --git a/Cargo.toml b/Cargo.toml index de7d527b..a924e200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.6.5" +version = "0.7.0" edition = "2021" authors = ["VoxCtrl Contributors"] license = "MIT" diff --git a/package-lock.json b/package-lock.json index 28e217d5..69b490ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voxctrl", - "version": "0.6.5", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voxctrl", - "version": "0.6.5", + "version": "0.7.0", "dependencies": { "@tauri-apps/api": "^2", "@tauri-apps/plugin-dialog": "^2", diff --git a/package.json b/package.json index dffb5aa7..34484008 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "voxctrl", "private": true, - "version": "0.6.5", + "version": "0.7.0", "type": "module", "scripts": { "predev": "pkill -x voxctrl 2>/dev/null || true; fuser -k 5173/tcp 2>/dev/null || true", diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 060fb926..e097d7e0 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "VoxCtrl", - "version": "0.6.5", + "version": "0.7.0", "identifier": "ai.voxctrl.app", "build": { "frontendDist": "../dist", From 0b03fda97484c19c2b51939055f2b71b947831b1 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 20:19:30 -0500 Subject: [PATCH 09/12] test(overlay): answer the status poll with the test's own status The status store polls get_status every second when no status-tick events arrive, which in tests is always. The mock replied with a fixed idle status, so whenever a run was slow enough for a poll to land inside a test's settle window (as on CI), speaking/recording were reset under it and 'hides target visualizer and displays SYSTEM RESPONDING' failed. The mock now returns the status store's current value, so a poll is a no-op. Co-Authored-By: Claude Opus 5.5 --- tests/svelte/Overlay.test.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index 9f284a64..230cfcc3 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -4,20 +4,18 @@ import { render } from "@testing-library/svelte"; // Mock tauri IPC used by the overlay components and the status store vi.mock("@tauri-apps/api/core", () => ({ - // `get_config` has no backend here: failing it leaves the config store on - // its defaults instead of overwriting it with this status-shaped reply. invoke: vi.fn(async (cmd: string) => { + // `get_config` has no backend here: failing it leaves the config store on + // its defaults. if (cmd === "get_config") throw new Error("no backend in tests"); - return { - recording: false, - processing: false, - speaking: false, - mcp_recording: false, - audio_ready: true, - word_count: 0, - active_target_id: "default", - active_target_label: "Focused Window", - }; + // The status store falls back to polling `get_status` every second when + // no status-tick events arrive, which in tests is always. Answer with + // whatever status the test has set, so a poll landing mid-test changes + // nothing — a fixed idle reply here reset `speaking`/`recording` under + // the tests whenever a run was slow enough for the poll to fire. + const { status } = await import("../../src/stores/status"); + const { get } = await import("svelte/store"); + return get(status); }), })); From 542f9955b5604543175dd223d35c4995db128b4c Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:48:20 -0500 Subject: [PATCH 10/12] feat(commands): detect voice commands while the user is still speaking Run short interim transcription passes over the opening of a recording so a spoken command ("Hey Vox, say ...") is recognised mid-speech: the command overlay appears early and the TTS model starts loading before release. Compared with the first cut of this feature: - Interim passes skip the hotkey's OpenAI post-processing, which otherwise made a billed request every ~400ms while recording. - Only the first 6s of audio is transcribed early, so passes stay short, the final transcription never waits long behind one, and long dictations don't keep the model busy for their whole length. - Interim passes run only when a command could match (a non-router target exists) and never for remote or CPU medium/large Whisper backends. - The early match is a head start only. Routing is decided by the final transcript alone, which must carry the trigger itself, so a misheard interim ("hey folks" -> "hey box") can no longer send ordinary dictation to a command target. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-inference/src/lib.rs | 40 ++++++++++- src-tauri/src/lib.rs | 1 + src-tauri/src/pipeline.rs | 102 ++++++++++++++++++++++++++++ src-tauri/src/state.rs | 10 +++ src-tauri/src/tests.rs | 1 + 5 files changed, 152 insertions(+), 2 deletions(-) diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 0a670b31..c2fe0f1c 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -225,6 +225,10 @@ pub struct InferenceRequest { pub target_id: String, /// Hotkey binding ID (if triggered by a hotkey) pub binding_id: Option, + /// Whether this is an intermediate/live transcription during active speech + pub is_interim: bool, + /// Unique session identifier for the recording turn + pub session_id: u64, } /// Final output after transcription + post-processing. @@ -239,6 +243,8 @@ pub struct InferenceOutput { /// Set when transcription failed (model missing, backend error, ...). The /// UI layer surfaces this to the user; `text` is empty in that case. pub error: Option, + pub is_interim: bool, + pub session_id: u64, } // ── Engine ──────────────────────────────────────────────────────────────────── @@ -293,6 +299,9 @@ impl InferenceEngine { /// Transcribe and post-process. Returns the final text. pub fn process(&self, req: InferenceRequest) -> Result { + let is_interim = req.is_interim; + let session_id = req.session_id; + if req.audio.is_empty() { return Ok(InferenceOutput { text: String::new(), @@ -302,6 +311,8 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } @@ -345,6 +356,8 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } @@ -400,7 +413,10 @@ impl InferenceEngine { .and_then(|b| b.openai_enabled) .unwrap_or(false); - if binding_wants_openai && !processed.is_empty() { + // Interim passes only feed the voice-command trigger check and are + // thrown away; running the hotkey's (billed, slow) OpenAI rewrite on + // every one of them would cost a request per pass for nothing. + if binding_wants_openai && !is_interim && !processed.is_empty() { // Re-read the OpenAI settings from disk so changes made in the // Settings UI (model, endpoint, API key, prompts) take effect without // restarting the app. Targets and bindings above are already hot-read @@ -477,6 +493,8 @@ impl InferenceEngine { inference_ms: result.inference_ms, language: result.language, error: None, + is_interim, + session_id, }) } @@ -616,11 +634,25 @@ pub fn run_worker_with_config( loop { crossbeam_channel::select! { recv(rx) -> req_res => { - let req = match req_res { + let mut req = match req_res { Ok(r) => r, Err(_) => break, }; + // If an interim request arrived, drain any newer requests already in the queue + // to prioritize fresher audio or the final release request. + if req.is_interim { + while let Ok(newer) = rx.try_recv() { + req = newer; + if !req.is_interim { + break; + } + } + } + + let is_interim = req.is_interim; + let session_id = req.session_id; + if !loaded { match engine.load() { Ok(()) => { @@ -637,6 +669,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); continue; } @@ -657,6 +691,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: "".to_string(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 22b20399..74ba4ca5 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -350,6 +350,7 @@ pub fn run() { router: router.clone(), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(cfg_data.ui.show_overlay)), mcp_recording: Arc::new(AtomicBool::new(false)), diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 78460600..6d21b2dc 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -36,6 +36,8 @@ async fn process_remote_transcription( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim: false, + session_id: 0, }); return; } @@ -69,6 +71,8 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); return; } @@ -188,9 +192,21 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); } +/// Interim passes start once this much audio exists (0.5 s at 16 kHz)... +const INTERIM_MIN_SAMPLES: usize = 8_000; +/// ...and re-run after at least this much more (0.3 s). +const INTERIM_STEP_SAMPLES: usize = 4_800; +/// Only the opening of a recording is transcribed early: the trigger and the +/// target name come first ("Hey Vox, add this to my notes"), and capping the +/// window keeps each pass short, so the final transcription never waits long +/// behind one and long dictations don't keep the model busy the whole time. +const INTERIM_MAX_SAMPLES: usize = 6 * 16_000; + pub fn spawn_audio_coordinator( state_for_audio: Arc, audio_rx: crossbeam_channel::Receiver, @@ -205,6 +221,10 @@ pub fn spawn_audio_coordinator( let mut binding_id = String::new(); let mut remote_session: Option = None; let mut is_remote_backend = false; + let mut interim_enabled = false; + let mut session_id: u64 = 0; + let mut last_interim_sample_count: usize = 0; + let mut last_interim_instant = std::time::Instant::now(); while let Ok(chunk) = audio_rx.recv() { let is_recording = state_for_audio.is_recording(); @@ -215,8 +235,27 @@ pub fn spawn_audio_coordinator( target_id = state_for_audio.active_target.blocking_lock().clone(); binding_id = state_for_audio.active_binding_id.blocking_lock().clone(); was_recording = true; + session_id = session_id.wrapping_add(1); + last_interim_sample_count = 0; + last_interim_instant = std::time::Instant::now(); + state_for_audio.set_interim_in_flight(false); let cfg = state_for_audio.config.blocking_lock().data.clone(); + // Early command detection only pays off when a command + // could match (some non-router target exists) and the + // local model is fast enough to transcribe mid-recording. + let heavy_model = cfg.engine.backend == voxctrl_config::BackendChoice::WhisperCpp + && voxctrl_inference::whisper_gpu_backend().is_none() + && (cfg.engine.whisper_cpp.model_size.starts_with("medium") + || cfg.engine.whisper_cpp.model_size.starts_with("large")); + let has_command_targets = state_for_audio + .targets + .blocking_lock() + .iter() + .any(|t| t.delivery != voxctrl_routing::DeliveryType::Command); + interim_enabled = cfg.engine.backend != voxctrl_config::BackendChoice::RemoteOpenAi + && !heavy_model + && has_command_targets; if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { is_remote_backend = true; let mut merged_prompt = String::from( @@ -248,6 +287,27 @@ pub fn spawn_audio_coordinator( session.send_chunk(chunk.clone()); } accumulated_audio.extend(chunk); + + // Interim transcription of the opening of the recording, so a + // voice command is recognised while the user is still talking. + let window = accumulated_audio.len().min(INTERIM_MAX_SAMPLES); + if interim_enabled + && window >= INTERIM_MIN_SAMPLES + && window.saturating_sub(last_interim_sample_count) >= INTERIM_STEP_SAMPLES + && last_interim_instant.elapsed() >= std::time::Duration::from_millis(400) + && !state_for_audio.is_interim_in_flight() + { + state_for_audio.set_interim_in_flight(true); + last_interim_sample_count = window; + last_interim_instant = std::time::Instant::now(); + let _ = inference_tx.send(voxctrl_inference::InferenceRequest { + audio: accumulated_audio[..window].to_vec(), + target_id: target_id.clone(), + binding_id: Some(binding_id.clone()), + is_interim: true, + session_id, + }); + } } else { if was_recording { if is_remote_backend { @@ -280,6 +340,8 @@ pub fn spawn_audio_coordinator( audio: std::mem::take(&mut accumulated_audio), target_id: target_id.clone(), binding_id: Some(binding_id.clone()), + is_interim: false, + session_id, }; state_for_audio.set_processing(true); let _ = inference_tx.send(req); @@ -298,7 +360,47 @@ pub fn spawn_text_delivery_worker( rt_handle: tokio::runtime::Handle, ) { std::thread::spawn(move || { + // (session, target) already announced from an interim pass, so the + // overlay and TTS preload fire once per command, not once per pass. + let mut announced: Option<(u64, String)> = None; + while let Ok(output) = text_rx.recv() { + if output.is_interim { + state.set_interim_in_flight(false); + if output.error.is_some() || output.text.trim().is_empty() { + continue; + } + // Early command detection: surface the command overlay and + // start loading the TTS model while the user is still talking. + // This is only a head start — routing is decided solely by the + // final transcript below, which must carry the trigger itself. + let dir = voxctrl_routing::config_dir(); + let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); + if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { + let key = (output.session_id, parsed.matched_target_id); + if announced.as_ref() != Some(&key) { + let matched_target = targets.iter().find(|t| t.id == key.1); + let matched_label = matched_target + .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) + .unwrap_or_else(|| key.1.clone()); + tracing::info!("Early command detected during speech: '{matched_label}' (target: {})", key.1); + voxctrl_routing::targets::notify_command_trigger(&matched_label, &parsed.payload); + + let leads_to_speech = matched_target.is_some_and(|t| { + t.delivery == voxctrl_routing::DeliveryType::Speak + || t.response_pipe.as_deref().is_some_and(|p| !p.trim().is_empty()) + }); + if leads_to_speech { + let state_c = state.clone(); + rt_handle.spawn(async move { state_c.preload_tts().await }); + } + announced = Some(key); + } + } + continue; + } + + state.set_interim_in_flight(false); state.set_processing(false); if let Some(ref err) = output.error { // Always surface transcription failures — without this a diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index 7e8f1637..d3461d99 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -14,6 +14,8 @@ pub struct AppState { pub recording: Arc, /// True while speech transcription/OpenAI post-processing is running pub processing: Arc, + /// True while an interim (mid-recording) transcription pass is running + pub interim_in_flight: Arc, /// True while TTS is playing back pub speaking: Arc, /// Live mirror of `ui.show_overlay` so the hot status-forwarding loops can @@ -150,6 +152,14 @@ impl AppState { self.processing.store(v, Ordering::SeqCst); } + pub fn is_interim_in_flight(&self) -> bool { + self.interim_in_flight.load(Ordering::SeqCst) + } + + pub fn set_interim_in_flight(&self, v: bool) { + self.interim_in_flight.store(v, Ordering::SeqCst); + } + pub fn is_audio_ready(&self) -> bool { self.audio_ready.load(Ordering::SeqCst) } diff --git a/src-tauri/src/tests.rs b/src-tauri/src/tests.rs index c5657570..340ad21c 100644 --- a/src-tauri/src/tests.rs +++ b/src-tauri/src/tests.rs @@ -66,6 +66,7 @@ fn make_test_state() -> AppState { router: Arc::new(OutputTargetRouter::new(Vec::new())), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(true)), mcp_recording: Arc::new(AtomicBool::new(false)), From 62f8997b6d162d9be11e0b22673284f3c6e258ad Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 20:03:06 -0500 Subject: [PATCH 11/12] feat(commands): withdraw unconfirmed early commands; setting to turn detection off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - When an interim pass announced a command but the final transcript doesn't confirm it (no command, empty result or a transcription error), the command overlay is taken down right away through a new "command-withdrawn" event instead of lingering for its full duration. - New features.early_command_detection setting (default on; older configs read as on), with a Settings → Features → Voice Commands toggle and docs. Co-Authored-By: Claude Opus 5.5 --- crates/voxctrl-config/src/lib.rs | 22 ++++++++++++++++++++++ crates/voxctrl-routing/src/targets.rs | 15 +++++++++++++++ docs/api.md | 1 + docs/configuration.md | 4 +++- src-tauri/src/pipeline.rs | 16 +++++++++++++++- src-tauri/src/services.rs | 8 ++++++++ src-tauri/src/state.rs | 5 +++++ src/lib/Overlay/Overlay.svelte | 10 ++++++++++ src/lib/Settings/FeaturesTab.svelte | 14 ++++++++++++++ src/stores/config.ts | 2 ++ 10 files changed, 95 insertions(+), 2 deletions(-) diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index 40e37e30..06825fb5 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -286,6 +286,16 @@ pub struct FeaturesConfig { pub show_notification: Option, /// Map of trigger → expansion, e.g. {"addr" → "123 Main St"} pub snippets: std::collections::HashMap, + /// Transcribe the opening seconds of a recording while it is still going, + /// so a spoken voice command shows its overlay and starts loading the TTS + /// model before the user releases the hotkey. Costs some extra + /// transcription work at the start of each recording. + #[serde(default = "default_early_command_detection")] + pub early_command_detection: bool, +} + +fn default_early_command_detection() -> bool { + true } impl Default for FeaturesConfig { @@ -297,6 +307,7 @@ impl Default for FeaturesConfig { auto_format_lists: true, show_notification: None, snippets: std::collections::HashMap::new(), + early_command_detection: true, } } } @@ -1647,4 +1658,15 @@ mod tests { serde_json::from_str::(&with_leftover).expect("older configs must still load"); } + + #[test] + fn early_command_detection_defaults_on_for_older_configs() { + use super::FeaturesConfig; + let features: FeaturesConfig = serde_json::from_str( + r#"{"remove_fillers": true, "custom_vocabulary": [], "spoken_punctuation": true, + "auto_format_lists": true, "snippets": {}}"#, + ) + .unwrap(); + assert!(features.early_command_detection); + } } diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 4d3f9307..d79afd86 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -29,6 +29,21 @@ pub fn notify_command_trigger(command_name: &str, text_summary: &str) { } } +pub type CommandWithdrawnCallback = Arc; +static COMMAND_WITHDRAWN_CALLBACK: OnceLock = OnceLock::new(); + +pub fn set_command_withdrawn_callback(callback: CommandWithdrawnCallback) { + let _ = COMMAND_WITHDRAWN_CALLBACK.set(callback); +} + +/// Take back a command announced early (from a partial transcript) that the +/// final transcript did not confirm, so its overlay does not linger. +pub fn notify_command_withdrawn() { + if let Some(cb) = COMMAND_WITHDRAWN_CALLBACK.get() { + cb(); + } +} + // Shared HTTP client — built once, reused for connection pooling. fn http_client() -> &'static reqwest::Client { static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/docs/api.md b/docs/api.md index 77d353b7..d70b711a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -794,6 +794,7 @@ interface FeaturesConfig { spoken_punctuation: boolean; auto_format_lists: boolean; snippets: Record; + early_command_detection: boolean; } interface OpenAiConfig { diff --git a/docs/configuration.md b/docs/configuration.md index f1581999..17a4dffc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,7 +70,8 @@ Full schema with defaults: "custom_vocabulary": [], "spoken_punctuation": true, "auto_format_lists": true, - "snippets": {} + "snippets": {}, + "early_command_detection": true }, "openai": { "enabled": false, @@ -237,6 +238,7 @@ The `.en` variants are English-only but slightly faster. `large-v3-turbo` is a d | `auto_format_lists` | bool | `true` | Detect "first/second/third" patterns and reformat as a numbered list | | `custom_vocabulary` | string[] | `[]` | Custom words; VoxCtrl uses fuzzy Levenshtein matching to correct near-matches post-transcription | | `snippets` | object | `{}` | Short code → expansion map | +| `early_command_detection` | bool | `true` | Transcribe the first 6 seconds of a recording while it is still going, so a voice command shows its overlay and starts loading the TTS model before the hotkey is released. Routing is still decided by the final transcript. Skipped for the remote backend and CPU-only medium/large Whisper models | Example with snippets: ```json diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 6d21b2dc..83f9cd79 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -253,7 +253,8 @@ pub fn spawn_audio_coordinator( .blocking_lock() .iter() .any(|t| t.delivery != voxctrl_routing::DeliveryType::Command); - interim_enabled = cfg.engine.backend != voxctrl_config::BackendChoice::RemoteOpenAi + interim_enabled = cfg.features.early_command_detection + && cfg.engine.backend != voxctrl_config::BackendChoice::RemoteOpenAi && !heavy_model && has_command_targets; if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { @@ -402,7 +403,18 @@ pub fn spawn_text_delivery_worker( state.set_interim_in_flight(false); state.set_processing(false); + + // A command announced from an interim pass stands only if the + // final transcript confirms it; otherwise take its overlay down. + let announced_early = announced.take().is_some_and(|(session, _)| session == output.session_id); + let withdraw_early_command = || { + if announced_early { + voxctrl_routing::targets::notify_command_withdrawn(); + } + }; + if let Some(ref err) = output.error { + withdraw_early_command(); // Always surface transcription failures — without this a // fresh install with no Whisper model records audio and // then silently drops it, which reads as "hotkeys broken". @@ -411,6 +423,7 @@ pub fn spawn_text_delivery_worker( continue; } if output.text.trim().is_empty() { + withdraw_early_command(); continue; } @@ -451,6 +464,7 @@ pub fn spawn_text_delivery_worker( }; (matched_id, cleaned_payload) } else { + withdraw_early_command(); let cleaned_text = if s1_mini_enabled && !output.text.trim().is_empty() { voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) } else { diff --git a/src-tauri/src/services.rs b/src-tauri/src/services.rs index 1fe74eda..05fd694e 100644 --- a/src-tauri/src/services.rs +++ b/src-tauri/src/services.rs @@ -240,6 +240,14 @@ pub fn register_speak_target(app_handle: &tauri::AppHandle) { pub fn register_command_trigger_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); let app_handle_clone = app_handle.clone(); + + let withdraw_state = state.clone(); + let withdraw_handle = app_handle.clone(); + voxctrl_routing::targets::set_command_withdrawn_callback(std::sync::Arc::new(move || { + withdraw_state.clear_command_overlay(); + let _ = withdraw_handle.emit("command-withdrawn", ()); + })); + voxctrl_routing::targets::set_command_trigger_callback(std::sync::Arc::new( move |command_name, text_summary| { let (show_overlay, duration_secs) = if let Ok(cfg) = state.config.try_lock() { diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index d3461d99..a21bf70e 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -252,6 +252,11 @@ impl AppState { *self.command_overlay_until.lock().unwrap() = Some(std::time::Instant::now() + duration); } + /// Stop showing the command-executed overlay pill now. + pub fn clear_command_overlay(&self) { + *self.command_overlay_until.lock().unwrap() = None; + } + /// Whether the command-executed overlay pill should still be showing. pub fn is_command_overlay_active(&self) -> bool { self.command_overlay_until diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index 1dbaeb7e..4289c0fd 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -37,6 +37,7 @@ let commandOverlayText = $state(""); let commandTimerId: any = null; let unlistenCommandExecuted: (() => void) | null = null; + let unlistenCommandWithdrawn: (() => void) | null = null; let unlistenOverlayStyleSelected: (() => void) | null = null; // Delay unmounting the visualizer when recording/speaking/command stops to allow CSS outro animation to finish @@ -276,11 +277,20 @@ unlistenCommandExecuted = unlisten; }); + // A command recognised mid-speech that the final transcript didn't confirm. + listen("command-withdrawn", () => { + if (commandTimerId) clearTimeout(commandTimerId); + commandOverlayActive = false; + }).then((unlisten) => { + unlistenCommandWithdrawn = unlisten; + }); + return () => { document.documentElement.classList.remove("overlay-window"); document.body.classList.remove("overlay-window"); if (unlistenAudioLevel) unlistenAudioLevel(); if (unlistenCommandExecuted) unlistenCommandExecuted(); + if (unlistenCommandWithdrawn) unlistenCommandWithdrawn(); if (unlistenOverlayStyleSelected) unlistenOverlayStyleSelected(); if (commandTimerId) clearTimeout(commandTimerId); if (animationFrameId !== null) cancelAnimationFrame(animationFrameId); diff --git a/src/lib/Settings/FeaturesTab.svelte b/src/lib/Settings/FeaturesTab.svelte index 2c79a471..47384ccc 100644 --- a/src/lib/Settings/FeaturesTab.svelte +++ b/src/lib/Settings/FeaturesTab.svelte @@ -259,6 +259,20 @@ +
+

Voice Commands

+ +

+ Transcribes the first few seconds of each recording early, so a command like + “Hey Vox, say …” shows its overlay and starts loading the voice before you let go + of the hotkey. Where the text goes is still decided by the full transcript. Turn + this off to save the extra transcription work. +

+
+

Custom Dictionary

Provide a comma-separated list of words (e.g. names or jargon like "Waylin, Rufer, Enola, Kenz") that are hard to spell. The transcription process will correct these in the final text.

diff --git a/src/stores/config.ts b/src/stores/config.ts index f048c32e..38428dab 100644 --- a/src/stores/config.ts +++ b/src/stores/config.ts @@ -78,6 +78,7 @@ export interface FeaturesConfig { spoken_punctuation: boolean; auto_format_lists: boolean; snippets: Record; + early_command_detection: boolean; } export interface OpenAiConfig { @@ -203,6 +204,7 @@ const defaultConfig: AppConfig = { spoken_punctuation: true, auto_format_lists: true, snippets: {}, + early_command_detection: true, }, openai: { enabled: false, From 8ead8385fbf4f17d46673875f197e1aab448ea87 Mon Sep 17 00:00:00 2001 From: Josh Rufer Date: Wed, 23 Sep 2026 19:51:01 -0500 Subject: [PATCH 12/12] feat(pipeline): skip S1-mini cleanup for text bound for Speak targets Speak targets read their text aloud verbatim, so the S1-mini rewrite only adds latency before the first word and can change what the user meant to have spoken. Command resolution now picks the target and raw text first and runs S1-mini once afterwards, instead of repeating the check per branch. Co-Authored-By: Claude Opus 5.5 --- src-tauri/src/pipeline.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 83f9cd79..4d77ad2a 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -447,7 +447,7 @@ pub fn spawn_text_delivery_worker( .unwrap_or(global_s1_mini_enabled); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let (target_id, text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { + let (target_id, raw_text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { let matched_id = parsed.matched_target_id.clone(); let payload = parsed.payload; let matched_label = targets @@ -456,21 +456,22 @@ pub fn spawn_text_delivery_worker( .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) .unwrap_or_else(|| matched_id.clone()); voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); - - let cleaned_payload = if s1_mini_enabled && !payload.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) - } else { - payload - }; - (matched_id, cleaned_payload) + (matched_id, payload) } else { withdraw_early_command(); - let cleaned_text = if s1_mini_enabled && !output.text.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) - } else { - output.text.clone() - }; - (output.target_id.clone(), cleaned_text) + (output.target_id.clone(), output.text.clone()) + }; + + // Text bound for a Speak target is read aloud verbatim: S1-mini + // would add latency before the first word and may rewrite what + // the user meant to have spoken. + let is_speak_target = targets + .iter() + .any(|t| t.id == target_id && t.delivery == voxctrl_routing::DeliveryType::Speak); + let text = if !is_speak_target && s1_mini_enabled && !raw_text.trim().is_empty() { + voxctrl_inference::s1_mini::clean_dictation(&raw_text, &s1_mini_styling, None) + } else { + raw_text }; if text.trim().is_empty() {