diff --git a/Cargo.lock b/Cargo.lock index 5b459709..40051222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6689,7 +6689,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "voxctrl-app" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -6728,7 +6728,7 @@ dependencies = [ [[package]] name = "voxctrl-audio" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "cpal", @@ -6743,7 +6743,7 @@ dependencies = [ [[package]] name = "voxctrl-bugreport" -version = "0.6.5" +version = "0.7.0" dependencies = [ "chrono", "dirs 5.0.1", @@ -6761,7 +6761,7 @@ dependencies = [ [[package]] name = "voxctrl-config" -version = "0.6.5" +version = "0.7.0" dependencies = [ "dirs 5.0.1", "serde", @@ -6774,7 +6774,7 @@ dependencies = [ [[package]] name = "voxctrl-dbus" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "serde", @@ -6786,7 +6786,7 @@ dependencies = [ [[package]] name = "voxctrl-hotkeys" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "ashpd", @@ -6808,7 +6808,7 @@ dependencies = [ [[package]] name = "voxctrl-inference" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "bytes", @@ -6839,7 +6839,7 @@ dependencies = [ [[package]] name = "voxctrl-inject" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", @@ -6853,7 +6853,7 @@ dependencies = [ [[package]] name = "voxctrl-llm" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "reqwest 0.12.28", @@ -6867,7 +6867,7 @@ dependencies = [ [[package]] name = "voxctrl-llm-sidecar" -version = "0.6.5" +version = "0.7.0" dependencies = [ "encoding_rs", "llama-cpp-2", @@ -6879,7 +6879,7 @@ dependencies = [ [[package]] name = "voxctrl-mcp" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "serde", @@ -6891,7 +6891,7 @@ dependencies = [ [[package]] name = "voxctrl-routing" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", @@ -6916,14 +6916,14 @@ dependencies = [ [[package]] name = "voxctrl-text" -version = "0.6.5" +version = "0.7.0" dependencies = [ "regex", ] [[package]] name = "voxctrl-tts" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "cc", @@ -6947,7 +6947,7 @@ dependencies = [ [[package]] name = "voxctrl-update" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "futures-util", @@ -6964,7 +6964,7 @@ dependencies = [ [[package]] name = "voxctrl-winput" -version = "0.6.5" +version = "0.7.0" dependencies = [ "anyhow", "arboard", diff --git a/Cargo.toml b/Cargo.toml index de7d527b..a924e200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.6.5" +version = "0.7.0" edition = "2021" authors = ["VoxCtrl Contributors"] license = "MIT" diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index 87c54a0a..06825fb5 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -286,17 +286,28 @@ pub struct FeaturesConfig { pub show_notification: Option, /// Map of trigger → expansion, e.g. {"addr" → "123 Main St"} pub snippets: std::collections::HashMap, + /// Transcribe the opening seconds of a recording while it is still going, + /// so a spoken voice command shows its overlay and starts loading the TTS + /// model before the user releases the hotkey. Costs some extra + /// transcription work at the start of each recording. + #[serde(default = "default_early_command_detection")] + pub early_command_detection: bool, +} + +fn default_early_command_detection() -> bool { + true } impl Default for FeaturesConfig { fn default() -> Self { Self { remove_fillers: true, - custom_vocabulary: vec!["VoxCtrl".into()], + custom_vocabulary: vec!["VoxCtrl".into(), "Hey Vox".into()], spoken_punctuation: true, auto_format_lists: true, show_notification: None, snippets: std::collections::HashMap::new(), + early_command_detection: true, } } } @@ -1647,4 +1658,15 @@ mod tests { serde_json::from_str::(&with_leftover).expect("older configs must still load"); } + + #[test] + fn early_command_detection_defaults_on_for_older_configs() { + use super::FeaturesConfig; + let features: FeaturesConfig = serde_json::from_str( + r#"{"remove_fillers": true, "custom_vocabulary": [], "spoken_punctuation": true, + "auto_format_lists": true, "snippets": {}}"#, + ) + .unwrap(); + assert!(features.early_command_detection); + } } diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 74e75b98..c2fe0f1c 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -225,6 +225,10 @@ pub struct InferenceRequest { pub target_id: String, /// Hotkey binding ID (if triggered by a hotkey) pub binding_id: Option, + /// Whether this is an intermediate/live transcription during active speech + pub is_interim: bool, + /// Unique session identifier for the recording turn + pub session_id: u64, } /// Final output after transcription + post-processing. @@ -239,6 +243,8 @@ pub struct InferenceOutput { /// Set when transcription failed (model missing, backend error, ...). The /// UI layer surfaces this to the user; `text` is empty in that case. pub error: Option, + pub is_interim: bool, + pub session_id: u64, } // ── Engine ──────────────────────────────────────────────────────────────────── @@ -293,6 +299,9 @@ impl InferenceEngine { /// Transcribe and post-process. Returns the final text. pub fn process(&self, req: InferenceRequest) -> Result { + let is_interim = req.is_interim; + let session_id = req.session_id; + if req.audio.is_empty() { return Ok(InferenceOutput { text: String::new(), @@ -302,6 +311,8 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } @@ -345,13 +356,15 @@ impl InferenceEngine { inference_ms: 0, language: "en".into(), error: None, + is_interim, + session_id, }); } let dir = voxctrl_routing::config_dir(); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. "); + let mut merged_prompt = String::from("VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. "); // Custom vocabulary words from features config if !app_config.features.custom_vocabulary.is_empty() { @@ -400,7 +413,10 @@ impl InferenceEngine { .and_then(|b| b.openai_enabled) .unwrap_or(false); - if binding_wants_openai && !processed.is_empty() { + // Interim passes only feed the voice-command trigger check and are + // thrown away; running the hotkey's (billed, slow) OpenAI rewrite on + // every one of them would cost a request per pass for nothing. + if binding_wants_openai && !is_interim && !processed.is_empty() { // Re-read the OpenAI settings from disk so changes made in the // Settings UI (model, endpoint, API key, prompts) take effect without // restarting the app. Targets and bindings above are already hot-read @@ -477,6 +493,8 @@ impl InferenceEngine { inference_ms: result.inference_ms, language: result.language, error: None, + is_interim, + session_id, }) } @@ -616,11 +634,25 @@ pub fn run_worker_with_config( loop { crossbeam_channel::select! { recv(rx) -> req_res => { - let req = match req_res { + let mut req = match req_res { Ok(r) => r, Err(_) => break, }; + // If an interim request arrived, drain any newer requests already in the queue + // to prioritize fresher audio or the final release request. + if req.is_interim { + while let Ok(newer) = rx.try_recv() { + req = newer; + if !req.is_interim { + break; + } + } + } + + let is_interim = req.is_interim; + let session_id = req.session_id; + if !loaded { match engine.load() { Ok(()) => { @@ -637,6 +669,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); continue; } @@ -657,6 +691,8 @@ pub fn run_worker_with_config( inference_ms: 0, language: "".to_string(), error: Some(format!("{e:#}")), + is_interim, + session_id, }); } } diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 892f28c4..d79afd86 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -29,6 +29,21 @@ pub fn notify_command_trigger(command_name: &str, text_summary: &str) { } } +pub type CommandWithdrawnCallback = Arc; +static COMMAND_WITHDRAWN_CALLBACK: OnceLock = OnceLock::new(); + +pub fn set_command_withdrawn_callback(callback: CommandWithdrawnCallback) { + let _ = COMMAND_WITHDRAWN_CALLBACK.set(callback); +} + +/// Take back a command announced early (from a partial transcript) that the +/// final transcript did not confirm, so its overlay does not linger. +pub fn notify_command_withdrawn() { + if let Some(cb) = COMMAND_WITHDRAWN_CALLBACK.get() { + cb(); + } +} + // Shared HTTP client — built once, reused for connection pooling. fn http_client() -> &'static reqwest::Client { static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -1310,44 +1325,87 @@ fn levenshtein_distance(s1: &str, s2: &str) -> usize { dp[len1][len2] } -/// Parse text for keyword "VoxCtrl" and target name/label matching. -/// Supports both direct commands (e.g. "VoxCtrl notes Hi there") and natural, -/// conversational phrasing (e.g. "VoxCtrl add this to my notes. What are you doing here?"). -/// Returns `Some(VoiceCommandParseResult)` if the trigger keyword was found AND a target matched; -/// otherwise returns `None`. +/// Lowercase `s` without changing any character's byte length, so byte offsets +/// found in the result are valid in `s` too. The few characters whose lowercase +/// form is longer or shorter in UTF-8 (e.g. `İ`, `ẞ`) are left as they are, +/// which only costs them case-insensitive matching. +fn fold_case(s: &str) -> String { + s.chars() + .map(|c| { + let mut lower = c.to_lowercase(); + match (lower.next(), lower.next()) { + (Some(l), None) if l.len_utf8() == c.len_utf8() => l, + _ => c, + } + }) + .collect() +} + +/// True when `word` sounds like "vox" as speech recognition tends to mishear it +/// ("box", "fox", "walks", "locks"): short and ending in an x/ks sound. +fn sounds_like_vox(word: &str) -> bool { + word.len() <= 6 && (word.ends_with('x') || word.ends_with("ks") || word.ends_with("cs")) +} + +/// Parse text for a trigger keyword ("VoxCtrl", "Vox Control", "Hey Vox", ...) +/// followed by a target name/label. Supports both direct commands +/// (e.g. "Hey Vox notes Hi there") and natural, conversational phrasing +/// (e.g. "Hey Vox, add this to my notes. What are you doing here?"). +/// Without an explicit trigger nothing matches, so normal dictation is never +/// misclassified as a command. Returns `Some(VoiceCommandParseResult)` if the +/// trigger was found AND a target matched; otherwise `None`. pub fn parse_voice_command( text: &str, targets: &[OutputTarget], ) -> Option { - let lower_text = text.to_lowercase(); + let lower_text = fold_case(text); let mut found_pos = None; let mut trigger_len = 0; // 1. Exact & standard triggers - let exact_triggers = ["voxctrl", "vox ctrl", "vox-ctrl", "vox control"]; + let exact_triggers = [ + "voxctrl", "vox ctrl", "vox-ctrl", "vox control", + "hey vox", "hey, vox", "hey-vox", "hey_vox", + ]; for trigger in &exact_triggers { - if let Some(pos) = lower_text.find(trigger) { - if found_pos.map_or(true, |p| pos < p) { - found_pos = Some(pos); - trigger_len = trigger.len(); + let mut search_start = 0; + while let Some(match_idx) = lower_text[search_start..].find(trigger) { + let pos = search_start + match_idx; + let end_idx = pos + trigger.len(); + let is_boundary_start = pos == 0 || { + let prev = lower_text[..pos].chars().last().unwrap(); + prev.is_whitespace() || prev.is_ascii_punctuation() + }; + let is_boundary_end = end_idx == lower_text.len() || { + let next = lower_text[end_idx..].chars().next().unwrap(); + next.is_whitespace() || next.is_ascii_punctuation() + }; + if is_boundary_start && is_boundary_end { + if found_pos.map_or(true, |p| pos < p) { + found_pos = Some(pos); + trigger_len = trigger.len(); + } + break; } + search_start = pos + 1; } } - // 2. Dynamic pattern trigger for any " control" or " ctrl" phrase + // 2. Misheard "vox control" at the very start ("box control", "walks control"). + // Only the leading two words count, and the first must sound like "vox": + // "take control, say hello" or "I lost control" is ordinary dictation. if found_pos.is_none() { let words: Vec<&str> = lower_text.split_whitespace().collect(); - for (i, word) in words.iter().enumerate() { - let clean_w = word.trim_matches(|c: char| c.is_ascii_punctuation()); - if clean_w == "control" || clean_w == "ctrl" || clean_w == "ctl" || clean_w == "kontrol" { - if i > 0 { - let start_idx = lower_text.find(words[0]).unwrap_or(0); - let ctrl_pos = lower_text.find(word).unwrap_or(0); - let end_pos = ctrl_pos + word.len(); - found_pos = Some(start_idx); - trigger_len = end_pos - start_idx; - break; - } + if words.len() >= 2 { + let first = words[0].trim_matches(|c: char| c.is_ascii_punctuation()); + let second = words[1].trim_matches(|c: char| c.is_ascii_punctuation()); + if sounds_like_vox(first) && matches!(second, "control" | "ctrl" | "ctl" | "kontrol") { + let start_idx = lower_text.find(words[0]).unwrap_or(0); + let first_end = start_idx + words[0].len(); + let second_end = + first_end + lower_text[first_end..].find(words[1]).unwrap_or(0) + words[1].len(); + found_pos = Some(start_idx); + trigger_len = second_end - start_idx; } } } @@ -1361,12 +1419,32 @@ pub fn parse_voice_command( let clean_cand = candidate.trim_matches(|c: char| c.is_ascii_punctuation()); let dist1 = levenshtein_distance(clean_cand, "voxctrl"); let dist2 = levenshtein_distance(clean_cand, "vox control"); - if dist1 <= 2 || dist2 <= 3 { - if let Some(pos) = lower_text.find(clean_cand) { - found_pos = Some(pos); - trigger_len = clean_cand.len(); - break; - } + + // Check "hey vox" fuzzy match (distance <= 1) on normalized tokens + let clean_tokens: Vec<&str> = words[..len] + .iter() + .map(|w| w.trim_matches(|c: char| c.is_ascii_punctuation())) + .filter(|w| !w.is_empty()) + .collect(); + let hey_vox_cand = clean_tokens.join(" "); + let dist_hey_vox = if !hey_vox_cand.is_empty() { + levenshtein_distance(&hey_vox_cand, "hey vox") + } else { + usize::MAX + }; + + if dist1 <= 2 || dist2 <= 3 || dist_hey_vox <= 1 { + let first_word = words[0]; + let last_word = words[len - 1]; + let start_idx = lower_text.find(first_word).unwrap_or(0); + let last_pos = lower_text[start_idx..] + .find(last_word) + .map(|p| start_idx + p) + .unwrap_or(start_idx); + let end_idx = last_pos + last_word.len(); + found_pos = Some(start_idx); + trigger_len = end_idx - start_idx; + break; } } } @@ -1392,10 +1470,10 @@ pub fn parse_voice_command( candidate_entries.sort_by(|a, b| b.1.len().cmp(&a.1.len())); - let after_trigger_lower = after_trigger.to_lowercase(); + let after_trigger_lower = fold_case(after_trigger); for (target_id, candidate) in candidate_entries { - let cand_lower = candidate.to_lowercase(); + let cand_lower = fold_case(candidate); let mut search_start = 0; while let Some(match_idx) = after_trigger_lower[search_start..].find(&cand_lower) { @@ -1425,9 +1503,12 @@ pub fn parse_voice_command( } } - search_start = abs_match_start + 1; + // Step past the whole first character: target names need not be ASCII. + search_start = abs_match_start + + after_trigger_lower[abs_match_start..].chars().next().map_or(1, char::len_utf8); } } None } + diff --git a/crates/voxctrl-routing/src/tests.rs b/crates/voxctrl-routing/src/tests.rs index cd68ab76..3d092930 100644 --- a/crates/voxctrl-routing/src/tests.rs +++ b/crates/voxctrl-routing/src/tests.rs @@ -457,6 +457,11 @@ async fn test_clipboard_target_linux_cli() { #[tokio::test] async fn test_clipboard_target_failure_empty_text() { let _env = env_lock().lock().await; + // Without the mock this ran the machine's real `wl-copy`, which replaced + // the developer's clipboard and forked a background server that kept the + // test binary's stdout open, so a piped `cargo test` never exited. + #[cfg(target_os = "linux")] + let _mock = MockClipboardTool::install(); let mut config = OutputTarget::default_inject(); config.delivery = DeliveryType::Clipboard; let target = build_target(config); @@ -1768,6 +1773,12 @@ fn test_parse_voice_command_no_keyword() { ]; assert!(parse_voice_command("hi there", &targets).is_none()); + assert!(parse_voice_command("Hey folks, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey boss, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("Hey guys, notes about meeting", &targets).is_none()); + assert!(parse_voice_command("They voxelize notes and models", &targets).is_none()); + assert!(parse_voice_command("heavy box notes something", &targets).is_none()); + assert!(parse_voice_command("the box notes something", &targets).is_none()); } #[test] @@ -1854,6 +1865,60 @@ fn test_parse_voice_command_matched_target() { let res8 = parse_voice_command("Walks control speak how are you?", &speak_targets).expect("should match Walks control dynamic trigger pattern"); assert_eq!(res8.matched_target_id, "speak"); assert_eq!(res8.payload, "how are you?"); + + // Hey Vox commands + let res9 = parse_voice_command("Hey Vox, notes Hi there", &targets).expect("should match Hey Vox with comma"); + assert_eq!(res9.matched_target_id, "notes"); + assert_eq!(res9.payload, "Hi there"); + + let res10 = parse_voice_command("Hey Vox notes Hi there", &targets).expect("should match Hey Vox without comma"); + assert_eq!(res10.matched_target_id, "notes"); + assert_eq!(res10.payload, "Hi there"); + + let res11 = parse_voice_command("Hey, Vox, notes Hi there", &targets).expect("should match Hey, Vox, with internal comma"); + assert_eq!(res11.matched_target_id, "notes"); + assert_eq!(res11.payload, "Hi there"); + + let res12 = parse_voice_command("Hey Vox: Notes, please write this down", &targets).expect("should match Hey Vox with colon"); + assert_eq!(res12.matched_target_id, "notes"); + assert_eq!(res12.payload, "please write this down"); + + let res13 = parse_voice_command("Hey, Vox: Notes, please write this down", &targets).expect("should match Hey, Vox: with colon"); + assert_eq!(res13.matched_target_id, "notes"); + assert_eq!(res13.payload, "please write this down"); + + let res14 = parse_voice_command("hey vox add this to my notes. What are you doing here?", &targets).expect("should match conversational hey vox notes"); + assert_eq!(res14.matched_target_id, "notes"); + assert_eq!(res14.payload, "What are you doing here?"); + + let res15 = parse_voice_command("Hey Vox put this into my Notes: What are you doing here?", &targets).expect("should match conversational put into notes"); + assert_eq!(res15.matched_target_id, "notes"); + assert_eq!(res15.payload, "What are you doing here?"); + + let res16 = parse_voice_command("Hey, Vox, send us to my notes. I love you.", &targets).expect("should match conversational send us to my notes"); + assert_eq!(res16.matched_target_id, "notes"); + assert_eq!(res16.payload, "I love you."); + + // Hey Vox homophones (Acoustic mis-recognitions within distance <= 1) + let res17 = parse_voice_command("Hey Box speak text", &speak_targets).expect("should match Hey Box homophone"); + assert_eq!(res17.matched_target_id, "speak"); + assert_eq!(res17.payload, "text"); + + let res18 = parse_voice_command("hay vox speak text", &speak_targets).expect("should match hay vox homophone"); + assert_eq!(res18.matched_target_id, "speak"); + assert_eq!(res18.payload, "text"); + + let res19 = parse_voice_command("hey fox speak text", &speak_targets).expect("should match hey fox homophone"); + assert_eq!(res19.matched_target_id, "speak"); + assert_eq!(res19.payload, "text"); + + let res20 = parse_voice_command("hey vax speak text", &speak_targets).expect("should match hey vax homophone"); + assert_eq!(res20.matched_target_id, "speak"); + assert_eq!(res20.payload, "text"); + + let res21 = parse_voice_command("Hey, Box, speak, eat my ass.", &speak_targets).expect("should match Hey, Box, homophone with punctuation"); + assert_eq!(res21.matched_target_id, "speak"); + assert_eq!(res21.payload, "eat my ass."); } #[test] @@ -2218,3 +2283,69 @@ async fn command_targets_keep_newlines_when_not_stripping() { let text = delivered.expect("delivery produced no text"); assert_eq!(text, "first line\nsecond line "); } + +#[test] +fn test_parse_voice_command_requires_wake_word() { + use crate::targets::parse_voice_command; + + let targets = vec![ + OutputTarget { + id: "say_target".into(), + label: "Say".into(), + delivery: DeliveryType::Speak, + ..OutputTarget::default_inject() + }, + OutputTarget { + id: "notes".into(), + label: "Notes".into(), + delivery: DeliveryType::File, + ..OutputTarget::default_inject() + }, + ]; + + // Normal speech must NEVER trigger a command without wake words + assert!(parse_voice_command("say hello world", &targets).is_none()); + assert!(parse_voice_command("notes: buy groceries", &targets).is_none()); + assert!(parse_voice_command("what the hell is going on. And why is it happening?", &targets).is_none()); + assert!(parse_voice_command("There have been several changes made in this branch.", &targets).is_none()); + assert!(parse_voice_command("Do you hear me? Do you know what's happening?", &targets).is_none()); + // " control" only triggers at the start, and only when sounds like "vox" + assert!(parse_voice_command("Take control, say hello", &targets).is_none()); + assert!(parse_voice_command("I lost control notes about the meeting", &targets).is_none()); + assert!(parse_voice_command("The remote control, say what you will", &targets).is_none()); + + // Valid commands with Hey Vox or Vox Control trigger properly + let res1 = parse_voice_command("Hey Vox, say hello", &targets).expect("should match with hey vox trigger"); + assert_eq!(res1.matched_target_id, "say_target"); + assert_eq!(res1.payload, "hello"); + + let res2 = parse_voice_command("Vox Control, say hello world", &targets).expect("should match with vox control trigger"); + assert_eq!(res2.matched_target_id, "say_target"); + assert_eq!(res2.payload, "hello world"); + + let res3 = parse_voice_command("Hey Vox, add this to my notes: buy groceries", &targets).expect("should match notes with filler"); + assert_eq!(res3.matched_target_id, "notes"); + assert_eq!(res3.payload, "buy groceries"); +} + +#[test] +fn test_parse_voice_command_non_ascii_text_and_targets() { + use crate::targets::parse_voice_command; + + let targets = vec![OutputTarget { + id: "uber".into(), + label: "Überblick".into(), + delivery: DeliveryType::File, + ..OutputTarget::default_inject() + }]; + + // `İ` lowercases to a longer UTF-8 sequence; offsets must still line up. + let res = parse_voice_command("İİ Hey Vox, Überblick grüße aus İstanbul", &targets) + .expect("should match non-ASCII label after non-ASCII prefix"); + assert_eq!(res.matched_target_id, "uber"); + assert_eq!(res.payload, "grüße aus İstanbul"); + + // A non-boundary occurrence of a label starting with a multibyte char + // must be skipped without slicing mid-character. + assert!(parse_voice_command("Hey Vox, xÜberblick hallo", &targets).is_none()); +} diff --git a/crates/voxctrl-tts/src/engine.rs b/crates/voxctrl-tts/src/engine.rs index 7b928876..995ba0c8 100644 --- a/crates/voxctrl-tts/src/engine.rs +++ b/crates/voxctrl-tts/src/engine.rs @@ -123,6 +123,38 @@ impl TtsEngineHandle { } } +/// Whether the selected engine's `prewarm` setting asks for its model to be +/// loaded up front. +fn prewarm_enabled(config: &TtsConfig) -> bool { + match config.engine { + TtsEngine::PocketTts => config.pocket_tts.prewarm, + TtsEngine::InflectMicro => config.inflect_micro.prewarm, + TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, + TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, + _ => false, + } +} + +/// Load the selected engine's model if it is not resident yet. `None` when +/// there was nothing to do: already loaded, or an engine that holds no model. +fn load_selected_engine( + config: &TtsConfig, + inflect_model: &mut InflectModelSlot, + audiocpp_session: &mut Option, +) -> Option> { + match config.engine { + TtsEngine::InflectMicro if inflect_model.is_none() => { + Some(ensure_inflect_micro_loaded(config, inflect_model)) + } + TtsEngine::PocketTts if audiocpp_session.is_none() => Some(ensure_pocket_tts_loaded(config, audiocpp_session)), + TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { + Some(ensure_breeze_tts_2_loaded(config, audiocpp_session)) + } + TtsEngine::VoxCpm2 if audiocpp_session.is_none() => Some(ensure_vox_cpm_2_loaded(config, audiocpp_session)), + _ => None, + } +} + // ── TTS engine worker ───────────────────────────────────────────────────────── pub type PlaybackCallback = Arc; @@ -158,13 +190,7 @@ impl TtsEngineWorker { model_loaded: model_loaded.clone(), }; - let prewarm = match config.engine { - TtsEngine::PocketTts => config.pocket_tts.prewarm, - TtsEngine::InflectMicro => config.inflect_micro.prewarm, - TtsEngine::BreezeTts2 => config.breeze_tts_2.prewarm, - TtsEngine::VoxCpm2 => config.vox_cpm_2.prewarm, - _ => false, - }; + let prewarm = prewarm_enabled(&config); // Pre-warming loads the model at startup and keeps it there, which is // exactly what the on-demand memory mode exists to avoid — so the two // settings do not fight: on-demand wins and the model waits for its @@ -283,10 +309,25 @@ impl TtsEngineWorker { "TTS worker config dynamically updated (engine={:?}, memory_mode={:?})", new_cfg.engine, new_cfg.memory_mode ); + let was_unloading = current_config.unloads_when_idle(); current_config = new_cfg; // Switching to on-demand starts the clock now rather than // dropping a model that may be about to be used again. last_used = Instant::now(); + + // Switching to always-loaded (e.g. from the tray) with + // pre-warming on: load now, as a startup in this mode would. + if was_unloading && !current_config.unloads_when_idle() && prewarm_enabled(¤t_config) { + if let Some(Err(e)) = + load_selected_engine(¤t_config, &mut inflect_model, &mut audiocpp_session) + { + warn!("TTS pre-warm after switching to always-loaded failed: {e:#}"); + } + self.model_loaded.store( + inflect_model.is_some() || audiocpp_session.is_some(), + Ordering::SeqCst, + ); + } } TtsCommand::Play { mut utterance, generation } => { // Synthesis is about to touch the model; hold off the idle @@ -427,22 +468,8 @@ impl TtsEngineWorker { // Speculative: failures are logged, not surfaced. The same // error is reported properly (with a toast) if an utterance // actually arrives. - let outcome = match current_config.engine { - TtsEngine::InflectMicro if inflect_model.is_none() => { - Some(ensure_inflect_micro_loaded(¤t_config, &mut inflect_model)) - } - TtsEngine::PocketTts if audiocpp_session.is_none() => { - Some(ensure_pocket_tts_loaded(¤t_config, &mut audiocpp_session)) - } - TtsEngine::BreezeTts2 if audiocpp_session.is_none() => { - Some(ensure_breeze_tts_2_loaded(¤t_config, &mut audiocpp_session)) - } - TtsEngine::VoxCpm2 if audiocpp_session.is_none() => { - Some(ensure_vox_cpm_2_loaded(¤t_config, &mut audiocpp_session)) - } - // Already resident, or an engine that holds no model. - _ => None, - }; + let outcome = + load_selected_engine(¤t_config, &mut inflect_model, &mut audiocpp_session); match outcome { Some(Err(e)) => debug!("TTS preload skipped: {e:#}"), Some(Ok(())) => debug!("TTS model pre-loaded and primed"), diff --git a/docs/api.md b/docs/api.md index 77d353b7..d70b711a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -794,6 +794,7 @@ interface FeaturesConfig { spoken_punctuation: boolean; auto_format_lists: boolean; snippets: Record; + early_command_detection: boolean; } interface OpenAiConfig { diff --git a/docs/configuration.md b/docs/configuration.md index f1581999..17a4dffc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,7 +70,8 @@ Full schema with defaults: "custom_vocabulary": [], "spoken_punctuation": true, "auto_format_lists": true, - "snippets": {} + "snippets": {}, + "early_command_detection": true }, "openai": { "enabled": false, @@ -237,6 +238,7 @@ The `.en` variants are English-only but slightly faster. `large-v3-turbo` is a d | `auto_format_lists` | bool | `true` | Detect "first/second/third" patterns and reformat as a numbered list | | `custom_vocabulary` | string[] | `[]` | Custom words; VoxCtrl uses fuzzy Levenshtein matching to correct near-matches post-transcription | | `snippets` | object | `{}` | Short code → expansion map | +| `early_command_detection` | bool | `true` | Transcribe the first 6 seconds of a recording while it is still going, so a voice command shows its overlay and starts loading the TTS model before the hotkey is released. Routing is still decided by the final transcript. Skipped for the remote backend and CPU-only medium/large Whisper models | Example with snippets: ```json diff --git a/docs/routing.md b/docs/routing.md index 64945727..02e55f33 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -23,8 +23,8 @@ Both files are hot-reloaded when changed on disk. Defined in `~/.config/voxctrl/targets.toml`. Each `[[target]]` block describes one destination. -**Saying one by name.** Start dictation and say *"VoxCtrl"*, then the command's -name, then the text: *"VoxCtrl notes, remember to call the plumber"* delivers +**Saying one by name.** Start dictation and say *"Hey Vox"* (or *"Vox Control"*), then the command's +name, then the text: *"Hey Vox, notes, remember to call the plumber"* (or *"Vox Control notes, remember to call the plumber"*) delivers *remember to call the plumber* to the command named **notes**, whatever the active hotkey was pointed at. Everything after the name is the payload. The full matching rules — conversational lead-ins, fuzzy name matching, the overlay — @@ -262,13 +262,13 @@ typing into the focused window — so voice command routing works the day anothe target is added, with no re-binding. **How Voice Command Routing Works:** -- **Trigger Keyword**: Listens for the `"VoxCtrl"` keyword (case-insensitive, supporting `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, and optional punctuation like `VoxCtrl:`). +- **Trigger Keyword**: Listens for the `"Hey Vox"` or `"Vox Control"` keywords (case-insensitive, supporting `Hey Vox`, `Hey, Vox`, `Hey-Vox`, `VoxCtrl`, `voxctrl`, `vox ctrl`, `vox-ctrl`, `vox control`, and trailing punctuation like `Hey Vox:`). - **Target Resolution**: Matches spoken target names against all configured target IDs and Labels (case-insensitively). Longest candidate target names take precedence (e.g. `"Personal Notes"` is matched before `"Notes"`). -- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"VoxCtrl send this to my notes. I love you."*, *"VoxCtrl add this to my personal notes, help"*, *"VoxCtrl put this into my Notes: hello"*, or *"VoxCtrl send us to my notes. I love you."*. +- **Conversational Lead-in Support**: Supports natural lead-in command phrases between the trigger keyword and the target name, such as *"Hey Vox, send this to my notes. I love you."*, *"Hey Vox, add this to my personal notes, help"*, *"Vox Control put this into my Notes: hello"*, or *"Hey, Vox, send us to my notes. I love you."*. - **Payload Extraction**: Strips transition punctuation (`.`, `:`, `,`, `;`) and connector words (`saying`, `that`, `with text`) and routes the remaining text payload to the matched target. - **Command UI Overlay**: When a voice command trigger is matched and executed, VoxCtrl automatically displays a temporary purple/indigo HUD overlay pill displaying the command name and text payload summary for a configurable duration (default: 3 seconds). -- **Auto-Trigger Detection**: Dictating with the `"VoxCtrl"` keyword (e.g. *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. -- **Fallback**: If no `"VoxCtrl"` keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). +- **Auto-Trigger Detection**: Dictating with the `"Hey Vox"` or `"Vox Control"` keyword (e.g. *"Hey Vox, notes Help me!"* or *"VoxCtrl notes Help me!"*) automatically activates voice command routing and surfaces the overlay pill regardless of whether the active hotkey target is set to `command` or `inject`. +- **Fallback**: If no trigger keyword is spoken or if no target matches, it falls back to direct text injection into the active application (identical to the `inject` target). --- diff --git a/docs/tts.md b/docs/tts.md index e87d3167..a9bec782 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -333,7 +333,7 @@ survives being idle: | Mode | Behaviour | |---|---| -| `"always_loaded"` (default) | The model is loaded on first use and stays resident for the life of the TTS worker. Fastest, highest memory. | +| `"always_loaded"` (default) | The model is loaded on first use (or at startup with `prewarm`) and stays resident for the life of the TTS worker. Fastest, highest memory. Recording against a speech target, or a spoken command to one, starts that first load early, as in `"on_demand"` mode. Switching into this mode with `prewarm` on loads the model right away. | | `"on_demand"` | The model is loaded when VoxCtrl knows it is needed, kept primed while it keeps being used, and dropped after `idle_unload_secs` (default 900 = 15 minutes) of inactivity. | In `"on_demand"` mode TTS itself stays enabled the whole time — only the weights come and go: diff --git a/package-lock.json b/package-lock.json index 28e217d5..69b490ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voxctrl", - "version": "0.6.5", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voxctrl", - "version": "0.6.5", + "version": "0.7.0", "dependencies": { "@tauri-apps/api": "^2", "@tauri-apps/plugin-dialog": "^2", diff --git a/package.json b/package.json index dffb5aa7..34484008 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "voxctrl", "private": true, - "version": "0.6.5", + "version": "0.7.0", "type": "module", "scripts": { "predev": "pkill -x voxctrl 2>/dev/null || true; fuser -k 5173/tcp 2>/dev/null || true", diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 22b20399..43d11c86 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -345,11 +345,25 @@ pub fn run() { let hotkey_health = Arc::new(voxctrl_hotkeys::ListenerHealth::default()); + // TTS initial worker + let initial_tts_handle = if cfg_data.tts.enabled { + Some(voxctrl_tts::TtsEngineWorker::start( + cfg_data.tts.clone(), + cfg_data.features.custom_vocabulary.clone(), + None, + None, + None, + )) + } else { + None + }; + let app_state = Arc::new(AppState { config: config.clone(), router: router.clone(), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(cfg_data.ui.show_overlay)), mcp_recording: Arc::new(AtomicBool::new(false)), @@ -373,7 +387,7 @@ pub fn run() { audio_tx: audio_tx.clone(), audio_wake: audio_wake_tx, inference_config_tx: inference_cfg_tx, - tts_handle: Arc::new(Mutex::new(None)), + tts_handle: Arc::new(Mutex::new(initial_tts_handle.clone())), active_fifos: Arc::new(Mutex::new(std::collections::HashSet::new())), stop_key_held: Arc::new(AtomicBool::new(false)), speaking_tx: Arc::new(std::sync::OnceLock::new()), @@ -424,26 +438,9 @@ pub fn run() { inference_cfg_rx, ); - // TTS initial worker - let _tts_handle = if cfg_data.tts.enabled { - Some(voxctrl_tts::TtsEngineWorker::start( - cfg_data.tts.clone(), - cfg_data.features.custom_vocabulary.clone(), - None, - None, - None, - )) - } else { - None - }; - let state_for_tts = app_state.clone(); - let tts_handle_clone = _tts_handle.clone(); + let tts_handle_clone = initial_tts_handle.clone(); tokio::spawn(async move { - { - let mut handle = state_for_tts.tts_handle.lock().await; - *handle = tts_handle_clone.clone(); - } if let Some(tts) = tts_handle_clone { state_for_tts.spawn_fifo_responders(tts).await; } diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 37486331..4d77ad2a 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -36,6 +36,8 @@ async fn process_remote_transcription( inference_ms: 0, language: String::new(), error: Some(format!("{e:#}")), + is_interim: false, + session_id: 0, }); return; } @@ -69,6 +71,8 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); return; } @@ -188,9 +192,21 @@ async fn process_remote_transcription( inference_ms: result.inference_ms, language: result.language, error: None, + is_interim: false, + session_id: 0, }); } +/// Interim passes start once this much audio exists (0.5 s at 16 kHz)... +const INTERIM_MIN_SAMPLES: usize = 8_000; +/// ...and re-run after at least this much more (0.3 s). +const INTERIM_STEP_SAMPLES: usize = 4_800; +/// Only the opening of a recording is transcribed early: the trigger and the +/// target name come first ("Hey Vox, add this to my notes"), and capping the +/// window keeps each pass short, so the final transcription never waits long +/// behind one and long dictations don't keep the model busy the whole time. +const INTERIM_MAX_SAMPLES: usize = 6 * 16_000; + pub fn spawn_audio_coordinator( state_for_audio: Arc, audio_rx: crossbeam_channel::Receiver, @@ -205,6 +221,10 @@ pub fn spawn_audio_coordinator( let mut binding_id = String::new(); let mut remote_session: Option = None; let mut is_remote_backend = false; + let mut interim_enabled = false; + let mut session_id: u64 = 0; + let mut last_interim_sample_count: usize = 0; + let mut last_interim_instant = std::time::Instant::now(); while let Ok(chunk) = audio_rx.recv() { let is_recording = state_for_audio.is_recording(); @@ -215,12 +235,32 @@ pub fn spawn_audio_coordinator( target_id = state_for_audio.active_target.blocking_lock().clone(); binding_id = state_for_audio.active_binding_id.blocking_lock().clone(); was_recording = true; + session_id = session_id.wrapping_add(1); + last_interim_sample_count = 0; + last_interim_instant = std::time::Instant::now(); + state_for_audio.set_interim_in_flight(false); let cfg = state_for_audio.config.blocking_lock().data.clone(); + // Early command detection only pays off when a command + // could match (some non-router target exists) and the + // local model is fast enough to transcribe mid-recording. + let heavy_model = cfg.engine.backend == voxctrl_config::BackendChoice::WhisperCpp + && voxctrl_inference::whisper_gpu_backend().is_none() + && (cfg.engine.whisper_cpp.model_size.starts_with("medium") + || cfg.engine.whisper_cpp.model_size.starts_with("large")); + let has_command_targets = state_for_audio + .targets + .blocking_lock() + .iter() + .any(|t| t.delivery != voxctrl_routing::DeliveryType::Command); + interim_enabled = cfg.features.early_command_detection + && cfg.engine.backend != voxctrl_config::BackendChoice::RemoteOpenAi + && !heavy_model + && has_command_targets; if cfg.engine.backend == voxctrl_config::BackendChoice::RemoteOpenAi { is_remote_backend = true; let mut merged_prompt = String::from( - "VoxCtrl is a voice control assistant application. VoxCtrl commands start with VoxCtrl. ", + "VoxCtrl is a voice control assistant application. VoxCtrl commands start with Vox Control or Hey Vox. ", ); if !cfg.features.custom_vocabulary.is_empty() { merged_prompt.push_str("Vocabulary: "); @@ -248,6 +288,27 @@ pub fn spawn_audio_coordinator( session.send_chunk(chunk.clone()); } accumulated_audio.extend(chunk); + + // Interim transcription of the opening of the recording, so a + // voice command is recognised while the user is still talking. + let window = accumulated_audio.len().min(INTERIM_MAX_SAMPLES); + if interim_enabled + && window >= INTERIM_MIN_SAMPLES + && window.saturating_sub(last_interim_sample_count) >= INTERIM_STEP_SAMPLES + && last_interim_instant.elapsed() >= std::time::Duration::from_millis(400) + && !state_for_audio.is_interim_in_flight() + { + state_for_audio.set_interim_in_flight(true); + last_interim_sample_count = window; + last_interim_instant = std::time::Instant::now(); + let _ = inference_tx.send(voxctrl_inference::InferenceRequest { + audio: accumulated_audio[..window].to_vec(), + target_id: target_id.clone(), + binding_id: Some(binding_id.clone()), + is_interim: true, + session_id, + }); + } } else { if was_recording { if is_remote_backend { @@ -280,6 +341,8 @@ pub fn spawn_audio_coordinator( audio: std::mem::take(&mut accumulated_audio), target_id: target_id.clone(), binding_id: Some(binding_id.clone()), + is_interim: false, + session_id, }; state_for_audio.set_processing(true); let _ = inference_tx.send(req); @@ -298,9 +361,60 @@ pub fn spawn_text_delivery_worker( rt_handle: tokio::runtime::Handle, ) { std::thread::spawn(move || { + // (session, target) already announced from an interim pass, so the + // overlay and TTS preload fire once per command, not once per pass. + let mut announced: Option<(u64, String)> = None; + while let Ok(output) = text_rx.recv() { + if output.is_interim { + state.set_interim_in_flight(false); + if output.error.is_some() || output.text.trim().is_empty() { + continue; + } + // Early command detection: surface the command overlay and + // start loading the TTS model while the user is still talking. + // This is only a head start — routing is decided solely by the + // final transcript below, which must carry the trigger itself. + let dir = voxctrl_routing::config_dir(); + let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); + if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { + let key = (output.session_id, parsed.matched_target_id); + if announced.as_ref() != Some(&key) { + let matched_target = targets.iter().find(|t| t.id == key.1); + let matched_label = matched_target + .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) + .unwrap_or_else(|| key.1.clone()); + tracing::info!("Early command detected during speech: '{matched_label}' (target: {})", key.1); + voxctrl_routing::targets::notify_command_trigger(&matched_label, &parsed.payload); + + let leads_to_speech = matched_target.is_some_and(|t| { + t.delivery == voxctrl_routing::DeliveryType::Speak + || t.response_pipe.as_deref().is_some_and(|p| !p.trim().is_empty()) + }); + if leads_to_speech { + let state_c = state.clone(); + rt_handle.spawn(async move { state_c.preload_tts().await }); + } + announced = Some(key); + } + } + continue; + } + + state.set_interim_in_flight(false); state.set_processing(false); + + // A command announced from an interim pass stands only if the + // final transcript confirms it; otherwise take its overlay down. + let announced_early = announced.take().is_some_and(|(session, _)| session == output.session_id); + let withdraw_early_command = || { + if announced_early { + voxctrl_routing::targets::notify_command_withdrawn(); + } + }; + if let Some(ref err) = output.error { + withdraw_early_command(); // Always surface transcription failures — without this a // fresh install with no Whisper model records audio and // then silently drops it, which reads as "hotkeys broken". @@ -309,6 +423,7 @@ pub fn spawn_text_delivery_worker( continue; } if output.text.trim().is_empty() { + withdraw_early_command(); continue; } @@ -332,7 +447,7 @@ pub fn spawn_text_delivery_worker( .unwrap_or(global_s1_mini_enabled); let targets = voxctrl_routing::load_targets(&dir).unwrap_or_default(); - let (target_id, text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { + let (target_id, raw_text) = if let Some(parsed) = voxctrl_routing::targets::parse_voice_command(&output.text, &targets) { let matched_id = parsed.matched_target_id.clone(); let payload = parsed.payload; let matched_label = targets @@ -341,20 +456,22 @@ pub fn spawn_text_delivery_worker( .map(|t| if t.label.is_empty() { t.id.clone() } else { t.label.clone() }) .unwrap_or_else(|| matched_id.clone()); voxctrl_routing::targets::notify_command_trigger(&matched_label, &payload); + (matched_id, payload) + } else { + withdraw_early_command(); + (output.target_id.clone(), output.text.clone()) + }; - let cleaned_payload = if s1_mini_enabled && !payload.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&payload, &s1_mini_styling, None) - } else { - payload - }; - (matched_id, cleaned_payload) + // Text bound for a Speak target is read aloud verbatim: S1-mini + // would add latency before the first word and may rewrite what + // the user meant to have spoken. + let is_speak_target = targets + .iter() + .any(|t| t.id == target_id && t.delivery == voxctrl_routing::DeliveryType::Speak); + let text = if !is_speak_target && s1_mini_enabled && !raw_text.trim().is_empty() { + voxctrl_inference::s1_mini::clean_dictation(&raw_text, &s1_mini_styling, None) } else { - let cleaned_text = if s1_mini_enabled && !output.text.trim().is_empty() { - voxctrl_inference::s1_mini::clean_dictation(&output.text, &s1_mini_styling, None) - } else { - output.text.clone() - }; - (output.target_id.clone(), cleaned_text) + raw_text }; if text.trim().is_empty() { diff --git a/src-tauri/src/services.rs b/src-tauri/src/services.rs index 1fe74eda..7eb51d61 100644 --- a/src-tauri/src/services.rs +++ b/src-tauri/src/services.rs @@ -224,8 +224,14 @@ pub fn setup_tts_and_fifos(app_handle: &tauri::AppHandle, state: Arc) pub fn register_speak_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); voxctrl_routing::targets::set_speak_callback(std::sync::Arc::new(move |text| { - let state = state.clone(); let text_str = text.to_string(); + if let Ok(handle) = state.tts_handle.try_lock() { + if let Some(ref tts) = *handle { + tts.speak(text_str); + return; + } + } + let state = state.clone(); tauri::async_runtime::spawn(async move { let handle = state.tts_handle.lock().await; if let Some(ref tts) = *handle { @@ -240,6 +246,14 @@ pub fn register_speak_target(app_handle: &tauri::AppHandle) { pub fn register_command_trigger_target(app_handle: &tauri::AppHandle) { let state = app_handle.state::>().inner().clone(); let app_handle_clone = app_handle.clone(); + + let withdraw_state = state.clone(); + let withdraw_handle = app_handle.clone(); + voxctrl_routing::targets::set_command_withdrawn_callback(std::sync::Arc::new(move || { + withdraw_state.clear_command_overlay(); + let _ = withdraw_handle.emit("command-withdrawn", ()); + })); + voxctrl_routing::targets::set_command_trigger_callback(std::sync::Arc::new( move |command_name, text_summary| { let (show_overlay, duration_secs) = if let Ok(cfg) = state.config.try_lock() { diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index 7e8f1637..51c384bc 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -14,6 +14,8 @@ pub struct AppState { pub recording: Arc, /// True while speech transcription/OpenAI post-processing is running pub processing: Arc, + /// True while an interim (mid-recording) transcription pass is running + pub interim_in_flight: Arc, /// True while TTS is playing back pub speaking: Arc, /// Live mirror of `ui.show_overlay` so the hot status-forwarding loops can @@ -150,6 +152,14 @@ impl AppState { self.processing.store(v, Ordering::SeqCst); } + pub fn is_interim_in_flight(&self) -> bool { + self.interim_in_flight.load(Ordering::SeqCst) + } + + pub fn set_interim_in_flight(&self, v: bool) { + self.interim_in_flight.store(v, Ordering::SeqCst); + } + pub fn is_audio_ready(&self) -> bool { self.audio_ready.load(Ordering::SeqCst) } @@ -242,6 +252,11 @@ impl AppState { *self.command_overlay_until.lock().unwrap() = Some(std::time::Instant::now() + duration); } + /// Stop showing the command-executed overlay pill now. + pub fn clear_command_overlay(&self) { + *self.command_overlay_until.lock().unwrap() = None; + } + /// Whether the command-executed overlay pill should still be showing. pub fn is_command_overlay_active(&self) -> bool { self.command_overlay_until @@ -285,17 +300,14 @@ impl AppState { /// Start loading the TTS model now, before anything asks it to speak. /// - /// Only meaningful in the on-demand memory mode, where the model is not - /// resident between uses: the load takes seconds, so kicking it off the - /// moment we know speech is coming (the user has just started dictating) - /// hides most of that behind the time they spend talking. In always-loaded - /// mode the model is already there and this is a no-op. + /// The load takes seconds, so kicking it off the moment we know speech is + /// coming (the user has just started dictating, or a spoken command was + /// recognised mid-speech) hides most of that behind the time they spend + /// talking. That holds in both memory modes: on-demand drops the model + /// when idle, and always-loaded without pre-warming loads it lazily. When + /// the model is already resident the worker treats this as a no-op. pub async fn preload_tts(&self) { - let unloads_when_idle = { - let cfg = self.config.lock().await; - cfg.data.tts.enabled && cfg.data.tts.unloads_when_idle() - }; - if !unloads_when_idle { + if !self.config.lock().await.data.tts.enabled { return; } if let Some(tts) = self.tts_handle.lock().await.as_ref() { diff --git a/src-tauri/src/tests.rs b/src-tauri/src/tests.rs index c5657570..340ad21c 100644 --- a/src-tauri/src/tests.rs +++ b/src-tauri/src/tests.rs @@ -66,6 +66,7 @@ fn make_test_state() -> AppState { router: Arc::new(OutputTargetRouter::new(Vec::new())), recording: Arc::new(AtomicBool::new(false)), processing: Arc::new(AtomicBool::new(false)), + interim_in_flight: Arc::new(AtomicBool::new(false)), speaking: Arc::new(AtomicBool::new(false)), overlay_enabled: Arc::new(AtomicBool::new(true)), mcp_recording: Arc::new(AtomicBool::new(false)), diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 060fb926..e097d7e0 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "VoxCtrl", - "version": "0.6.5", + "version": "0.7.0", "identifier": "ai.voxctrl.app", "build": { "frontendDist": "../dist", diff --git a/src/lib/Overlay/Overlay.svelte b/src/lib/Overlay/Overlay.svelte index 1dbaeb7e..438c1f08 100644 --- a/src/lib/Overlay/Overlay.svelte +++ b/src/lib/Overlay/Overlay.svelte @@ -37,6 +37,7 @@ let commandOverlayText = $state(""); let commandTimerId: any = null; let unlistenCommandExecuted: (() => void) | null = null; + let unlistenCommandWithdrawn: (() => void) | null = null; let unlistenOverlayStyleSelected: (() => void) | null = null; // Delay unmounting the visualizer when recording/speaking/command stops to allow CSS outro animation to finish @@ -46,7 +47,32 @@ ($mcpRecording && $config.mcp.visual_feedback) || (commandOverlayActive && $config.ui.show_command_overlay) ); + + // Recording wins: starting to dictate interrupts playback (begin_recording + // stops TTS), and until the speaking flag catches up the user must still + // see that they are being recorded rather than a stale SYSTEM RESPONDING. + let isSystemResponding = $derived( + $speaking && $config.tts.enabled && $config.tts.response_overlay && !$recording + ); let renderOverlay = $state(false); + + // The target visualizer belongs to recording. It gives way to the SYSTEM + // RESPONDING pill so the two never overlap, and is dropped once recording + // ends if the overlay stays up only for a command/MCP pill. When the whole + // overlay is fading out it stays mounted, so its outro transition can play. + let primaryVisualizerOn = $state(false); + $effect(() => { + if (!renderOverlay || isSystemResponding) { + primaryVisualizerOn = false; + } else if ($recording || $status.processing) { + primaryVisualizerOn = true; + } else if (isRecordingOrSpeaking) { + primaryVisualizerOn = false; + } + }); + let showPrimaryVisualizer = $derived( + $config.ui.overlay_style !== "none" && $config.ui.show_overlay && primaryVisualizerOn + ); let animateActive = $state(false); let timeoutId: any; let animateTimeoutId: any; @@ -276,11 +302,20 @@ unlistenCommandExecuted = unlisten; }); + // A command recognised mid-speech that the final transcript didn't confirm. + listen("command-withdrawn", () => { + if (commandTimerId) clearTimeout(commandTimerId); + commandOverlayActive = false; + }).then((unlisten) => { + unlistenCommandWithdrawn = unlisten; + }); + return () => { document.documentElement.classList.remove("overlay-window"); document.body.classList.remove("overlay-window"); if (unlistenAudioLevel) unlistenAudioLevel(); if (unlistenCommandExecuted) unlistenCommandExecuted(); + if (unlistenCommandWithdrawn) unlistenCommandWithdrawn(); if (unlistenOverlayStyleSelected) unlistenOverlayStyleSelected(); if (commandTimerId) clearTimeout(commandTimerId); if (animationFrameId !== null) cancelAnimationFrame(animationFrameId); @@ -311,58 +346,68 @@
{#if renderOverlay && visible} - {#if $config.ui.overlay_style === "waveform"} - - {:else if $config.ui.overlay_style === "pulse"} - - {:else if $config.ui.overlay_style === "blue_wave"} - - {:else if $config.ui.overlay_style === "mono_bars"} - - {:else if $config.ui.overlay_style === "spectrum"} - - {:else if $config.ui.overlay_style === "terminal"} - - {:else if $config.ui.overlay_style === "vinyl"} - - {:else if activeCustomOverlay} - {@html ``} -
- {@html processedHtml} -
- {:else if $config.ui.overlay_style !== "none"} - - {/if} - - {#if $speaking} -
- - {#each [0, 1, 2, 3, 4] as i} - - {/each} - - - SYSTEM RESPONDING - ▸ {targetLabel} - -
- {:else if commandOverlayActive} -
- ⚡ - - {commandOverlayName.toUpperCase()} - ▸ {commandOverlayText} - -
- {:else if $mcpRecording} -
- - - RECORDING - ▸ {targetLabel} - -
- {/if} +
+ {#if showPrimaryVisualizer} +
+ {#if $config.ui.overlay_style === "waveform"} + + {:else if $config.ui.overlay_style === "pulse"} + + {:else if $config.ui.overlay_style === "blue_wave"} + + {:else if $config.ui.overlay_style === "mono_bars"} + + {:else if $config.ui.overlay_style === "spectrum"} + + {:else if $config.ui.overlay_style === "terminal"} + + {:else if $config.ui.overlay_style === "vinyl"} + + {:else if activeCustomOverlay} + {@html ``} +
+ {@html processedHtml} +
+ {:else if $config.ui.overlay_style !== "none"} + + {/if} +
+ {/if} + + {#if commandOverlayActive && $config.ui.show_command_overlay && !isSystemResponding} +
+ ⚡ + + {commandOverlayName.toUpperCase()} + ▸ {commandOverlayText} + +
+ {/if} + + {#if isSystemResponding} +
+ + {#each [0, 1, 2, 3, 4] as i} + + {/each} + + + SYSTEM RESPONDING + ▸ {targetLabel} + +
+ {/if} + + {#if $mcpRecording && $config.mcp.visual_feedback} +
+ + + RECORDING + ▸ {targetLabel} + +
+ {/if} +
{/if}
@@ -395,8 +440,33 @@ overflow: hidden !important; } + .overlay-stack { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + max-width: 100%; + max-height: 100%; + pointer-events: none; + } + + .visualizer-container { + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + /* Built-in visualizers are at most ~152px, leaving room in the 444px + window for two pills below. A custom overlay may be taller: let this + box give way (trimmed, not scaled) so the pills are never pushed out + of the window. */ + min-height: 0; + flex-shrink: 1; + overflow: hidden; + } + .system-response-box { - position: absolute; + position: relative; z-index: 10; display: flex; align-items: center; @@ -406,10 +476,12 @@ border-radius: 23px; font-family: 'Outfit', 'Inter', system-ui, sans-serif; opacity: 0; - transform: translateY(20px); + transform: translateY(10px); transition: transform 0.34s cubic-bezier(0.175, 0.885, 0.32, 1.25), opacity 0.28s ease; + pointer-events: none; + flex-shrink: 0; } .system-response-box.on { diff --git a/src/lib/Settings/AudioTab.svelte b/src/lib/Settings/AudioTab.svelte index e7c943a5..cbec8441 100644 --- a/src/lib/Settings/AudioTab.svelte +++ b/src/lib/Settings/AudioTab.svelte @@ -139,7 +139,7 @@

- Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "VoxCtrl" — is not lost. + Only opens the microphone stream when recording is triggered, which saves power but cannot capture anything spoken while the microphone is still opening. Turn this off to keep the stream open: the last 300ms before you trigger recording is then kept and included, so a first word spoken early — the one carrying a voice command's "Hey Vox" or "Vox Control" — is not lost.

diff --git a/src/lib/Settings/CommandsTab.svelte b/src/lib/Settings/CommandsTab.svelte index 6540f745..26135d16 100644 --- a/src/lib/Settings/CommandsTab.svelte +++ b/src/lib/Settings/CommandsTab.svelte @@ -139,11 +139,11 @@

Saying a command by name. Start dictation and say - “VoxCtrl”, then the command's name, then what you want to send — for - example “VoxCtrl notes, remember to call the plumber” routes + “Hey Vox,” (or “VoxCtrl” / “Vox Control”), then the command's name, then what you want to send — for + example “Hey Vox, notes, remember to call the plumber” or “VoxCtrl notes, remember to call the plumber” routes remember to call the plumber to the command named notes. Everything after the name is the text. Natural phrasing works too - (“VoxCtrl, add this to my notes: …”). Say nothing of the sort and + (“Hey Vox, add this to my notes: …”). Say nothing of the sort and dictation goes wherever your hotkey already points.

diff --git a/src/lib/Settings/FeaturesTab.svelte b/src/lib/Settings/FeaturesTab.svelte index 2c79a471..47384ccc 100644 --- a/src/lib/Settings/FeaturesTab.svelte +++ b/src/lib/Settings/FeaturesTab.svelte @@ -259,6 +259,20 @@ +
+

Voice Commands

+ +

+ Transcribes the first few seconds of each recording early, so a command like + “Hey Vox, say …” shows its overlay and starts loading the voice before you let go + of the hotkey. Where the text goes is still decided by the full transcript. Turn + this off to save the extra transcription work. +

+
+

Custom Dictionary

Provide a comma-separated list of words (e.g. names or jargon like "Waylin, Rufer, Enola, Kenz") that are hard to spell. The transcription process will correct these in the final text.

diff --git a/src/lib/Settings/TargetEditorModal.svelte b/src/lib/Settings/TargetEditorModal.svelte index 21fe2b7a..30ba9572 100644 --- a/src/lib/Settings/TargetEditorModal.svelte +++ b/src/lib/Settings/TargetEditorModal.svelte @@ -320,8 +320,8 @@

Names this target in the app, and is what you say to send dictation here through a - Voice Command Router target: “VoxCtrl, {editingTarget.label || "Obsidian Notes"}, - buy milk tomorrow”. The Target ID works as a spoken name too, so pick something + Voice Command Router target: “Hey Vox, {editingTarget.label || "Obsidian Notes"}, + buy milk tomorrow” (or “VoxCtrl, {editingTarget.label || "Obsidian Notes"}...”). The Target ID works as a spoken name too, so pick something easy to say and easy for speech recognition to catch.

@@ -330,7 +330,7 @@ @@ -352,7 +352,7 @@
Voice Command Router Settings

- Types dictated text into your active application by default. If your dictation contains VoxCtrl <target_name> <text> (for example, "VoxCtrl Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead. + Types dictated text into your active application by default. If your dictation contains Hey Vox <target_name> <text> or Vox Control <target_name> <text> (for example, "Hey Vox, Notes Hello world" or "Vox Control Notes Hello world"), VoxCtrl dynamically reroutes the text to that target instead.

{/if} diff --git a/src/stores/config.ts b/src/stores/config.ts index 5346f8de..38428dab 100644 --- a/src/stores/config.ts +++ b/src/stores/config.ts @@ -78,6 +78,7 @@ export interface FeaturesConfig { spoken_punctuation: boolean; auto_format_lists: boolean; snippets: Record; + early_command_detection: boolean; } export interface OpenAiConfig { @@ -199,10 +200,11 @@ const defaultConfig: AppConfig = { }, features: { remove_fillers: true, - custom_vocabulary: ["VoxCtrl"], + custom_vocabulary: ["VoxCtrl", "Hey Vox"], spoken_punctuation: true, auto_format_lists: true, snippets: {}, + early_command_detection: true, }, openai: { enabled: false, diff --git a/tests/svelte/Overlay.test.ts b/tests/svelte/Overlay.test.ts index b1a935b8..230cfcc3 100644 --- a/tests/svelte/Overlay.test.ts +++ b/tests/svelte/Overlay.test.ts @@ -1,18 +1,22 @@ -import { describe, test, expect, vi, beforeEach } from "vitest"; +import { describe, test, expect, vi, beforeEach, afterEach } from "vitest"; +import { get } from "svelte/store"; import { render } from "@testing-library/svelte"; // Mock tauri IPC used by the overlay components and the status store vi.mock("@tauri-apps/api/core", () => ({ - invoke: vi.fn(async () => ({ - recording: false, - processing: false, - speaking: false, - mcp_recording: false, - audio_ready: true, - word_count: 0, - active_target_id: "default", - active_target_label: "Focused Window", - })), + invoke: vi.fn(async (cmd: string) => { + // `get_config` has no backend here: failing it leaves the config store on + // its defaults. + if (cmd === "get_config") throw new Error("no backend in tests"); + // The status store falls back to polling `get_status` every second when + // no status-tick events arrive, which in tests is always. Answer with + // whatever status the test has set, so a poll landing mid-test changes + // nothing — a fixed idle reply here reset `speaking`/`recording` under + // the tests whenever a run was slow enough for the poll to fire. + const { status } = await import("../../src/stores/status"); + const { get } = await import("svelte/store"); + return get(status); + }), })); vi.mock("@tauri-apps/api/event", () => ({ @@ -29,6 +33,8 @@ vi.stubGlobal( vi.stubGlobal("cancelAnimationFrame", (id: number) => clearTimeout(id)); import { status, type AppStatus } from "../../src/stores/status"; +import { config } from "../../src/stores/config"; +import Overlay from "../../src/lib/Overlay/Overlay.svelte"; import Waveform from "../../src/lib/Overlay/Waveform.svelte"; import Pulse from "../../src/lib/Overlay/Pulse.svelte"; import BlueWave from "../../src/lib/Overlay/BlueWave.svelte"; @@ -189,3 +195,76 @@ describe("VoiceCard.svelte (membership card)", () => { expect(procContainer.querySelector(".field-value")?.textContent).toContain("Reading the card…"); }); }); + +describe("Overlay.svelte (root layout)", () => { + // These tests reconfigure the shared config store; put it back afterwards + // so later suites see the defaults. + let savedConfig: ReturnType>; + beforeEach(() => { + savedConfig = get(config); + config.update((c) => ({ + ...c, + ui: { ...c.ui, show_overlay: true, overlay_style: "waveform" }, + tts: { ...c.tts, enabled: true, response_overlay: true }, + })); + }); + afterEach(() => { + config.set(savedConfig); + }); + + // The overlay mounts its content after a 25ms repaint delay (see the + // overlay_style effect), so wait past that rather than a single tick. + const settle = (ms = 50) => new Promise((r) => setTimeout(r, ms)); + + test("hides target visualizer and displays SYSTEM RESPONDING when speaking", async () => { + setStatus({ speaking: true, recording: false, active_target_label: "Kitty Terminal" }); + + const { container } = render(Overlay); + await settle(); + + expect(container.textContent).toContain("SYSTEM RESPONDING"); + expect(container.textContent).toContain("Kitty Terminal"); + // The target visualizer (Waveform) must not be rendered alongside it + expect(container.querySelector(".scope")).toBeNull(); + expect(container.textContent).not.toContain("WAVEFORM // OSC-01"); + }); + + test("shows target visualizer when recording", async () => { + setStatus({ speaking: false, recording: true, active_target_label: "Code Editor" }); + + const { container } = render(Overlay); + await settle(); + + expect(container.querySelector(".scope")).not.toBeNull(); + expect(container.textContent).toContain("WAVEFORM // OSC-01"); + expect(container.textContent).not.toContain("SYSTEM RESPONDING"); + }); + + test("recording over a spoken reply shows the visualizer, not SYSTEM RESPONDING", async () => { + // begin_recording stops playback, but the speaking flag can lag behind it. + setStatus({ speaking: true, recording: true }); + + const { container } = render(Overlay); + await settle(); + + expect(container.querySelector(".scope")).not.toBeNull(); + expect(container.textContent).not.toContain("SYSTEM RESPONDING"); + }); + + test("keeps the visualizer mounted while the overlay fades out after recording", async () => { + setStatus({ recording: true }); + const { container } = render(Overlay); + await settle(); + expect(container.querySelector(".scope")).not.toBeNull(); + + setStatus({ recording: false }); + await settle(); + // Still inside the 450ms outro window: the visualizer animates out rather + // than vanishing the instant recording stops... + expect(container.querySelector(".scope")).not.toBeNull(); + + // ...and is gone once the outro has finished. + await settle(500); + expect(container.querySelector(".scope")).toBeNull(); + }); +});