diff --git a/justfile b/justfile index 350803bac..ed303c451 100644 --- a/justfile +++ b/justfile @@ -228,6 +228,8 @@ test-siri-tts-stream-regression: [unix] _tauri-test-unix: + # rust-cache can restore Sherpa's generated cache directory without its native libraries. + if [ "$(uname -s)" = "Linux" ]; then rm -rf src-tauri/target/sherpa-onnx-prebuilt; fi just _tauri-cargo-unix test -p tauri-plugin-berdctl --features server just _tauri-cargo-unix test -p berdctl just _tauri-cargo-unix test --lib telemetry diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a44d22ff3..539f0d573 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -46,6 +46,7 @@ dependencies = [ "reqwest 0.13.4", "rodio", "rubato", + "rustls", "semver", "serde", "serde_json", @@ -73,6 +74,7 @@ dependencies = [ "tempfile", "time", "tokio", + "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -1145,7 +1147,7 @@ dependencies = [ "bitflags 2.13.1", "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1428,6 +1430,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -1439,6 +1447,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "dbus", + "openssl", + "zeroize", +] + [[package]] name = "defmt" version = "1.1.1" @@ -2027,6 +2046,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2034,7 +2062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2048,6 +2076,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -3214,9 +3248,13 @@ version = "3.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" dependencies = [ + "byteorder", + "dbus-secret-service", "log", + "openssl", "security-framework 2.11.1", "security-framework 3.7.0", + "windows-sys 0.60.2", "zeroize", ] @@ -3271,6 +3309,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" dependencies = [ + "cc", "pkg-config", ] @@ -4175,12 +4214,59 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -7147,6 +7233,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -7407,6 +7509,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.20", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -7616,6 +7737,12 @@ dependencies = [ "url", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -8998,6 +9125,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index d32e2a577..64cf1a73b 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -47,7 +47,6 @@ hex = "0.4" ignore = "0.4.25" fern = "0.7" infer = "0.19.0" -keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } libc = "0.2" log = "0.4.29" mime_guess = "2" @@ -101,11 +100,14 @@ tauri-plugin-updater = "2" tauri-plugin-window-state = "2" time = { version = "0.3", features = ["formatting"] } tokio = { version = "1.50.0", features = ["full"] } +tokio-tungstenite = { version = "0.27", features = ["rustls-tls-webpki-roots"] } +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } url = "2" uuid = { version = "1", features = ["v4", "serde"] } zip = { version = "2", default-features = false, features = ["deflate"] } [target.'cfg(windows)'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["windows-native"] } windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_Globalization", @@ -118,6 +120,7 @@ windows-sys = { version = "0.59", features = [ [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.6" coreaudio-rs = "0.14.2" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSRunningApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } @@ -126,6 +129,9 @@ objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", " objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } +[target.'cfg(target_os = "linux")'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["sync-secret-service", "vendored"] } + [features] default = [] diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 2337baaf5..de66441de 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -41,6 +41,10 @@ const SILENCE_FLUSH_FRAMES: usize = 75; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS: u64 = 5; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT: Duration = Duration::from_secs(FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS); +const OPENAI_NETWORK_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +const OPENAI_FINAL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); +const OPENAI_STARTUP_TIMEOUT: Duration = Duration::from_secs(12); +const OPENAI_PRE_ROLL_CHUNKS: usize = 15; const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = mac_speech::RECOGNITION_FINISH_TIMEOUT_SECONDS + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS + 1; @@ -49,6 +53,7 @@ const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = pub enum VoiceInputBackend { Parakeet, Macos, + Openai, } fn active_vad_threshold_for_speech( @@ -603,10 +608,13 @@ impl NativeVoiceState { } // Native input mute is authoritative when installed so a hardware // unmute cannot be masked by a stale renderer fallback latch. - self.microphone_muted.store( - software_microphone_mute(native_microphone_mute_control, muted), - Ordering::SeqCst, - ); + let software_muted = software_microphone_mute(native_microphone_mute_control, muted); + let previous_software_muted = + self.microphone_muted.swap(software_muted, Ordering::SeqCst); + if !native_microphone_mute_control && previous_software_muted != software_muted { + self.input_muted.store(software_muted, Ordering::Release); + self.input_mute_epoch.fetch_add(1, Ordering::AcqRel); + } owner_window_label }; Ok(Some(owner_window_label)) @@ -775,6 +783,12 @@ struct AudioBatch { mute_epoch: u64, } +type OpenAiSttPipelineStart = ( + SttPipeline, + tokio_mpsc::Receiver, + mpsc::Receiver>, +); + impl SttPipeline { fn new_parakeet( model_dir: PathBuf, @@ -827,6 +841,64 @@ impl SttPipeline { )) } + fn new_openai( + api_key: String, + input_muted: Arc, + input_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, + ) -> Result { + let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); + let (event_tx, event_rx) = tokio_mpsc::channel(64); + let (startup_tx, startup_rx) = mpsc::sync_channel(0); + let endpoint = super::openai_audio::realtime_endpoint()?; + let model = super::openai_audio::transcription_model(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); + let worker_shutdown = Arc::clone(&shutdown); + let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); + let worker_input_muted = Arc::clone(&input_muted); + let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); + let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); + let thread = thread::Builder::new() + .name("berd-openai-stt".into()) + .spawn(move || { + openai_stt_worker( + api_key, + endpoint, + model, + audio_rx, + event_tx, + worker_shutdown, + worker_discard_on_shutdown, + worker_input_muted, + worker_input_mute_epoch, + worker_shutdown_mute_epoch, + assistant_speaking, + assistant_vad_threshold, + speech_vad_threshold, + startup_tx, + ) + }) + .map_err(|error| format!("start OpenAI speech recognition: {error}"))?; + Ok(( + Self { + audio_tx, + audio_seen: AtomicBool::new(false), + shutdown, + discard_on_shutdown, + input_muted, + input_mute_epoch, + shutdown_mute_epoch, + thread: Some(thread), + }, + event_rx, + startup_rx, + )) + } + fn new_macos( input_muted: Arc, input_mute_epoch: Arc, @@ -1073,7 +1145,10 @@ where } async fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { - let parakeet_available = parakeet_model_dir(app).is_ok(); + let parakeet_available = parakeet_model_dir(app).is_ok() + || app + .state::() + .is_configured(); #[cfg(target_os = "macos")] let macos_status = || async { mac_speech::status_async() @@ -1231,6 +1306,14 @@ pub async fn start_native_voice_conversation( "Download the macOS speech recognition model before starting a call.".to_string(), ); } + if input_backend == VoiceInputBackend::Openai { + let deadline = tokio::time::Instant::now() + Duration::from_secs(1); + while !webview_window.is_focused().unwrap_or(false) + && tokio::time::Instant::now() < deadline + { + tokio::time::sleep(Duration::from_millis(25)).await; + } + } let window_label = webview_window.label().to_string(); let owner_id = native_owner_id(&session_id); let lifecycle_guard = state @@ -1246,6 +1329,11 @@ pub async fn start_native_voice_conversation( ) }) .await?; + let openai_api_key = if input_backend == VoiceInputBackend::Openai { + Some(super::openai_audio::stt_api_key()?) + } else { + None + }; let mut microphone_claimed = capture.claim_microphone( window_label.clone(), renderer_id.clone(), @@ -1271,6 +1359,40 @@ pub async fn start_native_voice_conversation( Arc::clone(&state.assistant_vad_threshold), speech_vad_threshold, ), + VoiceInputBackend::Openai => { + let startup = SttPipeline::new_openai( + openai_api_key.expect("OpenAI key resolved for OpenAI input"), + Arc::clone(&state.input_muted), + Arc::clone(&state.input_mute_epoch), + Arc::clone(&state.assistant_speaking), + Arc::clone(&state.assistant_vad_threshold), + speech_vad_threshold, + ); + match startup { + Err(error) => Err(error), + Ok((pipeline, events, startup)) => { + let startup_result = tokio::task::spawn_blocking(move || { + startup.recv_timeout(OPENAI_STARTUP_TIMEOUT) + }) + .await; + match startup_result { + Ok(Ok(Ok(()))) => Ok((pipeline, events)), + Ok(Ok(Err(error))) => { + drop(pipeline); + Err(error) + } + Ok(Err(error)) => { + drop(pipeline); + Err(format!("OpenAI transcription did not start: {error}")) + } + Err(error) => { + drop(pipeline); + Err(format!("wait for OpenAI transcription startup: {error}")) + } + } + } + } + } }; let (pipeline, mut events) = match pipeline { Ok(result) => result, @@ -1795,6 +1917,20 @@ impl NativeVoiceState { self.stop_active_inner(app, capture, None).await.map(|_| ()) } + pub(crate) async fn stop_active_then( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + action: F, + ) -> Result + where + F: FnOnce() -> Result, + { + let _stop_guard = self.stop_serial.lock().await; + self.stop_active_inner_locked(app, capture, None).await?; + action() + } + pub async fn stop_active_for_lifecycle( &self, app: &AppHandle, @@ -2473,6 +2609,658 @@ fn macos_stt_worker( let _ = forward_macos_events(&mut recognition_events, &event_tx, Some(delivery_deadline)); } +#[derive(Debug, PartialEq, Eq)] +struct OpenAiCommittedTurn { + item_id: String, + mute_epoch: u64, +} + +fn block_on_openai_operation( + runtime: &tokio::runtime::Runtime, + shutdown: &AtomicBool, + future: F, + action: &str, +) -> Result, String> +where + F: std::future::Future>, + E: std::fmt::Display, +{ + runtime.block_on(async { + let wait_for_shutdown = async { + while !shutdown.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }; + tokio::select! { + result = tokio::time::timeout(OPENAI_NETWORK_OPERATION_TIMEOUT, future) => { + match result { + Ok(Ok(value)) => Ok(Some(value)), + Ok(Err(error)) => Err(format!("{action}: {error}")), + Err(_) => Err(format!("{action}: operation timed out")), + } + } + () = wait_for_shutdown => Ok(None), + } + }) +} + +async fn wait_for_openai_transcription_ready(socket: &mut S) -> Result<(), String> +where + S: futures_util::Stream> + Unpin, + E: std::fmt::Display, +{ + use futures_util::StreamExt; + use tokio_tungstenite::tungstenite::Message; + + loop { + match socket.next().await { + Some(Ok(Message::Text(text))) => { + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + match value.get("type").and_then(|value| value.as_str()) { + Some("session.updated") => return Ok(()), + Some("error") => { + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI rejected the transcription session configuration.") + .to_string()); + } + _ => {} + } + } + Some(Ok(Message::Close(_))) | None => { + return Err( + "OpenAI realtime transcription disconnected before it was ready.".to_string(), + ); + } + Some(Ok(_)) => {} + Some(Err(error)) => { + return Err(format!( + "OpenAI realtime transcription failed before it was ready: {error}" + )); + } + } + } +} + +fn push_openai_pre_roll(pre_roll: &mut VecDeque>, pcm: Vec) { + pre_roll.push_back(pcm); + while pre_roll.len() > OPENAI_PRE_ROLL_CHUNKS { + pre_roll.pop_front(); + } +} + +fn openai_turn_reached_limit(samples_16k: usize) -> bool { + samples_16k >= MAX_SPEECH_SAMPLES +} + +fn record_openai_transcription_event( + value: &serde_json::Value, + current_mute_epoch: u64, + pending_commit_epochs: &mut VecDeque, + committed: &mut VecDeque, + completed: &mut HashMap, +) -> Result, String> { + let mut newly_committed = None; + match value.get("type").and_then(|value| value.as_str()) { + Some("input_audio_buffer.committed") => { + if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { + let turn = OpenAiCommittedTurn { + item_id: item_id.to_string(), + mute_epoch: pending_commit_epochs + .pop_front() + .unwrap_or(current_mute_epoch), + }; + if turn.mute_epoch == current_mute_epoch { + committed.push_back(OpenAiCommittedTurn { + item_id: turn.item_id.clone(), + mute_epoch: turn.mute_epoch, + }); + } + newly_committed = Some(turn); + } + } + Some("conversation.item.input_audio_transcription.completed") => { + if let (Some(item_id), Some(transcript)) = ( + value.get("item_id").and_then(|value| value.as_str()), + value.get("transcript").and_then(|value| value.as_str()), + ) { + if committed + .iter() + .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_mute_epoch) + { + completed.insert(item_id.to_string(), transcript.trim().to_string()); + } + } + } + Some("conversation.item.input_audio_transcription.failed") => { + if value + .get("item_id") + .and_then(|value| value.as_str()) + .is_some_and(|item_id| { + !committed.iter().any(|turn| { + turn.item_id == item_id && turn.mute_epoch == current_mute_epoch + }) + }) + { + return Ok(None); + } + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") + .to_string()); + } + Some("error") => { + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") + .to_string()); + } + _ => {} + } + Ok(newly_committed) +} + +fn deliver_completed_openai_turns( + committed: &mut VecDeque, + completed: &mut HashMap, + event_tx: &tokio_mpsc::Sender, + current_mute_epoch: u64, + final_item_id: Option<&str>, + final_delivery: &mut Option>, +) { + while committed + .front() + .is_some_and(|turn| completed.contains_key(&turn.item_id)) + { + let turn = committed.pop_front().expect("checked front"); + let text = completed.remove(&turn.item_id).unwrap_or_default(); + if turn.mute_epoch != current_mute_epoch { + continue; + } + let delivered = (Some(turn.item_id.as_str()) == final_item_id) + .then(|| final_delivery.take()) + .flatten(); + deliver_recognition_result(text, event_tx, delivered); + } +} + +#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. +fn openai_stt_worker( + key: String, + endpoint: String, + model: String, + audio_rx: Receiver, + event_tx: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + input_muted: Arc, + input_mute_epoch: Arc, + shutdown_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, + startup_tx: SyncSender>, +) { + use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; + use futures_util::{SinkExt, StreamExt}; + use rubato::{Fft, FixedSync, Resampler}; + use tokio_tungstenite::tungstenite::{client::IntoClientRequest, Message}; + + macro_rules! fail_openai_startup { + ($error:expr) => {{ + let error = $error; + let _ = startup_tx.send(Err(error.clone())); + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + }}; + } + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + fail_openai_startup!(format!( + "Could not initialize OpenAI realtime transcription: {error}" + )); + } + }; + if let Err(existing) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + // Another dependency may have installed the same process-wide provider first. + drop(existing); + } + let mut request = match endpoint.into_client_request() { + Ok(request) => request, + Err(error) => { + fail_openai_startup!(format!("prepare OpenAI realtime connection: {error}")); + } + }; + let authorization = match format!("Bearer {key}").parse() { + Ok(authorization) => authorization, + Err(_) => { + fail_openai_startup!("OpenAI API key is not a valid header value".to_string()); + } + }; + request.headers_mut().insert("Authorization", authorization); + let connection = block_on_openai_operation( + &runtime, + &shutdown, + tokio_tungstenite::connect_async(request), + "connect to OpenAI realtime transcription", + ); + let mut socket = match connection { + Ok(Some((socket, _))) => socket, + Ok(None) => return, + Err(error) => { + fail_openai_startup!(error); + } + }; + + macro_rules! send_openai_startup_message { + ($message:expr, $action:literal) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => return, + Err(error) => { + fail_openai_startup!(error); + } + } + }; + } + let session_update = serde_json::json!({ + "type": "session.update", + "session": { + "type": "transcription", + "audio": { "input": { + "format": { "type": "audio/pcm", "rate": 24000 }, + "transcription": { "model": model, "delay": "low" }, + "turn_detection": null + }} + } + }); + send_openai_startup_message!( + Message::Text(session_update.to_string().into()), + "configure OpenAI realtime transcription" + ); + + let mut resampler = match Fft::::new(48_000, 24_000, 960, 2, 1, FixedSync::Input) { + Ok(resampler) => resampler, + Err(error) => { + fail_openai_startup!(format!( + "Could not initialize OpenAI audio resampling: {error}" + )); + } + }; + let readiness = block_on_openai_operation( + &runtime, + &shutdown, + wait_for_openai_transcription_ready(&mut socket), + "wait for OpenAI realtime transcription readiness", + ); + match readiness { + Ok(Some(())) => {} + Ok(None) => return, + Err(error) => fail_openai_startup!(error), + } + if startup_tx.send(Ok(())).is_err() { + return; + } + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut vad_16k = Vec::new(); + let mut silence_frames = 0_usize; + let mut in_speech = false; + let mut turn_has_audio = false; + let mut turn_samples_16k = 0_usize; + let mut pre_roll = VecDeque::>::new(); + let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + let mut pending_commit_epochs = VecDeque::::new(); + let mut committed_turns = VecDeque::::new(); + let mut completed_turns = HashMap::::new(); + + macro_rules! send_openai_stream_message { + ($message:expr, $action:literal, $on_shutdown:block) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => $on_shutdown + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + } + }; + } + + 'worker: loop { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + observed_mute_epoch = current_mute_epoch; + input_48k.clear(); + vad_16k.clear(); + silence_frames = 0; + turn_has_audio = false; + turn_samples_16k = 0; + pre_roll.clear(); + committed_turns.clear(); + completed_turns.clear(); + if std::mem::take(&mut in_speech) { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.clear"}) + .to_string() + .into() + ), + "clear muted OpenAI transcription audio", + { break 'worker } + ); + } + while let Some(event) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(1), socket.next()) + .await + .ok() + .flatten() + }) { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break 'worker; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + let message = match event { + Ok(Message::Text(text)) => text, + Ok(Message::Close(_)) => { + let _ = event_tx.blocking_send(SttMessage::Failed( + "OpenAI realtime transcription disconnected.".to_string(), + )); + return; + } + Ok(_) => continue, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "OpenAI realtime transcription failed: {error}" + ))); + return; + } + }; + let Ok(value) = serde_json::from_str::(&message) else { + continue; + }; + let recorded_turn = match record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + Ok(turn) => turn, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + if let Some(turn) = recorded_turn.filter(|turn| turn.mute_epoch != observed_mute_epoch) + { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "conversation.item.delete", + "item_id": turn.item_id, + }) + .to_string() + .into() + ), + "discard muted OpenAI transcription turn", + { break 'worker } + ); + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + input_mute_epoch.load(Ordering::Acquire), + None, + &mut None, + ); + } + + let batch = match audio_rx.recv_timeout(Duration::from_millis(20)) { + Ok(batch) => Some(batch), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + // Stop consuming queued microphone batches as soon as shutdown begins. + // The bounded finalization below commits only audio already accepted by + // this worker, so network backpressure cannot extend the owner timeout. + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + if input_muted.load(Ordering::Acquire) { + continue; + } + let Some(batch) = batch else { continue }; + if batch.mute_epoch != observed_mute_epoch { + continue; + } + input_48k.extend( + batch + .bytes + .chunks_exact(4) + .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), + ); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + let pcm_24k = resample(&mut resampler, &chunk); + let pcm_bytes: Vec = pcm_24k + .iter() + .flat_map(|sample| { + ((sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16).to_le_bytes() + }) + .collect(); + // Earshot requires 16 kHz; use every third 48 kHz source sample for activity only. + vad_16k.extend(chunk.iter().step_by(3).copied()); + let mut speech_started = false; + let mut should_commit = false; + while vad_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = vad_16k.drain(..VAD_FRAME_SAMPLES).collect(); + let threshold = active_vad_threshold_for_speech( + &assistant_speaking, + &assistant_vad_threshold, + speech_vad_threshold, + ); + if vad.predict_f32(&frame) > threshold { + silence_frames = 0; + if !in_speech { + in_speech = true; + speech_started = true; + let _ = event_tx.blocking_send(SttMessage::Speaking(true)); + } + } else if in_speech { + silence_frames += 1; + if silence_frames >= SILENCE_FLUSH_FRAMES { + should_commit = true; + silence_frames = 0; + in_speech = false; + } + } + } + + if speech_started { + while let Some(pre_roll_bytes) = pre_roll.pop_front() { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pre_roll_bytes) + }) + .to_string() + .into() + ), + "stream OpenAI transcription pre-roll", + { break 'worker } + ); + } + } + + if speech_started || in_speech || should_commit { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pcm_bytes) + }) + .to_string() + .into() + ), + "stream audio to OpenAI realtime transcription", + { break 'worker } + ); + turn_has_audio = true; + turn_samples_16k = turn_samples_16k.saturating_add(chunk.len() / 3); + } else { + push_openai_pre_roll(&mut pre_roll, pcm_bytes); + } + + if should_commit || openai_turn_reached_limit(turn_samples_16k) { + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.commit"}) + .to_string() + .into() + ), + "commit OpenAI transcription turn", + { break 'worker } + ); + pending_commit_epochs.push_back(observed_mute_epoch); + turn_has_audio = false; + turn_samples_16k = 0; + silence_frames = 0; + if std::mem::take(&mut in_speech) || should_commit { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + } + } + } + if discard_on_shutdown.load(Ordering::Acquire) { + return; + } + let mut final_item_id = None::; + let mut final_delivery = None::>; + let mut final_delivered = None::>; + if turn_has_audio { + let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); + let commit = serde_json::json!({"type":"input_audio_buffer.commit"}); + let final_write = runtime.block_on(tokio::time::timeout( + OPENAI_FINAL_WRITE_TIMEOUT, + socket.send(Message::Text(commit.to_string().into())), + )); + if matches!(final_write, Ok(Ok(()))) { + pending_commit_epochs.push_back(observed_mute_epoch); + final_delivery = Some(delivered_tx); + final_delivered = Some(delivered_rx); + } + } + if final_delivered.is_none() + && (!committed_turns.is_empty() || !pending_commit_epochs.is_empty()) + { + let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); + final_item_id = pending_commit_epochs + .is_empty() + .then(|| committed_turns.back().map(|turn| turn.item_id.clone())) + .flatten(); + final_delivery = Some(delivered_tx); + final_delivered = Some(delivered_rx); + } + + // Keep receiving until the shutdown commit itself completes. Earlier turns + // are delivered from the queue first, and already committed transcripts are + // drained even when there was no partial turn to commit at shutdown. + let deadline = std::time::Instant::now() + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT; + while std::time::Instant::now() < deadline { + if final_delivered + .as_ref() + .is_some_and(|receiver| receiver.try_recv().is_ok()) + || (final_delivered.is_none() + && committed_turns.is_empty() + && pending_commit_epochs.is_empty()) + { + break; + } + let Some(message) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(50), socket.next()) + .await + .ok() + .flatten() + }) else { + continue; + }; + let Ok(Message::Text(text)) = message else { + continue; + }; + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + let event_type = value.get("type").and_then(|value| value.as_str()); + match record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + Ok(Some(turn)) + if turn.mute_epoch == observed_mute_epoch + && pending_commit_epochs.is_empty() + && final_delivered.is_some() => + { + final_item_id = Some(turn.item_id); + } + Ok(_) => {} + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + break; + } + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + observed_mute_epoch, + final_item_id.as_deref(), + &mut final_delivery, + ); + if event_type == Some("conversation.item.input_audio_transcription.completed") + && final_item_id.as_deref() == value.get("item_id").and_then(|value| value.as_str()) + && final_delivery.is_none() + { + break; + } + } + // Do not await the peer's WebSocket close handshake here. OpenAI may leave + // it pending beyond the bounded voice-stop window; dropping the socket + // closes the connection after final transcript delivery has been drained. + drop(socket); +} + #[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. fn stt_worker( model_dir: PathBuf, @@ -3308,6 +4096,32 @@ mod tests { assert!(state.microphone_is_muted()); } + #[test] + fn software_microphone_mute_advances_the_audio_epoch() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.native_microphone_mute_control = false; + } + + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 0); + state + .set_microphone_muted_target("main", "session-1", 4, true) + .expect("mute"); + assert!(state.input_muted.load(Ordering::Acquire)); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 1); + state + .set_microphone_muted_target("main", "session-1", 4, false) + .expect("unmute"); + assert!(!state.input_muted.load(Ordering::Acquire)); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 2); + } + #[test] fn owner_stop_authorization_is_lifecycle_bound() { let state = NativeVoiceState::default(); @@ -3971,6 +4785,393 @@ mod tests { assert!(delivered_rx.try_recv().is_ok()); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn openai_startup_waits_for_session_updated_before_consuming_audio() { + use futures_util::{SinkExt, StreamExt}; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + use tokio_tungstenite::{accept_async, tungstenite::Message}; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let endpoint = format!( + "ws://{}/v1/realtime?intent=transcription", + listener.local_addr().unwrap() + ); + let (update_seen_tx, update_seen_rx) = oneshot::channel(); + let (ack_tx, ack_rx) = oneshot::channel(); + let (finish_tx, finish_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = accept_async(stream).await.expect("websocket"); + let update = socket + .next() + .await + .expect("session update") + .expect("session update frame"); + let Message::Text(update) = update else { + panic!("expected text session update"); + }; + let value: serde_json::Value = serde_json::from_str(&update).expect("json update"); + assert_eq!( + value.get("type").and_then(|value| value.as_str()), + Some("session.update") + ); + update_seen_tx.send(()).expect("report update"); + ack_rx.await.expect("release acknowledgement"); + socket + .send(Message::Text( + serde_json::json!({"type":"session.updated"}) + .to_string() + .into(), + )) + .await + .expect("send acknowledgement"); + let _ = finish_rx.await; + }); + + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx + .try_send(AudioBatch { + bytes: vec![0; 4], + mute_epoch: 0, + }) + .expect("queue initial audio"); + let (event_tx, _event_rx) = tokio_mpsc::channel(4); + let (startup_tx, startup_rx) = mpsc::sync_channel(0); + let shutdown = Arc::new(AtomicBool::new(false)); + let worker_shutdown = Arc::clone(&shutdown); + let worker = thread::spawn(move || { + openai_stt_worker( + "test-key".to_string(), + endpoint, + "gpt-live-transcribe".to_string(), + audio_rx, + event_tx, + worker_shutdown, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU64::new(0)), + Arc::new(AtomicU64::new(0)), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU32::new(VAD_THRESHOLD.to_bits())), + VAD_THRESHOLD, + startup_tx, + ); + }); + + tokio::time::timeout(Duration::from_secs(2), update_seen_rx) + .await + .expect("session update timeout") + .expect("session update signal"); + assert!(matches!( + startup_rx.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + assert!(matches!( + audio_tx.try_send(AudioBatch { + bytes: vec![0; 4], + mute_epoch: 0, + }), + Err(TrySendError::Full(_)) + )); + + ack_tx.send(()).expect("release acknowledgement"); + let startup = + tokio::task::spawn_blocking(move || startup_rx.recv_timeout(Duration::from_secs(2))) + .await + .expect("join startup wait") + .expect("startup signal"); + assert_eq!(startup, Ok(())); + + shutdown.store(true, Ordering::Release); + let _ = finish_tx.send(()); + drop(audio_tx); + tokio::task::spawn_blocking(move || worker.join().expect("worker")) + .await + .expect("join worker task"); + server.await.expect("server"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn openai_startup_surfaces_error_before_session_updated() { + use futures_util::{SinkExt, StreamExt}; + use tokio::net::TcpListener; + use tokio_tungstenite::{accept_async, tungstenite::Message}; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let endpoint = format!( + "ws://{}/v1/realtime?intent=transcription", + listener.local_addr().unwrap() + ); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = accept_async(stream).await.expect("websocket"); + let update = socket.next().await.expect("session update").expect("frame"); + assert!(matches!(update, Message::Text(_))); + socket + .send(Message::Text( + serde_json::json!({ + "type":"error", + "error":{"message":"model rejected"} + }) + .to_string() + .into(), + )) + .await + .expect("send error"); + }); + + let (audio_tx, audio_rx) = mpsc::sync_channel(1); + audio_tx + .try_send(AudioBatch { + bytes: vec![0; 4], + mute_epoch: 0, + }) + .expect("queue initial audio"); + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + let (startup_tx, startup_rx) = mpsc::sync_channel(0); + let worker = thread::spawn(move || { + openai_stt_worker( + "test-key".to_string(), + endpoint, + "bad-model".to_string(), + audio_rx, + event_tx, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU64::new(0)), + Arc::new(AtomicU64::new(0)), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU32::new(VAD_THRESHOLD.to_bits())), + VAD_THRESHOLD, + startup_tx, + ); + }); + + let startup = + tokio::task::spawn_blocking(move || startup_rx.recv_timeout(Duration::from_secs(2))) + .await + .expect("join startup wait") + .expect("startup signal") + .expect_err("startup must fail"); + assert!(startup.contains("model rejected")); + match event_rx.recv().await.expect("failure event") { + SttMessage::Failed(error) => assert!(error.contains("model rejected")), + _ => panic!("expected startup failure event"), + } + + drop(audio_tx); + tokio::task::spawn_blocking(move || worker.join().expect("worker")) + .await + .expect("join worker task"); + server.await.expect("server"); + } + + #[test] + fn openai_transcripts_are_delivered_in_commit_order() { + let mut pending_epochs = VecDeque::from([7, 7]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + for item_id in ["first", "second"] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": item_id, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event"); + } + for (item_id, transcript) in [("second", "two"), ("first", "one")] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.completed", + "item_id": item_id, + "transcript": transcript, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("completion event"); + } + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 7, + None, + &mut None, + ); + + let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { + Ok(SttMessage::Final { text, .. }) => text, + _ => panic!("expected finalized transcript"), + }); + assert_eq!(texts, ["one", "two"]); + } + + #[test] + fn openai_transcription_ignores_commits_from_stale_mute_epochs() { + let mut pending_epochs = VecDeque::from([1]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + + let turn = record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": "stale", + }), + 2, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event") + .expect("recorded commit"); + + assert_eq!(turn.mute_epoch, 1); + assert!(committed.is_empty()); + } + + #[test] + fn openai_transcription_discards_completed_turns_after_mute_changes() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "stale".to_string(), + mute_epoch: 1, + }]); + let mut completed = HashMap::from([("stale".to_string(), "ignore me".to_string())]); + let (event_tx, mut event_rx) = tokio_mpsc::channel(1); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 2, + None, + &mut None, + ); + + assert!(committed.is_empty()); + assert!(completed.is_empty()); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn openai_transcription_surfaces_protocol_failures() { + let error = record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.failed", + "error": { "message": "bad audio" }, + }), + 0, + &mut VecDeque::new(), + &mut VecDeque::new(), + &mut HashMap::new(), + ) + .expect_err("failure event"); + + assert_eq!(error, "bad audio"); + } + + #[test] + fn openai_transcription_ignores_failures_for_discarded_turns() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "current".to_string(), + mute_epoch: 2, + }]); + + let result = record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.failed", + "item_id": "stale", + "error": { "message": "discarded turn failed" }, + }), + 2, + &mut VecDeque::new(), + &mut committed, + &mut HashMap::new(), + ); + + assert_eq!(result, Ok(None)); + } + + #[test] + fn empty_openai_final_acknowledges_shutdown_delivery() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "final".to_string(), + mute_epoch: 3, + }]); + let mut completed = HashMap::from([("final".to_string(), String::new())]); + let (event_tx, mut event_rx) = tokio_mpsc::channel(1); + let (delivered_tx, delivered_rx) = mpsc::sync_channel(1); + let mut final_delivery = Some(delivered_tx); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 3, + Some("final"), + &mut final_delivery, + ); + + assert!(delivered_rx.try_recv().is_ok()); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn stalled_openai_operation_observes_shutdown() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_for_thread = shutdown.clone(); + let signal = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(30)); + shutdown_for_thread.store(true, Ordering::Release); + }); + + let result = block_on_openai_operation( + &runtime, + shutdown.as_ref(), + std::future::pending::>(), + "stalled operation", + ) + .expect("shutdown is not an error"); + + signal.join().expect("shutdown signal"); + assert_eq!(result, None); + } + + #[test] + fn openai_idle_audio_keeps_only_bounded_pre_roll() { + let mut pre_roll = VecDeque::new(); + for index in 0..(OPENAI_PRE_ROLL_CHUNKS * 4) { + push_openai_pre_roll(&mut pre_roll, vec![index as u8]); + } + + assert_eq!(pre_roll.len(), OPENAI_PRE_ROLL_CHUNKS); + assert_eq!( + pre_roll.front(), + Some(&vec![(OPENAI_PRE_ROLL_CHUNKS * 3) as u8]) + ); + } + + #[test] + fn openai_continuous_speech_has_a_turn_limit() { + assert!(!openai_turn_reached_limit(MAX_SPEECH_SAMPLES - 1)); + assert!(openai_turn_reached_limit(MAX_SPEECH_SAMPLES)); + } + #[test] fn native_voice_events_use_renderer_field_names() { let event = NativeVoiceEvent::User { @@ -4017,5 +5218,10 @@ mod tests { .expect("deserialize macOS backend"), VoiceInputBackend::Macos, ); + assert_eq!( + serde_json::from_str::("\"openai\"") + .expect("deserialize OpenAI backend"), + VoiceInputBackend::Openai, + ); } } diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 864dde555..e3b637ea8 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -1,7 +1,7 @@ -//! OpenAI streaming speech playback for voice conversations. +//! OpenAI realtime transcription configuration and streaming speech playback. use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, mpsc, Arc, Mutex, }; #[cfg(any(test, target_os = "macos"))] @@ -31,15 +31,17 @@ use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, openai_voice_credentials::{self, OpenAiVoiceCredential}, pocket_voice::VoiceInterruptionMode, + voice_capture::VoiceCaptureState, }; #[cfg(any(test, target_os = "macos"))] use std::time::Instant; -#[cfg(target_os = "macos")] const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-live-transcribe"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; +const STT_MODEL_ENV: &str = "BERD_OPENAI_STT_MODEL"; const TTS_MODEL_ENV: &str = "BERD_OPENAI_TTS_MODEL"; const TTS_VOICE_ENV: &str = "BERD_OPENAI_TTS_VOICE"; const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; @@ -63,6 +65,14 @@ const MAX_FINAL_PLAYBACK_DRAIN: Duration = Duration::from_secs(600); #[derive(Clone, Debug, Default)] pub struct OpenAiVoiceState { playback: Arc>, + configured: Arc, + credential_revision: Arc, +} + +impl OpenAiVoiceState { + pub(crate) fn is_configured(&self) -> bool { + self.configured.load(Ordering::Acquire) + } } #[derive(Debug)] @@ -101,8 +111,13 @@ enum OpenAiStreamCommand { #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { + stt_configured: bool, tts_configured: bool, + stt_configuration_source: OpenAiVoiceConfigurationSource, tts_configuration_source: OpenAiVoiceConfigurationSource, + stt_unavailable_reason: Option, + tts_unavailable_reason: Option, + transcription_model: String, speech_model: String, speech_voice: String, playback_speed: f32, @@ -168,7 +183,10 @@ fn tts_api_key() -> Result { openai_voice_credentials::require(OpenAiVoiceCredential::TextToSpeech) } -#[cfg(any(test, target_os = "macos"))] +pub(crate) fn stt_api_key() -> Result { + openai_voice_credentials::require(OpenAiVoiceCredential::SpeechToText) +} + fn normalize_openai_base_url(raw_url: String) -> Result { let mut url = reqwest::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -190,7 +208,6 @@ fn normalize_openai_base_url(raw_url: String) -> Result { Ok(url.to_string().trim_end_matches('/').to_string()) } -#[cfg(target_os = "macos")] fn base_url() -> Result { if let Some(base_url) = env_trimmed(BASE_URL_ENV) { return normalize_openai_base_url(base_url); @@ -198,6 +215,27 @@ fn base_url() -> Result { Ok(DEFAULT_BASE_URL.to_string()) } +pub(crate) fn realtime_endpoint() -> Result { + let mut url = reqwest::Url::parse(&endpoint("realtime")?) + .map_err(|error| format!("OpenAI realtime endpoint is invalid: {error}"))?; + url.query_pairs_mut().append_pair("intent", "transcription"); + match url.scheme() { + "http" => url.set_scheme("ws").expect("compatible scheme"), + "https" => url.set_scheme("wss").expect("compatible scheme"), + "ws" | "wss" => {} + scheme => { + return Err(format!( + "OpenAI realtime endpoint has unsupported scheme: {scheme}" + )) + } + } + Ok(url.to_string()) +} + +pub(crate) fn transcription_model() -> String { + env_trimmed(STT_MODEL_ENV).unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) +} + fn speech_model() -> String { env_trimmed(TTS_MODEL_ENV).unwrap_or_else(|| DEFAULT_TTS_MODEL.to_string()) } @@ -217,12 +255,21 @@ fn tts_configuration_source() -> OpenAiVoiceConfigurationSource { } } -#[cfg(target_os = "macos")] +fn stt_configuration_source() -> OpenAiVoiceConfigurationSource { + if [BASE_URL_ENV, STT_MODEL_ENV] + .iter() + .any(|name| env_trimmed(name).is_some()) + { + OpenAiVoiceConfigurationSource::Environment + } else { + OpenAiVoiceConfigurationSource::Default + } +} + fn endpoint(path: &str) -> Result { endpoint_for_base_url(&base_url()?, path) } -#[cfg(any(test, target_os = "macos"))] fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { let mut url = reqwest::Url::parse(base_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -291,19 +338,28 @@ pub async fn get_openai_voice_status( .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? .speed; let tts_available = cfg!(target_os = "macos"); - let tts_configured = if tts_available { - tauri::async_runtime::spawn_blocking(|| { - openai_voice_credentials::read(OpenAiVoiceCredential::TextToSpeech) - }) - .await - .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?? - .is_some() - } else { - false - }; + let credential_revision = state.credential_revision.load(Ordering::Acquire); + let credential_result = tauri::async_runtime::spawn_blocking(move || { + openai_voice_credentials::read(OpenAiVoiceCredential::SpeechToText) + }) + .await + .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?; + let credential_error = credential_result.as_ref().err().cloned(); + let stt_error = credential_error.clone(); + let tts_error = tts_available.then_some(credential_error).flatten(); + let stt_configured = credential_result.unwrap_or(None).is_some(); + let tts_configured = tts_available && stt_configured; + if state.credential_revision.load(Ordering::Acquire) == credential_revision { + state.configured.store(stt_configured, Ordering::Release); + } Ok(OpenAiVoiceStatus { + stt_configured, tts_configured, + stt_configuration_source: stt_configuration_source(), tts_configuration_source: tts_configuration_source(), + stt_unavailable_reason: stt_error, + tts_unavailable_reason: tts_error, + transcription_model: transcription_model(), speech_model: speech_model(), speech_voice: speech_voice(), playback_speed, @@ -319,32 +375,89 @@ pub async fn get_openai_voice_status( } #[tauri::command] -pub fn set_openai_tts_api_key( +pub async fn set_openai_stt_api_key( app: AppHandle, state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + api_key: String, +) -> Result<(), String> { + let api_key = api_key.trim(); + if api_key.is_empty() { + return Err("OpenAI speech-to-text API key cannot be empty".to_string()); + } + native_voice + .stop_active_then(&app, &capture, || { + stop_openai_voice_inner(&state)?; + openai_voice_credentials::store(OpenAiVoiceCredential::SpeechToText, api_key)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(true, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) + }) + .await +} + +#[tauri::command] +pub async fn clear_openai_stt_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, +) -> Result<(), String> { + native_voice + .stop_active_then(&app, &capture, || { + stop_openai_voice_inner(&state)?; + openai_voice_credentials::clear(OpenAiVoiceCredential::SpeechToText)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(false, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) + }) + .await +} + +#[tauri::command] +pub async fn set_openai_tts_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, api_key: String, ) -> Result<(), String> { let api_key = api_key.trim(); if api_key.is_empty() { return Err("OpenAI text-to-speech API key cannot be empty".to_string()); } - stop_openai_voice_inner(&state)?; - openai_voice_credentials::store(OpenAiVoiceCredential::TextToSpeech, api_key)?; - app.emit(SETTINGS_CHANGED_EVENT, ()) - .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; - Ok(()) + native_voice + .stop_active_then(&app, &capture, || { + stop_openai_voice_inner(&state)?; + openai_voice_credentials::store(OpenAiVoiceCredential::TextToSpeech, api_key)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(true, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) + }) + .await } #[tauri::command] -pub fn clear_openai_tts_api_key( +pub async fn clear_openai_tts_api_key( app: AppHandle, state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, ) -> Result<(), String> { - stop_openai_voice_inner(&state)?; - openai_voice_credentials::clear(OpenAiVoiceCredential::TextToSpeech)?; - app.emit(SETTINGS_CHANGED_EVENT, ()) - .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; - Ok(()) + native_voice + .stop_active_then(&app, &capture, || { + stop_openai_voice_inner(&state)?; + openai_voice_credentials::clear(OpenAiVoiceCredential::TextToSpeech)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(false, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) + }) + .await } #[tauri::command] @@ -1202,6 +1315,7 @@ mod tests { #[test] fn voice_configuration_uses_berd_scoped_environment_names() { assert_eq!(BASE_URL_ENV, "BERD_OPENAI_VOICE_BASE_URL"); + assert_eq!(STT_MODEL_ENV, "BERD_OPENAI_STT_MODEL"); assert_eq!(TTS_MODEL_ENV, "BERD_OPENAI_TTS_MODEL"); assert_eq!(TTS_VOICE_ENV, "BERD_OPENAI_TTS_VOICE"); } @@ -1236,6 +1350,16 @@ mod tests { assert_eq!(drained_at, None); } + #[test] + fn configured_readiness_does_not_require_reading_the_secret() { + let state = OpenAiVoiceState::default(); + assert!(!state.is_configured()); + + state.configured.store(true, Ordering::Release); + + assert!(state.is_configured()); + } + #[cfg(target_os = "macos")] #[test] fn chunks_tts_text_on_char_boundaries() { diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 1015b14ab..3e35a4ea4 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -1,42 +1,39 @@ //! Berd-owned credentials for OpenAI voice services. -#[cfg(target_os = "macos")] const KEYCHAIN_SERVICE: &str = "berd-openai-voice"; -#[cfg(any(test, target_os = "macos"))] const KEYCHAIN_ACCOUNT: &str = "api-key"; -#[cfg(target_os = "macos")] const LEGACY_TTS_KEYCHAIN_ACCOUNT: &str = "tts-api-key"; #[derive(Clone, Copy)] pub(crate) enum OpenAiVoiceCredential { + SpeechToText, TextToSpeech, } impl OpenAiVoiceCredential { - #[cfg(any(test, target_os = "macos"))] const fn account(self) -> &'static str { match self { - Self::TextToSpeech => KEYCHAIN_ACCOUNT, + Self::SpeechToText | Self::TextToSpeech => KEYCHAIN_ACCOUNT, } } - #[cfg(target_os = "macos")] const fn missing_message(self) -> &'static str { match self { + Self::SpeechToText => { + "OpenAI speech-to-text is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + } Self::TextToSpeech => { - "OpenAI text-to-speech is not configured. Add its API key in Voice settings, then try again." + "OpenAI text-to-speech is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." } } } } -#[cfg(target_os = "macos")] fn entry(account: &str) -> Result { keyring::Entry::new(KEYCHAIN_SERVICE, account) .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}")) } -#[cfg(target_os = "macos")] fn read_account(account: &str) -> Result, String> { let entry = entry(account)?; match entry.get_password() { @@ -48,7 +45,6 @@ fn read_account(account: &str) -> Result, String> { } } -#[cfg(target_os = "macos")] fn clear_account(account: &str) -> Result<(), String> { let entry = entry(account)?; match entry.delete_credential() { @@ -59,7 +55,6 @@ fn clear_account(account: &str) -> Result<(), String> { } } -#[cfg(any(test, target_os = "macos"))] fn canonical_mutation_with_legacy_cleanup( canonical_mutation: impl FnOnce() -> Result, legacy_cleanup: impl FnOnce() -> Result<(), String>, @@ -71,7 +66,6 @@ fn canonical_mutation_with_legacy_cleanup( Ok(value) } -#[cfg(target_os = "macos")] pub(crate) fn read(credential: OpenAiVoiceCredential) -> Result, String> { if let Some(api_key) = read_account(credential.account())? { return Ok(Some(api_key)); @@ -83,12 +77,6 @@ pub(crate) fn read(credential: OpenAiVoiceCredential) -> Result, Ok(Some(api_key)) } -#[cfg(not(target_os = "macos"))] -pub(crate) fn read(_credential: OpenAiVoiceCredential) -> Result, String> { - Ok(None) -} - -#[cfg(target_os = "macos")] pub(crate) fn store(credential: OpenAiVoiceCredential, api_key: &str) -> Result<(), String> { let entry = entry(credential.account())?; canonical_mutation_with_legacy_cleanup( @@ -101,12 +89,6 @@ pub(crate) fn store(credential: OpenAiVoiceCredential, api_key: &str) -> Result< ) } -#[cfg(not(target_os = "macos"))] -pub(crate) fn store(_credential: OpenAiVoiceCredential, _api_key: &str) -> Result<(), String> { - Err("OpenAI voice credentials are unsupported on this platform".to_string()) -} - -#[cfg(target_os = "macos")] pub(crate) fn clear(credential: OpenAiVoiceCredential) -> Result<(), String> { canonical_mutation_with_legacy_cleanup( || clear_account(credential.account()), @@ -114,12 +96,6 @@ pub(crate) fn clear(credential: OpenAiVoiceCredential) -> Result<(), String> { ) } -#[cfg(not(target_os = "macos"))] -pub(crate) fn clear(_credential: OpenAiVoiceCredential) -> Result<(), String> { - Err("OpenAI voice credentials are unsupported on this platform".to_string()) -} - -#[cfg(target_os = "macos")] pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result { read(credential)?.ok_or_else(|| credential.missing_message().to_string()) } @@ -130,7 +106,8 @@ mod tests { use std::cell::RefCell; #[test] - fn text_to_speech_uses_the_shared_voice_keychain_account() { + fn speech_services_use_the_shared_voice_keychain_account() { + assert_eq!(OpenAiVoiceCredential::SpeechToText.account(), "api-key"); assert_eq!(OpenAiVoiceCredential::TextToSpeech.account(), "api-key"); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 2e41e9a23..b95c0b06a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -656,6 +656,8 @@ pub fn run() { commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, commands::openai_audio::get_openai_voice_status, + commands::openai_audio::set_openai_stt_api_key, + commands::openai_audio::clear_openai_stt_api_key, commands::openai_audio::set_openai_tts_api_key, commands::openai_audio::clear_openai_tts_api_key, commands::openai_audio::start_openai_voice_stream, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 2face1bbf..a73c2d299 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -745,7 +745,9 @@ export function AppShell({ ); const globalVoiceOutput = useVoiceOutputPreference(); const globalOpenAiVoiceSetup = useOpenAiVoiceSetup( - capabilities.voiceConversation && globalVoiceOutput.backend === "openai", + capabilities.voiceConversation && + (globalVoiceInput.backend === "openai" || + globalVoiceOutput.backend === "openai"), ); const globalSiriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && globalVoiceOutput.backend === "siri", diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index 4ea061b45..b1ffe2f9e 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -236,7 +236,8 @@ export function ChatView({ ); const voiceOutput = useVoiceOutputPreference(); const openAiVoiceSetup = useOpenAiVoiceSetup( - capabilities.voiceConversation && voiceOutput.backend === "openai", + capabilities.voiceConversation && + (voiceInput.backend === "openai" || voiceOutput.backend === "openai"), ); const siriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && voiceOutput.backend === "siri", diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 92a64deb5..8fe696a63 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -7,8 +7,13 @@ import type { } from "../lib/voiceInterruptionPreference"; export interface OpenAiVoiceStatus { + sttConfigured: boolean; ttsConfigured: boolean; + sttConfigurationSource: "default" | "environment"; ttsConfigurationSource: "default" | "environment"; + sttUnavailableReason: string | null; + ttsUnavailableReason: string | null; + transcriptionModel: string; speechModel: string; speechVoice: string; playbackSpeed: number; @@ -31,6 +36,14 @@ export function setOpenAiTtsApiKey(apiKey: string): Promise { return invoke("set_openai_tts_api_key", { apiKey }); } +export function setOpenAiSttApiKey(apiKey: string): Promise { + return invoke("set_openai_stt_api_key", { apiKey }); +} + +export function clearOpenAiSttApiKey(): Promise { + return invoke("clear_openai_stt_api_key"); +} + export function clearOpenAiTtsApiKey(): Promise { return invoke("clear_openai_tts_api_key"); } diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index df1520925..9c4846053 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -526,7 +526,7 @@ export function rejectVoiceConversationTranscript( export async function startVoiceConversation( sessionId: string, - inputBackend: "parakeet" | "macos" = "parakeet", + inputBackend: "parakeet" | "macos" | "openai" = "parakeet", foregroundGeneration = 0, ): Promise { resetMicrophoneMuteState(); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index 68e4b6153..d6a4c4ad8 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -33,8 +33,13 @@ function deferred() { function status(configured: boolean): OpenAiVoiceStatus { return { + sttConfigured: configured, ttsConfigured: configured, + sttConfigurationSource: "default", ttsConfigurationSource: "default", + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -65,13 +70,13 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.settingsChanged?.()); refreshed.resolve(status(true)); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); initial.resolve(status(false)); await act(async () => Promise.resolve()); - expect(result.current.status?.ttsConfigured).toBe(true); + expect(result.current.status?.sttConfigured).toBe(true); }); it("refreshes after listener registration captures credential changes", async () => { @@ -84,7 +89,7 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.finishListening?.()); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); }); @@ -95,7 +100,7 @@ describe("useOpenAiVoiceSetup", () => { const { result } = renderHook(() => useOpenAiVoiceSetup()); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); }); diff --git a/src/features/voice-conversation/lib/voiceInputPreference.ts b/src/features/voice-conversation/lib/voiceInputPreference.ts index 4743634b0..c47fe7f28 100644 --- a/src/features/voice-conversation/lib/voiceInputPreference.ts +++ b/src/features/voice-conversation/lib/voiceInputPreference.ts @@ -1,14 +1,16 @@ import { useCallback, useSyncExternalStore } from "react"; import type { MacSpeechStatus } from "../api/macSpeech"; -export type VoiceInputBackend = "parakeet" | "macos"; +export type VoiceInputBackend = "parakeet" | "macos" | "openai"; const STORAGE_KEY = "goose:voice-input-backend"; const CHANGED_EVENT = "goose:voice-input-backend-changed"; let inMemoryBackend: VoiceInputBackend | null = null; function normalizeStored(value: unknown): VoiceInputBackend | null { - return value === "parakeet" || value === "macos" ? value : null; + return value === "parakeet" || value === "macos" || value === "openai" + ? value + : null; } export function getStoredVoiceInputBackend(): VoiceInputBackend | null { @@ -28,6 +30,7 @@ export function resolveVoiceInputBackend( ): VoiceInputBackend | null { if (macSpeechAvailable === null) return null; if (stored === "parakeet") return "parakeet"; + if (stored === "openai") return "openai"; if (stored === "macos" && macSpeechAvailable) return "macos"; return macSpeechAvailable ? "macos" : "parakeet"; } diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index 6a35f946e..d373ae189 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -53,7 +53,7 @@ describe("voice setup readiness", () => { ).toBe(false); }); - it("requires the dedicated OpenAI text-to-speech key for OpenAI output", () => { + it("requires the configured OpenAI TTS key and TTS availability for OpenAI output", () => { const configured = { ttsConfigured: true, ttsAvailable: true } as never; expect( isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), @@ -63,6 +63,16 @@ describe("voice setup readiness", () => { ); }); + it("allows configured OpenAI input when TTS is unavailable", () => { + const configuredStt = { sttConfigured: true, ttsAvailable: false } as never; + expect( + isVoiceSetupReady(pocket, null, null, "openai", "pocket", configuredStt), + ).toBe(true); + expect(isVoiceSetupReady(pocket, null, null, "openai", "pocket")).toBe( + false, + ); + }); + it("uses native macOS speech readiness instead of Parakeet when selected", () => { expect( isVoiceSetupReady( diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts index 196c853d9..fae141b14 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -15,13 +15,15 @@ export function isVoiceSetupReady( ): boolean { if (inputBackend === null) return false; const inputReady = - inputBackend === "macos" - ? Boolean( - macSpeech?.supported && - macSpeech.localeSupported && - macSpeech.modelInstalled, - ) - : Boolean(pocket?.parakeetInstalled); + inputBackend === "openai" + ? Boolean(openAi?.sttConfigured) + : inputBackend === "macos" + ? Boolean( + macSpeech?.supported && + macSpeech.localeSupported && + macSpeech.modelInstalled, + ) + : Boolean(pocket?.parakeetInstalled); if (!inputReady) return false; if (outputBackend === "openai") return Boolean(openAi?.ttsConfigured && openAi.ttsAvailable); diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 08e9f39bd..01baa584d 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -52,8 +52,13 @@ const microphonePermissionState = vi.hoisted(() => ({ })); const openAiStatusState = vi.hoisted(() => ({ current: { + sttConfigured: true, ttsConfigured: true, + sttConfigurationSource: "default" as "default" | "environment", ttsConfigurationSource: "default" as "default" | "environment", + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -62,12 +67,16 @@ const openAiStatusState = vi.hoisted(() => ({ }, })); const openAiApiMocks = vi.hoisted(() => ({ + setSttApiKey: vi.fn(() => Promise.resolve()), + clearSttApiKey: vi.fn(() => Promise.resolve()), setTtsApiKey: vi.fn(() => Promise.resolve()), clearTtsApiKey: vi.fn(() => Promise.resolve()), })); vi.mock("../api/openAiVoice", () => ({ setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), + setOpenAiSttApiKey: openAiApiMocks.setSttApiKey, + clearOpenAiSttApiKey: openAiApiMocks.clearSttApiKey, setOpenAiTtsApiKey: openAiApiMocks.setTtsApiKey, clearOpenAiTtsApiKey: openAiApiMocks.clearTtsApiKey, })); @@ -218,8 +227,13 @@ describe("VoiceSettings", () => { interruptionState.mode = "automatic"; siriSetupState.current = siriSetup(); openAiStatusState.current = { + sttConfigured: true, ttsConfigured: true, + sttConfigurationSource: "default", ttsConfigurationSource: "default", + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -228,17 +242,43 @@ describe("VoiceSettings", () => { }; openAiApiMocks.setTtsApiKey.mockClear(); openAiApiMocks.clearTtsApiKey.mockClear(); + openAiApiMocks.setSttApiKey.mockClear(); + openAiApiMocks.clearSttApiKey.mockClear(); }); - it("renders selected OpenAI output settings", async () => { + it("renders independently selected OpenAI input and output settings", async () => { + inputState.backend = "openai"; outputState.backend = "openai"; setupState.current = setup(pocketStatus()); renderWithProviders(); expect( - await screen.findByText(/gpt-4o-mini-tts.*marin voice/), + await screen.findByText("Uses gpt-live-transcribe."), + ).toBeInTheDocument(); + expect( + screen.getByText(/gpt-4o-mini-tts.*marin voice/), ).toBeInTheDocument(); expect(screen.getByText("Playback speed")).toBeInTheDocument(); + expect( + screen.getAllByText( + "Saved securely and shared by OpenAI transcription and voice playback.", + ), + ).toHaveLength(2); + }); + + it("saves the shared OpenAI voice key from the speech-to-text settings", async () => { + inputState.backend = "openai"; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + const user = userEvent.setup(); + await user.type( + screen.getByLabelText("OpenAI speech-to-text API key"), + "stt-secret", + ); + await user.click(screen.getAllByRole("button", { name: "Save key" })[0]); + + expect(openAiApiMocks.setSttApiKey).toHaveBeenCalledWith("stt-secret"); }); it("labels purpose-specific environment overrides", async () => { @@ -257,7 +297,23 @@ describe("VoiceSettings", () => { ).toBeInTheDocument(); }); - it("saves a dedicated OpenAI text-to-speech API key", async () => { + it("labels speech-to-text environment overrides", async () => { + inputState.backend = "openai"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigurationSource: "environment", + }; + setupState.current = setup(pocketStatus()); + renderWithProviders(); + + expect( + await screen.findByText( + "Development configuration is overridden by the Berd process environment.", + ), + ).toBeInTheDocument(); + }); + + it("saves the shared OpenAI voice key from the text-to-speech settings", async () => { outputState.backend = "openai"; setupState.current = setup(pocketStatus({ parakeetInstalled: true })); renderWithProviders(); @@ -272,20 +328,103 @@ describe("VoiceSettings", () => { expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); }); + it("uses OpenAI guidance when only the selected OpenAI input is not ready", async () => { + inputState.backend = "openai"; + outputState.backend = "pocket"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigured: false, + }; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText( + "OpenAI transcription is not ready. Add the shared OpenAI voice API key below, then try again.", + ), + ).toBeInTheDocument(); + expect( + screen.queryByText(/Parakeet STT is not installed/), + ).not.toBeInTheDocument(); + }); + + it("reports missing OpenAI input and Pocket output together", async () => { + inputState.backend = "openai"; + outputState.backend = "pocket"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigured: false, + }; + setupState.current = setup(pocketStatus({ pocketInstalled: false })); + + renderWithProviders(); + + expect( + await screen.findByText( + "The shared OpenAI voice API key is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", + ), + ).toBeInTheDocument(); + }); + + it("reports missing OpenAI input and Siri output together", async () => { + inputState.backend = "openai"; + outputState.backend = "siri"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigured: false, + }; + const current = siriSetup(); + siriSetupState.current = { + ...current, + status: current.status + ? { + ...current.status, + selectedVoice: null, + selectedVoiceInstalled: false, + } + : null, + }; + + renderWithProviders(); + + expect( + await screen.findByText( + "The shared OpenAI voice API key is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + ), + ).toBeInTheDocument(); + }); + + it("does not show the TTS platform restriction for configured OpenAI input", async () => { + inputState.backend = "openai"; + outputState.backend = "pocket"; + openAiStatusState.current = { + ...openAiStatusState.current, + ttsAvailable: false, + unavailableReason: "unsupportedPlatform", + }; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText("Uses gpt-live-transcribe."), + ).toBeInTheDocument(); + expect( + screen.queryByText(/playback is currently supported on macOS only/), + ).not.toBeInTheDocument(); + }); + it("uses OpenAI guidance when the selected OpenAI output is not ready", async () => { outputState.backend = "openai"; openAiStatusState.current = { ...openAiStatusState.current, ttsConfigured: false, - unavailableReason: - "Add an OpenAI text-to-speech API key in Voice settings.", }; setupState.current = setup(pocketStatus({ parakeetInstalled: true })); renderWithProviders(); expect( await screen.findByText( - "OpenAI voice is not ready. Add the required API key below, then try again.", + "OpenAI voice playback is not ready. Add the shared OpenAI voice API key below, then try again.", ), ).toBeInTheDocument(); expect( diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index ec5b45b01..ecc7b97fd 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -16,7 +16,9 @@ import { } from "@/shared/ui/select"; import { useEffect, useState } from "react"; import { + clearOpenAiSttApiKey, clearOpenAiTtsApiKey, + setOpenAiSttApiKey, setOpenAiPlaybackSpeed, setOpenAiTtsApiKey, } from "../api/openAiVoice"; @@ -54,6 +56,12 @@ function readinessDescriptionKey( ): string | null { if (inputReady && outputReady) return null; if (!inputReady && !outputReady) { + if (inputBackend === "openai") { + if (backend === "openai") return "voice.notReadyOpenAiSttAndTts"; + return backend === "siri" + ? "voice.notReadyOpenAiSttAndSiriOutput" + : "voice.notReadyOpenAiSttAndPocketOutput"; + } if (backend === "openai") { return inputBackend === "macos" ? "voice.notReadyMacInputAndOpenAiOutput" @@ -69,11 +77,12 @@ function readinessDescriptionKey( : "voice.notReadyInputAndPocketOutput"; } if (!inputReady) { + if (inputBackend === "openai") return "voice.notReadyOpenAiStt"; return inputBackend === "macos" ? "voice.notReadyMacInput" : "voice.notReadyInput"; } - if (backend === "openai") return "voice.notReadyOpenAi"; + if (backend === "openai") return "voice.notReadyOpenAiTts"; return backend === "siri" ? "voice.notReadySiriOutput" : "voice.notReadyPocketOutput"; @@ -104,13 +113,15 @@ export function VoiceSettings() { const interruptionHeadingId = useId(); const interruptionDescriptionId = useId(); const inputReady = - input.backend === "macos" - ? Boolean( - macSpeechSetup.status?.supported && - macSpeechSetup.status.localeSupported && - macSpeechSetup.status.modelInstalled, - ) - : (setup.status?.parakeetInstalled ?? false); + input.backend === "openai" + ? (openAiStatus?.sttConfigured ?? false) + : input.backend === "macos" + ? Boolean( + macSpeechSetup.status?.supported && + macSpeechSetup.status.localeSupported && + macSpeechSetup.status.modelInstalled, + ) + : (setup.status?.parakeetInstalled ?? false); const outputReady = output.backend === "openai" ? Boolean(openAiStatus?.ttsConfigured && openAiStatus.ttsAvailable) @@ -204,6 +215,9 @@ export function VoiceSettings() { {t("voice.backendParakeet")} + + {t("voice.backendOpenAiStt")} + {macSpeechSetup.status?.supported && macSpeechSetup.status.localeSupported ? ( @@ -214,7 +228,32 @@ export function VoiceSettings() { )} details={ - input.backend === "macos" ? ( + input.backend === "openai" ? ( +
+ +

+ {openAiError ?? + openAiStatus?.sttUnavailableReason ?? + (openAiStatus + ? openAiStatus.sttConfigured + ? t("voice.openAiSttConfigured", { + model: openAiStatus.transcriptionModel, + }) + : t("voice.openAiSttNotConfigured") + : t("voice.openAiChecking"))} +

+ {openAiStatus?.sttConfigurationSource === "environment" ? ( +

+ {t("voice.openAiEnvironmentOverride")} +

+ ) : null} +
+ ) : input.backend === "macos" ? ( ) : input.backend === "parakeet" ? (

{openAiError ?? + openAiStatus?.ttsUnavailableReason ?? (openAiStatus?.unavailableReason === "unsupportedPlatform" ? t("voice.openAiTtsUnsupportedPlatform") : openAiStatus?.unavailableReason === "missingApiKey" diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 7abd0fdfa..ac1618334 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -870,6 +870,7 @@ }, "voice": { "backendMacSpeech": "Apple speech recognition", + "backendOpenAiStt": "OpenAI speech-to-text", "backendOpenAiTts": "OpenAI text-to-speech", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -913,14 +914,19 @@ "modelMissingSize": "Not installed · {{size}} download", "modelNotInstalled": "Not installed", "notReadyInput": "Parakeet STT is not installed. Download it below to use Voice Conversation.", - "notReadyInputAndOpenAiOutput": "Parakeet STT is not installed, and the OpenAI text-to-speech key is missing. Complete both steps below to use Voice Conversation.", + "notReadyInputAndOpenAiOutput": "Parakeet STT is not installed, and the shared OpenAI voice API key is missing. Complete both steps below to use Voice Conversation.", "notReadyInputAndPocketOutput": "Parakeet STT and Pocket TTS are not installed. Download both below to use Voice Conversation.", "notReadyInputAndSiriOutput": "Parakeet STT is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyMacInput": "Apple's on-device dictation model is not installed. Download it below to use Voice Conversation.", - "notReadyMacInputAndOpenAiOutput": "Apple's on-device dictation model is not installed, and the OpenAI text-to-speech key is missing. Complete both steps below to use Voice Conversation.", + "notReadyMacInputAndOpenAiOutput": "Apple's on-device dictation model is not installed, and the shared OpenAI voice API key is missing. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyOpenAi": "OpenAI voice is not ready. Add the required API key below, then try again.", + "notReadyOpenAiStt": "OpenAI transcription is not ready. Add the shared OpenAI voice API key below, then try again.", + "notReadyOpenAiSttAndPocketOutput": "The shared OpenAI voice API key is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", + "notReadyOpenAiSttAndSiriOutput": "The shared OpenAI voice API key is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + "notReadyOpenAiSttAndTts": "OpenAI transcription and voice playback are not ready. Add their shared OpenAI voice API key below, then try again.", + "notReadyOpenAiTts": "OpenAI voice playback is not ready. Add the shared OpenAI voice API key below, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -929,14 +935,17 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", - "openAiApiKeyConfigured": "Saved securely for this voice service.", - "openAiApiKeyNotConfigured": "This key is separate from provider credentials used by Goose.", + "openAiApiKeyConfigured": "Saved securely and shared by OpenAI transcription and voice playback.", + "openAiApiKeyNotConfigured": "This key is shared by OpenAI transcription and voice playback. It is separate from provider credentials used by Goose.", "openAiApiKeySaved": "API key saved", "openAiChecking": "Checking OpenAI voice settings…", "openAiEnvironmentOverride": "Development configuration is overridden by the Berd process environment.", + "openAiSttApiKey": "OpenAI speech-to-text API key", + "openAiSttConfigured": "Uses {{model}}.", + "openAiSttNotConfigured": "Add the shared OpenAI voice API key to use OpenAI transcription.", "openAiTtsApiKey": "OpenAI text-to-speech API key", "openAiTtsConfigured": "Uses {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", - "openAiTtsNeedsKey": "Add an OpenAI text-to-speech API key to use this voice.", + "openAiTtsNeedsKey": "Add the shared OpenAI voice API key to use this voice.", "openAiTtsUnsupportedPlatform": "OpenAI voice playback is currently supported on macOS only.", "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index fbd541916..2a7682c56 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -873,6 +873,7 @@ }, "voice": { "backendMacSpeech": "Reconocimiento de voz de Apple", + "backendOpenAiStt": "Voz a texto de OpenAI", "backendOpenAiTts": "Texto a voz de OpenAI", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -916,14 +917,19 @@ "modelMissingSize": "No instalado · descarga de {{size}}", "modelNotInstalled": "No instalado", "notReadyInput": "Parakeet STT no está instalado. Descárgalo abajo para usar la conversación por voz.", - "notReadyInputAndOpenAiOutput": "Parakeet STT no está instalado y falta la clave de texto a voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyInputAndOpenAiOutput": "Parakeet STT no está instalado y falta la clave API compartida de voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyInputAndPocketOutput": "Parakeet STT y Pocket TTS no están instalados. Descarga ambos abajo para usar la conversación por voz.", "notReadyInputAndSiriOutput": "Parakeet STT no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInput": "El modelo de dictado de Apple no está instalado. Descárgalo abajo para usar la conversación por voz.", - "notReadyMacInputAndOpenAiOutput": "El modelo de dictado de Apple no está instalado y falta la clave de texto a voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyMacInputAndOpenAiOutput": "El modelo de dictado de Apple no está instalado y falta la clave API compartida de voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyOpenAi": "La voz de OpenAI no está lista. Añade abajo la clave API necesaria e inténtalo de nuevo.", + "notReadyOpenAiStt": "La transcripción de OpenAI no está lista. Añade abajo la clave API compartida de voz de OpenAI e inténtalo de nuevo.", + "notReadyOpenAiSttAndPocketOutput": "Falta la clave API compartida de voz de OpenAI y Pocket TTS no está instalado. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAiSttAndSiriOutput": "Falta la clave API compartida de voz de OpenAI y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAiSttAndTts": "La transcripción y la reproducción de voz de OpenAI no están listas. Añade abajo su clave API compartida de voz de OpenAI e inténtalo de nuevo.", + "notReadyOpenAiTts": "La reproducción de voz de OpenAI no está lista. Añade abajo la clave API compartida de voz de OpenAI e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -932,14 +938,17 @@ "noVoiceSelected": "No hay ninguna voz seleccionada", "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", - "openAiApiKeyConfigured": "Guardada de forma segura para este servicio de voz.", - "openAiApiKeyNotConfigured": "Esta clave es independiente de las credenciales de proveedor que usa Goose.", + "openAiApiKeyConfigured": "Guardada de forma segura y compartida por la transcripción y la reproducción de voz de OpenAI.", + "openAiApiKeyNotConfigured": "Esta clave se comparte entre la transcripción y la reproducción de voz de OpenAI. Es independiente de las credenciales de proveedor que usa Goose.", "openAiApiKeySaved": "Clave API guardada", "openAiChecking": "Comprobando los ajustes de voz de OpenAI…", "openAiEnvironmentOverride": "La configuración de desarrollo está reemplazada por el entorno del proceso de Berd.", + "openAiSttApiKey": "Clave API de voz a texto de OpenAI", + "openAiSttConfigured": "Usa {{model}}.", + "openAiSttNotConfigured": "Añade la clave API compartida de voz de OpenAI para usar la transcripción de OpenAI.", "openAiTtsApiKey": "Clave API de texto a voz de OpenAI", "openAiTtsConfigured": "Usa {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", - "openAiTtsNeedsKey": "Añade una clave API de texto a voz de OpenAI para usar esta voz.", + "openAiTtsNeedsKey": "Añade la clave API compartida de voz de OpenAI para usar esta voz.", "openAiTtsUnsupportedPlatform": "La reproducción de voz de OpenAI solo es compatible actualmente con macOS.", "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción",