From 3022dc6f88b64f812079b4d83687c8769986f107 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Sun, 30 Aug 2026 00:25:05 -0400 Subject: [PATCH 01/41] feat(voice): add realtime master emissary mode --- docs/app-e2e.md | 44 + src-tauri/Cargo.toml | 5 +- .../crates/berdctl/api-surface-feedback.json | 55 +- src-tauri/crates/berdctl/api-surface.json | 55 +- .../crates/berdctl/cli-surface-feedback.json | 7 +- src-tauri/crates/berdctl/cli-surface.json | 7 +- src-tauri/crates/berdctl/src/main.rs | 8 + src-tauri/src/commands/mod.rs | 1 - src-tauri/src/commands/openai_realtime.rs | 243 ++- .../src/commands/openai_voice_credentials.rs | 7 +- src-tauri/src/lib.rs | 6 +- src/app/AppShell.tsx | 20 +- .../__tests__/commands/commands.test.ts | 5 + .../commands/impl/sendToEmissarySession.ts | 86 ++ src/features/berdctl/commands/registry.ts | 7 +- .../__tests__/acpNotificationHandler.test.ts | 35 + .../chat/acp/acpNotificationHandler.ts | 11 +- .../ConversationComposerCapability.test.tsx | 36 + .../ConversationComposerCapability.tsx | 27 +- src/features/chat/hooks/useChat.ts | 1 + .../chat/lib/__tests__/steerCore.test.ts | 77 + src/features/chat/lib/sendCore.test.ts | 336 +++++ src/features/chat/lib/sendCore.ts | 163 +- src/features/chat/lib/steerCore.ts | 35 +- .../chat/lib/voiceConversationNoop.ts | 11 + .../projection/buildTranscriptItems.test.ts | 104 ++ .../projection/buildTranscriptItems.ts | 51 +- src/features/chat/types.ts | 4 + src/features/chat/ui/ChatView.tsx | 27 +- .../useOpenAiRealtimeConversation.test.ts | 1343 +++++++++++++++++ .../hooks/useOpenAiRealtimeConversation.ts | 1057 +++++++++++++ .../lib/realtimeEmissaryBridge.test.ts | 48 + .../lib/realtimeEmissaryBridge.ts | 61 + .../lib/realtimeEmissaryProtocol.test.ts | 918 +++++++++++ .../lib/realtimeEmissaryProtocol.ts | 931 ++++++++++++ .../lib/realtimeVoicePreference.test.ts | 50 + .../lib/realtimeVoicePreference.ts | 104 ++ .../voiceConversationModePreference.test.ts | 18 + .../lib/voiceConversationModePreference.ts | 78 + .../ui/RealtimeVoiceSettings.tsx | 205 +++ .../voice-conversation/ui/VoiceSettings.tsx | 468 +++--- src/shared/api/openaiRealtime.ts | 13 + src/shared/i18n/locales/en/settings.json | 19 + src/shared/i18n/locales/es/settings.json | 19 + tests/app-e2e/lib/setup.ts | 63 +- tests/app-e2e/lib/test-driver-client.ts | 117 +- .../realtime-master-emissary.eval.test.ts | 296 ++++ 47 files changed, 6853 insertions(+), 429 deletions(-) create mode 100644 src/features/berdctl/commands/impl/sendToEmissarySession.ts create mode 100644 src/features/chat/lib/voiceConversationNoop.ts create mode 100644 src/features/chat/transcript/projection/buildTranscriptItems.test.ts create mode 100644 src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts create mode 100644 src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts create mode 100644 src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts create mode 100644 src/features/voice-conversation/lib/realtimeEmissaryBridge.ts create mode 100644 src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts create mode 100644 src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts create mode 100644 src/features/voice-conversation/lib/realtimeVoicePreference.test.ts create mode 100644 src/features/voice-conversation/lib/realtimeVoicePreference.ts create mode 100644 src/features/voice-conversation/lib/voiceConversationModePreference.test.ts create mode 100644 src/features/voice-conversation/lib/voiceConversationModePreference.ts create mode 100644 src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx create mode 100644 tests/app-e2e/realtime-master-emissary.eval.test.ts diff --git a/docs/app-e2e.md b/docs/app-e2e.md index ff6a91a31..acffaba1d 100644 --- a/docs/app-e2e.md +++ b/docs/app-e2e.md @@ -44,3 +44,47 @@ Windows uses the same runtime contract through `scripts/windows/Dev-Windows.ps1` Set `BERD_E2E_MODE=1`, `BERD_E2E_RUN_ROOT`, and the optional provider bootstrap environment above before invoking it; the app owns its random driver port and publishes readiness under the run root. + +## Live Realtime Master–Emissary evaluation + +`tests/app-e2e/realtime-master-emissary.eval.test.ts` is an opt-in live +evaluation driven by typed chat messages. It starts a fresh Realtime voice +conversation, mutes its microphone so ambient audio cannot affect the run, asks +how many repositories are in the user's Development folder, then asks whether +any are symbolic links. It verifies that each typed question is followed in +order by visible Master-to-Emissary coordination and a visible terminal Master +turn. Each turn may contain one finalized Emissary answer or a brief +acknowledgement followed by the answer; more than two finalized utterances fails +the evaluation as a likely coordination loop. + +The legacy app-test-driver protocol serves one command per TCP connection, so +the client opens a fresh authenticated connection for every command. Home +navigation and promotion of its composer draft may temporarily replace the app +webview; this eval waits for the expected destination after those two known +boundaries without replaying the navigation or call-button click. Mutating +actions remain single-shot. + +This scenario intentionally uses the normal local dev profile, not isolated +E2E mode: Realtime needs the Berd-owned API key stored from Voice settings, and +the master needs the normal configured agent/tool environment for inspecting the +real Development folder. Before running it, select **OpenAI Realtime** as the +Voice mode and save the Realtime API key in Berd. + +Start the instrumented app in one terminal: + +```bash +APP_TEST_DRIVER_TOKEN=local-realtime-eval just dev-e2e +``` + +Then run only the live scenario in another terminal: + +```bash +APP_TEST_DRIVER_TOKEN=local-realtime-eval \ +BERD_E2E_REALTIME_EVAL=1 \ +pnpm exec vitest run --config vitest.app-e2e.config.ts \ + tests/app-e2e/realtime-master-emissary.eval.test.ts +``` + +Without `BERD_E2E_REALTIME_EVAL=1`, the live scenario is skipped so the normal +app E2E suite never makes network/model calls or depends on a developer's local +filesystem and credentials. diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 08505f3e4..7dcec08ba 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -142,8 +142,9 @@ devtools = ["tauri/devtools"] # restricted build is expressed by ADDING these (never with # --no-default-features + re-listing everything you want to keep). # -# no-voice-dictation: the realtime client secret is never requested -# (`get_openai_realtime_status` reports `configured: false`). +# no-voice-dictation: disables Berd's native chained dictation implementation. +# Browser-owned OpenAI Realtime voice remains available because it uses +# getUserMedia/WebRTC and does not depend on the native dictation engine. no-voice-dictation = [] # no-bb-cli-install: the app never auto-installs or offers to symlink the # bundled berdctl into /usr/local/bin/bb. The bundled binary still ships for diff --git a/src-tauri/crates/berdctl/api-surface-feedback.json b/src-tauri/crates/berdctl/api-surface-feedback.json index 7ed5474dd..763fb5512 100644 --- a/src-tauri/crates/berdctl/api-surface-feedback.json +++ b/src-tauri/crates/berdctl/api-surface-feedback.json @@ -3,7 +3,7 @@ "protocolVersion": 4, "groups": { "sessions": { - "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, fork, archive.", + "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, send to a live voice emissary, fork, archive.", "actions": { "create": { "description": "Create a new chat session on any installed agent harness and send the prompt in it. Fire-and-forget: returns the session id immediately and the session runs in the background without changing what the user sees; the user can open it themselves. Use --from to give the delegating session or tool a concise visible label on the initial message. Only check on it later (action \"get\") if the user asks.", @@ -427,6 +427,59 @@ "additionalProperties": false } }, + "send_to_emissary": { + "description": "Inject a private coordination message into the OpenAI Realtime voice emissary owned by an existing Berd session. The emissary receives the message immediately and starts a response; active speech may be interrupted. The command fails when the target session has no live Realtime voice conversation.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime emissary.", + "min": 1 + }, + { + "name": "message", + "required": true, + "kind": "string", + "description": "Private coordination message to inject into the emissary.", + "min": 1, + "max": 20000 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Latest direct-message cursor returned by the voice bridge.", + "min": 0, + "max": 4294967295 + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime emissary." + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 20000, + "description": "Private coordination message to inject into the emissary." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Latest direct-message cursor returned by the voice bridge." + } + }, + "required": ["session_id", "message", "cursor"], + "additionalProperties": false + } + }, "fork": { "description": "Duplicate an existing chat session, copying its full conversation history into a new session the user can continue down an independent path. The fork appears in the app's session list; the user's current view does not change.", "fields": [ diff --git a/src-tauri/crates/berdctl/api-surface.json b/src-tauri/crates/berdctl/api-surface.json index b7daa844d..180385380 100644 --- a/src-tauri/crates/berdctl/api-surface.json +++ b/src-tauri/crates/berdctl/api-surface.json @@ -3,7 +3,7 @@ "protocolVersion": 4, "groups": { "sessions": { - "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, fork, archive.", + "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, send to a live voice emissary, fork, archive.", "actions": { "create": { "description": "Create a new chat session on any installed agent harness and send the prompt in it. Fire-and-forget: returns the session id immediately and the session runs in the background without changing what the user sees; the user can open it themselves. Use --from to give the delegating session or tool a concise visible label on the initial message. Only check on it later (action \"get\") if the user asks.", @@ -427,6 +427,59 @@ "additionalProperties": false } }, + "send_to_emissary": { + "description": "Inject a private coordination message into the OpenAI Realtime voice emissary owned by an existing Berd session. The emissary receives the message immediately and starts a response; active speech may be interrupted. The command fails when the target session has no live Realtime voice conversation.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime emissary.", + "min": 1 + }, + { + "name": "message", + "required": true, + "kind": "string", + "description": "Private coordination message to inject into the emissary.", + "min": 1, + "max": 20000 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Latest direct-message cursor returned by the voice bridge.", + "min": 0, + "max": 4294967295 + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime emissary." + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 20000, + "description": "Private coordination message to inject into the emissary." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Latest direct-message cursor returned by the voice bridge." + } + }, + "required": ["session_id", "message", "cursor"], + "additionalProperties": false + } + }, "fork": { "description": "Duplicate an existing chat session, copying its full conversation history into a new session the user can continue down an independent path. The fork appears in the app's session list; the user's current view does not change.", "fields": [ diff --git a/src-tauri/crates/berdctl/cli-surface-feedback.json b/src-tauri/crates/berdctl/cli-surface-feedback.json index 59438fb52..a58bad256 100644 --- a/src-tauri/crates/berdctl/cli-surface-feedback.json +++ b/src-tauri/crates/berdctl/cli-surface-feedback.json @@ -3,7 +3,7 @@ "nouns": { "session": { "group": "sessions", - "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to emissary, fork, archive", "verbs": { "create": { "action": "create", @@ -50,6 +50,11 @@ "about": "Move a chat session out of any project", "afterHelp": "Example:\n berdctl session clear-project --session-id \n\nResult:\n {\"ok\": true} — the app's session list regroups immediately." }, + "send-to-emissary": { + "action": "send_to_emissary", + "about": "Send private guidance to a session's live voice emissary", + "afterHelp": "Example:\n berdctl session send-to-emissary --session-id --cursor 0 \\\n --message \"The build failed because the signing certificate expired.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":0,\"delivery_status\":\"sent\"|\"interrupting\"|\"queued\"}\n\nA send while the pipe is carrying emissary-to-master coordination fails with\nreason \"pipe_busy\" without consuming that pending message. Wait for Berd to\ndeliver it normally, then retry with the cursor included in that message." + }, "fork": { "action": "fork", "about": "Fork a chat session into an independent copy with its history", diff --git a/src-tauri/crates/berdctl/cli-surface.json b/src-tauri/crates/berdctl/cli-surface.json index 1d297f14b..534e6292e 100644 --- a/src-tauri/crates/berdctl/cli-surface.json +++ b/src-tauri/crates/berdctl/cli-surface.json @@ -3,7 +3,7 @@ "nouns": { "session": { "group": "sessions", - "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to emissary, fork, archive", "verbs": { "create": { "action": "create", @@ -50,6 +50,11 @@ "about": "Move a chat session out of any project", "afterHelp": "Example:\n berdctl session clear-project --session-id \n\nResult:\n {\"ok\": true} — the app's session list regroups immediately." }, + "send-to-emissary": { + "action": "send_to_emissary", + "about": "Send private guidance to a session's live voice emissary", + "afterHelp": "Example:\n berdctl session send-to-emissary --session-id --cursor 0 \\\n --message \"The build failed because the signing certificate expired.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":0,\"delivery_status\":\"sent\"|\"interrupting\"|\"queued\"}\n\nA send while the pipe is carrying emissary-to-master coordination fails with\nreason \"pipe_busy\" without consuming that pending message. Wait for Berd to\ndeliver it normally, then retry with the cursor included in that message." + }, "fork": { "action": "fork", "about": "Fork a chat session into an independent copy with its history", diff --git a/src-tauri/crates/berdctl/src/main.rs b/src-tauri/crates/berdctl/src/main.rs index e762ffeec..99b496422 100644 --- a/src-tauri/crates/berdctl/src/main.rs +++ b/src-tauri/crates/berdctl/src/main.rs @@ -276,6 +276,14 @@ mod tests { ("session", "move") => vec!["--session-id", "s", "--project-id", "p"], ("session", "move-to-group") => vec!["--session-id", "s", "--group-id", "g"], ("session", "clear-project") => vec!["--session-id", "s"], + ("session", "send-to-emissary") => vec![ + "--session-id", + "s", + "--cursor", + "0", + "--message", + "status", + ], ("folder", "attach") | ("folder", "detach") | ("folder", "set-cwd") => { vec!["--session-id", "s", "--path", "/w"] } diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4b88b1b00..e11766c5b 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -38,7 +38,6 @@ mod native_input_mute; pub mod native_voice; pub mod notifications; pub mod openai_audio; -#[cfg(feature = "block-voice-dictation")] pub mod openai_realtime; mod openai_voice_credentials; pub mod path_resolver; diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index c7524c4fb..33e3575c2 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -1,24 +1,20 @@ -#[cfg(not(feature = "no-voice-dictation"))] -use crate::services::kgoose::KgooseContext; -use crate::{ - commands::runtime_config::RuntimeConfigState, - services::{distro_bundle::DistroBundleState, kgoose}, -}; -use serde::Serialize; -#[cfg(not(feature = "no-voice-dictation"))] +use serde::{Deserialize, Serialize}; use serde_json::json; use tauri::{State, WebviewWindow}; +use super::openai_voice_credentials::{self, OpenAiVoiceCredential}; use super::voice_capture::VoiceCaptureState; -#[cfg(not(feature = "no-voice-dictation"))] -const OPENAI_REALTIME_CLIENT_SECRETS_ENDPOINT: &str = "transcribe/v1/realtime-client-secret"; const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-realtime-whisper"; +const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime"; +const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = + "https://api.openai.com/v1/realtime/client_secrets"; #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiRealtimeStatus { configured: bool, + voice_configured: bool, transcription_model: String, } @@ -29,72 +25,89 @@ pub struct OpenAiRealtimeSession { transcription_model: String, } -fn non_empty_env(name: &str) -> Option { - std::env::var(name) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SaveOpenAiRealtimeApiKeyRequest { + api_key: String, } fn transcription_model() -> String { - non_empty_env("OPENAI_REALTIME_TRANSCRIPTION_MODEL") - .unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) + DEFAULT_TRANSCRIPTION_MODEL.to_string() } -fn openai_realtime_configured( - runtime_config: &crate::commands::runtime_config::RuntimeConfig, - distro_state: &DistroBundleState, -) -> bool { - cfg!(not(feature = "no-voice-dictation")) - && kgoose::is_configured(runtime_config.kgoose.as_ref(), distro_state.kgoose_config()) +fn stored_openai_api_key() -> Result, String> { + openai_voice_credentials::read(OpenAiVoiceCredential::Realtime) } #[tauri::command] -pub async fn get_openai_realtime_status( - distro_state: State<'_, DistroBundleState>, - runtime_config_state: State<'_, RuntimeConfigState>, -) -> Result { - let runtime_config = runtime_config_state - .ready_config(distro_state.inner()) - .await?; +pub async fn get_openai_realtime_status() -> Result { + let configured = stored_openai_api_key()?.is_some(); Ok(OpenAiRealtimeStatus { - configured: openai_realtime_configured(&runtime_config, distro_state.inner()), + configured, + voice_configured: configured, transcription_model: transcription_model(), }) } #[tauri::command] -pub async fn create_openai_realtime_session( - _distro_state: State<'_, DistroBundleState>, - _runtime_config_state: State<'_, RuntimeConfigState>, +pub async fn save_openai_realtime_api_key( + request: SaveOpenAiRealtimeApiKeyRequest, +) -> Result<(), String> { + openai_voice_credentials::store(OpenAiVoiceCredential::Realtime, &request.api_key) +} + +#[tauri::command] +pub async fn create_openai_realtime_voice_session( + model: Option, ) -> Result { - #[cfg(feature = "no-voice-dictation")] - { - Err("OpenAI realtime sessions are unsupported because voice dictation is disabled in this build.".to_string()) + let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; + let model = model + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_REALTIME_MODEL.to_string()); + let response = realtime_client_secret_request(&reqwest::Client::new(), &api_key, &model) + .send() + .await + .map_err(|error| format!("Failed to create OpenAI Realtime voice session: {error}"))?; + let status = response.status(); + let body = response + .text() + .await + .map_err(|error| format!("Failed to read OpenAI Realtime response: {error}"))?; + if !status.is_success() { + return Err(format!( + "OpenAI Realtime session creation failed ({status}): {body}" + )); } + let value: serde_json::Value = serde_json::from_str(&body) + .map_err(|error| format!("OpenAI Realtime returned invalid JSON: {error}"))?; - #[cfg(not(feature = "no-voice-dictation"))] - { - let transcription_model = transcription_model(); - let runtime_config = _runtime_config_state - .ready_config(_distro_state.inner()) - .await?; - let kgoose = KgooseContext::new(_distro_state.inner(), &runtime_config); - let value = kgoose - .post_json( - OPENAI_REALTIME_CLIENT_SECRETS_ENDPOINT, - json!({ "language": "en" }), - ) - .await - .map_err(|error| format!("Failed to create OpenAI realtime session: {error}"))?; - let client_secret = parse_client_secret(&value)?; + Ok(OpenAiRealtimeSession { + client_secret: parse_client_secret(&value)?, + transcription_model: transcription_model(), + }) +} - Ok(OpenAiRealtimeSession { - client_secret, - transcription_model, - }) - } +#[tauri::command] +pub async fn create_openai_realtime_session() -> Result { + create_openai_realtime_voice_session(None).await +} + +fn realtime_client_secret_request( + client: &reqwest::Client, + api_key: &str, + model: &str, +) -> reqwest::RequestBuilder { + client + .post(OPENAI_REALTIME_CLIENT_SECRETS_URL) + .bearer_auth(api_key) + .json(&json!({ + "session": { + "type": "realtime", + "model": model, + } + })) } #[tauri::command] @@ -132,7 +145,6 @@ pub fn release_voice_dictation_microphone( Ok(()) } -#[cfg(not(feature = "no-voice-dictation"))] fn parse_client_secret(value: &serde_json::Value) -> Result { let client_secret = value.get("client_secret").and_then(client_secret_value); let top_level_value = value.get("value").and_then(|value| value.as_str()); @@ -143,13 +155,11 @@ fn parse_client_secret(value: &serde_json::Value) -> Result { .or(top_level_secret) .map(ToString::to_string) .ok_or_else(|| { - format!( - "OpenAI realtime client secret response did not include a recognized secret field: {value}" - ) + "OpenAI realtime client secret response did not include a recognized secret field." + .to_string() }) } -#[cfg(not(feature = "no-voice-dictation"))] fn client_secret_value(value: &serde_json::Value) -> Option<&str> { value .get("value") @@ -159,76 +169,9 @@ fn client_secret_value(value: &serde_json::Value) -> Option<&str> { #[cfg(test)] mod tests { - use super::openai_realtime_configured; - #[cfg(not(feature = "no-voice-dictation"))] - use super::parse_client_secret; - use crate::{ - commands::runtime_config::{default_runtime_config, RuntimeConfig, RuntimeKgooseConfig}, - services::distro_bundle::{DistroBundleState, KgooseDistroConfig}, - test_support::env_lock, - }; - #[cfg(not(feature = "no-voice-dictation"))] + use super::{parse_client_secret, realtime_client_secret_request}; use serde_json::json; - use std::env; - #[test] - fn status_is_unconfigured_without_explicit_kgoose_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let runtime_config = default_runtime_config(); - - assert!(!openai_realtime_configured( - &runtime_config, - &DistroBundleState::empty_for_tests(), - )); - } - - #[test] - fn status_tracks_explicit_runtime_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let mut runtime_config = default_runtime_config(); - runtime_config.kgoose = Some(RuntimeKgooseConfig { - base_url: Some("https://kgoose.example.test/".to_string()), - path: None, - }); - - assert_eq!( - openai_realtime_configured(&runtime_config, &DistroBundleState::empty_for_tests(),), - cfg!(not(feature = "no-voice-dictation")), - ); - } - - #[test] - fn status_tracks_explicit_distro_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let runtime_config: RuntimeConfig = default_runtime_config(); - let distro_state = DistroBundleState::with_kgoose_for_tests(KgooseDistroConfig { - base_url: Some("https://kgoose.example.test/".to_string()), - path: None, - }); - - assert_eq!( - openai_realtime_configured(&runtime_config, &distro_state), - cfg!(not(feature = "no-voice-dictation")), - ); - } - - #[test] - fn status_tracks_explicit_environment_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::set_var("KGOOSE_BASE_URL", "https://kgoose.example.test/"); - let runtime_config = default_runtime_config(); - - assert_eq!( - openai_realtime_configured(&runtime_config, &DistroBundleState::empty_for_tests(),), - cfg!(not(feature = "no-voice-dictation")), - ); - env::remove_var("KGOOSE_BASE_URL"); - } - - #[cfg(not(feature = "no-voice-dictation"))] #[test] fn parses_supported_client_secret_shapes() { assert_eq!( @@ -249,9 +192,47 @@ mod tests { ); } - #[cfg(not(feature = "no-voice-dictation"))] #[test] fn rejects_missing_client_secret() { assert!(parse_client_secret(&json!({ "ok": true })).is_err()); } + + #[test] + fn client_secret_request_uses_only_the_standard_openai_endpoint() { + let request = realtime_client_secret_request( + &reqwest::Client::new(), + "sk-test-secret", + "gpt-realtime-test", + ) + .build() + .expect("build request"); + + assert_eq!( + request.url().as_str(), + "https://api.openai.com/v1/realtime/client_secrets" + ); + assert_eq!( + request + .headers() + .get(reqwest::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer sk-test-secret") + ); + let body: serde_json::Value = serde_json::from_slice( + request + .body() + .and_then(|body| body.as_bytes()) + .expect("JSON body"), + ) + .expect("parse request body"); + assert_eq!( + body, + json!({ + "session": { + "type": "realtime", + "model": "gpt-realtime-test", + } + }) + ); + } } diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 3e35a4ea4..926d46326 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -8,12 +8,13 @@ const LEGACY_TTS_KEYCHAIN_ACCOUNT: &str = "tts-api-key"; pub(crate) enum OpenAiVoiceCredential { SpeechToText, TextToSpeech, + Realtime, } impl OpenAiVoiceCredential { const fn account(self) -> &'static str { match self { - Self::SpeechToText | Self::TextToSpeech => KEYCHAIN_ACCOUNT, + Self::SpeechToText | Self::TextToSpeech | Self::Realtime => KEYCHAIN_ACCOUNT, } } @@ -25,6 +26,9 @@ impl OpenAiVoiceCredential { Self::TextToSpeech => { "OpenAI text-to-speech is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." } + Self::Realtime => { + "OpenAI Realtime voice is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + } } } } @@ -109,6 +113,7 @@ mod tests { fn speech_services_use_the_shared_voice_keychain_account() { assert_eq!(OpenAiVoiceCredential::SpeechToText.account(), "api-key"); assert_eq!(OpenAiVoiceCredential::TextToSpeech.account(), "api-key"); + assert_eq!(OpenAiVoiceCredential::Realtime.account(), "api-key"); } #[test] diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 43a2ff65b..9ee0d355c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -602,13 +602,11 @@ pub fn run() { commands::model_setup::list_model_setup_status, commands::model_setup::clear_model_setup_status, commands::notifications::show_completion_notification, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::get_openai_realtime_status, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::create_openai_realtime_session, - #[cfg(feature = "block-voice-dictation")] + commands::openai_realtime::create_openai_realtime_voice_session, + commands::openai_realtime::save_openai_realtime_api_key, commands::openai_realtime::claim_voice_dictation_microphone, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::release_voice_dictation_microphone, commands::agent_setup::start_agent_setup, commands::agent_setup::get_agent_setup_status, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index ea701fc71..8a60c7ad8 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -250,6 +250,11 @@ import { } from "@/features/voice-conversation/lib/voiceInputPreference"; import { useVoiceOutputPreference } from "@/features/voice-conversation/lib/voiceOutputPreference"; import { isVoiceSetupReady } from "@/features/voice-conversation/lib/voiceSetupReadiness"; +import { getVoiceConversationMode } from "@/features/voice-conversation/lib/voiceConversationModePreference"; +import { + requestOpenAiRealtimeConversationStart, + stopOpenAiRealtimeConversation, +} from "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation"; import { useProfileCapabilities } from "@/shared/profile/capabilities"; import { getOptimisticArtifactCwd } from "@/shared/artifacts/sessionArtifactLocation"; import { @@ -3384,7 +3389,8 @@ export function AppShell({ const handleGlobalVoiceConversationStart = useCallback( (payload: GlobalComposerExpandPayload): Promise => { if (!capabilities.voiceConversation) return Promise.resolve(false); - if (!globalVoiceReady) { + const realtimeMode = getVoiceConversationMode() === "openai-realtime"; + if (!realtimeMode && !globalVoiceReady) { return new Promise((resolve) => { guardAppNavigation( () => { @@ -3402,7 +3408,11 @@ export function AppShell({ ? projects.find((candidate) => candidate.id === options.projectId) : undefined; const chatOptions = { - activate: false, + // Realtime voice belongs to the chat the user is about to see. Use + // the ordinary optimistic chat lifecycle so the mounted transcript + // owns the same ACP notification stream as a normal Berd session. + // The realtime runtime follows the draft id through promotion. + activate: realtimeMode, reuseExistingDraft: false, executionTarget: options?.executionTarget, reasoningEffort: options?.reasoningEffort, @@ -3411,6 +3421,9 @@ export function AppShell({ }; const createAndStart = async () => { + if (realtimeMode) { + await stopOpenAiRealtimeConversation(); + } const voice = useVoiceConversationStore.getState(); if ( voice.status.lifecycle === "starting" || @@ -3453,7 +3466,8 @@ export function AppShell({ chatState.setDraft(sessionId, payload.text); chatState.setSkillDrafts(sessionId, payload.selectedSkills); chatState.setDraftAttachments(sessionId, options?.attachments ?? []); - requestVoiceConversationStart(sessionId); + if (realtimeMode) requestOpenAiRealtimeConversationStart(session.id); + else requestVoiceConversationStart(sessionId); handleNavigateToSession(sessionId); resetGlobalComposerTransition(); return true; diff --git a/src/features/berdctl/__tests__/commands/commands.test.ts b/src/features/berdctl/__tests__/commands/commands.test.ts index 27565c68c..e6d175da4 100644 --- a/src/features/berdctl/__tests__/commands/commands.test.ts +++ b/src/features/berdctl/__tests__/commands/commands.test.ts @@ -600,6 +600,11 @@ describe("action schemas", () => { const validArgs: Record> = { "sessions.create": { prompt: "hi" }, "sessions.send": { session_id: "s1", prompt: "hi" }, + "sessions.send_to_emissary": { + session_id: "s1", + cursor: 0, + message: "Status update", + }, "sessions.open": { session_id: "s1" }, "sessions.list": {}, "sessions.get": { session_id: "s1" }, diff --git a/src/features/berdctl/commands/impl/sendToEmissarySession.ts b/src/features/berdctl/commands/impl/sendToEmissarySession.ts new file mode 100644 index 000000000..56b2ca56d --- /dev/null +++ b/src/features/berdctl/commands/impl/sendToEmissarySession.ts @@ -0,0 +1,86 @@ +import { z } from "zod/v4"; + +import { CommandError, defineCommand } from "../types"; + +const sendToEmissarySessionSchema = z + .object({ + session_id: z + .string() + .min(1) + .describe("Id of the session that owns the live Realtime emissary."), + message: z + .string() + .trim() + .min(1) + .max(20_000) + .describe("Private coordination message to inject into the emissary."), + cursor: z + .number() + .int() + .min(0) + .max(4_294_967_295) + .describe("Latest direct-message cursor returned by the voice bridge."), + }) + .strict(); + +interface SendToEmissarySessionResult { + session_id: string; + cursor: number; + delivery_status: "sent" | "interrupting" | "queued"; +} + +export const sendToEmissarySessionCommand = defineCommand({ + effect: "update", + visibility: "immediate", + destructive: false, + summary: "Send private guidance to a session's live voice emissary", + description: + "Inject a private coordination message into the OpenAI Realtime voice " + + "emissary owned by an existing Berd session. The emissary receives the " + + "message immediately and starts a response; active speech may be interrupted. " + + "The command fails when the target session has no live Realtime voice conversation.", + helpFooter: `Example: + berdctl session send-to-emissary --session-id --cursor 0 \\ + --message "The build failed because the signing certificate expired." --json + +Result: + {"session_id":"...","cursor":0,"delivery_status":"sent"|"interrupting"|"queued"} + +A send while the pipe is carrying emissary-to-master coordination fails with +reason "pipe_busy" without consuming that pending message. Wait for Berd to +deliver it normally, then retry with the cursor included in that message.`, + schema: sendToEmissarySessionSchema, + execute: async (args): Promise => { + const { getActiveRealtimeEmissary } = await import( + "@/features/voice-conversation/lib/realtimeEmissaryBridge" + ); + const emissary = getActiveRealtimeEmissary(); + if (!emissary || emissary.sessionId !== args.session_id) { + throw new CommandError( + "invalid_args", + `Session "${args.session_id}" has no live OpenAI Realtime voice emissary. Start Realtime voice in that session and retry.`, + ); + } + + const delivery = await emissary.sendMasterMessage( + args.message, + args.cursor, + ); + if (!delivery.accepted) { + throw new CommandError( + "invalid_args", + JSON.stringify({ + reason: delivery.reason, + cursor: delivery.cursor, + unread_peer_messages: delivery.unreadPeerMessages, + }), + ); + } + + return { + session_id: args.session_id, + cursor: delivery.cursor, + delivery_status: delivery.deliveryStatus, + }; + }, +}); diff --git a/src/features/berdctl/commands/registry.ts b/src/features/berdctl/commands/registry.ts index 7e73fdc37..c2444d6ca 100644 --- a/src/features/berdctl/commands/registry.ts +++ b/src/features/berdctl/commands/registry.ts @@ -31,6 +31,7 @@ import { openFeedbackCommand } from "./impl/openFeedback"; import { openSessionCommand } from "./impl/openSession"; import { renameSessionCommand } from "./impl/renameSession"; import { sendSessionCommand } from "./impl/sendSession"; +import { sendToEmissarySessionCommand } from "./impl/sendToEmissarySession"; import { setProjectStartupModeCommand } from "./impl/setProjectStartupMode"; import { submitFeedbackCommand } from "./impl/submitFeedback"; import { commandBridgeTimeoutMs } from "./timeouts"; @@ -57,11 +58,11 @@ export const ALL_TOOL_GROUPS = { description: "Manage the user's chat sessions: create (fire-and-forget, on any " + "installed agent harness), send, open, list, get, rename, move, " + - "move to group, clear project, fork, archive.", + "move to group, clear project, send to a live voice emissary, fork, archive.", cli: { noun: "session", about: - "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to emissary, fork, archive", verbs: { create: "create", send: "send", @@ -72,6 +73,7 @@ export const ALL_TOOL_GROUPS = { move: "move", "move-to-group": "move_to_group", "clear-project": "clear_project", + "send-to-emissary": "send_to_emissary", fork: "fork", archive: "archive", }, @@ -86,6 +88,7 @@ export const ALL_TOOL_GROUPS = { move: moveSessionCommand, move_to_group: moveSessionToGroupCommand, clear_project: clearSessionProjectCommand, + send_to_emissary: sendToEmissarySessionCommand, fork: forkSessionCommand, archive: archiveSessionCommand, }, diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index c9a89b71e..1ce646783 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -2390,6 +2390,41 @@ describe("acpNotificationHandler", () => { }); }); + it("streams an owned local prompt live while session history is loading", async () => { + const sessionId = "loading-with-live-prompt"; + markSessionReplayLoading(sessionId); + claimSessionPrompt(sessionId); + setActiveMessageId(sessionId, "master-live", {}); + + await handleSessionNotification({ + sessionId, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: "Inspecting the workspace" }, + }, + } as never); + await handleSessionNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "I found the answer." }, + }, + } as never); + flushBufferedStreamingUpdatesForSession(sessionId); + + expect(getReplayBuffer(sessionId)).toBeUndefined(); + expect(useChatStore.getState().messagesBySession[sessionId]).toMatchObject([ + { + id: "master-live", + role: "assistant", + content: [ + { type: "thinking", text: "Inspecting the workspace" }, + { type: "text", text: "I found the answer." }, + ], + }, + ]); + }); + it("replay preserves timestamps from goose metadata on user and assistant chunks", async () => { const replaySessionId = "replay-timestamp-session"; const userCreated = 1_700_000_000; diff --git a/src/features/chat/acp/acpNotificationHandler.ts b/src/features/chat/acp/acpNotificationHandler.ts index 8474e1619..e2da1f650 100644 --- a/src/features/chat/acp/acpNotificationHandler.ts +++ b/src/features/chat/acp/acpNotificationHandler.ts @@ -60,6 +60,7 @@ import { getSubagentToolCallContext, resolveSubagentContext, } from "@/features/chat/lib/subagentToolCalls"; +import { getSessionPromptOwner } from "@/features/chat/lib/sessionPromptOwnership"; import { applyChatSessionConfigOptionsSnapshot } from "./sessionConfigSnapshotAdapter"; import { perfLog } from "@/shared/lib/perfLog"; import { @@ -230,8 +231,14 @@ export async function handleSessionNotification( ): Promise { const sessionId = notification.sessionId; const { update } = notification; - const isReplay = useChatStore.getState().loadingSessionIds.has(sessionId); - + // A newly-created session can still be hydrating when its first local prompt + // starts. Notifications from that owned prompt are live output, even while + // the history-loading flag is set. Routing them into the replay buffer loses + // them when replay has already taken its snapshot (realtime voice makes this + // race easy to hit because it can dispatch immediately after session start). + const isReplay = + useChatStore.getState().loadingSessionIds.has(sessionId) && + getSessionPromptOwner(sessionId) === null; if (isReplay) { const sid = sessionId.slice(0, 8); let perf = replayPerf.get(sessionId); diff --git a/src/features/chat/capabilities/ConversationComposerCapability.test.tsx b/src/features/chat/capabilities/ConversationComposerCapability.test.tsx index 78c1e080f..315a6a8d0 100644 --- a/src/features/chat/capabilities/ConversationComposerCapability.test.tsx +++ b/src/features/chat/capabilities/ConversationComposerCapability.test.tsx @@ -151,6 +151,42 @@ function createBinding( describe("ConversationComposerCapability surface parity", () => { beforeEach(() => chatInputSpy.mockClear()); + it("notifies an active voice frontend after typed sends commit", async () => { + const controller = createController(); + const notify = vi.fn(); + controller.handleSend.mockImplementation( + (_text, _personaId, _attachments, options) => { + options?.onUserMessageCommitted?.(); + return true; + }, + ); + controller.steerDraftMessage.mockImplementation( + async (_text, _personaId, _attachments, options) => { + options?.onUserMessageCommitted?.(); + return true; + }, + ); + render( + , + ); + + const actions = latestProps().composerActions; + expect(actions.onSend("typed message")).toBe(true); + expect(notify).toHaveBeenCalledWith("typed message"); + + await actions.onSteerMessage?.("typed steer"); + expect(notify).toHaveBeenCalledWith("typed steer"); + }); + it("preserves Home deferred-queue policy while sharing draft and selection behavior", () => { const controller = createController(); render( diff --git a/src/features/chat/capabilities/ConversationComposerCapability.tsx b/src/features/chat/capabilities/ConversationComposerCapability.tsx index cadeb606a..35bc41022 100644 --- a/src/features/chat/capabilities/ConversationComposerCapability.tsx +++ b/src/features/chat/capabilities/ConversationComposerCapability.tsx @@ -143,6 +143,7 @@ interface ConversationComposerCapabilityProps { onRecallLastUserMessage?: () => string | null; attachmentDropTargetRef?: RefObject; onAttachmentDragOverChange?: (isDragOver: boolean) => void; + onUserTextCommitted?: (text: string) => void; } export function ConversationComposerCapability({ @@ -152,6 +153,7 @@ export function ConversationComposerCapability({ onRecallLastUserMessage, attachmentDropTargetRef, onAttachmentDragOverChange, + onUserTextCommitted, }: ConversationComposerCapabilityProps) { const { t } = useTranslation("chat"); const { @@ -179,6 +181,19 @@ export function ConversationComposerCapability({ : undefined; const isReadOnly = Boolean(readOnlyReason); const lifecycle = renderingPolicy.lifecycleConstraints; + const sendWithCommitNotification = useCallback( + (...args: Parameters) => { + const [text, personaId, attachments, options] = args; + return onSend(text, personaId, attachments, { + ...options, + onUserMessageCommitted: () => { + options?.onUserMessageCommitted?.(); + onUserTextCommitted?.(text); + }, + }); + }, + [onSend, onUserTextCommitted], + ); const securityConfirmationPending = target.kind === "existingSession" && target.admission.securityConfirmationPending; @@ -260,7 +275,7 @@ export function ConversationComposerCapability({ isPendingConversation ? undefined : controller.selectedProvider } composerActions={{ - onSend, + onSend: onUserTextCommitted ? sendWithCommitNotification : onSend, onSteerMessage: isPendingConversation || admissionBlocked ? undefined @@ -269,7 +284,15 @@ export function ConversationComposerCapability({ text, personaId ?? undefined, attachments, - options, + onUserTextCommitted + ? { + ...options, + onUserMessageCommitted: () => { + options?.onUserMessageCommitted?.(); + onUserTextCommitted(text); + }, + } + : options, ), canSteerMessage: isPendingConversation || admissionBlocked diff --git a/src/features/chat/hooks/useChat.ts b/src/features/chat/hooks/useChat.ts index aee7ea014..93a2fabe5 100644 --- a/src/features/chat/hooks/useChat.ts +++ b/src/features/chat/hooks/useChat.ts @@ -207,6 +207,7 @@ export function useChat( attachments, assistantPrompt: sendOptions?.assistantPrompt, displayText: sendOptions?.displayText, + userMessageId: sendOptions?.userMessageId, chips: sendOptions?.chips, userMessageMetadata: sendOptions?.userMessageMetadata, acpGooseMetadata: sendOptions?.acpGooseMetadata, diff --git a/src/features/chat/lib/__tests__/steerCore.test.ts b/src/features/chat/lib/__tests__/steerCore.test.ts index 8f5602679..1f710891a 100644 --- a/src/features/chat/lib/__tests__/steerCore.test.ts +++ b/src/features/chat/lib/__tests__/steerCore.test.ts @@ -16,6 +16,7 @@ vi.mock("@/shared/i18n", () => ({ })); import { steerPromptInSession } from "../steerCore"; +import { VOICE_CONVERSATION_EMPTY_RESPONSE } from "../voiceConversationNoop"; function oversizedImageDraft() { return { @@ -178,3 +179,79 @@ describe("steerPromptInSession commit callback", () => { expect(onUserMessageCommitted).toHaveBeenCalledTimes(1); }); }); + +describe("steerPromptInSession voice no-op", () => { + beforeEach(() => { + vi.clearAllMocks(); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + activeSessionId: null, + isConnected: true, + }); + }); + + it("preserves a provisional voice transcript's original ordering timestamp", async () => { + useChatStore.getState().addMessage("session-1", { + id: "voice-user", + role: "user", + created: 100, + content: [{ type: "text", text: "provisional" }], + metadata: { origin: "voice_conversation" }, + }); + mockAcpSteerMessage.mockResolvedValue({ + runId: "run-1", + messageId: "voice-user", + }); + + const accepted = await steerPromptInSession( + "session-1", + "final transcript", + undefined, + { + displayText: "final transcript", + userMessageId: "voice-user", + userMessageMetadata: { origin: "voice_conversation" }, + }, + { throwOnError: true }, + ); + + expect(accepted).toBe(true); + expect( + useChatStore + .getState() + .messagesBySession["session-1"]?.find( + (message) => message.id === "voice-user", + ), + ).toMatchObject({ created: 100 }); + }); + + it("keeps the transcript and suppresses the known empty master response", async () => { + mockAcpSteerMessage.mockRejectedValue( + new Error(VOICE_CONVERSATION_EMPTY_RESPONSE), + ); + + const accepted = await steerPromptInSession( + "session-1", + "[Voice transcript] User said: Nice weather today.", + undefined, + { + displayText: "User said: Nice weather today.", + userMessageMetadata: { origin: "voice_conversation" }, + }, + { throwOnError: true }, + ); + + expect(accepted).toBe(true); + const messages = + useChatStore.getState().messagesBySession["session-1"] ?? []; + expect(messages).toHaveLength(1); + expect(messages[0]).toMatchObject({ + role: "user", + metadata: { origin: "voice_conversation", delivery: "steer" }, + }); + expect(messages[0].content[0]).toMatchObject({ + text: "User said: Nice weather today.", + }); + }); +}); diff --git a/src/features/chat/lib/sendCore.test.ts b/src/features/chat/lib/sendCore.test.ts index cfb7ddd38..ed4c663cd 100644 --- a/src/features/chat/lib/sendCore.test.ts +++ b/src/features/chat/lib/sendCore.test.ts @@ -4,6 +4,7 @@ import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; import type { SessionChatRuntime } from "@/shared/types/chat"; import { QueuedMessageOwnershipLostError } from "./preCommitSendRejection"; import { dispatchPrompt } from "./sendCore"; +import { registerRealtimeEmissary } from "@/features/voice-conversation/lib/realtimeEmissaryBridge"; const mocks = vi.hoisted(() => ({ acpSendMessage: vi.fn(), @@ -117,3 +118,338 @@ describe("dispatchPrompt pre-commit rejection", () => { ); }); }); + +describe("dispatchPrompt voice conversation no-op", () => { + const emptyResponseError = + "The model returned an empty response. Please resend your message to continue."; + + beforeEach(() => { + vi.clearAllMocks(); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + queuedMessageBySession: {}, + draftsBySession: {}, + activeSessionId: null, + isConnected: false, + }); + }); + + function rejectCommittedPrompt(message: string): void { + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { onPromptDispatching(): void }, + ) => { + options.onPromptDispatching(); + const store = useChatStore.getState(); + store.addMessage(sessionId, { + id: "empty-assistant", + role: "assistant", + created: Date.now(), + content: [], + metadata: { completionStatus: "inProgress" }, + }); + store.setStreamingMessageId(sessionId, "empty-assistant"); + return Promise.reject(new Error(message)); + }, + ); + } + + it("preserves a provisional voice transcript's original ordering timestamp", async () => { + useChatStore.getState().addMessage("session-1", { + id: "voice-user", + role: "user", + created: 100, + content: [{ type: "text", text: "provisional" }], + metadata: { origin: "voice_conversation" }, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + _sessionId: string, + _prompt: string, + options: { onPromptDispatching(): void }, + ) => { + options.onPromptDispatching(); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "final transcript", { + displayText: "final transcript", + userMessageId: "voice-user", + userMessageMetadata: { origin: "voice_conversation" }, + }); + + expect( + useChatStore + .getState() + .messagesBySession["session-1"]?.find( + (message) => message.id === "voice-user", + ), + ).toMatchObject({ created: 100 }); + }); + + it("treats a committed voice empty response as a clean semantic no-op", async () => { + rejectCommittedPrompt(emptyResponseError); + + await expect( + dispatchPrompt("session-1", "Emissary said: Hello", { + userMessageMetadata: { origin: "voice_conversation" }, + }), + ).resolves.toBeUndefined(); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages).toHaveLength(2); + expect(messages[0]).toMatchObject({ + role: "user", + metadata: { origin: "voice_conversation" }, + }); + expect(messages[1]).toMatchObject({ + id: "empty-assistant", + role: "assistant", + metadata: { completionStatus: "completed" }, + }); + expect(messages.some((message) => message.role === "system")).toBe(false); + + const runtime = useChatStore.getState().getSessionRuntime("session-1"); + expect(runtime.chatState).toBe("idle"); + expect(runtime.error).toBeNull(); + expect(runtime.streamingMessageId).toBeNull(); + expect(runtime.pendingAssistantProviderId).toBeNull(); + }); + + it("does not suppress a different error for a voice turn", async () => { + rejectCommittedPrompt("Provider authentication failed"); + + await expect( + dispatchPrompt("session-1", "User said: Hello", { + userMessageMetadata: { origin: "voice_conversation" }, + }), + ).rejects.toThrow("Provider authentication failed"); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages.at(-1)).toMatchObject({ role: "system" }); + expect(useChatStore.getState().getSessionRuntime("session-1").error).toBe( + "Provider authentication failed", + ); + }); + + it("does not suppress the empty-response error for a non-voice turn", async () => { + rejectCommittedPrompt(emptyResponseError); + + await expect(dispatchPrompt("session-1", "Hello", {})).rejects.toThrow( + emptyResponseError, + ); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages.at(-1)).toMatchObject({ role: "system" }); + expect(useChatStore.getState().getSessionRuntime("session-1").error).toBe( + emptyResponseError, + ); + }); +}); + +describe("dispatchPrompt realtime Master turn lifecycle", () => { + beforeEach(() => { + vi.clearAllMocks(); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + queuedMessageBySession: {}, + draftsBySession: {}, + activeSessionId: null, + isConnected: false, + }); + }); + + it("publishes the normal final Master text at the terminal prompt boundary", async () => { + const beginMasterTurn = vi.fn(); + const endMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + beginMasterTurn, + endMasterTurn, + sendMasterMessage: vi.fn(), + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore.getState().addMessage(sessionId, { + id: "master-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "There are 20 repositories." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Count repositories", {}); + + expect(beginMasterTurn).toHaveBeenCalledOnce(); + const turnId = beginMasterTurn.mock.calls[0]?.[0]; + expect(turnId).toEqual(expect.any(String)); + expect(endMasterTurn).toHaveBeenCalledWith({ + turnId, + status: "completed", + finalText: "There are 20 repositories.", + }); + release(); + }); + + it("publishes final text appended to a reused streaming assistant row", async () => { + const beginMasterTurn = vi.fn(); + const endMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + beginMasterTurn, + endMasterTurn, + sendMasterMessage: vi.fn(), + }); + useChatStore.getState().addMessage("session-1", { + id: "reused-stream", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "" }], + metadata: { agentVisible: true, userVisible: true }, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore + .getState() + .updateMessage(sessionId, "reused-stream", (message) => ({ + ...message, + content: [{ type: "text", text: "There are 20 repositories." }], + })); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Count repositories", {}); + + expect(endMasterTurn).toHaveBeenCalledWith({ + turnId: expect.any(String), + status: "completed", + finalText: "There are 20 repositories.", + }); + release(); + }); + + it("includes a final Master notification delivered just after prompt resolution", async () => { + const endMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + beginMasterTurn: vi.fn(), + endMasterTurn, + sendMasterMessage: vi.fn(), + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + window.setTimeout(() => { + useChatStore.getState().addMessage(sessionId, { + id: "late-master-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "The late final answer." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + }, 0); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Check the answer", {}); + + expect(endMasterTurn).toHaveBeenCalledWith({ + turnId: expect.any(String), + status: "completed", + finalText: "The late final answer.", + }); + release(); + }); + + it("does not forward the backend empty-response placeholder as Master output", async () => { + const beginMasterTurn = vi.fn(); + const endMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + beginMasterTurn, + endMasterTurn, + sendMasterMessage: vi.fn(), + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore.getState().addMessage(sessionId, { + id: "master-empty-fallback", + role: "assistant", + created: Date.now(), + content: [ + { + type: "text", + text: "The model returned an empty response. Please resend your message to continue.", + }, + ], + metadata: { agentVisible: true, userVisible: true }, + }); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "[Voice transcript] User said: hello", { + userMessageMetadata: { origin: "voice_conversation" }, + }); + + expect(endMasterTurn).toHaveBeenCalledWith({ + turnId: expect.any(String), + status: "completed", + finalText: undefined, + }); + release(); + }); +}); diff --git a/src/features/chat/lib/sendCore.ts b/src/features/chat/lib/sendCore.ts index a490b916d..1cd678df3 100644 --- a/src/features/chat/lib/sendCore.ts +++ b/src/features/chat/lib/sendCore.ts @@ -33,6 +33,11 @@ import { } from "@/features/chat/lib/sessionPromptOwnership"; import { perfLog } from "@/shared/lib/perfLog"; import { completeAssistantMessage } from "@/features/chat/lib/messageCompletion"; +import { isVoiceConversationEmptyResponse } from "@/features/chat/lib/voiceConversationNoop"; +import { + beginActiveRealtimeMasterTurn, + endActiveRealtimeMasterTurn, +} from "@/features/voice-conversation/lib/realtimeEmissaryBridge"; import { type ChatAttachmentDraft, type MessageMetadata, @@ -55,6 +60,8 @@ export interface SendCoreOptions { assistantPrompt?: string; /** Text shown in the transcript when it differs from the prompt sent. */ displayText?: string; + /** Reuses a provisional local transcript row when the send commits. */ + userMessageId?: string; /** User-visible chips stored on the user message's metadata. */ chips?: MessageChip[]; /** Extra renderer-only metadata to stamp on the local user message. */ @@ -98,6 +105,58 @@ function throwIfAborted(signal?: AbortSignal): void { throw new DOMException("The operation was aborted.", "AbortError"); } +function finalMasterTextSince( + sessionId: string, + existingAssistantTextById: ReadonlyMap, +): string | undefined { + const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if ( + !message || + message.role !== "assistant" || + message.metadata?.origin === "voice_conversation" || + message.metadata?.agentVisible === false + ) { + continue; + } + const text = message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(); + if (isVoiceConversationEmptyResponse(text)) continue; + if (text && existingAssistantTextById.get(message.id) !== text) return text; + } + return undefined; +} + +function assistantTextSnapshot(sessionId: string): ReadonlyMap { + const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + return new Map( + messages + .filter( + (message) => + message.role === "assistant" && + message.metadata?.origin !== "voice_conversation" && + message.metadata?.agentVisible !== false, + ) + .map((message) => [ + message.id, + message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(), + ]), + ); +} + +async function settleMasterTranscriptNotifications(): Promise { + // ACP may resolve session/prompt immediately before dispatching the final + // session/update already read from the same transport. Yield one macrotask + // so the terminal Master notification sees that last visible text block. + await new Promise((resolve) => window.setTimeout(resolve, 0)); +} + type AssistantPromptOutcome = "completed" | "error"; interface AssistantCancellationRace { @@ -211,6 +270,7 @@ export async function dispatchPrompt( signal, systemPrompt, userMessageMetadata, + userMessageId, } = opts; const sessionRunsRemotely = Boolean( useChatSessionStore.getState().getSession(sessionId)?.remoteHost, @@ -232,6 +292,21 @@ export async function dispatchPrompt( } const promptOwner = claimSessionPrompt(sessionId); + const realtimeMasterTurnId = crypto.randomUUID(); + const assistantTextBeforeTurn = assistantTextSnapshot(sessionId); + let realtimeMasterTurnStarted = false; + let realtimeMasterTurnEnded = false; + const endRealtimeMasterTurn = ( + status: "completed" | "cancelled" | "failed", + ) => { + if (!realtimeMasterTurnStarted || realtimeMasterTurnEnded) return; + realtimeMasterTurnEnded = true; + endActiveRealtimeMasterTurn(sessionId, { + turnId: realtimeMasterTurnId, + status, + finalText: finalMasterTextSince(sessionId, assistantTextBeforeTurn), + }); + }; const isCurrent = () => ownsSessionPrompt(sessionId, promptOwner); let userMessageCommitted = false; let preCommitRejected = false; @@ -245,6 +320,29 @@ export async function dispatchPrompt( setPendingAssistantProvider(sessionId, pendingAssistantProvider); clearLiveSubtitleUpdate(sessionId); + const finishPromptSuccessfully = () => { + const cancellationRace = assistantCancellationRaces.get(promptOwner); + if (cancellationRace) { + flushBufferedStreamingUpdatesForSession(sessionId, { + flushSubtitle: true, + owner: promptOwner, + }); + recordAssistantPromptOutcome(promptOwner, "completed"); + } else if (isCurrent()) { + const ownedStreamingMessageId = useChatStore + .getState() + .getSessionRuntime(sessionId).streamingMessageId; + flushBufferedStreamingUpdatesForSession(sessionId, { + flushSubtitle: true, + owner: promptOwner, + }); + completeAssistantMessageById(sessionId, ownedStreamingMessageId); + if (isCurrent()) { + setChatState(sessionId, "idle"); + } + } + }; + try { // Preparation can be superseded or aborted. Complete it before committing // local transcript state so a retained queued record can retry without @@ -261,6 +359,7 @@ export async function dispatchPrompt( buildMessageAttachments(dispatchAttachments), chips, ); + if (userMessageId) userMessage.id = userMessageId; if (persona) { userMessage.metadata = { ...userMessage.metadata, @@ -284,7 +383,23 @@ export async function dispatchPrompt( }); } } - addMessage(sessionId, userMessage); + const provisionalMessage = userMessageId + ? useChatStore + .getState() + .messagesBySession[sessionId]?.some( + (message) => message.id === userMessageId, + ) + : false; + if (provisionalMessage) { + useChatStore + .getState() + .updateMessage(sessionId, userMessage.id, (existing) => ({ + ...userMessage, + created: existing.created, + })); + } else { + addMessage(sessionId, userMessage); + } userMessageCommitted = true; setChatState(sessionId, "thinking"); setError(sessionId, null); @@ -329,7 +444,13 @@ export async function dispatchPrompt( (img) => [img.base64, img.mimeType] as [string, string], ), onPromptDispatching: commitUserMessage, - onPromptDispatched, + onPromptDispatched: () => { + realtimeMasterTurnStarted = beginActiveRealtimeMasterTurn( + sessionId, + realtimeMasterTurnId, + ); + onPromptDispatched?.(); + }, }); await promptPromise; if (!background) { @@ -338,27 +459,29 @@ export async function dispatchPrompt( ); } - const cancellationRace = assistantCancellationRaces.get(promptOwner); - if (cancellationRace) { - flushBufferedStreamingUpdatesForSession(sessionId, { - flushSubtitle: true, - owner: promptOwner, - }); - recordAssistantPromptOutcome(promptOwner, "completed"); - } else if (isCurrent()) { - const ownedStreamingMessageId = useChatStore - .getState() - .getSessionRuntime(sessionId).streamingMessageId; - flushBufferedStreamingUpdatesForSession(sessionId, { - flushSubtitle: true, - owner: promptOwner, - }); - completeAssistantMessageById(sessionId, ownedStreamingMessageId); + finishPromptSuccessfully(); + await settleMasterTranscriptNotifications(); + endRealtimeMasterTurn("completed"); + } catch (err) { + const isVoiceConversationNoop = + userMessageCommitted && + userMessageMetadata?.origin === "voice_conversation" && + isVoiceConversationEmptyResponse(formatAcpErrorMessage(err)); + if (isVoiceConversationNoop) { + finishPromptSuccessfully(); + await settleMasterTranscriptNotifications(); + endRealtimeMasterTurn("completed"); if (isCurrent()) { - setChatState(sessionId, "idle"); + setError(sessionId, null); + setPendingAssistantProvider(sessionId, null); } + return; } - } catch (err) { + endRealtimeMasterTurn( + err instanceof DOMException && err.name === "AbortError" + ? "cancelled" + : "failed", + ); preCommitRejected = err instanceof PreCommitSendRejectedError; if (!preCommitRejected) { const cancellationRace = assistantCancellationRaces.get(promptOwner); diff --git a/src/features/chat/lib/steerCore.ts b/src/features/chat/lib/steerCore.ts index e4de2198c..b471a9a92 100644 --- a/src/features/chat/lib/steerCore.ts +++ b/src/features/chat/lib/steerCore.ts @@ -22,6 +22,7 @@ import { } from "./attachments"; import { isSessionRunning } from "./sessionActivity"; import { getSessionPromptOwner } from "./sessionPromptOwnership"; +import { isVoiceConversationEmptyResponse } from "./voiceConversationNoop"; import { i18n } from "@/shared/i18n"; function formatSteerErrorMessage(error: unknown): string { @@ -80,6 +81,7 @@ export async function steerPromptInSession( buildMessageAttachments(dispatchAttachments), sendOptions?.chips, ); + if (sendOptions?.userMessageId) userMessage.id = sendOptions.userMessageId; userMessage.metadata = { ...userMessage.metadata, ...sendOptions?.userMessageMetadata, @@ -103,7 +105,19 @@ export async function steerPromptInSession( ); const acpPrompt = promptWithPaths || (images?.length ? " " : promptWithPaths); const chatStore = useChatStore.getState(); - chatStore.addMessage(sessionId, userMessage); + if ( + sendOptions?.userMessageId && + chatStore.messagesBySession[sessionId]?.some( + (message) => message.id === sendOptions.userMessageId, + ) + ) { + chatStore.updateMessage(sessionId, userMessage.id, (existing) => ({ + ...userMessage, + created: existing.created, + })); + } else { + chatStore.addMessage(sessionId, userMessage); + } chatStore.setPendingInterventionBoundary(sessionId, { interventionMessageId: userMessage.id, }); @@ -175,13 +189,29 @@ export async function steerPromptInSession( }); } catch (err) { const liveStore = useChatStore.getState(); + const errorMessage = formatSteerErrorMessage(err); const liveMessage = liveStore.messagesBySession[sessionId]?.find( (message) => message.id === userMessage.id || message.metadata?.steeringRequestId === userMessage.id, ); const deliveryWasEstablished = liveMessage?.metadata?.delivery === "steer"; - if (!deliveryWasEstablished) { + const emptyVoiceTurnIsNoop = + sendOptions?.userMessageMetadata?.origin === "voice_conversation" && + isVoiceConversationEmptyResponse(errorMessage); + if (emptyVoiceTurnIsNoop) { + const liveMessageId = liveMessage?.id ?? userMessage.id; + liveStore.updateMessage(sessionId, liveMessageId, (message) => ({ + ...message, + metadata: { ...message.metadata, delivery: "steer" }, + })); + if ( + liveStore.getSessionRuntime(sessionId).pendingInterventionBoundary + ?.interventionMessageId === liveMessageId + ) { + liveStore.setPendingInterventionBoundary(sessionId, null); + } + } else if (!deliveryWasEstablished) { const liveMessageId = liveMessage?.id ?? userMessage.id; liveStore.removeMessage(sessionId, liveMessageId); if ( @@ -190,7 +220,6 @@ export async function steerPromptInSession( ) { liveStore.setPendingInterventionBoundary(sessionId, null); } - const errorMessage = formatSteerErrorMessage(err); liveStore.addMessage( sessionId, createSystemNotificationMessage(errorMessage, "error"), diff --git a/src/features/chat/lib/voiceConversationNoop.ts b/src/features/chat/lib/voiceConversationNoop.ts new file mode 100644 index 000000000..b7be4d873 --- /dev/null +++ b/src/features/chat/lib/voiceConversationNoop.ts @@ -0,0 +1,11 @@ +export const VOICE_CONVERSATION_EMPTY_RESPONSE = + "The model returned an empty response. Please resend your message to continue."; + +const VOICE_CONVERSATION_EMPTY_RESPONSES = new Set([ + VOICE_CONVERSATION_EMPTY_RESPONSE, + "Le modèle a renvoyé une réponse vide. Veuillez renvoyer votre message pour continuer.", +]); + +export function isVoiceConversationEmptyResponse(text: string): boolean { + return VOICE_CONVERSATION_EMPTY_RESPONSES.has(text.trim()); +} diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.test.ts b/src/features/chat/transcript/projection/buildTranscriptItems.test.ts new file mode 100644 index 000000000..30cce59e4 --- /dev/null +++ b/src/features/chat/transcript/projection/buildTranscriptItems.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import type { Message } from "@/shared/types/messages"; +import { VOICE_CONVERSATION_EMPTY_RESPONSE } from "@/features/chat/lib/voiceConversationNoop"; +import { getVisibleTranscriptMessages } from "./buildTranscriptItems"; + +function message( + id: string, + role: Message["role"], + text: string, + origin?: "voice_conversation", +): Message { + return { + id, + role, + created: 1, + content: [{ type: "text", text }], + metadata: origin ? { origin } : undefined, + }; +} + +describe("getVisibleTranscriptMessages voice no-op", () => { + it("hides the backend empty-response fallback after a voice turn", () => { + const voice = message( + "voice", + "user", + "Emissary said: Hello", + "voice_conversation", + ); + const fallback = message( + "fallback", + "assistant", + VOICE_CONVERSATION_EMPTY_RESPONSE, + ); + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("hides an empty-response system notification after a voice turn", () => { + const voice = message( + "voice", + "user", + "Emissary said: Hello", + "voice_conversation", + ); + const fallback: Message = { + id: "fallback", + role: "system", + created: 1, + content: [ + { + type: "systemNotification", + notificationType: "error", + text: VOICE_CONVERSATION_EMPTY_RESPONSE, + }, + ], + }; + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("hides replayed and localized empty-response fallbacks after a voice transcript", () => { + const voice = message( + "voice", + "user", + "[Voice transcript] Emissary said: Bonjour", + ); + const fallback = message( + "fallback", + "assistant", + "Le modèle a renvoyé une réponse vide. Veuillez renvoyer votre message pour continuer.", + ); + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("keeps the same fallback visible after a normal chat turn", () => { + const user = message("user", "user", "Hello"); + const fallback = message( + "fallback", + "assistant", + VOICE_CONVERSATION_EMPTY_RESPONSE, + ); + + expect(getVisibleTranscriptMessages([user, fallback])).toEqual([ + user, + fallback, + ]); + }); + + it("keeps real assistant errors visible after a voice turn", () => { + const voice = message( + "voice", + "user", + "User said: Hello", + "voice_conversation", + ); + const error = message("error", "assistant", "Authentication failed"); + + expect(getVisibleTranscriptMessages([voice, error])).toEqual([ + voice, + error, + ]); + }); +}); diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.ts b/src/features/chat/transcript/projection/buildTranscriptItems.ts index ac0a43b76..9266afa8d 100644 --- a/src/features/chat/transcript/projection/buildTranscriptItems.ts +++ b/src/features/chat/transcript/projection/buildTranscriptItems.ts @@ -1,11 +1,13 @@ -import type { - Message, - MessageContent, - MessageMetadata, - ReasoningContent, - TextContent, - ThinkingContent, +import { + getTextContent, + type Message, + type MessageContent, + type MessageMetadata, + type ReasoningContent, + type TextContent, + type ThinkingContent, } from "@/shared/types/messages"; +import { isVoiceConversationEmptyResponse } from "@/features/chat/lib/voiceConversationNoop"; import { classifyTranscriptMeasurementPolicy, type TranscriptMeasurementPolicyDecision, @@ -125,11 +127,7 @@ export function buildTranscriptItems({ // of the assistant's work turn, so they should not reset this set. let displayedReasoningSignatures = new Set(); - for (const message of messages) { - if (!isVisibleTranscriptMessage(message)) { - continue; - } - + for (const message of getVisibleTranscriptMessages(messages)) { const visibleContent = expandReasoningContentSections( getUserVisibleMessageContent(message.content), ); @@ -1764,7 +1762,34 @@ function getAssistantFragmentChromeEstimate( export function getVisibleTranscriptMessages( messages: readonly Message[], ): readonly Message[] { - return messages.filter(isVisibleTranscriptMessage); + return messages.filter((message, index) => { + if (!isVisibleTranscriptMessage(message)) return false; + const isEmptyResponseFallback = + (message.role === "assistant" && + isVoiceConversationEmptyResponse(getTextContent(message))) || + message.content.some( + (content) => + content.type === "systemNotification" && + isVoiceConversationEmptyResponse(content.text), + ); + if (!isEmptyResponseFallback) { + return true; + } + + for (let prior = index - 1; prior >= 0; prior -= 1) { + const priorMessage = messages[prior]; + if (priorMessage?.role !== "user") continue; + return !isVoiceConversationUserTurn(priorMessage); + } + return true; + }); +} + +function isVoiceConversationUserTurn(message: Message): boolean { + return ( + message.metadata?.origin === "voice_conversation" || + getTextContent(message).trimStart().startsWith("[Voice transcript] ") + ); } function isVisibleTranscriptMessage(message: Message): boolean { diff --git a/src/features/chat/types.ts b/src/features/chat/types.ts index 4313411a1..140b37109 100644 --- a/src/features/chat/types.ts +++ b/src/features/chat/types.ts @@ -69,6 +69,8 @@ export interface ChatSendOptions { /** Persona-only prompt captured while workspace context is still loading. */ capturedPersonaSystemPrompt?: string; displayText?: string; + /** Reuses a provisional local transcript row when the send commits. */ + userMessageId?: string; assistantPrompt?: string; chips?: MessageChip[]; userMessageMetadata?: Partial; @@ -101,6 +103,8 @@ export interface ChatInputVoiceConversation { disabled?: boolean; onToggle: () => void | Promise; onMicrophoneMuteToggle: () => void | Promise; + /** Mirrors a committed typed user turn into an active voice frontend. */ + onTypedUserMessageCommitted?: (text: string) => void; } export interface ChatInputComposerActions { diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index 8df4f6359..ddafdb60b 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -65,6 +65,7 @@ import { import type { TranscriptSearchBackend } from "@/features/chat/lib/transcriptSearchBackend"; import type { GlobalComposerHandoffRect } from "@/shared/ui/GlobalComposerPill"; import { useVoiceConversationController } from "@/features/voice-conversation/hooks/useVoiceConversationController"; +import { useOpenAiRealtimeConversation } from "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; @@ -75,6 +76,7 @@ import { } from "@/features/voice-conversation/lib/voiceInputPreference"; import { useVoiceOutputPreference } from "@/features/voice-conversation/lib/voiceOutputPreference"; import { isVoiceSetupReady } from "@/features/voice-conversation/lib/voiceSetupReadiness"; +import { useVoiceConversationModePreference } from "@/features/voice-conversation/lib/voiceConversationModePreference"; import { useProfileCapabilities } from "@/shared/profile/capabilities"; import { requestOpenSettings } from "@/features/settings/lib/settingsEvents"; import { @@ -242,6 +244,7 @@ export function ChatView({ isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); const voiceOutput = useVoiceOutputPreference(); + const voiceMode = useVoiceConversationModePreference(); const openAiVoiceSetup = useOpenAiVoiceSetup( capabilities.voiceConversation && (voiceInput.backend === "openai" || voiceOutput.backend === "openai"), @@ -269,13 +272,13 @@ export function ChatView({ controller.isLoadingHistory || !controller.workspaceContextReady || controller.queue.queuedMessage !== null); - const voiceConversation = useVoiceConversationController({ + const chainedVoiceConversation = useVoiceConversationController({ sessionId, // Voice delivery only needs to wait for admission. Holding its per-session // queue through the full run would prevent later utterances from steering // the active run. onSend, - enabled: capabilities.voiceConversation, + enabled: capabilities.voiceConversation && voiceMode.mode === "chained", isGooseSession: controller.selectedProvider === "goose", pocketReady: voiceReady, inputBackend: voiceInput.backend, @@ -293,6 +296,20 @@ export function ChatView({ routeUnavailable: voiceAdmissionPermanentlyBlocked, disabled: admissionBlocked || voiceDeliveryTemporarilyBlocked, }); + const realtimeVoiceConversation = useOpenAiRealtimeConversation({ + sessionId, + onSend, + enabled: + capabilities.voiceConversation && + voiceMode.mode === "openai-realtime" && + controller.selectedProvider === "goose", + readOnly: Boolean(readOnlyStatus), + disabled: admissionBlocked || voiceDeliveryTemporarilyBlocked, + }); + const voiceConversation = + voiceMode.mode === "openai-realtime" + ? realtimeVoiceConversation + : chainedVoiceConversation; const isAgentBuilderOpen = agentBuilderOpenForLayout; const patchSession = useChatSessionStore((s) => s.patchSession); const agentBuilderContextState = effectiveSession?.agentBuilderContextState; @@ -703,6 +720,12 @@ export function ChatView({ ({ + appendSessionSystemPrompt: vi.fn(), + claimMicrophone: vi.fn(), + connectPeer: vi.fn(), + createSendToMasterToolOutput: vi.fn(), + createEndTurnToolOutput: vi.fn(), + createPeer: vi.fn(), + createSession: vi.fn(), + registerEmissary: vi.fn(), + activeEmissary: null as null | { + sessionId: string; + beginMasterTurn(turnId: string): void; + endMasterTurn(completion: { + turnId: string; + status: "completed" | "cancelled" | "failed"; + finalText?: string; + }): void; + sendMasterMessage(message: string, cursor: number): Promise; + }, + releaseBridge: vi.fn(), + releaseMicrophone: vi.fn(), + sendRealtimeEvents: vi.fn(), + steerPrompt: vi.fn(), + requestToolOutput: vi.fn(), + requestMasterMessage: vi.fn(), + requestTypedUserMessage: vi.fn(), +})); + +vi.mock("@/shared/api/acpApi", () => ({ + appendSessionSystemPrompt: mocks.appendSessionSystemPrompt, +})); + +vi.mock("@/shared/api/openaiRealtime", () => ({ + claimVoiceDictationMicrophone: mocks.claimMicrophone, + createOpenAiRealtimeVoiceSession: mocks.createSession, + releaseVoiceDictationMicrophone: mocks.releaseMicrophone, +})); + +vi.mock("@/features/chat/lib/openaiRealtimeAudio", () => ({ + connectOpenAiRealtimePeerConnection: mocks.connectPeer, + createOpenAiRealtimePeerConnection: mocks.createPeer, +})); + +vi.mock("@/features/chat/lib/steerCore", () => ({ + steerPromptInSession: mocks.steerPrompt, +})); + +vi.mock("../lib/realtimeEmissaryBridge", () => ({ + registerRealtimeEmissary: (emissary: typeof mocks.activeEmissary) => { + mocks.activeEmissary = emissary; + return mocks.registerEmissary(); + }, +})); + +vi.mock("../lib/realtimeVoicePreference", () => ({ + getRealtimeVoicePreference: () => ({ + model: "gpt-realtime", + sessionOverridesText: "{}", + speed: 1, + transcriptionModel: "gpt-4o-mini-transcribe", + voice: "marin", + }), + parseRealtimeSessionOverrides: () => ({}), +})); + +vi.mock("../lib/realtimeEmissaryProtocol", () => ({ + configureRealtimeEmissarySession: vi.fn(), + createEndTurnToolOutput: mocks.createEndTurnToolOutput, + createSendToMasterToolOutput: mocks.createSendToMasterToolOutput, + DirectMessagePipe: class { + cursor() { + return 0; + } + send(options: { sender: "master" | "emissary"; message: string }) { + return { + accepted: true, + cursor: 0, + unreadPeerMessages: [], + outbound: { + id: 1, + sender: options.sender, + recipient: options.sender === "master" ? "emissary" : "master", + senderCursor: 0, + message: options.message, + }, + }; + } + }, + REALTIME_MASTER_INSTRUCTIONS: "Master instructions", + RealtimeEmissaryProtocol: class { + handle(event: { type?: string }) { + if (event.type === "test.transcript") + return [ + { + interrupted: false, + itemId: "user-item-1", + speaker: "user", + text: "hello master", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_repository") + return [ + { + interrupted: false, + itemId: "user-item-repository", + speaker: "user", + text: "how many repos are in my development folder?", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_followup") + return [ + { + interrupted: false, + itemId: "user-item-2", + speaker: "user", + text: "are any of them symbolic links?", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_partial") + return [ + { + itemId: "user-item-1", + speaker: "user", + text: "hello", + type: "transcript.updated", + }, + ]; + if (event.type === "test.transcript_corrected") + return [ + { + itemId: "user-item-1", + speaker: "user", + text: "hello master", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary") + return [ + { + interrupted: false, + itemId: "emissary-item-1", + speaker: "emissary", + text: "hello user", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_result") + return [ + { + interrupted: false, + itemId: "emissary-item-2", + speaker: "emissary", + text: "You have 21 repositories.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_followup_ack") + return [ + { + interrupted: false, + itemId: "emissary-item-3", + speaker: "emissary", + text: "I'll verify that.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_symlink_result") + return [ + { + interrupted: false, + itemId: "emissary-item-4", + speaker: "emissary", + text: "None of those repositories are symbolic links.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_interrupted") + return [ + { + interrupted: true, + speaker: "emissary", + text: "partially heard", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.send_to_master") + return [ + { + callId: "call-1", + cursor: 0, + message: "Please inspect the disk.", + type: "send_to_master", + }, + ]; + if (event.type === "test.send_to_master_followup") + return [ + { + callId: "call-2", + cursor: 0, + message: "Please verify whether those repositories are symlinks.", + type: "send_to_master", + }, + ]; + if (event.type === "test.end_turn") + return [{ callId: "call-end", type: "end_turn" }]; + return []; + } + }, + RealtimeResponseCoordinator: class { + handle() { + return []; + } + requestMasterMessage(message: unknown) { + return mocks.requestMasterMessage(message); + } + requestToolOutput(event: unknown) { + return mocks.requestToolOutput(event); + } + requestTypedUserMessage(text: string) { + return mocks.requestTypedUserMessage(text); + } + }, + sendRealtimeEvents: mocks.sendRealtimeEvents, +})); + +class FakeDataChannel extends EventTarget { + readonly close = vi.fn(); + readyState: RTCDataChannelState = "open"; + readonly send = vi.fn(); +} + +class FakePeer extends EventTarget { + readonly addTrack = vi.fn(); + readonly close = vi.fn(); + readonly createDataChannel = vi.fn(); + + constructor(channel: FakeDataChannel) { + super(); + this.createDataChannel.mockReturnValue(channel); + } +} + +class FakeAudio { + autoplay = false; + readonly pause = vi.fn(); + readonly play = vi.fn().mockResolvedValue(undefined); + srcObject: MediaStream | null = null; +} + +const originalAudio = globalThis.Audio; +const originalMediaDevices = navigator.mediaDevices; +let channel: FakeDataChannel; +let peer: FakePeer; +let track: MediaStreamTrack & { stop: ReturnType }; + +function renderConversation(sessionId: string, onSend = vi.fn()) { + return renderHook(() => + useOpenAiRealtimeConversation({ enabled: true, onSend, sessionId }), + ); +} + +describe("createRealtimeTranscriptReplayEvents", () => { + it("reconstructs a compact ordinary transcript without realtime state", () => { + expect( + createRealtimeTranscriptReplayEvents([ + { + id: "u1", + role: "user", + created: 1, + content: [{ type: "text", text: "What is in this folder?" }], + }, + { + id: "progress", + role: "assistant", + created: 2, + content: [{ type: "text", text: "I am checking." }], + metadata: { completionStatus: "completed" }, + }, + { + id: "final", + role: "assistant", + created: 3, + content: [{ type: "text", text: "There are 25 directories." }], + metadata: { completionStatus: "completed" }, + }, + { + id: "coordination", + role: "assistant", + created: 4, + content: [{ type: "text", text: "Private coordination" }], + metadata: { + completionStatus: "completed", + personaName: "Master → Emissary", + }, + }, + { + id: "u2", + role: "user", + created: 5, + content: [{ type: "text", text: "Are any symlinks?" }], + }, + ]), + ).toEqual([ + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "What is in this folder?" }], + }, + }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "There are 25 directories." }], + }, + }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Are any symlinks?" }], + }, + }, + ]); + }); +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.activeEmissary = null; + useChatStore.setState({ messagesBySession: {}, sessionStateById: {} }); + useChatSessionStore.setState({ sessions: [] }); + channel = new FakeDataChannel(); + peer = new FakePeer(channel); + track = { + enabled: true, + stop: vi.fn(), + } as unknown as MediaStreamTrack & { stop: ReturnType }; + const stream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia: vi.fn().mockResolvedValue(stream) }, + }); + globalThis.Audio = FakeAudio as unknown as typeof Audio; + mocks.appendSessionSystemPrompt.mockResolvedValue(undefined); + mocks.claimMicrophone.mockResolvedValue(undefined); + mocks.connectPeer.mockResolvedValue(undefined); + mocks.createSendToMasterToolOutput.mockReturnValue({ + type: "conversation.item.create", + item: { type: "function_call_output" }, + }); + mocks.createEndTurnToolOutput.mockReturnValue({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-end", + output: '{"status":"ended"}', + }, + }); + mocks.createPeer.mockReturnValue(peer); + mocks.createSession.mockResolvedValue({ clientSecret: "test-secret" }); + mocks.registerEmissary.mockReturnValue(mocks.releaseBridge); + mocks.releaseMicrophone.mockResolvedValue(undefined); + mocks.requestToolOutput.mockImplementation((event) => ({ + status: "queued", + events: [event], + })); + mocks.requestMasterMessage.mockImplementation((message) => ({ + status: "sent", + events: [{ type: "conversation.item.create", message }], + })); + mocks.requestTypedUserMessage.mockReturnValue({ + status: "interrupting", + events: [{ type: "response.cancel" }, { type: "conversation.item.create" }], + }); +}); + +afterEach(async () => { + await resetOpenAiRealtimeConversationRuntimeForTests(); + globalThis.Audio = originalAudio; + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: originalMediaDevices, + }); +}); + +describe("useOpenAiRealtimeConversation lifecycle", () => { + it("starts after a newly created session mounts from a deferred call request", async () => { + act(() => requestOpenAiRealtimeConversationStart("session-a")); + const owner = renderConversation("session-a"); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.createSession).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("starts a promoted session from a deferred request for its client id", async () => { + useChatSessionStore.setState({ + sessions: [ + { + id: "backend-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + intent: null, + }, + ], + }); + act(() => requestOpenAiRealtimeConversationStart("draft-session")); + const owner = renderConversation("backend-session"); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.createSession).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("starts on an optimistic draft and defers the master prompt until promotion", async () => { + useChatSessionStore.setState({ + sessions: [ + { + id: "draft-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + creationState: "pending", + intent: null, + }, + ], + }); + mocks.appendSessionSystemPrompt.mockImplementation( + async (sessionId: string) => { + if (sessionId === "draft-session") + throw new Error("Resource not found"); + }, + ); + act(() => requestOpenAiRealtimeConversationStart("draft-session")); + const owner = renderHook( + ({ sessionId }) => + useOpenAiRealtimeConversation({ + enabled: true, + onSend: vi.fn(), + sessionId, + }), + { initialProps: { sessionId: "draft-session" } }, + ); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.appendSessionSystemPrompt).not.toHaveBeenCalledWith( + "draft-session", + expect.anything(), + expect.anything(), + ); + + act(() => { + useChatSessionStore + .getState() + .promoteDraftSession("draft-session", "backend-session"); + useChatStore + .getState() + .promoteSessionId("draft-session", "backend-session"); + owner.rerender({ sessionId: "backend-session" }); + }); + + await waitFor(() => + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining( + 'send-to-emissary --session-id "backend-session"', + ), + ), + ); + expect(owner.result.current.state).toBe("listening"); + expect(owner.result.current.boundSessionId).toBe("backend-session"); + + await act(async () => owner.result.current.onToggle()); + }); + + it("keeps the process-wide conversation alive across owner unmount and remount", async () => { + const originalOnSend = vi.fn().mockResolvedValue(true); + const remountedOnSend = vi.fn().mockResolvedValue(true); + const first = renderConversation("session-a", originalOnSend); + + await act(async () => first.result.current.onToggle()); + await waitFor(() => expect(first.result.current.state).toBe("listening")); + expect(first.result.current.ownsActiveConversation).toBe(true); + + first.unmount(); + + expect(channel.close).not.toHaveBeenCalled(); + expect(peer.close).not.toHaveBeenCalled(); + expect(track.stop).not.toHaveBeenCalled(); + expect(mocks.releaseBridge).not.toHaveBeenCalled(); + expect(mocks.releaseMicrophone).not.toHaveBeenCalled(); + + const remounted = renderConversation("session-a", remountedOnSend); + expect(remounted.result.current.state).toBe("listening"); + expect(remounted.result.current.ownsActiveConversation).toBe(true); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await waitFor(() => expect(remountedOnSend).toHaveBeenCalledOnce()); + expect(originalOnSend).not.toHaveBeenCalled(); + + await act(async () => remounted.result.current.onToggle()); + await waitFor(() => expect(remounted.result.current.state).toBe("off")); + expect(channel.close).toHaveBeenCalledOnce(); + expect(peer.close).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(mocks.releaseBridge).toHaveBeenCalledOnce(); + expect(mocks.releaseMicrophone).toHaveBeenCalledOnce(); + }); + + it("does not let another session steal the active conversation", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + const other = renderConversation("session-b"); + expect(other.result.current.boundSessionId).toBe("session-a"); + expect(other.result.current.ownsActiveConversation).toBe(false); + expect(other.result.current.disabled).toBe(true); + + await act(async () => other.result.current.onToggle()); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + expect(owner.result.current.state).toBe("listening"); + + await act(async () => owner.result.current.onToggle()); + }); + + it("moves the realtime owner and bridge when a draft session is promoted", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatSessionStore.setState({ + sessions: [ + { + id: "draft-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + creationState: "pending", + intent: null, + }, + ], + }); + const owner = renderHook( + ({ sessionId }) => + useOpenAiRealtimeConversation({ + enabled: true, + onSend, + sessionId, + }), + { initialProps: { sessionId: "draft-session" } }, + ); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + useChatSessionStore + .getState() + .promoteDraftSession("draft-session", "backend-session"); + useChatStore + .getState() + .promoteSessionId("draft-session", "backend-session"); + owner.rerender({ sessionId: "backend-session" }); + }); + + await waitFor(() => + expect(owner.result.current.boundSessionId).toBe("backend-session"), + ); + expect(owner.result.current.disabled).toBe(false); + expect(mocks.activeEmissary?.sessionId).toBe("backend-session"); + await waitFor(() => + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining( + 'send-to-emissary --session-id "backend-session"', + ), + ), + ); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + expect(onSend).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["backend-session"]?.[0], + ).toMatchObject({ metadata: { personaName: "Emissary" } }); + expect(useChatStore.getState().messagesBySession["draft-session"]).toBe( + undefined, + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("steers realtime deliveries while the master is running without using the composer queue", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("retries as a normal prompt when the master finishes before steer admission", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockRejectedValueOnce( + new Error("no active run to steer"), + ); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + act(() => { + useChatStore.getState().setChatState("session-a", "idle"); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + "[Voice transcript] User said: hello master", + undefined, + undefined, + expect.objectContaining({ displayText: "hello master" }), + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("shows accepted master-to-emissary coordination in the transcript", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage("There are 20 repos.", 0); + }); + + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + role: "assistant", + content: [{ type: "text", text: "There are 20 repos." }], + metadata: { + agentVisible: false, + personaName: "Master → Emissary", + }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("delivers every terminal master turn to the emissary for evaluation", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + mocks.activeEmissary?.beginMasterTurn("turn-1"); + mocks.activeEmissary?.endMasterTurn({ + turnId: "turn-1", + status: "completed", + finalText: "There are 20 repositories.", + }); + }); + + expect(mocks.requestMasterMessage).toHaveBeenCalledWith({ + eventId: "berd-master-turn-ended-turn-1", + message: expect.stringContaining( + "Final response:\nThere are 20 repositories.", + ), + }); + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + role: "assistant", + content: [{ type: "text", text: "There are 20 repositories." }], + metadata: { + agentVisible: false, + personaName: "Master ended turn", + }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("ends an emissary evaluation silently without scheduling a continuation", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.sendRealtimeEvents.mockClear(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.end_turn" }), + }), + ); + }); + + expect(mocks.createEndTurnToolOutput).toHaveBeenCalledWith("call-end"); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ type: "function_call_output" }), + }), + ]); + expect(mocks.requestToolOutput).not.toHaveBeenCalled(); + expect(mocks.requestMasterMessage).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("renders user speech as a normal user send", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + "[Voice transcript] User said: hello master", + undefined, + undefined, + expect.objectContaining({ + displayText: "hello master", + userMessageMetadata: { origin: "voice_conversation" }, + }), + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("edits a provisional user transcript in place when the final correction arrives", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_partial" }), + }), + ); + }); + const provisional = + useChatStore.getState().messagesBySession["session-a"]?.[0]; + expect(provisional).toMatchObject({ + role: "user", + content: [{ type: "text", text: "hello" }], + metadata: { completionStatus: "inProgress" }, + }); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_corrected" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + expect.stringContaining("hello master"), + undefined, + undefined, + expect.objectContaining({ userMessageId: provisional?.id }), + ); + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + id: provisional?.id, + content: [{ type: "text", text: "hello master" }], + metadata: { completionStatus: "completed" }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("waits for session hydration before dispatching a voice transcript", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatStore.getState().setSessionLoading("session-a", true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => useChatStore.getState().setSessionLoading("session-a", false)); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + + await act(async () => owner.result.current.onToggle()); + }); + + it("forwards committed typed user text to the realtime emissary", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + useChatStore.getState().setChatState("session-a", "thinking"); + }); + + act(() => { + owner.result.current.onTypedUserMessageCommitted?.( + "Please stop and check this.", + ); + }); + + expect(mocks.requestTypedUserMessage).toHaveBeenCalledWith( + "Please stop and check this.", + ); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { type: "response.cancel" }, + { type: "conversation.item.create" }, + ]); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(mocks.steerPrompt).toHaveBeenCalledWith( + "session-a", + "[Voice transcript] Emissary said: hello user", + undefined, + expect.objectContaining({ + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }), + { throwOnError: true }, + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("does not let a realtime transport failure abort the ordinary typed send", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.sendRealtimeEvents.mockImplementationOnce(() => { + throw new DOMException( + "The object is in an invalid state.", + "InvalidStateError", + ); + }); + + expect(() => { + owner.result.current.onTypedUserMessageCommitted?.("Still send this."); + }).not.toThrow(); + await waitFor(() => expect(owner.result.current.state).toBe("error")); + + await act(async () => owner.result.current.onToggle()); + }); + + it("renders emissary speech on the assistant side with spoken status", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toBeDefined(), + ); + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + role: "assistant", + content: [ + { + type: "text", + text: "hello user", + speech: { status: "spoken", spokenThrough: 10 }, + }, + ], + metadata: { + agentVisible: false, + origin: "voice_conversation", + personaName: "Emissary", + }, + }); + expect(onSend).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("handles a repository question and symlink follow-up without emissary-triggered master wakes", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_repository" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend.mock.calls[0]?.[0]).toBe( + "[Voice transcript] User said: how many repos are in my development folder?", + ); + act(() => useChatStore.getState().setChatState("session-a", "thinking")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledOnce(); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "The answer is 21 repositories.", + 0, + ); + }); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_result" }), + }), + ); + }); + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"], + ).toHaveLength(5), + ); + expect(onSend).toHaveBeenCalledOnce(); + + act(() => { + useChatStore.getState().setChatState("session-a", "idle"); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_followup" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + expect(onSend.mock.calls[1]?.[0]).toBe( + "[Voice transcript] Emissary said: hello user\n" + + "[Voice transcript] Emissary said: You have 21 repositories.\n" + + "[Voice transcript] User said: are any of them symbolic links?", + ); + act(() => useChatStore.getState().setChatState("session-a", "thinking")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_followup_ack" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master_followup" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledTimes(2)); + expect(onSend).toHaveBeenCalledTimes(2); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "None of the repositories are symbolic links.", + 0, + ); + }); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_symlink_result" }), + }), + ); + }); + await waitFor(() => + expect( + useChatStore + .getState() + .messagesBySession["session-a"]?.some( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === + "None of those repositories are symbolic links.", + ), + ).toBe(true), + ); + expect(onSend).toHaveBeenCalledTimes(2); + + const messages = + useChatStore.getState().messagesBySession["session-a"] ?? []; + expect( + messages.filter( + (message) => message.metadata?.personaName === "Master → Emissary", + ), + ).toHaveLength(2); + expect( + messages.filter( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === "You have 21 repositories.", + ), + ).toHaveLength(1); + expect( + messages.filter( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === + "None of those repositories are symbolic links.", + ), + ).toHaveLength(1); + + await act(async () => owner.result.current.onToggle()); + }); + + it("buffers emissary speech until the next user-triggered master turn", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + role: "assistant", + content: [{ type: "text", text: "hello user" }], + metadata: { personaName: "Emissary" }, + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + "[Voice transcript] Emissary said: hello user\n[Voice transcript] User said: hello master", + undefined, + undefined, + expect.objectContaining({ displayText: "hello master" }), + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("marks interrupted emissary speech without claiming a precise cutoff", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_interrupted" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toBeDefined(), + ); + const content = + useChatStore.getState().messagesBySession["session-a"]?.[0]?.content[0]; + if (content?.type !== "text") + throw new Error("expected an emissary text message"); + const speech = content.speech; + expect(speech).toEqual({ status: "interrupted", confidence: "low" }); + expect(onSend).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("shows accepted emissary-to-master coordination in the transcript", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + role: "assistant", + content: [{ type: "text", text: "Please inspect the disk." }], + metadata: { + agentVisible: false, + personaName: "Emissary → Master", + }, + }), + ); + expect(mocks.requestToolOutput).toHaveBeenCalledWith({ + type: "conversation.item.create", + item: { type: "function_call_output" }, + }); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { + type: "conversation.item.create", + item: { type: "function_call_output" }, + }, + ]); + + await act(async () => owner.result.current.onToggle()); + }); + + it("steers emissary-to-master coordination into an active master turn", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ metadata: { personaName: "Emissary → Master" } }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("uses one active-turn delivery for transcript coordination", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + act(() => useChatStore.getState().setChatState("session-a", "thinking")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ metadata: { personaName: "Emissary → Master" } }), + ); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("blocks acknowledgement loops until the user speaks again", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage("The result.", 0); + }); + mocks.createSendToMasterToolOutput.mockClear(); + mocks.sendRealtimeEvents.mockClear(); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + + await waitFor(() => + expect(mocks.createSendToMasterToolOutput).toHaveBeenCalledWith( + "call-1", + { + accepted: false, + reason: "awaiting_new_user_input", + unreadPeerMessages: [], + cursor: 0, + }, + ), + ); + expect(onSend).not.toHaveBeenCalled(); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { + type: "conversation.item.create", + item: { type: "function_call_output" }, + }, + ]); + expect(useChatStore.getState().messagesBySession["session-a"]).toHaveLength( + 1, + ); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.send_to_master" }), + }), + ); + }); + + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + await act(async () => owner.result.current.onToggle()); + }); +}); diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts new file mode 100644 index 000000000..71c98fdb8 --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -0,0 +1,1057 @@ +import { useCallback, useEffect, useSyncExternalStore } from "react"; +import { toast } from "sonner"; +import type { + ChatInputSendHandler, + ChatInputVoiceConversation, +} from "@/features/chat/types"; +import { steerPromptInSession } from "@/features/chat/lib/steerCore"; +import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { useChatStore } from "@/features/chat/stores/chatStore"; +import { appendSessionSystemPrompt } from "@/shared/api/acpApi"; +import { + claimVoiceDictationMicrophone, + createOpenAiRealtimeVoiceSession, + releaseVoiceDictationMicrophone, +} from "@/shared/api/openaiRealtime"; +import { + createSystemNotificationMessage, + type Message, +} from "@/shared/types/messages"; +import { + connectOpenAiRealtimePeerConnection, + createOpenAiRealtimePeerConnection, +} from "@/features/chat/lib/openaiRealtimeAudio"; +import { + type MasterMessageDelivery, + type MasterTurnCompletion, + registerRealtimeEmissary, +} from "../lib/realtimeEmissaryBridge"; +import { + createEndTurnToolOutput, + createSendToMasterToolOutput, + DirectMessagePipe, + REALTIME_MASTER_INSTRUCTIONS, + RealtimeEmissaryProtocol, + RealtimeResponseCoordinator, + sendRealtimeEvents, + configureRealtimeEmissarySession, +} from "../lib/realtimeEmissaryProtocol"; +import { + getRealtimeVoicePreference, + parseRealtimeSessionOverrides, +} from "../lib/realtimeVoicePreference"; + +const MASTER_PROMPT_KEY = "berd-realtime-voice-master"; +const MICROPHONE_OWNER_ID = "berd:realtime-voice-conversation"; +const MAX_REALTIME_REPLAY_ITEMS = 12; + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isUnavailableDevMicrophoneClaim(error: unknown): boolean { + return ( + import.meta.env.DEV && + errorText(error).includes("claim_voice_dictation_microphone not found") + ); +} + +function isMissingActiveRun(error: unknown): boolean { + return errorText(error).toLowerCase().includes("no active run to steer"); +} + +function waitForSessionHydration(sessionId: string): Promise { + if (!useChatStore.getState().loadingSessionIds.has(sessionId)) { + return Promise.resolve(); + } + + return new Promise((resolve) => { + const unsubscribe = useChatStore.subscribe((state) => { + if (state.loadingSessionIds.has(sessionId)) return; + unsubscribe(); + resolve(); + }); + }); +} + +function waitForMasterIdle(sessionId: string): Promise { + const isIdle = () => { + const runtime = useChatStore.getState().getSessionRuntime(sessionId); + return runtime.activeRunId === null && !isSessionRunning(runtime.chatState); + }; + if (isIdle()) return Promise.resolve(); + + return new Promise((resolve) => { + const unsubscribe = useChatStore.subscribe(() => { + if (!isIdle()) return; + unsubscribe(); + resolve(); + }); + }); +} + +function createEmissaryTranscriptMessage( + text: string, + interrupted: boolean, + id: string = crypto.randomUUID(), + provisional = false, +): Message { + return { + id, + role: "assistant", + created: Date.now(), + content: [ + { + type: "text", + text, + speech: provisional + ? { status: "speaking" } + : interrupted + ? { + status: "interrupted", + confidence: "low", + } + : { status: "spoken", spokenThrough: text.length }, + }, + ], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + personaName: "Emissary", + completionStatus: provisional ? "inProgress" : "completed", + }, + }; +} + +function createUserTranscriptMessage( + id: string, + text: string, + provisional: boolean, +): Message { + return { + id, + role: "user", + created: Date.now(), + content: [{ type: "text", text }], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + completionStatus: provisional ? "inProgress" : "completed", + }, + }; +} + +function createCoordinationMessage( + sender: "Emissary" | "Master", + recipient: "Emissary" | "Master", + text: string, +): Message { + return { + id: crypto.randomUUID(), + role: "assistant", + created: Date.now(), + content: [{ type: "text", text }], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + personaName: `${sender} → ${recipient}`, + completionStatus: "completed", + }, + }; +} + +function createMasterTurnEndedMessage( + status: "completed" | "cancelled" | "failed", + finalText?: string, +): Message { + const summary = finalText?.trim() + ? finalText.trim() + : status === "completed" + ? "No final response text." + : `The Master turn ${status}.`; + return { + id: crypto.randomUUID(), + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: summary }], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + personaName: "Master ended turn", + completionStatus: "completed", + }, + }; +} + +function visibleMessageText(message: Message): string { + return message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(); +} + +export function createRealtimeTranscriptReplayEvents( + messages: readonly Message[], + sessionId?: string, +): Record[] { + const turns: Array<{ role: "user" | "assistant"; text: string }> = []; + let pendingAssistant: { role: "assistant"; text: string } | null = null; + const flushAssistant = () => { + if (!pendingAssistant) return; + turns.push(pendingAssistant); + pendingAssistant = null; + }; + + for (const message of messages) { + if (message.metadata?.userVisible === false || message.role === "system") + continue; + const text = visibleMessageText(message); + if (!text) continue; + if (message.role === "user") { + flushAssistant(); + turns.push({ role: "user", text }); + continue; + } + if ( + message.metadata?.personaName?.includes("→") || + (message.metadata?.completionStatus && + message.metadata.completionStatus !== "completed") + ) + continue; + // Only the final visible assistant block before the next user turn is + // useful context. Progress narration and earlier replacements stay in the + // durable Master transcript but do not bloat a resumed voice frontend. + pendingAssistant = { role: "assistant", text }; + } + flushAssistant(); + + const tail = turns.slice(-MAX_REALTIME_REPLAY_ITEMS); + const firstUserIndex = tail.findIndex((turn) => turn.role === "user"); + if (firstUserIndex < 0) return []; + const replay = tail.slice(firstUserIndex).map((turn) => ({ + type: "conversation.item.create", + item: { + type: "message", + role: turn.role, + content: [ + { + type: turn.role === "assistant" ? "output_text" : "input_text", + text: turn.text, + }, + ], + }, + })); + if (!sessionId) return replay; + return [ + { + type: "conversation.item.create", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: `This voice conversation is being resumed from Berd session ${sessionId}. Durable session link: berd://session/${sessionId}. The following items are a compact recent transcript, not new turns. Ask the master to inspect the durable session when older context is needed.`, + }, + ], + }, + }, + ...replay, + ]; +} + +function waitForDataChannelOpen(channel: RTCDataChannel): Promise { + if (channel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + channel.removeEventListener("open", handleOpen); + channel.removeEventListener("error", handleError); + }; + const handleOpen = () => { + cleanup(); + resolve(); + }; + const handleError = () => { + cleanup(); + reject(new Error("OpenAI Realtime data channel failed to open.")); + }; + channel.addEventListener("open", handleOpen); + channel.addEventListener("error", handleError); + }); +} + +function masterPrompt(sessionId: string): string { + return `${REALTIME_MASTER_INSTRUCTIONS} + +Your send_to_emissary tool is the Berd CLI command below. The initial bridge cursor is 0. Always use the latest cursor returned by a successful command or stale-send error. + +berdctl session send-to-emissary --session-id ${JSON.stringify(sessionId)} --cursor --message --json`; +} + +type RuntimeState = ChatInputVoiceConversation["state"]; +interface Snapshot { + state: RuntimeState; + boundSessionId: string | null; + requestedStartSessionId: string | null; + microphoneMuted: boolean; + error: string | null; +} +interface StartOptions { + sessionId: string; + onSend: ChatInputSendHandler; +} +const OFF_SNAPSHOT: Snapshot = { + state: "off", + boundSessionId: null, + requestedStartSessionId: null, + microphoneMuted: false, + error: null, +}; + +class OpenAiRealtimeConversationRuntime { + private snapshot: Snapshot = OFF_SNAPSHOT; + private readonly listeners = new Set<() => void>(); + private peer: RTCPeerConnection | null = null; + private channel: RTCDataChannel | null = null; + private stream: MediaStream | null = null; + private audio: HTMLAudioElement | null = null; + private releaseBridge: (() => void) | null = null; + private bridgeSender: + | ((message: string, cursor: number) => Promise) + | null = null; + private bridgeMasterTurnBegin: ((turnId: string) => void) | null = null; + private bridgeMasterTurnEnd: + | ((completion: MasterTurnCompletion) => void) + | null = null; + private activeRun = 0; + private deliveryQueue = Promise.resolve(); + private boundOnSend: ChatInputSendHandler | null = null; + private typedUserMessageSink: ((text: string) => void) | null = null; + private pendingTypedUserMessages: string[] = []; + private failureInProgress = false; + private ownerMigration = Promise.resolve(); + private historyReplay = Promise.resolve(); + + readonly subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + readonly getSnapshot = (): Snapshot => this.snapshot; + + bindOwner(sessionId: string, onSend: ChatInputSendHandler): void { + if (this.snapshot.boundSessionId === sessionId) this.boundOnSend = onSend; + } + + requestStart(sessionId: string): void { + this.setSnapshot({ ...this.snapshot, requestedStartSessionId: sessionId }); + } + + rebindPromotedOwner(sessionId: string, onSend: ChatInputSendHandler): void { + const previousSessionId = this.snapshot.boundSessionId; + if (!previousSessionId || previousSessionId === sessionId) { + this.bindOwner(sessionId, onSend); + return; + } + + this.boundOnSend = onSend; + this.setSnapshot({ ...this.snapshot, boundSessionId: sessionId }); + this.registerBridge(sessionId); + this.ownerMigration = this.ownerMigration + .catch(() => undefined) + .then(async () => { + await appendSessionSystemPrompt( + previousSessionId, + MASTER_PROMPT_KEY, + "", + ).catch(() => undefined); + await appendSessionSystemPrompt( + sessionId, + MASTER_PROMPT_KEY, + masterPrompt(sessionId), + ); + }); + } + + async start({ sessionId, onSend }: StartOptions): Promise { + if ( + (this.snapshot.boundSessionId && + this.snapshot.boundSessionId !== sessionId) || + (this.snapshot.boundSessionId === sessionId && + this.snapshot.state !== "off" && + this.snapshot.state !== "error") + ) + return; + + const runId = ++this.activeRun; + this.failureInProgress = false; + this.boundOnSend = onSend; + this.pendingTypedUserMessages = []; + this.setSnapshot({ + state: "starting", + boundSessionId: sessionId, + requestedStartSessionId: null, + microphoneMuted: false, + error: null, + }); + const isStale = () => this.activeRun !== runId; + try { + await claimVoiceDictationMicrophone(MICROPHONE_OWNER_ID).catch( + (error) => { + if (!isUnavailableDevMicrophoneClaim(error)) throw error; + }, + ); + const preference = getRealtimeVoicePreference(); + const pendingDraft = + useChatSessionStore.getState().getSession(sessionId)?.creationState === + "pending"; + const [stream, session] = await Promise.all([ + navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }), + createOpenAiRealtimeVoiceSession(preference.model), + pendingDraft + ? Promise.resolve() + : appendSessionSystemPrompt( + sessionId, + MASTER_PROMPT_KEY, + masterPrompt(sessionId), + ), + ]).then(([stream, session]) => [stream, session] as const); + if (isStale()) { + stream.getTracks().forEach((track) => { + track.stop(); + }); + return; + } + + const peer = createOpenAiRealtimePeerConnection(); + const channel = peer.createDataChannel("oai-events"); + const audio = new Audio(); + audio.autoplay = true; + this.peer = peer; + this.channel = channel; + this.stream = stream; + this.audio = audio; + stream.getAudioTracks().forEach((track) => { + peer.addTrack(track, stream); + }); + peer.addEventListener("track", (event) => { + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); + void audio + .play() + .catch((error) => + this.fail(this.snapshot.boundSessionId ?? sessionId, error), + ); + }); + + const transport = { send: (data: string) => channel.send(data) }; + const protocol = new RealtimeEmissaryProtocol(); + const responses = new RealtimeResponseCoordinator(); + const pipe = new DirectMessagePipe(); + const transcriptMessageIds = new Map(); + const pendingEmissaryTranscripts: string[] = []; + const masterTurnHandoffs = new Map(); + let activeMasterTurnId: string | null = null; + let userTranscriptRevision = 0; + let masterDeliveryRevision: number | undefined; + const upsertTranscriptMessage = ( + ownerSessionId: string, + transcript: { + itemId: string; + speaker: "user" | "emissary"; + text: string; + interrupted?: true; + }, + provisional: boolean, + ): string => { + const existingId = transcriptMessageIds.get(transcript.itemId); + const messageId = existingId ?? crypto.randomUUID(); + transcriptMessageIds.set(transcript.itemId, messageId); + const nextMessage = + transcript.speaker === "user" + ? createUserTranscriptMessage( + messageId, + transcript.text, + provisional, + ) + : createEmissaryTranscriptMessage( + transcript.text, + transcript.interrupted === true, + messageId, + provisional, + ); + const store = useChatStore.getState(); + if (existingId) { + store.updateMessage(ownerSessionId, messageId, (existing) => ({ + ...nextMessage, + created: existing.created, + })); + } else { + store.addMessage(ownerSessionId, nextMessage); + } + return messageId; + }; + const forwardTypedUserMessage = (text: string) => { + userTranscriptRevision += 1; + const ownerSessionId = this.snapshot.boundSessionId; + if (ownerSessionId && pendingEmissaryTranscripts.length > 0) { + const priorEmissaryContext = pendingEmissaryTranscripts.splice(0); + const context = priorEmissaryContext.join("\n"); + this.deliverToMaster( + ownerSessionId, + context, + context, + undefined, + true, + ); + } + const request = responses.requestTypedUserMessage(text); + sendRealtimeEvents(transport, request.events); + }; + channel.addEventListener("message", (message) => { + try { + const ownerSessionId = this.snapshot.boundSessionId; + if (!ownerSessionId || isStale()) return; + const event: unknown = JSON.parse(String(message.data)); + sendRealtimeEvents(transport, responses.handle(event)); + for (const bridgeEvent of protocol.handle(event)) { + if (bridgeEvent.type === "transcript.started") { + upsertTranscriptMessage( + ownerSessionId, + { ...bridgeEvent, text: "" }, + true, + ); + } else if (bridgeEvent.type === "transcript.updated") { + upsertTranscriptMessage(ownerSessionId, bridgeEvent, true); + } else if (bridgeEvent.type === "transcript.finalized") { + const transcriptMessageId = upsertTranscriptMessage( + ownerSessionId, + bridgeEvent, + false, + ); + const interrupted = bridgeEvent.interrupted === true; + const transcriptLabel = + bridgeEvent.speaker === "user" + ? `User said: ${bridgeEvent.text}` + : `Emissary said${ + interrupted + ? " (interrupted; best-effort transcript)" + : "" + }: ${bridgeEvent.text}`; + const masterTranscript = `[Voice transcript] ${transcriptLabel}`; + if (bridgeEvent.speaker === "emissary") { + pendingEmissaryTranscripts.push(masterTranscript); + continue; + } + userTranscriptRevision += 1; + const priorEmissaryContext = pendingEmissaryTranscripts.splice(0); + this.deliverToMaster( + ownerSessionId, + [...priorEmissaryContext, masterTranscript].join("\n"), + bridgeEvent.text, + undefined, + false, + transcriptMessageId, + ); + } else if (bridgeEvent.type === "send_to_master") { + if (masterDeliveryRevision === userTranscriptRevision) { + sendRealtimeEvents(transport, [ + createSendToMasterToolOutput(bridgeEvent.callId, { + accepted: false, + reason: "awaiting_new_user_input", + unreadPeerMessages: [], + cursor: pipe.cursor("emissary"), + }), + ]); + continue; + } + const exchange = pipe.send({ + sender: "emissary", + cursor: bridgeEvent.cursor, + message: bridgeEvent.message, + }); + const toolFollowUp = responses.requestToolOutput( + createSendToMasterToolOutput(bridgeEvent.callId, exchange), + ); + sendRealtimeEvents(transport, toolFollowUp.events); + if (exchange.accepted) { + useChatStore + .getState() + .addMessage( + ownerSessionId, + createCoordinationMessage( + "Emissary", + "Master", + exchange.outbound.message, + ), + ); + this.deliverToMaster( + ownerSessionId, + `[Direct message from emissary; cursor ${exchange.outbound.id}] ${exchange.outbound.message}`, + exchange.outbound.message, + undefined, + true, + undefined, + false, + ); + } + } else if (bridgeEvent.type === "end_turn") { + sendRealtimeEvents(transport, [ + createEndTurnToolOutput(bridgeEvent.callId), + ]); + } + } + } catch (error) { + void this.fail(this.snapshot.boundSessionId ?? sessionId, error); + } + }); + + await connectOpenAiRealtimePeerConnection({ + peerConnection: peer, + clientSecret: session.clientSecret, + }); + await waitForDataChannelOpen(channel); + if (isStale()) return; + configureRealtimeEmissarySession(transport, { + transcriptionModel: preference.transcriptionModel, + voice: preference.voice, + speed: preference.speed, + sessionOverrides: parseRealtimeSessionOverrides( + preference.sessionOverridesText, + ), + }); + this.typedUserMessageSink = forwardTypedUserMessage; + for (const text of this.pendingTypedUserMessages.splice(0)) { + forwardTypedUserMessage(text); + } + const replaySessionId = this.snapshot.boundSessionId ?? sessionId; + this.historyReplay = waitForSessionHydration(replaySessionId).then(() => { + if (isStale() || this.snapshot.boundSessionId !== replaySessionId) + return; + sendRealtimeEvents( + transport, + createRealtimeTranscriptReplayEvents( + useChatStore.getState().messagesBySession[replaySessionId] ?? [], + replaySessionId, + ), + ); + }); + this.bridgeSender = async (message, cursor) => { + const exchange = pipe.send({ sender: "master", cursor, message }); + if (!exchange.accepted) return exchange; + if (activeMasterTurnId) { + masterTurnHandoffs.set( + activeMasterTurnId, + (masterTurnHandoffs.get(activeMasterTurnId) ?? 0) + 1, + ); + } + const request = responses.requestMasterMessage({ + message: `[bridge cursor ${exchange.outbound.id}] ${message}`, + eventId: `berd-master-${exchange.outbound.id}`, + }); + sendRealtimeEvents(transport, request.events); + masterDeliveryRevision = userTranscriptRevision; + const ownerSessionId = this.snapshot.boundSessionId; + if (!ownerSessionId) + throw new Error("The realtime voice owner is no longer available."); + useChatStore + .getState() + .addMessage( + ownerSessionId, + createCoordinationMessage("Master", "Emissary", message), + ); + return { ...exchange, deliveryStatus: request.status }; + }; + this.bridgeMasterTurnBegin = (turnId) => { + activeMasterTurnId = turnId; + masterTurnHandoffs.set(turnId, 0); + }; + this.bridgeMasterTurnEnd = (completion) => { + const handoffCount = masterTurnHandoffs.get(completion.turnId) ?? 0; + masterTurnHandoffs.delete(completion.turnId); + if (activeMasterTurnId === completion.turnId) { + activeMasterTurnId = null; + } + const finalText = completion.finalText?.trim(); + const notification = [ + `Master turn ended (${completion.status}).`, + handoffCount > 0 + ? `The Master sent ${handoffCount} direct message${handoffCount === 1 ? "" : "s"} during this turn.` + : "The Master sent no direct messages during this turn.", + finalText + ? `Final response:\n${finalText}` + : "The Master produced no final response text.", + "Evaluate whether the user still needs anything from this information. The Master's visible Berd output was not spoken. If you only gave a waiting acknowledgement and this notification now supplies the answer, speak the answer. If you already spoke the useful result, this is late or redundant, or there is no materially useful new information, call end_turn now. Do not speak filler, acknowledge receipt, offer more help, or repeat an answer.", + ].join("\n"); + const request = responses.requestMasterMessage({ + message: notification, + eventId: `berd-master-turn-ended-${completion.turnId}`, + }); + sendRealtimeEvents(transport, request.events); + const ownerSessionId = this.snapshot.boundSessionId; + if (ownerSessionId) { + useChatStore + .getState() + .addMessage( + ownerSessionId, + createMasterTurnEndedMessage( + completion.status, + completion.finalText, + ), + ); + } + }; + this.registerBridge(this.snapshot.boundSessionId ?? sessionId); + this.setSnapshot({ ...this.snapshot, state: "listening" }); + } catch (error) { + if (!isStale()) await this.fail(sessionId, error); + } + } + + async stop(sessionId: string): Promise { + if ( + this.snapshot.boundSessionId !== sessionId || + this.snapshot.state === "off" || + this.snapshot.state === "stopping" + ) + return; + this.setSnapshot({ ...this.snapshot, state: "stopping" }); + await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.failureInProgress = false; + this.setSnapshot(OFF_SNAPSHOT); + } + + toggleMute(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + const microphoneMuted = !this.snapshot.microphoneMuted; + this.stream?.getAudioTracks().forEach((track) => { + track.enabled = !microphoneMuted; + }); + this.setSnapshot({ ...this.snapshot, microphoneMuted }); + } + + forwardTypedUserMessage(sessionId: string, text: string): void { + if (this.snapshot.boundSessionId !== sessionId || !text.trim()) return; + if (!this.typedUserMessageSink) { + if (this.snapshot.state === "starting") + this.pendingTypedUserMessages.push(text); + return; + } + try { + this.typedUserMessageSink(text); + } catch (error) { + // Mirroring into the voice frontend is secondary to the ordinary Berd + // send that invoked this callback. Never let a synchronous WebRTC/data + // channel failure abort the user's Master turn. + void this.fail(sessionId, error); + } + } + + async dispose(): Promise { + const sessionId = this.snapshot.boundSessionId; + if (sessionId) await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.bridgeSender = null; + this.bridgeMasterTurnBegin = null; + this.bridgeMasterTurnEnd = null; + this.typedUserMessageSink = null; + this.pendingTypedUserMessages = []; + this.failureInProgress = false; + this.deliveryQueue = Promise.resolve(); + this.historyReplay = Promise.resolve(); + this.setSnapshot(OFF_SNAPSHOT); + } + + private deliverToMaster( + sessionId: string, + text: string, + displayText: string, + onDelivered?: () => void, + hidden = false, + userMessageId?: string, + queueUntilIdle = false, + ): void { + this.deliveryQueue = this.deliveryQueue + .catch(() => undefined) + .then(async () => { + // History replay replaces the transcript wholesale. Dispatching a + // realtime transcript while hydration is still active can therefore + // route the master's live ACP stream into the replay buffer, or let a + // subsequent replay replacement erase it. Preserve ordering in the + // delivery queue and wait for hydration to publish before sending. + await this.ownerMigration; + await this.historyReplay; + sessionId = this.snapshot.boundSessionId ?? sessionId; + await waitForSessionHydration(sessionId); + if (queueUntilIdle) await waitForMasterIdle(sessionId); + if (this.snapshot.boundSessionId !== sessionId || !this.boundOnSend) + throw new Error("The realtime voice owner is no longer available."); + const runtime = useChatStore.getState().getSessionRuntime(sessionId); + const sendOptions = { + displayText, + userMessageMetadata: { + origin: "voice_conversation" as const, + ...(hidden ? { userVisible: false } : {}), + }, + acpGooseMetadata: { origin: "voice_conversation" }, + ...(userMessageId ? { userMessageId } : {}), + }; + const sendAsPrompt = async () => { + const accepted = await this.boundOnSend?.( + text, + undefined, + undefined, + sendOptions, + ); + if (accepted === false) + throw new Error( + "The master session did not accept the voice transcript.", + ); + }; + this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + if ( + runtime.activeRunId !== null || + isSessionRunning(runtime.chatState) + ) { + try { + await steerPromptInSession( + sessionId, + text, + undefined, + sendOptions, + { throwOnError: true }, + ); + } catch (error) { + // The Master can finish between the runtime snapshot above and + // steer admission. That is an ordinary boundary race: retry as a + // new prompt after local completion catches up, so the transcript + // or coordination message is not dropped and the new run receives + // the same live-notification ownership as an ordinary send. + if (!isMissingActiveRun(error)) throw error; + await waitForMasterIdle(sessionId); + await sendAsPrompt(); + } + } else { + await sendAsPrompt(); + } + onDelivered?.(); + if (this.snapshot.boundSessionId === sessionId) + this.setSnapshot({ ...this.snapshot, state: "listening" }); + }) + .catch((error) => this.fail(sessionId, error)); + } + + private async fail(sessionId: string, error: unknown): Promise { + if ( + this.snapshot.boundSessionId !== sessionId || + this.failureInProgress || + this.snapshot.state === "error" + ) + return; + this.failureInProgress = true; + const message = errorText(error); + await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.setSnapshot({ + state: "error", + boundSessionId: sessionId, + requestedStartSessionId: null, + microphoneMuted: false, + error: message, + }); + useChatStore + .getState() + .addMessage(sessionId, createSystemNotificationMessage(message, "error")); + toast.error("OpenAI Realtime voice failed", { description: message }); + } + + private async cleanupResources(sessionId: string): Promise { + this.activeRun += 1; + this.releaseBridge?.(); + this.channel?.close(); + this.peer?.close(); + this.stream?.getTracks().forEach((track) => { + track.stop(); + }); + this.audio?.pause(); + this.releaseBridge = null; + this.bridgeSender = null; + this.bridgeMasterTurnBegin = null; + this.bridgeMasterTurnEnd = null; + this.typedUserMessageSink = null; + this.pendingTypedUserMessages = []; + this.channel = null; + this.peer = null; + this.stream = null; + this.audio = null; + await releaseVoiceDictationMicrophone(MICROPHONE_OWNER_ID).catch( + () => undefined, + ); + await appendSessionSystemPrompt(sessionId, MASTER_PROMPT_KEY, "").catch( + () => undefined, + ); + } + + private setSnapshot(snapshot: Snapshot): void { + this.snapshot = snapshot; + for (const listener of this.listeners) listener(); + } + + private registerBridge(sessionId: string): void { + if ( + !this.bridgeSender || + !this.bridgeMasterTurnBegin || + !this.bridgeMasterTurnEnd + ) + return; + this.releaseBridge?.(); + this.releaseBridge = registerRealtimeEmissary({ + sessionId, + beginMasterTurn: this.bridgeMasterTurnBegin, + endMasterTurn: this.bridgeMasterTurnEnd, + sendMasterMessage: this.bridgeSender, + }); + } +} + +const runtime = new OpenAiRealtimeConversationRuntime(); + +export function requestOpenAiRealtimeConversationStart( + sessionId: string, +): void { + runtime.requestStart(sessionId); +} + +export async function stopOpenAiRealtimeConversation(): Promise { + const sessionId = runtime.getSnapshot().boundSessionId; + if (sessionId) await runtime.stop(sessionId); +} + +if (import.meta.hot) { + import.meta.hot.dispose(() => { + void runtime.dispose(); + }); +} + +export async function resetOpenAiRealtimeConversationRuntimeForTests(): Promise { + await runtime.dispose(); +} + +export function useOpenAiRealtimeConversation(options: { + sessionId: string; + onSend: ChatInputSendHandler; + enabled: boolean; + readOnly?: boolean; + disabled?: boolean; +}): ChatInputVoiceConversation { + const { + sessionId, + onSend, + enabled, + readOnly = false, + disabled = false, + } = options; + const snapshot = useSyncExternalStore( + runtime.subscribe, + runtime.getSnapshot, + runtime.getSnapshot, + ); + const clientSessionId = useChatSessionStore( + (state) => + state.sessions?.find((candidate) => candidate.id === sessionId) + ?.clientSessionId, + ); + const ownsPromotedConversation = + snapshot.boundSessionId !== null && + snapshot.boundSessionId !== sessionId && + clientSessionId === snapshot.boundSessionId; + const ownsActiveConversation = snapshot.boundSessionId === sessionId; + const anotherSessionOwnsConversation = + snapshot.boundSessionId !== null && + !ownsActiveConversation && + !ownsPromotedConversation; + const requestedStartMatchesSession = + snapshot.requestedStartSessionId === sessionId || + (clientSessionId !== undefined && + snapshot.requestedStartSessionId === clientSessionId); + useEffect(() => { + if (ownsPromotedConversation) + runtime.rebindPromotedOwner(sessionId, onSend); + else if (ownsActiveConversation) runtime.bindOwner(sessionId, onSend); + }, [onSend, ownsActiveConversation, ownsPromotedConversation, sessionId]); + useEffect(() => { + if ( + !requestedStartMatchesSession || + !enabled || + disabled || + readOnly || + anotherSessionOwnsConversation + ) + return; + void runtime.start({ sessionId, onSend }); + }, [ + anotherSessionOwnsConversation, + disabled, + enabled, + onSend, + readOnly, + requestedStartMatchesSession, + sessionId, + ]); + const start = useCallback(async () => { + if (!enabled || disabled || readOnly || anotherSessionOwnsConversation) + return; + await runtime.start({ sessionId, onSend }); + }, [ + anotherSessionOwnsConversation, + disabled, + enabled, + onSend, + readOnly, + sessionId, + ]); + const stop = useCallback(async () => { + await runtime.stop(sessionId); + }, [sessionId]); + const toggleMute = useCallback( + () => runtime.toggleMute(sessionId), + [sessionId], + ); + const forwardTypedUserMessage = useCallback( + (text: string) => runtime.forwardTypedUserMessage(sessionId, text), + [sessionId], + ); + const shouldStart = + !ownsActiveConversation || + snapshot.state === "off" || + snapshot.state === "error"; + + return { + visible: enabled, + state: snapshot.state, + boundSessionId: snapshot.boundSessionId, + active: + snapshot.state !== "off" && + snapshot.state !== "error" && + snapshot.boundSessionId !== null, + ownsActiveConversation, + microphoneMuted: snapshot.microphoneMuted, + error: snapshot.error, + disabled: disabled || readOnly || anotherSessionOwnsConversation, + onToggle: shouldStart ? start : stop, + onMicrophoneMuteToggle: toggleMute, + onTypedUserMessageCommitted: forwardTypedUserMessage, + }; +} diff --git a/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts b/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts new file mode 100644 index 000000000..5178fa090 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from "vitest"; +import { + beginActiveRealtimeMasterTurn, + endActiveRealtimeMasterTurn, + getActiveRealtimeEmissary, + registerRealtimeEmissary, +} from "./realtimeEmissaryBridge"; + +describe("realtime emissary bridge registration", () => { + it("routes only to the current live session and releases by identity", async () => { + const sendMasterMessage = vi.fn().mockResolvedValue({ + accepted: false, + reason: "stale_cursor", + unreadPeerMessages: [], + cursor: 2, + }); + const beginMasterTurn = vi.fn(); + const endMasterTurn = vi.fn(); + const emissary = { + sessionId: "session-1", + beginMasterTurn, + endMasterTurn, + sendMasterMessage, + }; + const release = registerRealtimeEmissary(emissary); + + expect(getActiveRealtimeEmissary()).toBe(emissary); + await expect( + emissary.sendMasterMessage("update", 1), + ).resolves.toMatchObject({ accepted: false, cursor: 2 }); + expect(beginActiveRealtimeMasterTurn("session-1", "turn-1")).toBe(true); + expect(beginMasterTurn).toHaveBeenCalledWith("turn-1"); + endActiveRealtimeMasterTurn("session-1", { + turnId: "turn-1", + status: "completed", + finalText: "Finished.", + }); + expect(endMasterTurn).toHaveBeenCalledWith({ + turnId: "turn-1", + status: "completed", + finalText: "Finished.", + }); + + release(); + expect(getActiveRealtimeEmissary()).toBeNull(); + expect(beginActiveRealtimeMasterTurn("session-1", "turn-2")).toBe(false); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts b/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts new file mode 100644 index 000000000..94bf7ac6f --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts @@ -0,0 +1,61 @@ +import type { + DirectBridgeMessage, + DirectMessageExchange, +} from "./realtimeEmissaryProtocol"; + +export type MasterMessageDelivery = + | { + accepted: true; + cursor: number; + deliveryStatus: "sent" | "interrupting" | "queued"; + outbound: DirectBridgeMessage; + } + | Exclude; + +export interface ActiveRealtimeEmissary { + sessionId: string; + beginMasterTurn(turnId: string): void; + endMasterTurn(completion: MasterTurnCompletion): void; + sendMasterMessage( + message: string, + cursor: number, + ): Promise; +} + +export interface MasterTurnCompletion { + turnId: string; + status: "completed" | "cancelled" | "failed"; + finalText?: string; +} + +let activeEmissary: ActiveRealtimeEmissary | null = null; + +export function registerRealtimeEmissary( + emissary: ActiveRealtimeEmissary, +): () => void { + activeEmissary = emissary; + return () => { + if (activeEmissary === emissary) activeEmissary = null; + }; +} + +export function getActiveRealtimeEmissary(): ActiveRealtimeEmissary | null { + return activeEmissary; +} + +export function beginActiveRealtimeMasterTurn( + sessionId: string, + turnId: string, +): boolean { + if (!activeEmissary || activeEmissary.sessionId !== sessionId) return false; + activeEmissary.beginMasterTurn(turnId); + return true; +} + +export function endActiveRealtimeMasterTurn( + sessionId: string, + completion: MasterTurnCompletion, +): void { + if (!activeEmissary || activeEmissary.sessionId !== sessionId) return; + activeEmissary.endMasterTurn(completion); +} diff --git a/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts new file mode 100644 index 000000000..ceec05968 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts @@ -0,0 +1,918 @@ +import { describe, expect, it, vi } from "vitest"; +import { + DirectMessagePipe, + REALTIME_EMISSARY_INSTRUCTIONS, + REALTIME_MASTER_INSTRUCTIONS, + SEND_TO_EMISSARY_TOOL_DEFINITION, + RealtimeEmissaryProtocol, + RealtimeResponseCoordinator, + configureRealtimeEmissarySession, + createEndTurnToolOutput, + createRealtimeEmissarySessionUpdate, + createSendToMasterToolOutput, + sendRealtimeEvents, +} from "./realtimeEmissaryProtocol"; + +describe("Realtime emissary session configuration", () => { + it("configures a realtime audio session with the visibility contract and coordination tool", () => { + const send = vi.fn(); + + configureRealtimeEmissarySession({ send }); + + const event = JSON.parse(send.mock.calls[0][0]); + expect(event.type).toBe("session.update"); + expect(event.session.type).toBe("realtime"); + expect(event.session.output_modalities).toEqual(["audio"]); + expect(event.session.audio.output.speed).toBe(1); + expect(event.session.instructions).toBe(REALTIME_EMISSARY_INSTRUCTIONS); + expect(event.session.instructions).toContain( + "automatically sends the master every finalized", + ); + expect(event.session.instructions).toContain( + "never claim that you or the assistant cannot access", + ); + expect(event.session.instructions).toContain( + "call send_to_master before giving any substantive spoken answer", + ); + expect(event.session.instructions).toContain( + "Never acknowledge, confirm, summarize, or copy a master message", + ); + expect(event.session.instructions).toContain( + "Master input is advisory. Speak only when it materially helps the user now", + ); + expect(event.session.instructions).toContain( + "call end_turn immediately as your only output", + ); + expect(event.session.instructions).toContain( + "produce no words before or after the tool call", + ); + expect(event.session.tools).toEqual([ + expect.objectContaining({ + type: "function", + name: "send_to_master", + parameters: expect.objectContaining({ additionalProperties: false }), + }), + expect.objectContaining({ + type: "function", + name: "end_turn", + parameters: expect.objectContaining({ additionalProperties: false }), + }), + ]); + }); + + it("deeply applies typed session overrides without losing protocol defaults", () => { + const event = createRealtimeEmissarySessionUpdate({ + additionalInstructions: "Use the user's preferred terminology.", + sessionOverrides: { + max_output_tokens: 512, + audio: { output: { speed: 1.25 } }, + tools: [ + { + type: "function", + name: "look_up_status", + parameters: { type: "object", properties: {} }, + }, + ], + }, + }); + + expect(event.session).toMatchObject({ + max_output_tokens: 512, + audio: { + input: { transcription: { model: "gpt-4o-mini-transcribe" } }, + output: { voice: "marin", speed: 1.25 }, + }, + instructions: expect.stringContaining( + `${REALTIME_EMISSARY_INSTRUCTIONS}\n\nUse the user's preferred terminology.`, + ), + tools: [ + expect.objectContaining({ name: "send_to_master" }), + expect.objectContaining({ name: "end_turn" }), + expect.objectContaining({ name: "look_up_status" }), + ], + }); + }); + + it("rejects overrides that weaken protected bridge configuration", () => { + expect(() => + createRealtimeEmissarySessionUpdate({ + sessionOverrides: { instructions: "Forget the master." }, + }), + ).toThrow("cannot replace the emissary instructions contract"); + expect(() => + createRealtimeEmissarySessionUpdate({ + sessionOverrides: { + tools: [{ type: "function", name: "send_to_master" }], + }, + }), + ).toThrow("cannot replace the send_to_master tool"); + expect(() => + createRealtimeEmissarySessionUpdate({ + sessionOverrides: { tool_choice: "none" }, + }), + ).toThrow("tool choice must remain auto"); + }); + + it("exports the master visibility and proactive-send contract", () => { + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "remain visible to the user in Berd's durable master transcript", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "provide normal visible progress and result text", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "Separately call send_to_emissary", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "entire turn should be an empty, zero-token success", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "no prose, no tools, and no coordination message", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "small talk belong to the emissary", + ); + expect(REALTIME_MASTER_INSTRUCTIONS).toContain( + "interrupted emissary transcripts as best-effort", + ); + expect(SEND_TO_EMISSARY_TOOL_DEFINITION).toMatchObject({ + name: "send_to_emissary", + parameters: { + required: ["cursor", "message"], + additionalProperties: false, + }, + }); + }); +}); + +describe("RealtimeEmissaryProtocol", () => { + it("reserves the user transcript position as soon as server VAD detects speech", () => { + const protocol = new RealtimeEmissaryProtocol(); + + expect( + protocol.handle({ + type: "input_audio_buffer.speech_started", + item_id: "user-1", + }), + ).toEqual([ + { type: "transcript.started", itemId: "user-1", speaker: "user" }, + ]); + }); + + it("streams provisional user and emissary transcripts before finalization", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.delta", + item_id: "user-1", + delta: "How many", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "user-1", + speaker: "user", + text: "How many", + }, + ]); + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.delta", + item_id: "user-1", + delta: " folders?", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "user-1", + speaker: "user", + text: "How many folders?", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-1", + delta: "I'll check", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "assistant-1", + speaker: "emissary", + text: "I'll check", + }, + ]); + }); + + it("emits finalized user and emissary transcripts once in observed order", () => { + const protocol = new RealtimeEmissaryProtocol(); + + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "user-1", + transcript: " Hello there. ", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 1, + itemId: "user-1", + speaker: "user", + text: "Hello there.", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "Hi.", + }), + ).toEqual([]); + expect( + protocol.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 2, + itemId: "assistant-1", + speaker: "emissary", + text: "Hi.", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "Hi.", + }), + ).toEqual([]); + }); + + it("forwards interrupted streamed text as explicitly best-effort", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-1", + delta: "This part was heard", + }); + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "This part was never heard.", + }); + expect( + protocol.handle({ + type: "output_audio_buffer.cleared", + response_id: "response-1", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 1, + itemId: "assistant-1", + speaker: "emissary", + text: "This part was heard", + interrupted: true, + }, + { + type: "emissary.playback_interrupted", + responseId: "response-1", + }, + ]); + + // A late terminal transcript for the interrupted response is still + // generated text, not evidence that the user heard it. + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "This part was never heard.", + }); + + expect( + protocol.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([]); + }); + + it("fails loudly on Realtime server and transcription errors", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect(() => + protocol.handle({ + type: "error", + error: { message: "bad session configuration" }, + }), + ).toThrow("bad session configuration"); + expect(() => + protocol.handle({ + type: "conversation.item.input_audio_transcription.failed", + error: { message: "audio unintelligible" }, + }), + ).toThrow("audio unintelligible"); + }); + + it("ignores empty and non-terminal transcript events", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + item_id: "assistant-1", + delta: "partial", + }), + ).toEqual([]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + item_id: "assistant-1", + transcript: " ", + }), + ).toEqual([]); + }); + + it("assembles a send_to_master call from streamed arguments", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_item.added", + item: { + type: "function_call", + name: "send_to_master", + call_id: "call-1", + }, + }); + protocol.handle({ + type: "response.function_call_arguments.delta", + call_id: "call-1", + delta: '{"cursor":4,"message":"Please investigate', + }); + protocol.handle({ + type: "response.function_call_arguments.delta", + call_id: "call-1", + delta: ' this."}', + }); + + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + call_id: "call-1", + }), + ).toEqual([ + { + type: "send_to_master", + callId: "call-1", + cursor: 4, + message: "Please investigate this.", + }, + ]); + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + name: "send_to_master", + call_id: "call-1", + arguments: '{"cursor":4,"message":"duplicate"}', + }), + ).toEqual([]); + }); + + it("emits an explicit argument-free end_turn call once", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_item.added", + item: { + type: "function_call", + name: "end_turn", + call_id: "call-end", + }, + }); + + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + call_id: "call-end", + arguments: "{}", + }), + ).toEqual([{ type: "end_turn", callId: "call-end" }]); + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + name: "end_turn", + call_id: "call-end", + arguments: "{}", + }), + ).toEqual([]); + expect(createEndTurnToolOutput("call-end")).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-end", + output: '{"status":"ended"}', + }, + }); + }); + + it("rejects malformed send_to_master arguments", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect(() => + protocol.handle({ + type: "response.function_call_arguments.done", + name: "send_to_master", + call_id: "call-1", + arguments: '{"cursor":0,"message":"hello","unexpected":true}', + }), + ).toThrow("accepts only cursor and message arguments"); + }); +}); + +describe("master message injection", () => { + it("adds typed user text and creates a response while idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + + expect(coordinator.requestTypedUserMessage("Typed hello")).toEqual({ + status: "sent", + events: [ + { type: "input_audio_buffer.clear" }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Typed hello" }], + }, + }, + { type: "response.create" }, + ], + }); + }); + + it("interrupts active generation and playback for typed user text", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.requestMasterMessage({ message: "context" }); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect(coordinator.requestTypedUserMessage("New direction")).toEqual({ + status: "interrupting", + events: [ + { type: "response.cancel", response_id: "response-1" }, + { type: "output_audio_buffer.clear" }, + { type: "input_audio_buffer.clear" }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "New direction" }], + }, + }, + ], + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.cleared", + response_id: "response-1", + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("lets a server-VAD barge-in supersede a response before its terminal event", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + coordinator.requestMasterMessage({ message: "Queued master context." }); + + expect( + coordinator.handle({ + type: "response.created", + response: { id: "response-2" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "cancelled" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-2", status: "completed" }, + }), + ).toEqual([]); + + expect( + coordinator.requestMasterMessage({ message: "A later result." }), + ).toMatchObject({ status: "sent" }); + }); + + it("creates no emissary event for empty master output", () => { + const coordinator = new RealtimeResponseCoordinator(); + + expect(() => coordinator.requestMasterMessage({ message: " " })).toThrow( + "master message cannot be empty", + ); + + // Rejection leaves the coordinator idle; no hidden response lifecycle was + // created for the empty master turn. + expect( + coordinator.requestMasterMessage({ message: "Useful guidance." }).status, + ).toBe("sent"); + }); + + it("injects private master context and requests an emissary response", () => { + const coordinator = new RealtimeResponseCoordinator(); + const transport = { send: vi.fn() }; + const events = coordinator.requestMasterMessage({ + message: "Relay the result.", + eventId: "m1", + }).events; + sendRealtimeEvents(transport, events); + + expect( + transport.send.mock.calls.map(([event]) => JSON.parse(event)), + ).toEqual([ + { + type: "conversation.item.create", + event_id: "m1", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: "Private message from the master agent:\nRelay the result.", + }, + ], + }, + }, + { type: "response.create" }, + ]); + }); + + it("serializes a tool-output follow-up behind the response that called the tool", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + const toolOutput = { + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }; + + expect(coordinator.requestToolOutput(toolOutput)).toEqual({ + status: "queued", + events: [toolOutput], + }); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("coalesces a Master answer into the queued tool follow-up after playback", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + coordinator.requestToolOutput({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }); + expect( + coordinator.requestMasterMessage({ message: "The answer is 26." }), + ).toMatchObject({ status: "queued" }); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("requests a response immediately for a tool output while idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + const toolOutput = { + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }; + + expect(coordinator.requestToolOutput(toolOutput)).toEqual({ + status: "sent", + events: [toolOutput, { type: "response.create" }], + }); + }); + + it("sends immediately without cancelling when the session is idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + + const request = coordinator.requestMasterMessage({ + message: "Keep this in mind.", + }); + + expect(request.status).toBe("sent"); + expect(request.events.map((event) => event.type)).toEqual([ + "conversation.item.create", + "response.create", + ]); + expect(request.events).not.toContainEqual( + expect.objectContaining({ type: "response.cancel" }), + ); + }); + + it("lets completed generated audio finish playing when no master message is queued", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([]); + + expect( + coordinator.requestMasterMessage({ message: "A later result." }), + ).toMatchObject({ status: "sent" }); + }); + + it("injects master context immediately but waits for active playback before responding", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect( + coordinator.requestMasterMessage({ message: "First master message." }), + ).toEqual({ + status: "queued", + events: [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ + content: [ + expect.objectContaining({ + text: expect.stringContaining("First master message."), + }), + ], + }), + }), + ], + }); + expect( + coordinator.requestMasterMessage({ message: "Second master message." }), + ).toEqual({ + status: "queued", + events: [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ + content: [ + expect.objectContaining({ + text: expect.stringContaining("Second master message."), + }), + ], + }), + }), + ], + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("reports a busy reverse direction without consuming its message", () => { + expect( + createSendToMasterToolOutput("call-1", { + accepted: false, + reason: "pipe_busy", + cursor: 0, + unreadPeerMessages: [], + }), + ).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-1", + output: + '{"accepted":false,"reason":"pipe_busy","cursor":0,"unreadPeerMessages":[]}', + }, + }); + }); + + it("returns the coordination loop guard without requesting another reply", () => { + expect( + createSendToMasterToolOutput("call-2", { + accepted: false, + reason: "awaiting_new_user_input", + cursor: 4, + unreadPeerMessages: [], + }), + ).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-2", + output: + '{"accepted":false,"reason":"awaiting_new_user_input","cursor":4,"unreadPeerMessages":[]}', + }, + }); + }); +}); + +describe("DirectMessagePipe", () => { + it("allows the active sender to queue multiple messages", () => { + const pipe = new DirectMessagePipe(); + const first = pipe.send({ + sender: "emissary", + cursor: 0, + message: "First detail.", + }); + const second = pipe.send({ + sender: "emissary", + cursor: 0, + message: "Second detail.", + }); + expect(first).toMatchObject({ + accepted: true, + outbound: { id: 1, sender: "emissary" }, + }); + expect(second).toMatchObject({ + accepted: true, + outbound: { id: 2, sender: "emissary" }, + }); + if (!first.accepted || !second.accepted) + throw new Error("expected an accepted batch"); + + expect( + pipe.send({ sender: "master", cursor: 0, message: "Reply." }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + unreadPeerMessages: [], + cursor: 0, + }); + expect( + pipe.send({ sender: "master", cursor: 2, message: "Reply." }), + ).toMatchObject({ + accepted: true, + cursor: 2, + outbound: { id: 3, sender: "master", senderCursor: 2 }, + }); + expect(pipe.cursor("master")).toBe(2); + }); + + it("requires the cursor for the complete pending batch", () => { + const pipe = new DirectMessagePipe(); + const first = pipe.send({ sender: "master", cursor: 0, message: "One." }); + const second = pipe.send({ sender: "master", cursor: 0, message: "Two." }); + if (!first.accepted || !second.accepted) + throw new Error("expected an accepted batch"); + + expect( + pipe.send({ sender: "emissary", cursor: 1, message: "Too soon." }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + unreadPeerMessages: [], + cursor: 0, + }); + expect( + pipe.send({ sender: "emissary", cursor: 2, message: "Now reply." }), + ).toMatchObject({ + accepted: true, + cursor: 2, + outbound: { senderCursor: 2 }, + }); + expect(pipe.cursor("emissary")).toBe(2); + }); + + it("rejects a stale send without consuming the pending direction", () => { + const pipe = new DirectMessagePipe(); + const master = pipe.send({ + sender: "master", + cursor: 0, + message: "Result.", + }); + if (!master.accepted) throw new Error("expected accepted message"); + + expect( + pipe.send({ + sender: "emissary", + cursor: 0, + message: "Stale reply.", + }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + unreadPeerMessages: [], + cursor: 0, + }); + const reply = pipe.send({ + sender: "emissary", + cursor: master.outbound.id, + message: "Fresh reply.", + }); + expect(reply).toMatchObject({ + accepted: true, + unreadPeerMessages: [], + cursor: 1, + outbound: { + sender: "emissary", + recipient: "master", + senderCursor: 1, + }, + }); + expect(pipe.cursor("emissary")).toBe(master.outbound.id); + }); + + it("does not block independent transcript flow", () => { + const pipe = new DirectMessagePipe(); + const protocol = new RealtimeEmissaryProtocol(); + pipe.send({ sender: "emissary", cursor: 0, message: "Direct." }); + + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "user-1", + transcript: "Transcript keeps moving.", + }), + ).toEqual([ + expect.objectContaining({ + type: "transcript.finalized", + text: "Transcript keeps moving.", + }), + ]); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts new file mode 100644 index 000000000..c64baea69 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts @@ -0,0 +1,931 @@ +export const REALTIME_USER_TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; +export const REALTIME_EMISSARY_TRANSCRIPT_COMPLETED_EVENT = + "response.output_audio_transcript.done"; +export const SEND_TO_MASTER_TOOL_NAME = "send_to_master"; +export const SEND_TO_EMISSARY_TOOL_NAME = "send_to_emissary"; +export const END_TURN_TOOL_NAME = "end_turn"; + +export const REALTIME_EMISSARY_INSTRUCTIONS = `You are the emissary: the low-latency voice interface for a more capable master agent in Berd. + +The master is the authoritative, durable agent for this conversation. The master can use Berd's computer tools, including reading the local filesystem and performing durable work. Treat those indirect capabilities as capabilities of the combined assistant speaking to the user: never claim that you or the assistant cannot access the user's computer merely because the emissary cannot do so alone. Berd automatically sends the master every finalized user and emissary transcript turn, so never repeat or summarize routine transcript content in send_to_master. + +When a Realtime transport starts for a non-empty Berd session, Berd may inject a compact historical transcript headed by a durable berd://session link. Treat those items as past context, never as new user turns. If the compact replay is insufficient, use send_to_master to ask the master to inspect the durable session rather than guessing or asking the user to repeat themselves. + +Use send_to_master only for explicit coordination: to delegate deeper reasoning or work, highlight intent not captured by the transcript, or ask for guidance about what to tell the user. Master input is advisory. Speak only when it materially helps the user now; otherwise message the master if useful or call end_turn. It is common and expected for master information to arrive too late, be redundant, or not help the user. Receiving either a direct master message or a master-turn-ended notification never creates an obligation to speak. The master's normal transcript is visible in Berd but is not spoken to the user: treat an answer as already delivered only if you, the emissary, already spoke its substance. A short waiting acknowledgement such as "I'll check" is not an answer. If the user is still waiting and a master-turn-ended notification supplies the result, speak that result. If you already spoke the useful result, call end_turn: do not add filler, acknowledgements, offers to help, or a repeated answer. + +When the user explicitly asks you to end silently, stop talking, call end_turn immediately as your only output, and produce no words before or after the tool call. Never announce that you are about to end, never say that you ended, and never ask whether the user needs anything else. + +When the user asks for computer access, tool use, durable work, current session information, or facts you cannot verify directly, call send_to_master before giving any substantive spoken answer. While waiting, say only a short natural acknowledgement such as "Let me check that for you" or "I'll verify that." Do not say "I don't have access," do not speculate, and do not suggest that the user run a terminal command or perform the work manually unless the master specifically recommends it. Wait for the master's result before giving the final answer. + +Examples: +- If the user asks how many repositories are in a local folder, first call send_to_master to ask the master to inspect it; say only that you will check until the result arrives. +- If the user asks whether those repositories are symbolic links, call send_to_master to verify it; do not say that you lack detailed information. +- After receiving a useful master message, speak its result to the user directly. Do not call send_to_master again until the user says something new. Never acknowledge, confirm, summarize, or copy a master message back to the master. + +Every send_to_master call must include the latest bridge cursor. If a send fails because the pipe is busy in the other direction, do not retry yet: wait for Berd to deliver the pending master message normally, then retry with the cursor included in that message. The failed attempt did not send your message. + +Keep the spoken conversation natural and responsive. Represent the master's information accurately, and do not imply that you completed work performed by the master.`; + +export const REALTIME_MASTER_INSTRUCTIONS = `You are the master: the authoritative, durable agent for a Berd session whose live spoken conversation is conducted by a low-latency OpenAI Realtime emissary. + +Berd automatically sends you every finalized user and emissary transcript turn. Do not ask the emissary to repeat routine transcript content. + +While Realtime voice is active, Berd also delivers every ordinary typed user message directly to the emissary and interrupts any response currently being spoken. A typed message reaches you as an ordinary user turn; microphone transcripts are explicitly prefixed with "[Voice transcript]". Do not echo, paraphrase, or relay an ordinary typed user message through send_to_emissary unless you are adding genuinely new information the emissary needs. + +Your reasoning, ordinary assistant text, tool calls, and progress remain visible to the user in Berd's durable master transcript, but they are not visible to the emissary. On actionable turns, work normally in Berd: reason as needed, use the available tools, and provide normal visible progress and result text for the master transcript. Separately call send_to_emissary with the concise information that should influence what the emissary knows or says; do not assume your ordinary output was relayed. Each finalized transcript gives you an opportunity to act, not an obligation to react. When no work, correction, or useful emissary guidance is needed, your entire turn should be an empty, zero-token success: no prose, no tools, and no coordination message. Ordinary conversation and small talk belong to the emissary. Proactively send relevant facts, decisions, progress, constraints, and useful follow-up questions rather than waiting to be asked. Never call send_to_emissary merely to acknowledge, confirm, or echo a direct message from the emissary; acknowledgement-only coordination must be a zero-token no-op. + +Treat interrupted emissary transcripts as best-effort streamed text that may not exactly match the audio the user heard. Keep direct coordination concise. Every direct-message tool call must include the latest bridge cursor. If a send fails because the pipe is busy in the other direction, do not retry yet: wait for Berd to deliver the pending emissary message normally, then retry with the cursor included in that message.`; + +export interface RealtimeEventTransport { + send(data: string): void; +} + +export interface RealtimeEmissarySessionOptions { + /** Appended after the non-replaceable master/emissary contract. */ + additionalInstructions?: string; + transcriptionModel?: string; + voice?: string; + speed?: number; + /** + * Additional Realtime session fields. This deliberately remains an + * extensible JSON object so new API options do not require transport or + * protocol changes before Settings can expose them. + */ + sessionOverrides?: RealtimeSessionOverrides; +} + +export type RealtimeJsonValue = + | boolean + | number + | string + | null + | RealtimeJsonValue[] + | RealtimeJsonObject; + +export type RealtimeJsonObject = { + [key: string]: RealtimeJsonValue | undefined; +}; + +export type RealtimeSessionOverrides = RealtimeJsonObject; + +export type FinalizedRealtimeTranscript = { + type: "transcript.finalized"; + id: number; + itemId: string; + speaker: "user" | "emissary"; + text: string; + /** Best-effort streamed text; it may not exactly match the audio heard. */ + interrupted?: true; +}; + +export type UpdatedRealtimeTranscript = { + type: "transcript.updated"; + itemId: string; + speaker: "user" | "emissary"; + text: string; +}; + +export type StartedRealtimeTranscript = { + type: "transcript.started"; + itemId: string; + speaker: "user"; +}; + +export type SendToMasterCall = { + type: "send_to_master"; + callId: string; + cursor: number; + message: string; +}; + +export type EndTurnCall = { + type: "end_turn"; + callId: string; +}; + +export type RealtimePlaybackInterrupted = { + type: "emissary.playback_interrupted"; + responseId: string; +}; + +export type RealtimeEmissaryProtocolEvent = + | StartedRealtimeTranscript + | UpdatedRealtimeTranscript + | FinalizedRealtimeTranscript + | SendToMasterCall + | EndTurnCall + | RealtimePlaybackInterrupted; + +export type RealtimeClientEvent = Record; +type RealtimeServerEvent = Record; + +export const SEND_TO_EMISSARY_TOOL_DEFINITION: RealtimeJsonObject = { + type: "function", + name: SEND_TO_EMISSARY_TOOL_NAME, + description: + "Send concise private coordination to the realtime emissary. Include the latest bridge cursor and retry only after processing unread peer messages returned by a stale send.", + parameters: { + type: "object", + properties: { + cursor: { + type: "integer", + minimum: 0, + description: "Latest direct-message cursor returned by the bridge.", + }, + message: { type: "string" }, + }, + required: ["cursor", "message"], + additionalProperties: false, + }, +}; + +export function createRealtimeEmissarySessionUpdate( + options: RealtimeEmissarySessionOptions = {}, +): RealtimeServerEvent { + const overrides = options.sessionOverrides ?? {}; + assertSafeSessionOverrides(overrides); + const additionalTools = overrides.tools ?? []; + const mergeableOverrides = { ...overrides }; + delete mergeableOverrides.instructions; + delete mergeableOverrides.tools; + + const additionalInstructions = options.additionalInstructions?.trim(); + const defaults = { + type: "realtime", + output_modalities: ["audio"], + instructions: additionalInstructions + ? `${REALTIME_EMISSARY_INSTRUCTIONS}\n\n${additionalInstructions}` + : REALTIME_EMISSARY_INSTRUCTIONS, + audio: { + input: { + format: { type: "audio/pcm", rate: 24_000 }, + transcription: { + model: options.transcriptionModel ?? "gpt-4o-mini-transcribe", + }, + turn_detection: { + type: "server_vad", + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true, + }, + }, + output: { + format: { type: "audio/pcm", rate: 24_000 }, + voice: options.voice ?? "marin", + speed: options.speed ?? 1, + }, + }, + tools: [ + { + type: "function", + name: SEND_TO_MASTER_TOOL_NAME, + description: + "Send concise private coordination to the authoritative master agent. The master already receives the full finalized transcript, so do not repeat ordinary conversation turns.", + parameters: { + type: "object", + properties: { + cursor: { + type: "integer", + minimum: 0, + description: + "Latest direct-message cursor returned by the bridge.", + }, + message: { + type: "string", + description: + "A concise request, delegation, or important context not conveyed by the transcript alone.", + }, + }, + required: ["cursor", "message"], + additionalProperties: false, + }, + }, + { + type: "function", + name: END_TURN_TOOL_NAME, + description: + "Immediately end this emissary evaluation with no audio or follow-up response. Use as the sole output when the user asks you to end silently, or whenever speaking and further master coordination would not materially help now; never announce the call.", + parameters: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ...(additionalTools as RealtimeJsonValue[]), + ], + tool_choice: "auto", + } satisfies RealtimeSessionOverrides; + + return { + type: "session.update", + session: mergeRealtimeJson(defaults, mergeableOverrides), + }; +} + +export function configureRealtimeEmissarySession( + transport: RealtimeEventTransport, + options: RealtimeEmissarySessionOptions = {}, +): void { + sendRealtimeEvents(transport, [createRealtimeEmissarySessionUpdate(options)]); +} + +export function sendRealtimeEvents( + transport: RealtimeEventTransport, + events: readonly RealtimeClientEvent[], +): void { + for (const event of events) sendEvent(transport, event); +} + +function createMasterMessageItem(options: { + message: string; + eventId?: string; +}): RealtimeClientEvent { + const message = requireNonEmpty(options.message, "master message"); + const createItem: RealtimeServerEvent = { + type: "conversation.item.create", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: `Private message from the master agent:\n${message}`, + }, + ], + }, + }; + if (options.eventId) createItem.event_id = options.eventId; + + return createItem; +} + +function createMasterMessageEvents( + options: MasterMessage, +): RealtimeClientEvent[] { + return [createMasterMessageItem(options), { type: "response.create" }]; +} + +function createTypedUserMessageItem(text: string): RealtimeClientEvent { + return { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [ + { type: "input_text", text: requireNonEmpty(text, "user text") }, + ], + }, + }; +} + +export function createSendToMasterToolOutput( + callId: string, + exchange: SendToMasterToolResult, +): RealtimeServerEvent { + return { + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: requireNonEmpty(callId, "call id"), + output: JSON.stringify(exchange), + }, + }; +} + +export function createEndTurnToolOutput(callId: string): RealtimeServerEvent { + return { + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: requireNonEmpty(callId, "call id"), + output: JSON.stringify({ status: "ended" }), + }, + }; +} + +type MasterMessage = { message: string; eventId?: string }; + +export type MasterMessageRequest = { + status: "sent" | "interrupting" | "queued"; + events: RealtimeClientEvent[]; +}; + +type ActiveResponse = { + id?: string; + generationDone: boolean; + outputActive: boolean; +}; + +/** + * Serializes master-triggered responses with the default-conversation response + * lifecycle. Master context is injected immediately, but a follow-up response + * waits until the prior response and its WebRTC playback are terminal. This + * prevents late master results from cutting off the emissary mid-sentence. + */ +export class RealtimeResponseCoordinator { + private activeResponse: ActiveResponse | undefined; + private followUpResponsePending = false; + + requestMasterMessage(message: MasterMessage): MasterMessageRequest { + requireNonEmpty(message.message, "master message"); + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(); + return { status: "sent", events: createMasterMessageEvents(message) }; + } + + this.followUpResponsePending = true; + return { + status: "queued", + events: [createMasterMessageItem(message)], + }; + } + + requestToolOutput(event: RealtimeClientEvent): MasterMessageRequest { + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(); + return { + status: "sent", + events: [event, { type: "response.create" }], + }; + } + + this.followUpResponsePending = true; + return { status: "queued", events: [event] }; + } + + requestTypedUserMessage(text: string): MasterMessageRequest { + const item = createTypedUserMessageItem(text); + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(); + return { + status: "sent", + events: [ + { type: "input_audio_buffer.clear" }, + item, + { type: "response.create" }, + ], + }; + } + + this.followUpResponsePending = true; + const events: RealtimeClientEvent[] = []; + if (this.activeResponse.id && !this.activeResponse.generationDone) { + events.push({ + type: "response.cancel", + response_id: this.activeResponse.id, + }); + } + if (this.activeResponse.outputActive) { + events.push({ type: "output_audio_buffer.clear" }); + } + events.push({ type: "input_audio_buffer.clear" }, item); + return { + status: this.activeResponse.id ? "interrupting" : "queued", + events, + }; + } + + handle(event: unknown): RealtimeClientEvent[] { + if (!isRecord(event)) return []; + switch (event.type) { + case "response.created": { + const responseId = nestedResponseId(event); + if (!responseId) + throw new Error("response.created is missing response.id"); + if (this.activeResponse?.id) { + // Server VAD owns microphone barge-in and may create the replacement + // response before the cancelled response's terminal events arrive. + // Conversation items already queued for a follow-up are visible to + // this replacement response, so it also satisfies that pending wake. + this.followUpResponsePending = false; + } + this.activeResponse = { + id: responseId, + generationDone: false, + outputActive: false, + }; + return []; + } + case "output_audio_buffer.started": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.outputActive = true; + return []; + } + case "response.done": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.generationDone = true; + if (!active.outputActive) return this.finishActiveResponse(); + return []; + } + case "output_audio_buffer.stopped": + case "output_audio_buffer.cleared": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.outputActive = false; + return active.generationDone ? this.finishActiveResponse() : []; + } + default: + return []; + } + } + + private matchActiveResponse( + event: RealtimeServerEvent, + ): ActiveResponse | undefined { + const active = this.activeResponse; + if (!active) return undefined; + const responseId = + optionalString(event.response_id) ?? nestedResponseId(event); + return !responseId || !active.id || responseId === active.id + ? active + : undefined; + } + + private finishActiveResponse(): RealtimeClientEvent[] { + this.activeResponse = undefined; + if (!this.followUpResponsePending) return []; + this.followUpResponsePending = false; + this.activeResponse = awaitingCreatedResponse(); + return [{ type: "response.create" }]; + } +} + +function awaitingCreatedResponse(): ActiveResponse { + return { + generationDone: false, + outputActive: false, + }; +} + +export type DirectMessagePeer = "master" | "emissary"; + +export type DirectBridgeMessage = { + id: number; + sender: DirectMessagePeer; + recipient: DirectMessagePeer; + senderCursor: number; + message: string; +}; + +export type DirectMessageExchange = + | { + accepted: true; + unreadPeerMessages: []; + outbound: DirectBridgeMessage; + cursor: number; + } + | { + accepted: false; + reason: "pipe_busy" | "stale_cursor"; + unreadPeerMessages: []; + cursor: number; + }; + +export type SendToMasterToolResult = + | DirectMessageExchange + | { + accepted: false; + reason: "awaiting_new_user_input"; + unreadPeerMessages: []; + cursor: number; + }; + +/** + * One authoritative half-duplex direct-message pipe. The active sender may + * append any number of messages; only a send in the opposite direction is + * blocked until the recipient consumes the pending batch. Transcript events + * do not enter this state machine and therefore never block coordination. + * Ordinary delivery places pending messages into the recipient's context but + * does not mutate pipe state. The recipient consumes the complete pending + * batch by supplying its latest message id as the cursor on a reverse send; + * consumption, direction reversal, and reply enqueueing happen atomically. + * A stale reverse send neither exposes nor consumes pending messages. + */ +export class DirectMessagePipe { + private nextMessageId = 1; + private pending: DirectBridgeMessage[] = []; + private readonly consumedCursor: Record = { + master: 0, + emissary: 0, + }; + + send(options: { + sender: DirectMessagePeer; + cursor: number; + message: string; + }): DirectMessageExchange { + const message = requireNonEmpty(options.message, "direct message"); + const suppliedCursor = requireCursor(options.cursor); + const activeMessage = this.pending[0]; + if (activeMessage && activeMessage.sender !== options.sender) { + const latestPending = this.pending.at(-1); + if (!latestPending) + throw new Error("direct-message pending batch cannot be empty"); + if (suppliedCursor !== latestPending.id) { + return { + accepted: false, + reason: "pipe_busy", + unreadPeerMessages: [], + cursor: this.consumedCursor[options.sender], + }; + } + this.consumedCursor[options.sender] = latestPending.id; + this.pending = []; + } + const cursor = this.consumedCursor[options.sender]; + if (suppliedCursor !== cursor) { + return { + accepted: false, + reason: "stale_cursor", + unreadPeerMessages: [], + cursor, + }; + } + + const outbound: DirectBridgeMessage = { + id: this.nextMessageId++, + sender: options.sender, + recipient: otherPeer(options.sender), + senderCursor: cursor, + message, + }; + this.pending.push(outbound); + return { + accepted: true, + unreadPeerMessages: [], + outbound, + cursor, + }; + } + + cursor(peer: DirectMessagePeer): number { + return this.consumedCursor[peer]; + } +} + +function otherPeer(peer: DirectMessagePeer): DirectMessagePeer { + return peer === "master" ? "emissary" : "master"; +} + +/** + * Reduces Realtime server events to the durable bridge events Berd needs. + * One instance belongs to one Realtime session; it supplies local ordering and + * suppresses repeated terminal events by their stable OpenAI item/call ids. + */ +export class RealtimeEmissaryProtocol { + private nextTranscriptId = 1; + private readonly finalizedItemIds = new Set(); + private readonly completedCallIds = new Set(); + private readonly callNames = new Map(); + private readonly argumentDeltas = new Map(); + private readonly pendingUserTranscripts = new Map(); + private readonly pendingEmissaryTranscripts = new Map< + string, + { itemId: string; streamedText: string; finalText?: string } + >(); + private readonly interruptedResponseIds = new Set(); + + handle(event: unknown): RealtimeEmissaryProtocolEvent[] { + if (!isRecord(event)) return []; + + switch (event.type) { + case "error": + case "conversation.item.input_audio_transcription.failed": + throw new Error(realtimeErrorMessage(event)); + case "response.output_item.added": + this.captureFunctionCallName(event); + return []; + case "response.function_call_arguments.delta": + this.captureFunctionArguments(event); + return []; + case "response.function_call_arguments.done": { + const call = this.finishFunctionCall(event); + return call ? [call] : []; + } + case "input_audio_buffer.speech_started": { + const itemId = optionalString(event.item_id); + return itemId && !this.finalizedItemIds.has(itemId) + ? [{ type: "transcript.started", itemId, speaker: "user" }] + : []; + } + case REALTIME_USER_TRANSCRIPT_COMPLETED_EVENT: { + const transcript = this.finalizedTranscript(event, "user"); + return transcript ? [transcript] : []; + } + case "conversation.item.input_audio_transcription.delta": { + const transcript = this.captureUserTranscriptDelta(event); + return transcript ? [transcript] : []; + } + case REALTIME_EMISSARY_TRANSCRIPT_COMPLETED_EVENT: { + this.captureEmissaryTranscript(event); + return []; + } + case "response.output_audio_transcript.delta": + return this.captureEmissaryTranscriptDelta(event); + case "output_audio_buffer.stopped": { + const transcript = this.finishEmissaryPlayback(event); + return transcript ? [transcript] : []; + } + case "output_audio_buffer.cleared": { + const responseId = optionalString(event.response_id); + if (!responseId) return []; + this.interruptedResponseIds.add(responseId); + const transcript = this.finishInterruptedPlayback(responseId); + return [ + ...(transcript ? [transcript] : []), + { type: "emissary.playback_interrupted", responseId }, + ]; + } + default: + return []; + } + } + + private captureUserTranscriptDelta( + event: RealtimeServerEvent, + ): UpdatedRealtimeTranscript | undefined { + const itemId = optionalString(event.item_id); + const delta = optionalString(event.delta); + if (!itemId || delta === undefined || this.finalizedItemIds.has(itemId)) + return undefined; + const text = `${this.pendingUserTranscripts.get(itemId) ?? ""}${delta}`; + this.pendingUserTranscripts.set(itemId, text); + if (!text.trim()) return undefined; + return { type: "transcript.updated", itemId, speaker: "user", text }; + } + + private captureEmissaryTranscriptDelta( + event: RealtimeServerEvent, + ): UpdatedRealtimeTranscript[] { + const responseId = optionalString(event.response_id); + const itemId = optionalString(event.item_id); + const delta = optionalString(event.delta); + if ( + !responseId || + !itemId || + delta === undefined || + this.interruptedResponseIds.has(responseId) + ) { + return []; + } + const current = this.pendingEmissaryTranscripts.get(responseId); + const streamedText = (current?.streamedText ?? "") + delta; + this.pendingEmissaryTranscripts.set(responseId, { + itemId, + streamedText, + finalText: current?.finalText, + }); + return streamedText.trim() + ? [ + { + type: "transcript.updated", + itemId, + speaker: "emissary", + text: streamedText, + }, + ] + : []; + } + + private captureEmissaryTranscript(event: RealtimeServerEvent): void { + const responseId = optionalString(event.response_id); + const itemId = optionalString(event.item_id); + const text = optionalString(event.transcript)?.trim(); + if ( + !responseId || + !itemId || + !text || + this.finalizedItemIds.has(itemId) || + this.interruptedResponseIds.has(responseId) + ) { + return; + } + const current = this.pendingEmissaryTranscripts.get(responseId); + this.pendingEmissaryTranscripts.set(responseId, { + itemId, + streamedText: current?.streamedText ?? "", + finalText: text, + }); + } + + private finishEmissaryPlayback( + event: RealtimeServerEvent, + ): FinalizedRealtimeTranscript | undefined { + const responseId = optionalString(event.response_id); + if (!responseId) return undefined; + if (this.interruptedResponseIds.delete(responseId)) return undefined; + const pending = this.pendingEmissaryTranscripts.get(responseId); + this.pendingEmissaryTranscripts.delete(responseId); + const text = pending?.finalText ?? pending?.streamedText.trim(); + if (!pending || !text || this.finalizedItemIds.has(pending.itemId)) { + return undefined; + } + + this.finalizedItemIds.add(pending.itemId); + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId: pending.itemId, + speaker: "emissary", + text, + }; + } + + private finishInterruptedPlayback( + responseId: string, + ): FinalizedRealtimeTranscript | undefined { + const pending = this.pendingEmissaryTranscripts.get(responseId); + this.pendingEmissaryTranscripts.delete(responseId); + const text = pending?.streamedText.trim(); + if (!pending || !text || this.finalizedItemIds.has(pending.itemId)) { + return undefined; + } + + this.finalizedItemIds.add(pending.itemId); + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId: pending.itemId, + speaker: "emissary", + text, + interrupted: true, + }; + } + + private finalizedTranscript( + event: RealtimeServerEvent, + speaker: "user" | "emissary", + ): FinalizedRealtimeTranscript | undefined { + const itemId = optionalString(event.item_id); + const text = optionalString(event.transcript)?.trim(); + if (!itemId || !text || this.finalizedItemIds.has(itemId)) return undefined; + + this.pendingUserTranscripts.delete(itemId); + this.finalizedItemIds.add(itemId); + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId, + speaker, + text, + }; + } + + private captureFunctionCallName(event: RealtimeServerEvent): void { + const item = isRecord(event.item) ? event.item : undefined; + if (item?.type !== "function_call") return; + const callId = optionalString(item.call_id); + const name = optionalString(item.name); + if (callId && name) this.callNames.set(callId, name); + } + + private captureFunctionArguments(event: RealtimeServerEvent): void { + const callId = optionalString(event.call_id); + const delta = optionalString(event.delta); + if (!callId || delta === undefined) return; + this.argumentDeltas.set( + callId, + (this.argumentDeltas.get(callId) ?? "") + delta, + ); + } + + private finishFunctionCall( + event: RealtimeServerEvent, + ): SendToMasterCall | EndTurnCall | undefined { + const callId = optionalString(event.call_id); + if (!callId || this.completedCallIds.has(callId)) return undefined; + + const name = optionalString(event.name) ?? this.callNames.get(callId); + if (name === END_TURN_TOOL_NAME) { + const serializedArguments = + optionalString(event.arguments) ?? + this.argumentDeltas.get(callId) ?? + "{}"; + const parsed: unknown = JSON.parse(serializedArguments); + if (!isRecord(parsed) || Object.keys(parsed).length > 0) { + throw new Error("end_turn does not accept arguments"); + } + this.completedCallIds.add(callId); + this.argumentDeltas.delete(callId); + this.callNames.delete(callId); + return { type: "end_turn", callId }; + } + if (name !== SEND_TO_MASTER_TOOL_NAME) return undefined; + + const serializedArguments = + optionalString(event.arguments) ?? this.argumentDeltas.get(callId); + if (!serializedArguments) return undefined; + + const parsed: unknown = JSON.parse(serializedArguments); + if (!isRecord(parsed)) + throw new Error("send_to_master arguments must be an object"); + const keys = Object.keys(parsed).sort(); + if (keys.length !== 2 || keys[0] !== "cursor" || keys[1] !== "message") { + throw new Error( + "send_to_master accepts only cursor and message arguments", + ); + } + const cursor = requireCursor(parsed.cursor); + const message = requireNonEmpty(parsed.message, "send_to_master message"); + + this.completedCallIds.add(callId); + this.argumentDeltas.delete(callId); + this.callNames.delete(callId); + return { type: "send_to_master", callId, cursor, message }; + } +} + +function sendEvent( + transport: RealtimeEventTransport, + event: RealtimeClientEvent, +): void { + transport.send(JSON.stringify(event)); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function nestedResponseId(event: RealtimeServerEvent): string | undefined { + const response = isRecord(event.response) ? event.response : undefined; + return optionalString(response?.id); +} + +function requireNonEmpty(value: unknown, field: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`${field} cannot be empty`); + } + return value.trim(); +} + +function requireCursor(value: unknown): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error("direct-message cursor must be a non-negative integer"); + } + return value; +} + +function realtimeErrorMessage(event: RealtimeServerEvent): string { + const error = isRecord(event.error) ? event.error : undefined; + return ( + optionalString(error?.message) ?? + optionalString(event.message) ?? + "OpenAI Realtime reported an unknown error" + ); +} + +function assertSafeSessionOverrides(overrides: RealtimeSessionOverrides): void { + if (overrides.instructions !== undefined) { + throw new Error( + "sessionOverrides cannot replace the emissary instructions contract; use additionalInstructions", + ); + } + if (overrides.type !== undefined && overrides.type !== "realtime") { + throw new Error("emissary session type must remain realtime"); + } + if (overrides.tool_choice !== undefined && overrides.tool_choice !== "auto") { + throw new Error("emissary send_to_master tool choice must remain auto"); + } + if (overrides.tools === undefined) return; + if (!Array.isArray(overrides.tools)) { + throw new Error("sessionOverrides.tools must be an array"); + } + for (const tool of overrides.tools) { + if ( + isRecord(tool) && + optionalString(tool.name) === SEND_TO_MASTER_TOOL_NAME + ) { + throw new Error( + "sessionOverrides cannot replace the send_to_master tool", + ); + } + } +} + +function mergeRealtimeJson( + base: RealtimeSessionOverrides, + overrides: RealtimeSessionOverrides, +): RealtimeSessionOverrides { + const merged: RealtimeSessionOverrides = { ...base }; + for (const [key, value] of Object.entries(overrides)) { + if (value === undefined) continue; + const current = merged[key]; + merged[key] = + isRecord(current) && isRecord(value) + ? mergeRealtimeJson( + current as RealtimeSessionOverrides, + value as RealtimeSessionOverrides, + ) + : value; + } + return merged; +} diff --git a/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts b/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts new file mode 100644 index 000000000..d6cf3542b --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts @@ -0,0 +1,50 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + getRealtimeVoicePreference, + parseRealtimeSessionOverrides, + setRealtimeVoicePreference, +} from "./realtimeVoicePreference"; + +describe("realtime voice preferences", () => { + beforeEach(() => window.localStorage.clear()); + + it("returns a stable default snapshot", () => { + expect(getRealtimeVoicePreference()).toBe(getRealtimeVoicePreference()); + expect(getRealtimeVoicePreference()).toMatchObject({ + model: "gpt-realtime", + voice: "marin", + speed: 1, + }); + }); + + it("persists an updated configuration without storing a secret", () => { + const preference = { + model: "gpt-realtime-2.1", + transcriptionModel: "gpt-4o-mini-transcribe", + voice: "cedar", + speed: 1.25, + sessionOverridesText: '{"audio":{"input":{"turn_detection":null}}}', + }; + setRealtimeVoicePreference(preference); + expect(getRealtimeVoicePreference()).toBe(preference); + expect( + window.localStorage.getItem("goose:openai-realtime-voice-options"), + ).not.toContain("apiKey"); + }); + + it("falls back to normal speed when persisted speed is out of range", () => { + window.localStorage.setItem( + "goose:openai-realtime-voice-options", + JSON.stringify({ speed: 2 }), + ); + + expect(getRealtimeVoicePreference().speed).toBe(1); + }); + + it("accepts only JSON objects as advanced session overrides", () => { + expect(parseRealtimeSessionOverrides('{"max_output_tokens":128}')).toEqual({ + max_output_tokens: 128, + }); + expect(() => parseRealtimeSessionOverrides("[]")).toThrow("JSON object"); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeVoicePreference.ts b/src/features/voice-conversation/lib/realtimeVoicePreference.ts new file mode 100644 index 000000000..3b6bc0062 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePreference.ts @@ -0,0 +1,104 @@ +import { useCallback, useSyncExternalStore } from "react"; +import type { RealtimeSessionOverrides } from "./realtimeEmissaryProtocol"; + +export interface RealtimeVoicePreference { + model: string; + transcriptionModel: string; + voice: string; + speed: number; + sessionOverridesText: string; +} + +const DEFAULT_PREFERENCE: RealtimeVoicePreference = { + model: "gpt-realtime", + transcriptionModel: "gpt-4o-mini-transcribe", + voice: "marin", + speed: 1, + sessionOverridesText: "{}", +}; +const STORAGE_KEY = "goose:openai-realtime-voice-options"; +const CHANGED_EVENT = "goose:openai-realtime-voice-options-changed"; +const listeners = new Set<() => void>(); +let cachedRaw: string | null | undefined; +let cachedPreference = DEFAULT_PREFERENCE; + +export function getRealtimeVoicePreference(): RealtimeVoicePreference { + if (typeof window === "undefined") return DEFAULT_PREFERENCE; + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + if (raw === cachedRaw) return cachedPreference; + const parsed = JSON.parse(raw ?? "{}"); + cachedRaw = raw; + cachedPreference = { + model: + typeof parsed.model === "string" && parsed.model.trim() + ? parsed.model + : DEFAULT_PREFERENCE.model, + transcriptionModel: + typeof parsed.transcriptionModel === "string" && + parsed.transcriptionModel.trim() + ? parsed.transcriptionModel + : DEFAULT_PREFERENCE.transcriptionModel, + voice: + typeof parsed.voice === "string" && parsed.voice.trim() + ? parsed.voice + : DEFAULT_PREFERENCE.voice, + speed: + typeof parsed.speed === "number" && + Number.isFinite(parsed.speed) && + parsed.speed >= 0.25 && + parsed.speed <= 1.5 + ? parsed.speed + : DEFAULT_PREFERENCE.speed, + sessionOverridesText: + typeof parsed.sessionOverridesText === "string" + ? parsed.sessionOverridesText + : DEFAULT_PREFERENCE.sessionOverridesText, + }; + return cachedPreference; + } catch { + return DEFAULT_PREFERENCE; + } +} + +function subscribe(listener: () => void) { + listeners.add(listener); + const notify = () => listener(); + window.addEventListener(CHANGED_EVENT, notify); + return () => { + listeners.delete(listener); + window.removeEventListener(CHANGED_EVENT, notify); + }; +} + +export function setRealtimeVoicePreference( + preference: RealtimeVoicePreference, +): void { + const raw = JSON.stringify(preference); + window.localStorage.setItem(STORAGE_KEY, raw); + cachedRaw = raw; + cachedPreference = preference; + window.dispatchEvent(new Event(CHANGED_EVENT)); +} + +export function parseRealtimeSessionOverrides( + text: string, +): RealtimeSessionOverrides { + const parsed: unknown = JSON.parse(text || "{}"); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Realtime session overrides must be a JSON object."); + } + return parsed as RealtimeSessionOverrides; +} + +export function useRealtimeVoicePreference() { + const preference = useSyncExternalStore( + subscribe, + getRealtimeVoicePreference, + () => DEFAULT_PREFERENCE, + ); + const setPreference = useCallback((value: RealtimeVoicePreference) => { + setRealtimeVoicePreference(value); + }, []); + return { preference, setPreference }; +} diff --git a/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts b/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts new file mode 100644 index 000000000..644ab5367 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts @@ -0,0 +1,18 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + getVoiceConversationMode, + setVoiceConversationMode, +} from "./voiceConversationModePreference"; + +describe("voice conversation mode preference", () => { + beforeEach(() => window.localStorage.clear()); + + it("defaults to the existing chained pipeline", () => { + expect(getVoiceConversationMode()).toBe("chained"); + }); + + it("persists OpenAI Realtime mode", () => { + setVoiceConversationMode("openai-realtime"); + expect(getVoiceConversationMode()).toBe("openai-realtime"); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceConversationModePreference.ts b/src/features/voice-conversation/lib/voiceConversationModePreference.ts new file mode 100644 index 000000000..721a383ae --- /dev/null +++ b/src/features/voice-conversation/lib/voiceConversationModePreference.ts @@ -0,0 +1,78 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export type VoiceConversationMode = "chained" | "openai-realtime"; + +const STORAGE_KEY = "goose:voice-conversation-mode"; +const CHANGED_EVENT = "goose:voice-conversation-mode-changed"; +let inMemoryMode: VoiceConversationMode | null = null; + +function normalize(value: unknown): VoiceConversationMode { + return value === "openai-realtime" ? value : "chained"; +} + +export function getVoiceConversationMode(): VoiceConversationMode { + if (typeof window === "undefined") return "chained"; + if (inMemoryMode) return inMemoryMode; + try { + return normalize(window.localStorage.getItem(STORAGE_KEY)); + } catch { + return "chained"; + } +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + if (typeof window === "undefined") return () => undefined; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) { + inMemoryMode = null; + notify(); + } + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setVoiceConversationMode(mode: VoiceConversationMode): void { + if (typeof window === "undefined") return; + const value = normalize(mode); + inMemoryMode = value; + try { + window.localStorage.setItem(STORAGE_KEY, value); + } catch { + // Keep this renderer usable when persistent storage is unavailable. + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: { value } })); +} + +export function useVoiceConversationModePreference() { + const mode = useSyncExternalStore( + subscribe, + getVoiceConversationMode, + () => "chained" as const, + ); + const setMode = useCallback((value: VoiceConversationMode) => { + setVoiceConversationMode(value); + }, []); + return { mode, setMode }; +} diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx new file mode 100644 index 000000000..c8ea779f4 --- /dev/null +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx @@ -0,0 +1,205 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { toast } from "sonner"; +import { + getOpenAiRealtimeStatus, + saveOpenAiRealtimeApiKey, +} from "@/shared/api/openaiRealtime"; +import { Button } from "@/shared/ui/button"; +import { Input } from "@/shared/ui/input"; +import { Label } from "@/shared/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/shared/ui/select"; +import { Slider } from "@/shared/ui/slider"; +import { Textarea } from "@/shared/ui/textarea"; +import { + parseRealtimeSessionOverrides, + useRealtimeVoicePreference, +} from "../lib/realtimeVoicePreference"; + +const REALTIME_VOICES = [ + "marin", + "cedar", + "alloy", + "ash", + "ballad", + "coral", + "echo", + "sage", + "shimmer", + "verse", +] as const; + +function voiceLabel(voice: string): string { + return `${voice.charAt(0).toUpperCase()}${voice.slice(1)}`; +} + +export function RealtimeVoiceSettings() { + const { t } = useTranslation("settings"); + const { preference, setPreference } = useRealtimeVoicePreference(); + const [apiKey, setApiKey] = useState(""); + const [configured, setConfigured] = useState(false); + const [saving, setSaving] = useState(false); + + useEffect(() => { + void getOpenAiRealtimeStatus() + .then((status) => setConfigured(status.voiceConfigured)) + .catch(() => setConfigured(false)); + }, []); + + const saveKey = async () => { + setSaving(true); + try { + await saveOpenAiRealtimeApiKey(apiKey); + setApiKey(""); + setConfigured(true); + toast.success(t("voice.realtimeApiKeySaved")); + } catch (error) { + toast.error(t("voice.realtimeApiKeySaveFailed"), { + description: error instanceof Error ? error.message : String(error), + }); + } finally { + setSaving(false); + } + }; + + const update = (patch: Partial) => { + setPreference({ ...preference, ...patch }); + }; + + return ( +
+
+ +
+ setApiKey(event.target.value)} + /> + +
+

+ {t("voice.realtimeApiKeyDescription")} +

+
+
+
+ + update({ model: event.target.value })} + /> +
+
+ + + update({ transcriptionModel: event.target.value }) + } + /> +
+
+ + +
+
+
+
+ + + {preference.speed.toFixed(2)}× + +
+ update({ speed })} + aria-label={t("voice.realtimeSpeed")} + /> +

+ {t("voice.realtimeSpeedDescription")} +

+
+
+ +