diff --git a/docs/app-e2e.md b/docs/app-e2e.md index ff6a91a31..7d88857c8 100644 --- a/docs/app-e2e.md +++ b/docs/app-e2e.md @@ -44,3 +44,47 @@ Windows uses the same runtime contract through `scripts/windows/Dev-Windows.ps1` Set `BERD_E2E_MODE=1`, `BERD_E2E_RUN_ROOT`, and the optional provider bootstrap environment above before invoking it; the app owns its random driver port and publishes readiness under the run root. + +## Live Realtime Expert–Spokesperson evaluation + +`tests/app-e2e/realtime-expert-spokesperson.eval.test.ts` is an opt-in live +evaluation driven by typed chat messages. It starts a fresh Realtime voice +conversation, mutes its microphone so ambient audio cannot affect the run, asks +how many repositories are in the user's Development folder, then asks whether +any are symbolic links. It verifies that each typed question is followed in +order by visible Expert-to-Spokesperson coordination and a visible terminal Expert +turn. Each turn may contain one finalized Spokesperson answer, or an +acknowledgement and waiting update before the answer. More than three finalized +utterances fails the evaluation as a likely coordination loop. + +The app-test-driver protocol serves one command per TCP connection, so +the client opens a fresh authenticated connection for every command. Home +navigation and promotion of its composer draft may temporarily replace the app +webview; this eval waits for the expected destination after those two known +boundaries without replaying the navigation or call-button click. Mutating +actions remain single-shot. + +This scenario intentionally uses the normal local dev profile, not isolated +E2E mode: Realtime needs the Berd-owned API key stored from Voice settings, and +the Expert needs the normal configured agent/tool environment for inspecting the +real Development folder. Before running it, select **OpenAI Realtime** as the +Voice mode and save the Realtime API key in Berd. + +Start the instrumented app in one terminal: + +```bash +APP_TEST_DRIVER_TOKEN=local-realtime-eval just dev-e2e +``` + +Then run only the live scenario in another terminal: + +```bash +APP_TEST_DRIVER_TOKEN=local-realtime-eval \ +BERD_E2E_REALTIME_EVAL=1 \ +pnpm exec vitest run --config vitest.app-e2e.config.ts \ + tests/app-e2e/realtime-expert-spokesperson.eval.test.ts +``` + +Without `BERD_E2E_REALTIME_EVAL=1`, the live scenario is skipped so the normal +app E2E suite never makes network/model calls or depends on a developer's local +filesystem and credentials. diff --git a/scripts/block-feature-gates.sh b/scripts/block-feature-gates.sh index 2a9b6dc28..8de10acd1 100755 --- a/scripts/block-feature-gates.sh +++ b/scripts/block-feature-gates.sh @@ -21,10 +21,6 @@ done [[ "${VITE_MANAGED_CONNECTIONS:-0}" == "1" ]] && features+=(block-managed-connections) [[ "${VITE_SKILL_DISCOVERY:-0}" == "1" ]] && features+=(block-skill-discovery) [[ "${VITE_TELEMETRY_ENFORCED:-0}" == "1" ]] && features+=(block-telemetry-enforced) -if [[ "${VITE_VOICE_DICTATION:-0}" == "1" ]]; then - features+=(block-voice-dictation) -else - features+=(no-voice-dictation) -fi +[[ "${VITE_VOICE_DICTATION:-0}" == "1" ]] && features+=(block-voice-dictation) -(IFS=,; echo "${features[*]}") +(IFS=,; echo "${features[*]:-}") diff --git a/scripts/release/build-macos.sh b/scripts/release/build-macos.sh index 040620772..5ce772191 100755 --- a/scripts/release/build-macos.sh +++ b/scripts/release/build-macos.sh @@ -420,11 +420,7 @@ done # the user setting in Gate A and hides the toggle, the feature does the same # for the native Gate B in export_otel_logs. [[ "$VITE_TELEMETRY_ENFORCED_VALUE" == "1" ]] && CARGO_FEATURES="$CARGO_FEATURES,block-telemetry-enforced" -if [[ "$VITE_VOICE_DICTATION_VALUE" == "1" ]]; then - CARGO_FEATURES="$CARGO_FEATURES,block-voice-dictation" -else - CARGO_FEATURES="$CARGO_FEATURES,no-voice-dictation" -fi +[[ "$VITE_VOICE_DICTATION_VALUE" == "1" ]] && CARGO_FEATURES="$CARGO_FEATURES,block-voice-dictation" # bb CLI PATH install has no runtime-config representation; the custom pipeline # exposes a dedicated select that disables it via the Cargo feature. diff --git a/scripts/release/tests/release-scripts.test.mjs b/scripts/release/tests/release-scripts.test.mjs index ffbc6c78e..d9ca70af7 100644 --- a/scripts/release/tests/release-scripts.test.mjs +++ b/scripts/release/tests/release-scripts.test.mjs @@ -527,7 +527,6 @@ describe("build-macos Block-service feature seam", () => { expect(script).not.toContain( 'VITE_AUTH_GATE_VALUE="$VITE_BUILDERBOT_VALUE"', ); - expect(script).toContain("no-voice-dictation"); expect(script).toContain('if [[ "$VITE_AGENT_TOOLS_VALUE" == "1" ]]; then'); expect(script).toContain( 'jq \'.bundle.resources["../resources/bb"] = "bb"\'', @@ -2155,10 +2154,16 @@ async function canonicalGates() { } describe("Block feature gate propagation", () => { + it("supports an empty base feature set", () => { + const result = run("bash", ["scripts/block-feature-gates.sh"]); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(""); + }); + it("maps every updater-off default to the fail-closed Cargo posture", () => { const result = run("bash", ["scripts/block-feature-gates.sh", "berdctl"]); expect(result.status).toBe(0); - expect(result.stdout.trim()).toBe("berdctl,no-voice-dictation"); + expect(result.stdout.trim()).toBe("berdctl"); }); it("maps every renderer gate to its matching Cargo feature", () => { diff --git a/scripts/windows/Test-WindowsDev.ps1 b/scripts/windows/Test-WindowsDev.ps1 index 531a3e153..c2d1c6acd 100644 --- a/scripts/windows/Test-WindowsDev.ps1 +++ b/scripts/windows/Test-WindowsDev.ps1 @@ -77,7 +77,7 @@ try { Assert-Equal "process args: trailing backslash doubled inside quotes" (Join-WindowsProcessArguments -Arguments @("C:\Program Files\")) '"C:\Program Files\\"' Assert-Equal "process args: embedded quote escaped" (Join-WindowsProcessArguments -Arguments @('say "hi"')) '"say \"hi\""' - Assert-Equal "public app feature defaults fail closed" (Get-BerdAppFeatures) "berdctl,app-test-driver,no-voice-dictation" + Assert-Equal "public app feature defaults fail closed" (Get-BerdAppFeatures) "berdctl,app-test-driver" $featureGateNames = @("VITE_AGENT_TOOLS", "VITE_AUTOMATIONS", "VITE_BUILDERBOT", "VITE_FEEDBACK", "VITE_MANAGED_CONNECTIONS", "VITE_SKILL_DISCOVERY", "VITE_TELEMETRY_ENFORCED", "VITE_VOICE_DICTATION") $savedFeatureGates = @{} foreach ($name in $featureGateNames) { diff --git a/scripts/windows/WindowsDev.psm1 b/scripts/windows/WindowsDev.psm1 index a65688d8e..78a6b479c 100644 --- a/scripts/windows/WindowsDev.psm1 +++ b/scripts/windows/WindowsDev.psm1 @@ -513,9 +513,6 @@ function Get-BerdAppFeatures { } if ($value -eq "1") { $features.Add($gate.Feature) } } - if ([Environment]::GetEnvironmentVariable("VITE_VOICE_DICTATION", "Process") -ne "1") { - $features.Add("no-voice-dictation") - } return ($features -join ",") } diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 08505f3e4..602b59f4c 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -142,9 +142,6 @@ devtools = ["tauri/devtools"] # restricted build is expressed by ADDING these (never with # --no-default-features + re-listing everything you want to keep). # -# no-voice-dictation: the realtime client secret is never requested -# (`get_openai_realtime_status` reports `configured: false`). -no-voice-dictation = [] # no-bb-cli-install: the app never auto-installs or offers to symlink the # bundled berdctl into /usr/local/bin/bb. The bundled binary still ships for # direct `berdctl` use (gated separately by the `berdctl` protocol-server @@ -177,6 +174,8 @@ block-skill-discovery = [] # For managed internal distributions where consent is an employment-policy # fact, not a per-user choice. block-telemetry-enforced = [] +# Enables Block-service-backed voice dictation. Public Realtime voice is a +# separate, user-configured path and does not depend on Block services. block-voice-dictation = [] # Admin runtime-config endpoint fetch. Default-OFF: a normal build never # compiles the kgoose-backed fetch/cache path and instead loads the bundled diff --git a/src-tauri/crates/berdctl/api-surface-feedback.json b/src-tauri/crates/berdctl/api-surface-feedback.json index 7ed5474dd..c028af717 100644 --- a/src-tauri/crates/berdctl/api-surface-feedback.json +++ b/src-tauri/crates/berdctl/api-surface-feedback.json @@ -3,7 +3,7 @@ "protocolVersion": 4, "groups": { "sessions": { - "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, fork, archive.", + "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, send to a live voice Spokesperson, dismiss voice handoffs, fork, archive.", "actions": { "create": { "description": "Create a new chat session on any installed agent harness and send the prompt in it. Fire-and-forget: returns the session id immediately and the session runs in the background without changing what the user sees; the user can open it themselves. Use --from to give the delegating session or tool a concise visible label on the initial message. Only check on it later (action \"get\") if the user asks.", @@ -427,6 +427,159 @@ "additionalProperties": false } }, + "send_to_spokesperson": { + "description": "Inject a private coordination message into the OpenAI Realtime voice Spokesperson owned by an existing Berd session. The Spokesperson receives the message either as silent context for future turns or as a request to speak now. The command fails when the target session has no live Realtime voice conversation.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime Spokesperson.", + "min": 1 + }, + { + "name": "message", + "required": true, + "kind": "string", + "description": "Private coordination message to inject into the Spokesperson.", + "min": 1, + "max": 20000 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + "min": 0, + "max": 4294967295 + }, + { + "name": "mode", + "required": false, + "kind": "string", + "description": "Delivery mode: context updates future turns silently; say asks the Spokesperson to speak now.", + "values": ["context", "say"] + }, + { + "name": "resolves", + "required": false, + "kind": "string_array", + "description": "Open handoff id resolved by this say message; repeat for multiple handoffs." + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime Spokesperson." + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 20000, + "description": "Private coordination message to inject into the Spokesperson." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result." + }, + "mode": { + "default": "say", + "description": "Delivery mode: context updates future turns silently; say asks the Spokesperson to speak now.", + "type": "string", + "enum": ["context", "say"] + }, + "resolves": { + "default": [], + "description": "Open handoff id resolved by this say message; repeat for multiple handoffs.", + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 100 + } + } + }, + "required": ["session_id", "message", "cursor"], + "additionalProperties": false + } + }, + "dismiss_handoffs": { + "description": "Explicitly close one or more open Realtime Spokesperson handoffs and deliver the reason as silent context without waking the Spokesperson. Use this only when a spoken response is obsolete, superseded, or already handled. The command and its reason remain visible in the Expert's normal Berd activity.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime Spokesperson.", + "min": 1 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + "min": 0, + "max": 4294967295 + }, + { + "name": "handoff_id", + "required": true, + "kind": "string_array", + "description": "Open handoff id to dismiss; repeat for multiple handoffs." + }, + { + "name": "reason", + "required": true, + "kind": "string", + "description": "Why no spoken response is needed for these handoffs.", + "min": 1, + "max": 2000 + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime Spokesperson." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result." + }, + "handoff_id": { + "minItems": 1, + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": "Open handoff id to dismiss; repeat for multiple handoffs." + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 2000, + "description": "Why no spoken response is needed for these handoffs." + } + }, + "required": ["session_id", "cursor", "handoff_id", "reason"], + "additionalProperties": false + } + }, "fork": { "description": "Duplicate an existing chat session, copying its full conversation history into a new session the user can continue down an independent path. The fork appears in the app's session list; the user's current view does not change.", "fields": [ diff --git a/src-tauri/crates/berdctl/api-surface.json b/src-tauri/crates/berdctl/api-surface.json index b7daa844d..bb4b81b4a 100644 --- a/src-tauri/crates/berdctl/api-surface.json +++ b/src-tauri/crates/berdctl/api-surface.json @@ -3,7 +3,7 @@ "protocolVersion": 4, "groups": { "sessions": { - "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, fork, archive.", + "description": "Manage the user's chat sessions: create (fire-and-forget, on any installed agent harness), send, open, list, get, rename, move, move to group, clear project, send to a live voice Spokesperson, dismiss voice handoffs, fork, archive.", "actions": { "create": { "description": "Create a new chat session on any installed agent harness and send the prompt in it. Fire-and-forget: returns the session id immediately and the session runs in the background without changing what the user sees; the user can open it themselves. Use --from to give the delegating session or tool a concise visible label on the initial message. Only check on it later (action \"get\") if the user asks.", @@ -427,6 +427,159 @@ "additionalProperties": false } }, + "send_to_spokesperson": { + "description": "Inject a private coordination message into the OpenAI Realtime voice Spokesperson owned by an existing Berd session. The Spokesperson receives the message either as silent context for future turns or as a request to speak now. The command fails when the target session has no live Realtime voice conversation.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime Spokesperson.", + "min": 1 + }, + { + "name": "message", + "required": true, + "kind": "string", + "description": "Private coordination message to inject into the Spokesperson.", + "min": 1, + "max": 20000 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + "min": 0, + "max": 4294967295 + }, + { + "name": "mode", + "required": false, + "kind": "string", + "description": "Delivery mode: context updates future turns silently; say asks the Spokesperson to speak now.", + "values": ["context", "say"] + }, + { + "name": "resolves", + "required": false, + "kind": "string_array", + "description": "Open handoff id resolved by this say message; repeat for multiple handoffs." + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime Spokesperson." + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 20000, + "description": "Private coordination message to inject into the Spokesperson." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result." + }, + "mode": { + "default": "say", + "description": "Delivery mode: context updates future turns silently; say asks the Spokesperson to speak now.", + "type": "string", + "enum": ["context", "say"] + }, + "resolves": { + "default": [], + "description": "Open handoff id resolved by this say message; repeat for multiple handoffs.", + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 100 + } + } + }, + "required": ["session_id", "message", "cursor"], + "additionalProperties": false + } + }, + "dismiss_handoffs": { + "description": "Explicitly close one or more open Realtime Spokesperson handoffs and deliver the reason as silent context without waking the Spokesperson. Use this only when a spoken response is obsolete, superseded, or already handled. The command and its reason remain visible in the Expert's normal Berd activity.", + "fields": [ + { + "name": "session_id", + "required": true, + "kind": "string", + "description": "Id of the session that owns the live Realtime Spokesperson.", + "min": 1 + }, + { + "name": "cursor", + "required": true, + "kind": "number", + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + "min": 0, + "max": 4294967295 + }, + { + "name": "handoff_id", + "required": true, + "kind": "string_array", + "description": "Open handoff id to dismiss; repeat for multiple handoffs." + }, + { + "name": "reason", + "required": true, + "kind": "string", + "description": "Why no spoken response is needed for these handoffs.", + "min": 1, + "max": 2000 + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "session_id": { + "type": "string", + "minLength": 1, + "description": "Id of the session that owns the live Realtime Spokesperson." + }, + "cursor": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295, + "description": "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result." + }, + "handoff_id": { + "minItems": 1, + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": "Open handoff id to dismiss; repeat for multiple handoffs." + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 2000, + "description": "Why no spoken response is needed for these handoffs." + } + }, + "required": ["session_id", "cursor", "handoff_id", "reason"], + "additionalProperties": false + } + }, "fork": { "description": "Duplicate an existing chat session, copying its full conversation history into a new session the user can continue down an independent path. The fork appears in the app's session list; the user's current view does not change.", "fields": [ diff --git a/src-tauri/crates/berdctl/cli-surface-feedback.json b/src-tauri/crates/berdctl/cli-surface-feedback.json index 59438fb52..49449df33 100644 --- a/src-tauri/crates/berdctl/cli-surface-feedback.json +++ b/src-tauri/crates/berdctl/cli-surface-feedback.json @@ -3,7 +3,7 @@ "nouns": { "session": { "group": "sessions", - "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to Spokesperson, dismiss handoffs, fork, archive", "verbs": { "create": { "action": "create", @@ -50,6 +50,16 @@ "about": "Move a chat session out of any project", "afterHelp": "Example:\n berdctl session clear-project --session-id \n\nResult:\n {\"ok\": true} — the app's session list regroups immediately." }, + "send-to-spokesperson": { + "action": "send_to_spokesperson", + "about": "Send private guidance to a session's live voice Spokesperson", + "afterHelp": "Example:\n berdctl session send-to-spokesperson --session-id --cursor \\\n --mode say --resolves \\\n --message \"The build failed because the signing certificate expired.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":,\"delivery_status\":\"sent\"|\"interrupting\"|\"queued\",\"mode\":\"context\"|\"say\",\"resolved_handoff_ids\":[\"\"]}\n\nUse --mode context to update the Spokesperson's future context without starting a\nresponse. Use --mode say when the Spokesperson should speak the message now.\nRepeat --resolves to close every handoff answered by one say. Context messages\ncannot resolve handoffs. A say may omit --resolves when volunteering information.\n\nA send while the pipe contains a newer Expert-bound transcript, handoff, or\nreminder fails with reason \"pipe_busy\" without consuming that pending event.\nWait for Berd to deliver it normally, then retry with its cursor." + }, + "dismiss-handoffs": { + "action": "dismiss_handoffs", + "about": "Dismiss open voice handoffs without speaking", + "afterHelp": "Example:\n berdctl session dismiss-handoffs --session-id --cursor --handoff-id --reason \"The user's follow-up superseded both requests.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":,\"dismissed_handoff_ids\":[\"\"],\"context_delivery_status\":\"sent\"|\"interrupting\"|\"queued\"}\n\nEvery id must still be open. A dismissal consumes pending Spokesperson handoffs only\nwhen --cursor proves the Expert received the complete pending batch, then\natomically sends the dismissal reason back as silent context. Use send-to-spokesperson\n--mode say instead when the user still needs an answer." + }, "fork": { "action": "fork", "about": "Fork a chat session into an independent copy with its history", diff --git a/src-tauri/crates/berdctl/cli-surface.json b/src-tauri/crates/berdctl/cli-surface.json index 1d297f14b..e1b09aaa4 100644 --- a/src-tauri/crates/berdctl/cli-surface.json +++ b/src-tauri/crates/berdctl/cli-surface.json @@ -3,7 +3,7 @@ "nouns": { "session": { "group": "sessions", - "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "about": "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to Spokesperson, dismiss handoffs, fork, archive", "verbs": { "create": { "action": "create", @@ -50,6 +50,16 @@ "about": "Move a chat session out of any project", "afterHelp": "Example:\n berdctl session clear-project --session-id \n\nResult:\n {\"ok\": true} — the app's session list regroups immediately." }, + "send-to-spokesperson": { + "action": "send_to_spokesperson", + "about": "Send private guidance to a session's live voice Spokesperson", + "afterHelp": "Example:\n berdctl session send-to-spokesperson --session-id --cursor \\\n --mode say --resolves \\\n --message \"The build failed because the signing certificate expired.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":,\"delivery_status\":\"sent\"|\"interrupting\"|\"queued\",\"mode\":\"context\"|\"say\",\"resolved_handoff_ids\":[\"\"]}\n\nUse --mode context to update the Spokesperson's future context without starting a\nresponse. Use --mode say when the Spokesperson should speak the message now.\nRepeat --resolves to close every handoff answered by one say. Context messages\ncannot resolve handoffs. A say may omit --resolves when volunteering information.\n\nA send while the pipe contains a newer Expert-bound transcript, handoff, or\nreminder fails with reason \"pipe_busy\" without consuming that pending event.\nWait for Berd to deliver it normally, then retry with its cursor." + }, + "dismiss-handoffs": { + "action": "dismiss_handoffs", + "about": "Dismiss open voice handoffs without speaking", + "afterHelp": "Example:\n berdctl session dismiss-handoffs --session-id --cursor --handoff-id --reason \"The user's follow-up superseded both requests.\" --json\n\nResult:\n {\"session_id\":\"...\",\"cursor\":,\"dismissed_handoff_ids\":[\"\"],\"context_delivery_status\":\"sent\"|\"interrupting\"|\"queued\"}\n\nEvery id must still be open. A dismissal consumes pending Spokesperson handoffs only\nwhen --cursor proves the Expert received the complete pending batch, then\natomically sends the dismissal reason back as silent context. Use send-to-spokesperson\n--mode say instead when the user still needs an answer." + }, "fork": { "action": "fork", "about": "Fork a chat session into an independent copy with its history", diff --git a/src-tauri/crates/berdctl/src/main.rs b/src-tauri/crates/berdctl/src/main.rs index e762ffeec..4b4c09acf 100644 --- a/src-tauri/crates/berdctl/src/main.rs +++ b/src-tauri/crates/berdctl/src/main.rs @@ -276,6 +276,19 @@ mod tests { ("session", "move") => vec!["--session-id", "s", "--project-id", "p"], ("session", "move-to-group") => vec!["--session-id", "s", "--group-id", "g"], ("session", "clear-project") => vec!["--session-id", "s"], + ("session", "send-to-spokesperson") => { + vec!["--session-id", "s", "--cursor", "0", "--message", "status"] + } + ("session", "dismiss-handoffs") => vec![ + "--session-id", + "s", + "--cursor", + "1", + "--handoff-id", + "handoff-1", + "--reason", + "superseded", + ], ("folder", "attach") | ("folder", "detach") | ("folder", "set-cwd") => { vec!["--session-id", "s", "--path", "/w"] } diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4b88b1b00..e11766c5b 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -38,7 +38,6 @@ mod native_input_mute; pub mod native_voice; pub mod notifications; pub mod openai_audio; -#[cfg(feature = "block-voice-dictation")] pub mod openai_realtime; mod openai_voice_credentials; pub mod path_resolver; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index de66441de..042ccc754 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1299,6 +1299,12 @@ pub async fn start_native_voice_conversation( if session_id.is_empty() || session_id.len() > 256 { return Err("session id must be between 1 and 256 bytes".to_string()); } + if app + .try_state::() + .is_some_and(|state| state.active_target().is_some()) + { + return Err("An OpenAI Realtime voice conversation is already active.".to_string()); + } if input_backend == VoiceInputBackend::Macos && !mac_speech::status_async().await?.model_installed { diff --git a/src-tauri/src/commands/notifications.rs b/src-tauri/src/commands/notifications.rs index 4defc296d..c40b97a1b 100644 --- a/src-tauri/src/commands/notifications.rs +++ b/src-tauri/src/commands/notifications.rs @@ -12,6 +12,10 @@ struct CompletionNotificationRequest { sound: Option, } +fn voice_session_is_active(native_active: bool, realtime_active: bool) -> bool { + native_active || realtime_active +} + #[cfg(target_os = "macos")] #[derive(Clone, serde::Serialize)] #[serde(rename_all = "camelCase")] @@ -28,11 +32,15 @@ struct CompletionNotificationState { pub fn show_completion_notification( app: AppHandle, voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, + realtime_voice_state: State<'_, crate::commands::voice_buddy::RealtimeVoiceControlsState>, session_id: String, body: String, sound: Option, ) -> Result<(), String> { - if voice_state.is_active_for_session(&session_id) { + if voice_session_is_active( + voice_state.is_active_for_session(&session_id), + realtime_voice_state.is_active_for_session(&session_id), + ) { return Ok(()); } show_platform_completion_notification( @@ -48,9 +56,26 @@ pub fn show_completion_notification( #[tauri::command] pub fn should_suppress_completion_notification( voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, + realtime_voice_state: State<'_, crate::commands::voice_buddy::RealtimeVoiceControlsState>, session_id: String, ) -> bool { - voice_state.is_active_for_session(&session_id) + voice_session_is_active( + voice_state.is_active_for_session(&session_id), + realtime_voice_state.is_active_for_session(&session_id), + ) +} + +#[cfg(test)] +mod voice_presence_tests { + use super::voice_session_is_active; + + #[test] + fn either_voice_backend_suppresses_session_completion() { + assert!(voice_session_is_active(true, false)); + assert!(voice_session_is_active(false, true)); + assert!(voice_session_is_active(true, true)); + assert!(!voice_session_is_active(false, false)); + } } #[cfg(target_os = "macos")] diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index c7524c4fb..5ad4e7cc9 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -1,100 +1,121 @@ -#[cfg(not(feature = "no-voice-dictation"))] -use crate::services::kgoose::KgooseContext; -use crate::{ - commands::runtime_config::RuntimeConfigState, - services::{distro_bundle::DistroBundleState, kgoose}, -}; use serde::Serialize; -#[cfg(not(feature = "no-voice-dictation"))] use serde_json::json; use tauri::{State, WebviewWindow}; +use super::openai_voice_credentials::{self, OpenAiVoiceCredential}; use super::voice_capture::VoiceCaptureState; -#[cfg(not(feature = "no-voice-dictation"))] -const OPENAI_REALTIME_CLIENT_SECRETS_ENDPOINT: &str = "transcribe/v1/realtime-client-secret"; -const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-realtime-whisper"; +const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-2.1"; +const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = + "https://api.openai.com/v1/realtime/client_secrets"; #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiRealtimeStatus { configured: bool, - transcription_model: String, } #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiRealtimeSession { client_secret: String, - transcription_model: String, } -fn non_empty_env(name: &str) -> Option { - std::env::var(name) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) +fn stored_openai_api_key() -> Result, String> { + openai_voice_credentials::read(OpenAiVoiceCredential::Realtime) } -fn transcription_model() -> String { - non_empty_env("OPENAI_REALTIME_TRANSCRIPTION_MODEL") - .unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) -} +#[tauri::command] +pub async fn get_openai_realtime_status() -> Result { + let configured = stored_openai_api_key()?.is_some(); -fn openai_realtime_configured( - runtime_config: &crate::commands::runtime_config::RuntimeConfig, - distro_state: &DistroBundleState, -) -> bool { - cfg!(not(feature = "no-voice-dictation")) - && kgoose::is_configured(runtime_config.kgoose.as_ref(), distro_state.kgoose_config()) + Ok(OpenAiRealtimeStatus { configured }) } #[tauri::command] -pub async fn get_openai_realtime_status( - distro_state: State<'_, DistroBundleState>, - runtime_config_state: State<'_, RuntimeConfigState>, -) -> Result { - let runtime_config = runtime_config_state - .ready_config(distro_state.inner()) - .await?; - - Ok(OpenAiRealtimeStatus { - configured: openai_realtime_configured(&runtime_config, distro_state.inner()), - transcription_model: transcription_model(), - }) +pub async fn create_openai_realtime_voice_session( + model: Option, +) -> Result { + let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; + let model = model + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_REALTIME_MODEL.to_string()); + let response = realtime_client_secret_request(&reqwest::Client::new(), &api_key, &model) + .send() + .await + .map_err(|error| format!("Failed to create OpenAI Realtime voice session: {error}"))?; + parse_session_response(response, "voice").await } #[tauri::command] -pub async fn create_openai_realtime_session( - _distro_state: State<'_, DistroBundleState>, - _runtime_config_state: State<'_, RuntimeConfigState>, -) -> Result { - #[cfg(feature = "no-voice-dictation")] - { - Err("OpenAI realtime sessions are unsupported because voice dictation is disabled in this build.".to_string()) - } +pub async fn create_openai_realtime_session() -> Result { + let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; + let response = realtime_transcription_client_secret_request(&reqwest::Client::new(), &api_key) + .send() + .await + .map_err(|error| { + format!("Failed to create OpenAI Realtime transcription session: {error}") + })?; + parse_session_response(response, "transcription").await +} - #[cfg(not(feature = "no-voice-dictation"))] - { - let transcription_model = transcription_model(); - let runtime_config = _runtime_config_state - .ready_config(_distro_state.inner()) - .await?; - let kgoose = KgooseContext::new(_distro_state.inner(), &runtime_config); - let value = kgoose - .post_json( - OPENAI_REALTIME_CLIENT_SECRETS_ENDPOINT, - json!({ "language": "en" }), - ) - .await - .map_err(|error| format!("Failed to create OpenAI realtime session: {error}"))?; - let client_secret = parse_client_secret(&value)?; +fn realtime_client_secret_request( + client: &reqwest::Client, + api_key: &str, + model: &str, +) -> reqwest::RequestBuilder { + client + .post(OPENAI_REALTIME_CLIENT_SECRETS_URL) + .bearer_auth(api_key) + .json(&json!({ + "session": { + "type": "realtime", + "model": model, + } + })) +} - Ok(OpenAiRealtimeSession { - client_secret, - transcription_model, - }) +fn realtime_transcription_client_secret_request( + client: &reqwest::Client, + api_key: &str, +) -> reqwest::RequestBuilder { + client + .post(OPENAI_REALTIME_CLIENT_SECRETS_URL) + .bearer_auth(api_key) + .json(&json!({ + "session": { + "type": "transcription", + "audio": { + "input": { + "format": { "type": "audio/pcm", "rate": 24_000 }, + "transcription": { "model": "gpt-realtime-whisper" }, + "turn_detection": { "type": "server_vad" } + } + } + } + })) +} + +async fn parse_session_response( + response: reqwest::Response, + kind: &str, +) -> Result { + let status = response.status(); + let body = response + .text() + .await + .map_err(|error| format!("Failed to read OpenAI Realtime response: {error}"))?; + if !status.is_success() { + return Err(format!( + "OpenAI Realtime {kind} session creation failed ({status}): {body}" + )); } + let value: serde_json::Value = serde_json::from_str(&body) + .map_err(|error| format!("OpenAI Realtime returned invalid JSON: {error}"))?; + Ok(OpenAiRealtimeSession { + client_secret: parse_client_secret(&value)?, + }) } #[tauri::command] @@ -132,7 +153,6 @@ pub fn release_voice_dictation_microphone( Ok(()) } -#[cfg(not(feature = "no-voice-dictation"))] fn parse_client_secret(value: &serde_json::Value) -> Result { let client_secret = value.get("client_secret").and_then(client_secret_value); let top_level_value = value.get("value").and_then(|value| value.as_str()); @@ -143,13 +163,11 @@ fn parse_client_secret(value: &serde_json::Value) -> Result { .or(top_level_secret) .map(ToString::to_string) .ok_or_else(|| { - format!( - "OpenAI realtime client secret response did not include a recognized secret field: {value}" - ) + "OpenAI realtime client secret response did not include a recognized secret field." + .to_string() }) } -#[cfg(not(feature = "no-voice-dictation"))] fn client_secret_value(value: &serde_json::Value) -> Option<&str> { value .get("value") @@ -159,76 +177,12 @@ fn client_secret_value(value: &serde_json::Value) -> Option<&str> { #[cfg(test)] mod tests { - use super::openai_realtime_configured; - #[cfg(not(feature = "no-voice-dictation"))] - use super::parse_client_secret; - use crate::{ - commands::runtime_config::{default_runtime_config, RuntimeConfig, RuntimeKgooseConfig}, - services::distro_bundle::{DistroBundleState, KgooseDistroConfig}, - test_support::env_lock, + use super::{ + parse_client_secret, realtime_client_secret_request, + realtime_transcription_client_secret_request, }; - #[cfg(not(feature = "no-voice-dictation"))] use serde_json::json; - use std::env; - - #[test] - fn status_is_unconfigured_without_explicit_kgoose_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let runtime_config = default_runtime_config(); - - assert!(!openai_realtime_configured( - &runtime_config, - &DistroBundleState::empty_for_tests(), - )); - } - #[test] - fn status_tracks_explicit_runtime_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let mut runtime_config = default_runtime_config(); - runtime_config.kgoose = Some(RuntimeKgooseConfig { - base_url: Some("https://kgoose.example.test/".to_string()), - path: None, - }); - - assert_eq!( - openai_realtime_configured(&runtime_config, &DistroBundleState::empty_for_tests(),), - cfg!(not(feature = "no-voice-dictation")), - ); - } - - #[test] - fn status_tracks_explicit_distro_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::remove_var("KGOOSE_BASE_URL"); - let runtime_config: RuntimeConfig = default_runtime_config(); - let distro_state = DistroBundleState::with_kgoose_for_tests(KgooseDistroConfig { - base_url: Some("https://kgoose.example.test/".to_string()), - path: None, - }); - - assert_eq!( - openai_realtime_configured(&runtime_config, &distro_state), - cfg!(not(feature = "no-voice-dictation")), - ); - } - - #[test] - fn status_tracks_explicit_environment_endpoint() { - let _guard = env_lock().lock().expect("env lock"); - env::set_var("KGOOSE_BASE_URL", "https://kgoose.example.test/"); - let runtime_config = default_runtime_config(); - - assert_eq!( - openai_realtime_configured(&runtime_config, &DistroBundleState::empty_for_tests(),), - cfg!(not(feature = "no-voice-dictation")), - ); - env::remove_var("KGOOSE_BASE_URL"); - } - - #[cfg(not(feature = "no-voice-dictation"))] #[test] fn parses_supported_client_secret_shapes() { assert_eq!( @@ -249,9 +203,68 @@ mod tests { ); } - #[cfg(not(feature = "no-voice-dictation"))] #[test] fn rejects_missing_client_secret() { assert!(parse_client_secret(&json!({ "ok": true })).is_err()); } + + #[test] + fn client_secret_request_uses_only_the_standard_openai_endpoint() { + let request = realtime_client_secret_request( + &reqwest::Client::new(), + "sk-test-secret", + "gpt-realtime-test", + ) + .build() + .expect("build request"); + + assert_eq!( + request.url().as_str(), + "https://api.openai.com/v1/realtime/client_secrets" + ); + assert_eq!( + request + .headers() + .get(reqwest::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer sk-test-secret") + ); + let body: serde_json::Value = serde_json::from_slice( + request + .body() + .and_then(|body| body.as_bytes()) + .expect("JSON body"), + ) + .expect("parse request body"); + assert_eq!( + body, + json!({ + "session": { + "type": "realtime", + "model": "gpt-realtime-test", + } + }) + ); + } + + #[test] + fn dictation_client_secret_enables_input_transcription() { + let request = + realtime_transcription_client_secret_request(&reqwest::Client::new(), "sk-test-secret") + .build() + .expect("build request"); + let body: serde_json::Value = serde_json::from_slice( + request + .body() + .and_then(|body| body.as_bytes()) + .expect("JSON body"), + ) + .expect("parse request body"); + + assert_eq!(body["session"]["type"], "transcription"); + assert_eq!( + body["session"]["audio"]["input"]["transcription"]["model"], + "gpt-realtime-whisper" + ); + } } diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 3e35a4ea4..926d46326 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -8,12 +8,13 @@ const LEGACY_TTS_KEYCHAIN_ACCOUNT: &str = "tts-api-key"; pub(crate) enum OpenAiVoiceCredential { SpeechToText, TextToSpeech, + Realtime, } impl OpenAiVoiceCredential { const fn account(self) -> &'static str { match self { - Self::SpeechToText | Self::TextToSpeech => KEYCHAIN_ACCOUNT, + Self::SpeechToText | Self::TextToSpeech | Self::Realtime => KEYCHAIN_ACCOUNT, } } @@ -25,6 +26,9 @@ impl OpenAiVoiceCredential { Self::TextToSpeech => { "OpenAI text-to-speech is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." } + Self::Realtime => { + "OpenAI Realtime voice is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + } } } } @@ -109,6 +113,7 @@ mod tests { fn speech_services_use_the_shared_voice_keychain_account() { assert_eq!(OpenAiVoiceCredential::SpeechToText.account(), "api-key"); assert_eq!(OpenAiVoiceCredential::TextToSpeech.account(), "api-key"); + assert_eq!(OpenAiVoiceCredential::Realtime.account(), "api-key"); } #[test] diff --git a/src-tauri/src/commands/voice_buddy.rs b/src-tauri/src/commands/voice_buddy.rs index 4c2feeaa3..96c33f004 100644 --- a/src-tauri/src/commands/voice_buddy.rs +++ b/src-tauri/src/commands/voice_buddy.rs @@ -1,5 +1,7 @@ //! Cross-platform always-on-top controls for the process-wide voice conversation. +use std::sync::{Arc, Mutex}; + use serde::{Deserialize, Serialize}; use tauri::{ AppHandle, Emitter, Manager, PhysicalPosition, WebviewUrl, WebviewWindow, WebviewWindowBuilder, @@ -13,6 +15,7 @@ use super::{ pub const WINDOW_LABEL: &str = "voice-buddy"; pub const OPEN_SESSION_EVENT: &str = "voice-conversation:open-session"; +pub const REALTIME_CONTROL_EVENT: &str = "voice-conversation:realtime-control"; const WINDOW_WIDTH: f64 = 176.0; const WINDOW_HEIGHT: f64 = 56.0; const SCREEN_INSET: i32 = 24; @@ -21,6 +24,181 @@ fn controls_url(revision: u64) -> String { format!("index.html?voiceBuddy=1&voiceRevision={revision}") } +fn realtime_controls_url(revision: u64) -> String { + format!("index.html?voiceBuddy=1&voiceMode=realtime&voiceRevision={revision}") +} + +#[derive(Clone, Default)] +pub struct RealtimeVoiceControlsState { + runtime: Arc>, +} + +#[derive(Default)] +struct RealtimeVoiceControlsRuntime { + session_id: Option, + owner_window_label: Option, + revision: u64, + microphone_muted: bool, + controls_suppressed: bool, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeVoiceControlsStatus { + available: bool, + unavailable_reason: Option, + lifecycle: &'static str, + session_id: Option, + owner_window_label: Option, + microphone_muted: bool, + revision: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeControlsVisibilityRequest { + session_id: String, + expected_revision: u64, + suppressed: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeControlsActivityRequest { + session_id: String, + expected_revision: u64, + activity: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeControlsMuteRequest { + session_id: String, + expected_revision: u64, + muted: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeControlsRebindRequest { + previous_session_id: String, + session_id: String, + expected_revision: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeControlRequest { + session_id: String, + expected_revision: u64, + action: String, + muted: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct RealtimeControlPayload { + session_id: String, + revision: u64, + action: String, + muted: Option, +} + +impl RealtimeVoiceControlsState { + fn status(&self) -> RealtimeVoiceControlsStatus { + let runtime = self + .runtime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + RealtimeVoiceControlsStatus { + available: true, + unavailable_reason: None, + lifecycle: if runtime.session_id.is_some() { + "running" + } else { + "stopped" + }, + session_id: runtime.session_id.clone(), + owner_window_label: runtime.owner_window_label.clone(), + microphone_muted: runtime.microphone_muted, + revision: runtime.revision, + } + } + + pub(crate) fn active_target(&self) -> Option<(String, String, u64)> { + let runtime = self.runtime.lock().ok()?; + Some(( + runtime.session_id.clone()?, + runtime.owner_window_label.clone()?, + runtime.revision, + )) + } + + pub fn is_active_for_session(&self, session_id: &str) -> bool { + self.runtime + .lock() + .ok() + .and_then(|runtime| runtime.session_id.clone()) + .is_some_and(|active| active == session_id) + } + + fn begin(&self, session_id: String, owner_window_label: String) -> Result { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.is_some() { + return Err("Realtime voice controls are already active.".to_string()); + } + runtime.revision = runtime.revision.wrapping_add(1); + runtime.session_id = Some(session_id); + runtime.owner_window_label = Some(owner_window_label); + runtime.microphone_muted = false; + runtime.controls_suppressed = true; + Ok(runtime.revision) + } + + fn finish(&self, session_id: &str, expected_revision: u64) -> Result { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(false); + } + runtime.session_id = None; + runtime.owner_window_label = None; + runtime.microphone_muted = false; + runtime.controls_suppressed = false; + runtime.revision = runtime.revision.wrapping_add(1); + Ok(true) + } + + fn rebind( + &self, + owner_window_label: &str, + request: &RealtimeControlsRebindRequest, + ) -> Result { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(&request.previous_session_id) + || runtime.revision != request.expected_revision + { + return Ok(false); + } + if runtime.owner_window_label.as_deref() != Some(owner_window_label) { + return Err("Only the Realtime voice owner can move its session.".to_string()); + } + runtime.session_id = Some(request.session_id.clone()); + runtime.revision = runtime.revision.wrapping_add(1); + Ok(true) + } +} + #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] struct OpenSessionPayload { @@ -65,7 +243,12 @@ pub fn restore_hidden_owner(app: &AppHandle, owner_window_label: &str) { pub fn open_active_session(app: &AppHandle) -> Result<(), String> { let state = app.state::(); - let Some((session_id, owner_window_label)) = state.active_session_target() else { + let active_target = state.active_session_target().or_else(|| { + app.state::() + .active_target() + .map(|(session_id, owner_window_label, _)| (session_id, owner_window_label)) + }); + let Some((session_id, owner_window_label)) = active_target else { return Ok(()); }; let window = app @@ -80,11 +263,9 @@ pub fn open_active_session(app: &AppHandle) -> Result<(), String> { Ok(()) } -fn position_near_bottom_right(app: &AppHandle, window: &WebviewWindow) { +fn position_near_bottom_right(app: &AppHandle, window: &WebviewWindow, owner_window_label: &str) { let owner_monitor = app - .state::() - .active_session_target() - .and_then(|(_, label)| app.get_webview_window(&label)) + .get_webview_window(owner_window_label) .and_then(|owner| owner.current_monitor().ok().flatten()); let Some(monitor) = owner_monitor.or_else(|| window.primary_monitor().ok().flatten()) else { return; @@ -168,6 +349,41 @@ fn show_controls_without_activation(window: &WebviewWindow) -> Result<(), String window.show().map_err(|error| error.to_string()) } +fn build_controls_window( + app: &AppHandle, + url: String, + owner_window_label: &str, +) -> Result { + let builder = WebviewWindowBuilder::new(app, WINDOW_LABEL, WebviewUrl::App(url.into())) + .title("Berd voice conversation") + .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) + .resizable(false) + .maximizable(false) + .minimizable(false) + .decorations(false) + .shadow(false) + .always_on_top(true) + .skip_taskbar(true) + .focused(false) + .visible(false); + #[cfg(target_os = "macos")] + let builder = builder.accept_first_mouse(true); + #[cfg(not(target_os = "macos"))] + let builder = builder.transparent(true); + let window = builder.build().map_err(|error| error.to_string())?; + if let Err(error) = make_macos_transparent(&window) { + let _ = window.destroy(); + return Err(error); + } + window.on_window_event(|event| { + if let WindowEvent::CloseRequested { api, .. } = event { + api.prevent_close(); + } + }); + position_near_bottom_right(app, &window, owner_window_label); + Ok(window) +} + pub fn install(app: &AppHandle) -> Result<(), String> { let state = app.state::(); if let Some(window) = app.get_webview_window(WINDOW_LABEL) { @@ -185,41 +401,11 @@ pub fn install(app: &AppHandle) -> Result<(), String> { .active_session_lifecycle_target() .ok_or_else(|| "No native voice conversation is active.".to_string())?; - let builder = WebviewWindowBuilder::new( - app, - WINDOW_LABEL, - WebviewUrl::App(controls_url(revision).into()), - ) - .title("Berd voice conversation") - .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) - .resizable(false) - .maximizable(false) - .minimizable(false) - .decorations(false) - .shadow(false) - .always_on_top(true) - .skip_taskbar(true) - .focused(false) - .visible(false); - #[cfg(target_os = "macos")] - let builder = builder.accept_first_mouse(true); - #[cfg(not(target_os = "macos"))] - let builder = builder.transparent(true); - let window = builder.build().map_err(|error| error.to_string())?; - if let Err(error) = make_macos_transparent(&window) { - let _ = window.destroy(); - return Err(error); - } + let window = build_controls_window(app, controls_url(revision), &owner_window_label)?; if let Err(error) = state.register_controls_window(&session_id, revision) { let _ = window.destroy(); return Err(error); } - window.on_window_event(|event| { - if let WindowEvent::CloseRequested { api, .. } = event { - api.prevent_close(); - } - }); - position_near_bottom_right(app, &window); let fallback_app = app.clone(); tauri::async_runtime::spawn(async move { tokio::time::sleep(std::time::Duration::from_secs(5)).await; @@ -251,6 +437,26 @@ pub fn install(app: &AppHandle) -> Result<(), String> { Ok(()) } +fn install_realtime( + app: &AppHandle, + owner_window_label: &str, + revision: u64, +) -> Result<(), String> { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + window + .destroy() + .map_err(|error| format!("Could not replace stale floating voice controls: {error}"))?; + if app.get_webview_window(WINDOW_LABEL).is_some() { + return Err("Stale floating voice controls could not be replaced.".to_string()); + } + let native_state = app.state::(); + native_state.clear_controls_window_if_revision(native_state.controls_window_revision()); + } + + build_controls_window(app, realtime_controls_url(revision), owner_window_label)?; + Ok(()) +} + fn active_controls_match(active_revision: Option, controls_revision: Option) -> bool { active_revision.is_some() && active_revision == controls_revision } @@ -278,9 +484,13 @@ fn verify_stale_candidate_removed( pub fn matches_active_lifecycle(app: &AppHandle) -> bool { app.get_webview_window(WINDOW_LABEL).is_some() - && app + && (app .state::() .controls_window_matches_active_lifecycle() + || app + .state::() + .active_target() + .is_some()) } pub fn should_preserve_main_for_voice( @@ -291,6 +501,13 @@ pub fn should_preserve_main_for_voice( } pub fn destroy_stale_for_main_close(app: &AppHandle) -> Result<(), String> { + if app + .state::() + .active_target() + .is_some() + { + return Ok(()); + } let Some(window) = app.get_webview_window(WINDOW_LABEL) else { return Ok(()); }; @@ -319,6 +536,21 @@ pub fn destroy_stale_for_main_close(app: &AppHandle) -> Result<(), String> { result } +pub fn handle_realtime_voice_owner_window_destroyed(app: &AppHandle, window_label: &str) { + let state = app.state::(); + let Some((session_id, owner_window_label, revision)) = state.active_target() else { + return; + }; + if owner_window_label != window_label { + return; + } + if state.finish(&session_id, revision).unwrap_or(false) { + if let Some(controls) = app.get_webview_window(WINDOW_LABEL) { + let _ = controls.destroy(); + } + } +} + fn reconcile_terminal_controls( emit_terminal: impl FnOnce(), destroy: impl FnOnce() -> Result<(), String>, @@ -547,6 +779,260 @@ pub async fn stop_voice_conversation_from_buddy( Ok(()) } +#[tauri::command] +pub fn start_openai_realtime_voice_controls( + app: AppHandle, + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + native_state: tauri::State<'_, NativeVoiceState>, + session_id: String, +) -> Result { + if window.label() == WINDOW_LABEL { + return Err("Floating controls cannot own a Realtime voice conversation.".to_string()); + } + if native_state.active_session_target().is_some() { + return Err("A chained voice conversation is already active.".to_string()); + } + let owner_window_label = window.label().to_string(); + let revision = state.begin(session_id.clone(), owner_window_label.clone())?; + if let Err(error) = install_realtime(&app, &owner_window_label, revision) { + let _ = state.finish(&session_id, revision); + return Err(error); + } + Ok(state.status()) +} + +#[tauri::command] +pub fn get_openai_realtime_voice_controls_status( + state: tauri::State<'_, RealtimeVoiceControlsState>, +) -> RealtimeVoiceControlsStatus { + state.status() +} + +#[tauri::command] +pub fn rebind_openai_realtime_voice_controls( + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + request: RealtimeControlsRebindRequest, +) -> Result { + if state.rebind(window.label(), &request)? { + let status = state.status(); + emit( + window.app_handle(), + NativeVoiceEvent::Startup { + session_id: request.session_id, + owner_window_label: window.label().to_string(), + line: "Voice conversation resumed".to_string(), + revision: status.revision, + }, + ); + return Ok(status); + } + Err("The Realtime voice session changed before it could be moved.".to_string()) +} + +#[tauri::command] +pub fn show_openai_realtime_voice_controls( + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + session_id: String, + expected_revision: u64, +) -> Result<(), String> { + require_controls_window(window.label())?; + let suppressed = { + let runtime = state + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(&session_id) + || runtime.revision != expected_revision + { + return Ok(()); + } + runtime.controls_suppressed + }; + if suppressed { + window.hide().map_err(|error| error.to_string()) + } else { + show_controls_without_activation(&window) + } +} + +#[tauri::command] +pub fn set_openai_realtime_voice_controls_suppressed( + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + request: RealtimeControlsVisibilityRequest, +) -> Result<(), String> { + let should_show = { + let mut runtime = state + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(&request.session_id) + || runtime.revision != request.expected_revision + { + return Ok(()); + } + if runtime.owner_window_label.as_deref() != Some(window.label()) { + return Err("Only the Realtime voice owner can change control visibility.".to_string()); + } + runtime.controls_suppressed = request.suppressed; + !request.suppressed + }; + let Some(controls) = window.app_handle().get_webview_window(WINDOW_LABEL) else { + return Err("The floating voice controls are no longer available.".to_string()); + }; + if should_show { + show_controls_without_activation(&controls) + } else { + controls.hide().map_err(|error| error.to_string()) + } +} + +#[tauri::command] +pub fn publish_openai_realtime_voice_activity( + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + request: RealtimeControlsActivityRequest, +) -> Result<(), String> { + if !matches!( + request.activity.as_str(), + "user-speaking" | "user-idle" | "assistant-speaking" | "assistant-idle" + ) { + return Err("Unknown Realtime voice activity.".to_string()); + } + let active = state.active_target(); + if active.as_ref() + != Some(&( + request.session_id.clone(), + window.label().to_string(), + request.expected_revision, + )) + { + return Ok(()); + } + emit( + window.app_handle(), + NativeVoiceEvent::Activity { + session_id: request.session_id, + activity: match request.activity.as_str() { + "user-speaking" => "user-speaking", + "user-idle" => "user-idle", + "assistant-speaking" => "assistant-speaking", + _ => "assistant-idle", + }, + revision: request.expected_revision, + }, + ); + Ok(()) +} + +#[tauri::command] +pub fn publish_openai_realtime_voice_microphone_muted( + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + request: RealtimeControlsMuteRequest, +) -> Result<(), String> { + { + let mut runtime = state + .runtime + .lock() + .map_err(|_| "realtime voice controls state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(&request.session_id) + || runtime.owner_window_label.as_deref() != Some(window.label()) + || runtime.revision != request.expected_revision + { + return Ok(()); + } + runtime.microphone_muted = request.muted; + } + emit( + window.app_handle(), + NativeVoiceEvent::MicrophoneMute { + session_id: request.session_id, + muted: request.muted, + revision: request.expected_revision, + }, + ); + Ok(()) +} + +#[tauri::command] +pub fn request_openai_realtime_voice_control( + app: AppHandle, + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + request: RealtimeControlRequest, +) -> Result<(), String> { + require_controls_window(window.label())?; + if request.action != "stop" && request.action != "mute" { + return Err("Unknown Realtime voice control action.".to_string()); + } + if request.action == "mute" && request.muted.is_none() { + return Err("Realtime mute controls require the requested state.".to_string()); + } + let Some((session_id, owner_window_label, revision)) = state.active_target() else { + return Ok(()); + }; + if session_id != request.session_id || revision != request.expected_revision { + return Ok(()); + } + let owner = app + .get_webview_window(&owner_window_label) + .ok_or_else(|| "The Realtime voice owner is no longer available.".to_string())?; + owner + .emit( + REALTIME_CONTROL_EVENT, + RealtimeControlPayload { + session_id, + revision, + action: request.action, + muted: request.muted, + }, + ) + .map_err(|error| error.to_string()) +} + +#[tauri::command] +pub fn stop_openai_realtime_voice_controls( + app: AppHandle, + window: WebviewWindow, + state: tauri::State<'_, RealtimeVoiceControlsState>, + session_id: String, + expected_revision: u64, +) -> Result<(), String> { + let active = state.active_target(); + if active.as_ref() + != Some(&( + session_id.clone(), + window.label().to_string(), + expected_revision, + )) + { + return Ok(()); + } + let owner_window_label = active + .as_ref() + .map(|(_, owner_window_label, _)| owner_window_label.clone()) + .unwrap_or_default(); + if !state.finish(&session_id, expected_revision)? { + return Ok(()); + } + restore_hidden_owner(&app, &owner_window_label); + if let Some(controls) = app.get_webview_window(WINDOW_LABEL) { + let _ = controls.emit( + super::native_voice::EVENT_NAME, + NativeVoiceEvent::CleanShutdown { + session_id, + revision: expected_revision, + }, + ); + controls.destroy().map_err(|error| error.to_string())?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -622,4 +1108,34 @@ mod tests { assert!(emitted.get()); assert!(hidden.get()); } + + #[test] + fn realtime_voice_presence_follows_start_rebind_and_stop() { + let state = RealtimeVoiceControlsState::default(); + let revision = state + .begin("draft-session".to_string(), "main".to_string()) + .expect("start realtime controls"); + assert!(state.is_active_for_session("draft-session")); + assert!(!state.is_active_for_session("backend-session")); + + assert!(state + .rebind( + "main", + &RealtimeControlsRebindRequest { + previous_session_id: "draft-session".to_string(), + session_id: "backend-session".to_string(), + expected_revision: revision, + }, + ) + .expect("rebind realtime controls")); + assert!(!state.is_active_for_session("draft-session")); + assert!(state.is_active_for_session("backend-session")); + let rebound_revision = state.status().revision; + assert!(rebound_revision > revision); + + assert!(state + .finish("backend-session", rebound_revision) + .expect("stop realtime controls")); + assert!(!state.is_active_for_session("backend-session")); + } } diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index 298be24a2..149af8324 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -681,6 +681,10 @@ pub fn open_session_window( &app_for_close, &label_for_close, ); + crate::commands::voice_buddy::handle_realtime_voice_owner_window_destroyed( + &app_for_close, + &label_for_close, + ); reg_for_close.release_label(&label_for_close); let _ = emit_snapshot(&app_for_close, ®_for_close); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 43a2ff65b..f8086c2f1 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -236,6 +236,7 @@ pub fn run() { app.manage(commands::siri_voice::SiriVoiceState::default()); app.manage(commands::openai_audio::OpenAiVoiceState::default()); app.manage(commands::native_voice::NativeVoiceState::default()); + app.manage(commands::voice_buddy::RealtimeVoiceControlsState::default()); app.manage(commands::voice_capture::VoiceCaptureState::default()); app.manage(commands::telemetry::TelemetryAuthState::new( app_data_dir.clone(), @@ -602,13 +603,10 @@ pub fn run() { commands::model_setup::list_model_setup_status, commands::model_setup::clear_model_setup_status, commands::notifications::show_completion_notification, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::get_openai_realtime_status, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::create_openai_realtime_session, - #[cfg(feature = "block-voice-dictation")] + commands::openai_realtime::create_openai_realtime_voice_session, commands::openai_realtime::claim_voice_dictation_microphone, - #[cfg(feature = "block-voice-dictation")] commands::openai_realtime::release_voice_dictation_microphone, commands::agent_setup::start_agent_setup, commands::agent_setup::get_agent_setup_status, @@ -705,6 +703,15 @@ pub fn run() { commands::voice_buddy::show_voice_conversation_controls, commands::voice_buddy::set_voice_conversation_controls_suppressed, commands::voice_buddy::stop_voice_conversation_from_buddy, + commands::voice_buddy::start_openai_realtime_voice_controls, + commands::voice_buddy::get_openai_realtime_voice_controls_status, + commands::voice_buddy::rebind_openai_realtime_voice_controls, + commands::voice_buddy::show_openai_realtime_voice_controls, + commands::voice_buddy::set_openai_realtime_voice_controls_suppressed, + commands::voice_buddy::publish_openai_realtime_voice_activity, + commands::voice_buddy::publish_openai_realtime_voice_microphone_muted, + commands::voice_buddy::request_openai_realtime_voice_control, + commands::voice_buddy::stop_openai_realtime_voice_controls, commands::notifications::should_suppress_completion_notification, commands::voice_capture::register_voice_renderer_instance, commands::voice_capture::set_voice_renderer_foreground_session, @@ -786,6 +793,10 @@ fn attach_main_window_lifecycle(app: &tauri::App) { main.on_window_event(move |event| { if matches!(event, WindowEvent::Destroyed) { commands::native_voice::handle_voice_owner_window_destroyed(&app_handle, "main"); + commands::voice_buddy::handle_realtime_voice_owner_window_destroyed( + &app_handle, + "main", + ); return; } if let WindowEvent::CloseRequested { api, .. } = event { @@ -796,7 +807,13 @@ fn attach_main_window_lifecycle(app: &tauri::App) { let active_voice_owner_window_label = app_handle .state::() .active_session_lifecycle_target() - .map(|(_, owner_window_label, _)| owner_window_label); + .map(|(_, owner_window_label, _)| owner_window_label) + .or_else(|| { + app_handle + .state::() + .active_target() + .map(|(_, owner_window_label, _)| owner_window_label) + }); let controls_match_active_voice = commands::voice_buddy::matches_active_lifecycle(&app_handle); let preserve_for_voice = commands::voice_buddy::should_preserve_main_for_voice( diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index 53821fa94..6f04e061c 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -53,7 +53,9 @@ import { BUILDERBOT_SURFACE_EXPERIMENT_ID } from "@/features/experiments/experim import { EXPERIMENT_PREFERENCES_STORAGE_KEY, EXPERIMENT_PREFERENCES_STORAGE_VERSION, + setExperimentEnabled, } from "@/features/experiments/experimentPreferences"; +import { VOICE_CONVERSATION_EXPERIMENT_ID } from "@/features/experiments/experimentDefinitions"; import { ThemeProvider } from "@/shared/theme/ThemeProvider"; import { useDefaultProviderReadinessStore } from "@/features/providers/stores/defaultProviderReadinessStore"; import { useProviderModelCacheStore } from "@/features/providers/stores/providerModelCacheStore"; @@ -138,6 +140,17 @@ const mockVoiceSetupReadiness = vi.hoisted(() => ({ ready: false, })); const mockVoiceSettingsEnabled = vi.hoisted(() => ({ enabled: false })); +const mockStopOpenAiRealtimeConversation = vi.hoisted(() => vi.fn()); + +vi.mock( + "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/features/voice-conversation/hooks/useOpenAiRealtimeConversation") + >()), + stopOpenAiRealtimeConversation: mockStopOpenAiRealtimeConversation, + }), +); vi.mock("@/features/settings/ui/settingsSections", async (importOriginal) => { const actual = @@ -927,6 +940,22 @@ describe("AppShell global navigation", () => { ).toBe(false); }); + it("stops both voice pipelines when Voice is disabled", async () => { + mockBuildFeatures.voiceConversation = true; + const stopVoiceConversation = vi.fn().mockResolvedValue(undefined); + useVoiceConversationStore.setState({ stop: stopVoiceConversation }); + renderAppShell(); + + act(() => { + setExperimentEnabled(VOICE_CONVERSATION_EXPERIMENT_ID, false); + }); + + await waitFor(() => { + expect(stopVoiceConversation).toHaveBeenCalledOnce(); + expect(mockStopOpenAiRealtimeConversation).toHaveBeenCalledOnce(); + }); + }); + afterEach(cleanup); beforeEach(() => { @@ -950,6 +979,8 @@ describe("AppShell global navigation", () => { mockSessionWindowSupport.supported = false; mockVoiceSetupReadiness.ready = false; mockVoiceSettingsEnabled.enabled = false; + mockStopOpenAiRealtimeConversation.mockReset(); + mockStopOpenAiRealtimeConversation.mockResolvedValue(undefined); mockFocusSessionWindow.mockReset(); useSessionWindowStore.getState().setSnapshot([]); useVoiceConversationStore.setState({ diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index ea701fc71..24d6f9832 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -250,6 +250,11 @@ import { } from "@/features/voice-conversation/lib/voiceInputPreference"; import { useVoiceOutputPreference } from "@/features/voice-conversation/lib/voiceOutputPreference"; import { isVoiceSetupReady } from "@/features/voice-conversation/lib/voiceSetupReadiness"; +import { getVoiceConversationMode } from "@/features/voice-conversation/lib/voiceConversationModePreference"; +import { + requestOpenAiRealtimeConversationStart, + stopOpenAiRealtimeConversation, +} from "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation"; import { useProfileCapabilities } from "@/shared/profile/capabilities"; import { getOptimisticArtifactCwd } from "@/shared/artifacts/sessionArtifactLocation"; import { @@ -779,10 +784,13 @@ export function AppShell({ ) { return; } - // The native process survives renderer reloads and may be owned by another - // window, so an explicit on-to-off transition must clean up active use. - // Mounting with the experiment already off performs no Voice native work. - void stopVoiceConversation().catch(() => undefined); + // Voice resources can survive renderer navigation or be owned by another + // window, so an explicit on-to-off transition must stop both pipelines. + // Mounting with the experiment already off performs no voice cleanup. + void Promise.allSettled([ + stopVoiceConversation(), + stopOpenAiRealtimeConversation(), + ]); }, [capabilities.voiceConversation, stopVoiceConversation]); const sessions = useChatSessionStore(selectSessions); const activeSessionId = useChatSessionStore(selectActiveSessionId); @@ -3384,7 +3392,8 @@ export function AppShell({ const handleGlobalVoiceConversationStart = useCallback( (payload: GlobalComposerExpandPayload): Promise => { if (!capabilities.voiceConversation) return Promise.resolve(false); - if (!globalVoiceReady) { + const realtimeMode = getVoiceConversationMode() === "openai-realtime"; + if (!realtimeMode && !globalVoiceReady) { return new Promise((resolve) => { guardAppNavigation( () => { @@ -3402,7 +3411,11 @@ export function AppShell({ ? projects.find((candidate) => candidate.id === options.projectId) : undefined; const chatOptions = { - activate: false, + // Realtime voice belongs to the chat the user is about to see. Use + // the ordinary optimistic chat lifecycle so the mounted transcript + // owns the same ACP notification stream as a normal Berd session. + // The realtime runtime follows the draft id through promotion. + activate: realtimeMode, reuseExistingDraft: false, executionTarget: options?.executionTarget, reasoningEffort: options?.reasoningEffort, @@ -3411,6 +3424,9 @@ export function AppShell({ }; const createAndStart = async () => { + if (realtimeMode) { + await stopOpenAiRealtimeConversation(); + } const voice = useVoiceConversationStore.getState(); if ( voice.status.lifecycle === "starting" || @@ -3453,7 +3469,8 @@ export function AppShell({ chatState.setDraft(sessionId, payload.text); chatState.setSkillDrafts(sessionId, payload.selectedSkills); chatState.setDraftAttachments(sessionId, options?.attachments ?? []); - requestVoiceConversationStart(sessionId); + if (realtimeMode) requestOpenAiRealtimeConversationStart(session.id); + else requestVoiceConversationStart(sessionId); handleNavigateToSession(sessionId); resetGlobalComposerTransition(); return true; @@ -4234,10 +4251,7 @@ export function AppShell({ let cancelled = false; let unlisten: (() => void) | null = null; void listenToVoiceConversationOpenSession((sessionId) => { - const voice = useVoiceConversationStore.getState().status; - if (voice.lifecycle === "running" && voice.sessionId === sessionId) { - handleSelectSession(sessionId); - } + handleSelectSession(sessionId); }) .then((cleanup) => { if (cancelled) cleanup(); diff --git a/src/features/berdctl/__tests__/commands/commands.test.ts b/src/features/berdctl/__tests__/commands/commands.test.ts index 27565c68c..f95036a0b 100644 --- a/src/features/berdctl/__tests__/commands/commands.test.ts +++ b/src/features/berdctl/__tests__/commands/commands.test.ts @@ -600,6 +600,18 @@ describe("action schemas", () => { const validArgs: Record> = { "sessions.create": { prompt: "hi" }, "sessions.send": { session_id: "s1", prompt: "hi" }, + "sessions.send_to_spokesperson": { + session_id: "s1", + cursor: 0, + message: "Status update", + mode: "context", + }, + "sessions.dismiss_handoffs": { + session_id: "s1", + cursor: 1, + handoff_id: ["handoff-1"], + reason: "The request was superseded.", + }, "sessions.open": { session_id: "s1" }, "sessions.list": {}, "sessions.get": { session_id: "s1" }, diff --git a/src/features/berdctl/commands/impl/dismissHandoffsSession.ts b/src/features/berdctl/commands/impl/dismissHandoffsSession.ts new file mode 100644 index 000000000..fc1cdd3de --- /dev/null +++ b/src/features/berdctl/commands/impl/dismissHandoffsSession.ts @@ -0,0 +1,100 @@ +import { z } from "zod/v4"; + +import { CommandError, defineCommand } from "../types"; + +const dismissHandoffsSessionSchema = z + .object({ + session_id: z + .string() + .min(1) + .describe("Id of the session that owns the live Realtime Spokesperson."), + cursor: z + .number() + .int() + .min(0) + .max(4_294_967_295) + .describe( + "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + ), + handoff_id: z + .array(z.string().trim().min(1).max(100)) + .min(1) + .max(100) + .describe("Open handoff id to dismiss; repeat for multiple handoffs."), + reason: z + .string() + .trim() + .min(1) + .max(2_000) + .describe("Why no spoken response is needed for these handoffs."), + }) + .strict(); + +interface DismissHandoffsSessionResult { + session_id: string; + cursor: number; + dismissed_handoff_ids: string[]; + context_delivery_status: "sent" | "interrupting" | "queued"; +} + +export const dismissHandoffsSessionCommand = defineCommand({ + effect: "update", + visibility: "immediate", + destructive: false, + summary: "Dismiss open voice handoffs without speaking", + description: + "Explicitly close one or more open Realtime Spokesperson handoffs and deliver " + + "the reason as silent context without waking the Spokesperson. Use this only " + + "when a spoken response is obsolete, superseded, or already handled. The " + + "command and its reason remain visible in the Expert's normal Berd activity.", + helpFooter: `Example: + berdctl session dismiss-handoffs --session-id --cursor \ + --handoff-id \ + --reason "The user's follow-up superseded both requests." --json + +Result: + {"session_id":"...","cursor":,"dismissed_handoff_ids":[""],"context_delivery_status":"sent"|"interrupting"|"queued"} + +Every id must still be open. A dismissal consumes pending Spokesperson handoffs only +when --cursor proves the Expert received the complete pending batch, then +atomically sends the dismissal reason back as silent context. Use send-to-spokesperson +--mode say instead when the user still needs an answer.`, + schema: dismissHandoffsSessionSchema, + execute: async (args): Promise => { + const { dismissActiveRealtimeHandoffs } = await import( + "@/features/voice-conversation/lib/realtimeEmissaryBridge" + ); + const dismissal = await dismissActiveRealtimeHandoffs( + args.session_id, + args.cursor, + args.handoff_id, + args.reason, + ); + if (!dismissal) { + throw new CommandError( + "invalid_args", + `Session "${args.session_id}" has no live OpenAI Realtime voice Spokesperson. Start Realtime voice in that session and retry.`, + ); + } + + if (!dismissal.accepted) { + throw new CommandError( + "invalid_args", + JSON.stringify({ + reason: dismissal.reason, + cursor: dismissal.cursor, + ...(dismissal.reason === "unknown_handoff" + ? { handoff_ids: dismissal.handoffIds } + : {}), + }), + ); + } + + return { + session_id: args.session_id, + cursor: dismissal.cursor, + dismissed_handoff_ids: dismissal.dismissedHandoffIds, + context_delivery_status: dismissal.deliveryStatus, + }; + }, +}); diff --git a/src/features/berdctl/commands/impl/realtimeHandoffCommands.test.ts b/src/features/berdctl/commands/impl/realtimeHandoffCommands.test.ts new file mode 100644 index 000000000..f6fd211f6 --- /dev/null +++ b/src/features/berdctl/commands/impl/realtimeHandoffCommands.test.ts @@ -0,0 +1,128 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { registerRealtimeEmissary } from "@/features/voice-conversation/lib/realtimeEmissaryBridge"; +import { CommandError } from "../types"; +import { dismissHandoffsSessionCommand } from "./dismissHandoffsSession"; +import { sendToSpokespersonSessionCommand } from "./sendToSpokespersonSession"; + +let releaseBridge: (() => void) | undefined; + +afterEach(() => { + releaseBridge?.(); + releaseBridge = undefined; +}); + +describe("Realtime handoff commands", () => { + it("forwards every resolved handoff id through send-to-spokesperson", async () => { + const sendMasterMessage = vi.fn().mockResolvedValue({ + accepted: true, + cursor: 2, + deliveryStatus: "sent", + outbound: { + id: 3, + sender: "master", + recipient: "emissary", + senderCursor: 2, + message: "Both checks are complete.", + }, + }); + releaseBridge = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage, + dismissHandoffs: vi.fn(), + completeMasterTurn: vi.fn(), + }); + const args = sendToSpokespersonSessionCommand.schema.parse({ + session_id: "session-1", + cursor: 2, + mode: "say", + message: "Both checks are complete.", + resolves: ["handoff-1", "handoff-2"], + }); + + await expect( + sendToSpokespersonSessionCommand.execute(args, {}), + ).resolves.toEqual({ + session_id: "session-1", + cursor: 2, + delivery_status: "sent", + mode: "say", + resolved_handoff_ids: ["handoff-1", "handoff-2"], + }); + expect(sendMasterMessage).toHaveBeenCalledWith( + "Both checks are complete.", + 2, + "say", + ["handoff-1", "handoff-2"], + ); + }); + + it("reports unknown handoff ids from send-to-spokesperson", async () => { + releaseBridge = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn().mockResolvedValue({ + accepted: false, + reason: "unknown_handoff", + cursor: 2, + handoffIds: ["handoff-9"], + }), + dismissHandoffs: vi.fn(), + completeMasterTurn: vi.fn(), + }); + const args = sendToSpokespersonSessionCommand.schema.parse({ + session_id: "session-1", + cursor: 2, + mode: "say", + message: "Done.", + resolves: ["handoff-9"], + }); + + const error = await sendToSpokespersonSessionCommand + .execute(args, {}) + .catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(CommandError); + expect(error).toMatchObject({ code: "invalid_args" }); + expect(JSON.parse((error as Error).message)).toEqual({ + reason: "unknown_handoff", + cursor: 2, + handoff_ids: ["handoff-9"], + }); + }); + + it("dismisses multiple handoffs with silent context delivery status", async () => { + const sendMasterMessage = vi.fn(); + const dismissHandoffs = vi.fn().mockResolvedValue({ + accepted: true, + cursor: 2, + dismissedHandoffIds: ["handoff-1", "handoff-2"], + deliveryStatus: "sent", + }); + releaseBridge = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage, + dismissHandoffs, + completeMasterTurn: vi.fn(), + }); + const args = dismissHandoffsSessionCommand.schema.parse({ + session_id: "session-1", + cursor: 2, + handoff_id: ["handoff-1", "handoff-2"], + reason: "The user withdrew both requests.", + }); + + await expect( + dismissHandoffsSessionCommand.execute(args, {}), + ).resolves.toEqual({ + session_id: "session-1", + cursor: 2, + dismissed_handoff_ids: ["handoff-1", "handoff-2"], + context_delivery_status: "sent", + }); + expect(dismissHandoffs).toHaveBeenCalledWith( + 2, + ["handoff-1", "handoff-2"], + "The user withdrew both requests.", + ); + expect(sendMasterMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/berdctl/commands/impl/sendToSpokespersonSession.ts b/src/features/berdctl/commands/impl/sendToSpokespersonSession.ts new file mode 100644 index 000000000..d280a1180 --- /dev/null +++ b/src/features/berdctl/commands/impl/sendToSpokespersonSession.ts @@ -0,0 +1,118 @@ +import { z } from "zod/v4"; + +import { CommandError, defineCommand } from "../types"; + +const sendToSpokespersonSessionSchema = z + .object({ + session_id: z + .string() + .min(1) + .describe("Id of the session that owns the live Realtime Spokesperson."), + message: z + .string() + .trim() + .min(1) + .max(20_000) + .describe( + "Private coordination message to inject into the Spokesperson.", + ), + cursor: z + .number() + .int() + .min(0) + .max(4_294_967_295) + .describe( + "Newest cursor from any Expert-bound voice transcript, handoff, reminder, or bridge result.", + ), + mode: z + .enum(["context", "say"]) + .default("say") + .describe( + "Delivery mode: context updates future turns silently; say asks the Spokesperson to speak now.", + ), + resolves: z + .array(z.string().trim().min(1).max(100)) + .max(100) + .default([]) + .describe( + "Open handoff id resolved by this say message; repeat for multiple handoffs.", + ), + }) + .strict(); + +interface SendToSpokespersonSessionResult { + session_id: string; + cursor: number; + delivery_status: "sent" | "interrupting" | "queued"; + mode: "context" | "say"; + resolved_handoff_ids: string[]; +} + +export const sendToSpokespersonSessionCommand = defineCommand({ + effect: "update", + visibility: "immediate", + destructive: false, + summary: "Send private guidance to a session's live voice Spokesperson", + description: + "Inject a private coordination message into the OpenAI Realtime voice " + + "Spokesperson owned by an existing Berd session. The Spokesperson receives the " + + "message either as silent context for future turns or as a request to speak now. " + + "The command fails when the target session has no live Realtime voice conversation.", + helpFooter: `Example: + berdctl session send-to-spokesperson --session-id --cursor \\ + --mode say --resolves \\ + --message "The build failed because the signing certificate expired." --json + +Result: + {"session_id":"...","cursor":,"delivery_status":"sent"|"interrupting"|"queued","mode":"context"|"say","resolved_handoff_ids":[""]} + +Use --mode context to update the Spokesperson's future context without starting a +response. Use --mode say when the Spokesperson should speak the message now. +Repeat --resolves to close every handoff answered by one say. Context messages +cannot resolve handoffs. A say may omit --resolves when volunteering information. + +A send while the pipe contains a newer Expert-bound transcript, handoff, or +reminder fails with reason "pipe_busy" without consuming that pending event. +Wait for Berd to deliver it normally, then retry with its cursor.`, + schema: sendToSpokespersonSessionSchema, + execute: async (args): Promise => { + const { sendToActiveRealtimeSpokesperson } = await import( + "@/features/voice-conversation/lib/realtimeEmissaryBridge" + ); + const delivery = await sendToActiveRealtimeSpokesperson( + args.session_id, + args.message, + args.cursor, + args.mode, + args.resolves, + ); + if (!delivery) { + throw new CommandError( + "invalid_args", + `Session "${args.session_id}" has no live OpenAI Realtime voice Spokesperson. Start Realtime voice in that session and retry.`, + ); + } + + if (!delivery.accepted) { + throw new CommandError( + "invalid_args", + JSON.stringify({ + reason: delivery.reason, + cursor: delivery.cursor, + ...(delivery.reason === "unknown_handoff" || + delivery.reason === "context_cannot_resolve" + ? { handoff_ids: delivery.handoffIds } + : {}), + }), + ); + } + + return { + session_id: args.session_id, + cursor: delivery.cursor, + delivery_status: delivery.deliveryStatus, + mode: args.mode, + resolved_handoff_ids: args.resolves, + }; + }, +}); diff --git a/src/features/berdctl/commands/registry.ts b/src/features/berdctl/commands/registry.ts index 7e73fdc37..20abf9723 100644 --- a/src/features/berdctl/commands/registry.ts +++ b/src/features/berdctl/commands/registry.ts @@ -6,6 +6,7 @@ import { attachProjectFolderCommand } from "./impl/attachProjectFolder"; import { attachSessionFolderCommand } from "./impl/attachSessionFolder"; import { detachProjectFolderCommand } from "./impl/detachProjectFolder"; import { detachSessionFolderCommand } from "./impl/detachSessionFolder"; +import { dismissHandoffsSessionCommand } from "./impl/dismissHandoffsSession"; import { listSessionFoldersCommand } from "./impl/listSessionFolders"; import { replaceSessionFolderCommand } from "./impl/replaceSessionFolder"; import { setSessionFolderCwdCommand } from "./impl/setSessionFolderCwd"; @@ -31,6 +32,7 @@ import { openFeedbackCommand } from "./impl/openFeedback"; import { openSessionCommand } from "./impl/openSession"; import { renameSessionCommand } from "./impl/renameSession"; import { sendSessionCommand } from "./impl/sendSession"; +import { sendToSpokespersonSessionCommand } from "./impl/sendToSpokespersonSession"; import { setProjectStartupModeCommand } from "./impl/setProjectStartupMode"; import { submitFeedbackCommand } from "./impl/submitFeedback"; import { commandBridgeTimeoutMs } from "./timeouts"; @@ -57,11 +59,11 @@ export const ALL_TOOL_GROUPS = { description: "Manage the user's chat sessions: create (fire-and-forget, on any " + "installed agent harness), send, open, list, get, rename, move, " + - "move to group, clear project, fork, archive.", + "move to group, clear project, send to a live voice Spokesperson, dismiss voice handoffs, fork, archive.", cli: { noun: "session", about: - "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, fork, archive", + "Manage chat sessions: create, send, open, list, get, rename, move, move to group, clear project, send to Spokesperson, dismiss handoffs, fork, archive", verbs: { create: "create", send: "send", @@ -72,6 +74,8 @@ export const ALL_TOOL_GROUPS = { move: "move", "move-to-group": "move_to_group", "clear-project": "clear_project", + "send-to-spokesperson": "send_to_spokesperson", + "dismiss-handoffs": "dismiss_handoffs", fork: "fork", archive: "archive", }, @@ -86,6 +90,8 @@ export const ALL_TOOL_GROUPS = { move: moveSessionCommand, move_to_group: moveSessionToGroupCommand, clear_project: clearSessionProjectCommand, + send_to_spokesperson: sendToSpokespersonSessionCommand, + dismiss_handoffs: dismissHandoffsSessionCommand, fork: forkSessionCommand, archive: archiveSessionCommand, }, diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index c9a89b71e..1ce646783 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -2390,6 +2390,41 @@ describe("acpNotificationHandler", () => { }); }); + it("streams an owned local prompt live while session history is loading", async () => { + const sessionId = "loading-with-live-prompt"; + markSessionReplayLoading(sessionId); + claimSessionPrompt(sessionId); + setActiveMessageId(sessionId, "master-live", {}); + + await handleSessionNotification({ + sessionId, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: "Inspecting the workspace" }, + }, + } as never); + await handleSessionNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "I found the answer." }, + }, + } as never); + flushBufferedStreamingUpdatesForSession(sessionId); + + expect(getReplayBuffer(sessionId)).toBeUndefined(); + expect(useChatStore.getState().messagesBySession[sessionId]).toMatchObject([ + { + id: "master-live", + role: "assistant", + content: [ + { type: "thinking", text: "Inspecting the workspace" }, + { type: "text", text: "I found the answer." }, + ], + }, + ]); + }); + it("replay preserves timestamps from goose metadata on user and assistant chunks", async () => { const replaySessionId = "replay-timestamp-session"; const userCreated = 1_700_000_000; diff --git a/src/features/chat/acp/acpNotificationHandler.ts b/src/features/chat/acp/acpNotificationHandler.ts index 8474e1619..e2da1f650 100644 --- a/src/features/chat/acp/acpNotificationHandler.ts +++ b/src/features/chat/acp/acpNotificationHandler.ts @@ -60,6 +60,7 @@ import { getSubagentToolCallContext, resolveSubagentContext, } from "@/features/chat/lib/subagentToolCalls"; +import { getSessionPromptOwner } from "@/features/chat/lib/sessionPromptOwnership"; import { applyChatSessionConfigOptionsSnapshot } from "./sessionConfigSnapshotAdapter"; import { perfLog } from "@/shared/lib/perfLog"; import { @@ -230,8 +231,14 @@ export async function handleSessionNotification( ): Promise { const sessionId = notification.sessionId; const { update } = notification; - const isReplay = useChatStore.getState().loadingSessionIds.has(sessionId); - + // A newly-created session can still be hydrating when its first local prompt + // starts. Notifications from that owned prompt are live output, even while + // the history-loading flag is set. Routing them into the replay buffer loses + // them when replay has already taken its snapshot (realtime voice makes this + // race easy to hit because it can dispatch immediately after session start). + const isReplay = + useChatStore.getState().loadingSessionIds.has(sessionId) && + getSessionPromptOwner(sessionId) === null; if (isReplay) { const sid = sessionId.slice(0, 8); let perf = replayPerf.get(sessionId); diff --git a/src/features/chat/acp/acpSkillReplayChips.ts b/src/features/chat/acp/acpSkillReplayChips.ts index 81ff720aa..329b0ccb7 100644 --- a/src/features/chat/acp/acpSkillReplayChips.ts +++ b/src/features/chat/acp/acpSkillReplayChips.ts @@ -49,7 +49,10 @@ export function handleReplayUserMessageChunk( messageId: string, content: TextContent | ImageContent, created?: number, - metadata?: Pick, + metadata?: Pick< + MessageMetadata, + "delivery" | "origin" | "userVisible" | "agentVisible" + >, ): void { const buffer = ensureReplayBuffer(sessionId); const existing = getBufferedMessage(sessionId, messageId); diff --git a/src/features/chat/capabilities/ConversationComposerCapability.test.tsx b/src/features/chat/capabilities/ConversationComposerCapability.test.tsx index 78c1e080f..315a6a8d0 100644 --- a/src/features/chat/capabilities/ConversationComposerCapability.test.tsx +++ b/src/features/chat/capabilities/ConversationComposerCapability.test.tsx @@ -151,6 +151,42 @@ function createBinding( describe("ConversationComposerCapability surface parity", () => { beforeEach(() => chatInputSpy.mockClear()); + it("notifies an active voice frontend after typed sends commit", async () => { + const controller = createController(); + const notify = vi.fn(); + controller.handleSend.mockImplementation( + (_text, _personaId, _attachments, options) => { + options?.onUserMessageCommitted?.(); + return true; + }, + ); + controller.steerDraftMessage.mockImplementation( + async (_text, _personaId, _attachments, options) => { + options?.onUserMessageCommitted?.(); + return true; + }, + ); + render( + , + ); + + const actions = latestProps().composerActions; + expect(actions.onSend("typed message")).toBe(true); + expect(notify).toHaveBeenCalledWith("typed message"); + + await actions.onSteerMessage?.("typed steer"); + expect(notify).toHaveBeenCalledWith("typed steer"); + }); + it("preserves Home deferred-queue policy while sharing draft and selection behavior", () => { const controller = createController(); render( diff --git a/src/features/chat/capabilities/ConversationComposerCapability.tsx b/src/features/chat/capabilities/ConversationComposerCapability.tsx index cadeb606a..35bc41022 100644 --- a/src/features/chat/capabilities/ConversationComposerCapability.tsx +++ b/src/features/chat/capabilities/ConversationComposerCapability.tsx @@ -143,6 +143,7 @@ interface ConversationComposerCapabilityProps { onRecallLastUserMessage?: () => string | null; attachmentDropTargetRef?: RefObject; onAttachmentDragOverChange?: (isDragOver: boolean) => void; + onUserTextCommitted?: (text: string) => void; } export function ConversationComposerCapability({ @@ -152,6 +153,7 @@ export function ConversationComposerCapability({ onRecallLastUserMessage, attachmentDropTargetRef, onAttachmentDragOverChange, + onUserTextCommitted, }: ConversationComposerCapabilityProps) { const { t } = useTranslation("chat"); const { @@ -179,6 +181,19 @@ export function ConversationComposerCapability({ : undefined; const isReadOnly = Boolean(readOnlyReason); const lifecycle = renderingPolicy.lifecycleConstraints; + const sendWithCommitNotification = useCallback( + (...args: Parameters) => { + const [text, personaId, attachments, options] = args; + return onSend(text, personaId, attachments, { + ...options, + onUserMessageCommitted: () => { + options?.onUserMessageCommitted?.(); + onUserTextCommitted?.(text); + }, + }); + }, + [onSend, onUserTextCommitted], + ); const securityConfirmationPending = target.kind === "existingSession" && target.admission.securityConfirmationPending; @@ -260,7 +275,7 @@ export function ConversationComposerCapability({ isPendingConversation ? undefined : controller.selectedProvider } composerActions={{ - onSend, + onSend: onUserTextCommitted ? sendWithCommitNotification : onSend, onSteerMessage: isPendingConversation || admissionBlocked ? undefined @@ -269,7 +284,15 @@ export function ConversationComposerCapability({ text, personaId ?? undefined, attachments, - options, + onUserTextCommitted + ? { + ...options, + onUserMessageCommitted: () => { + options?.onUserMessageCommitted?.(); + onUserTextCommitted(text); + }, + } + : options, ), canSteerMessage: isPendingConversation || admissionBlocked diff --git a/src/features/chat/hooks/__tests__/useMessageQueue.test.ts b/src/features/chat/hooks/__tests__/useMessageQueue.test.ts index b0fde2442..aa531282c 100644 --- a/src/features/chat/hooks/__tests__/useMessageQueue.test.ts +++ b/src/features/chat/hooks/__tests__/useMessageQueue.test.ts @@ -12,15 +12,34 @@ import { import type { ChatSendOptions } from "../../types"; import { useChatStore } from "../../stores/chatStore"; import { useChatSessionStore } from "../../stores/chatSessionStore"; +import { loadCachedMessageQueues } from "../../stores/queuePersistence"; import { useMessageQueue } from "../useMessageQueue"; -const mockAcpPrepareSession = vi.fn().mockResolvedValue(undefined); +const mocks = vi.hoisted(() => ({ + acpPrepareSession: vi.fn().mockResolvedValue(undefined), + stopRealtimeForSession: vi.fn().mockResolvedValue(undefined), + toastError: vi.fn(), +})); vi.mock("@/shared/api/acp", async (importOriginal) => ({ ...(await importOriginal()), - acpPrepareSession: (...args: unknown[]) => mockAcpPrepareSession(...args), + acpPrepareSession: (...args: unknown[]) => mocks.acpPrepareSession(...args), +})); + +vi.mock("sonner", () => ({ + toast: { + error: (...args: unknown[]) => mocks.toastError(...args), + }, })); +vi.mock( + "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation", + () => ({ + stopOpenAiRealtimeConversationForSession: (...args: unknown[]) => + mocks.stopRealtimeForSession(...args), + }), +); + function deferred() { let resolve!: (value: T | PromiseLike) => void; let reject!: (reason?: unknown) => void; @@ -33,6 +52,10 @@ function deferred() { describe("useMessageQueue", () => { beforeEach(() => { + mocks.acpPrepareSession.mockClear(); + mocks.stopRealtimeForSession.mockClear(); + mocks.toastError.mockClear(); + window.localStorage.clear(); resetSessionTargetCoordinatorsForTests(); useChatSessionStore.setState({ sessions: [ @@ -1501,6 +1524,92 @@ describe("useMessageQueue", () => { vi.useRealTimers(); }); + it("keeps transport-only voice coordination hidden while queued", () => { + const sendMessage = vi.fn().mockReturnValue(true); + const { result } = renderHook(() => + useMessageQueue("s1", "streaming", sendMessage), + ); + + act(() => { + expect( + result.current.enqueue( + "[Handoff handoff-3 from spokesperson; cursor 3] Check the result", + undefined, + undefined, + { + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }, + ), + ).toBe(true); + }); + + expect( + useChatStore.getState().queuedMessageBySession.s1?.[0]?.payload, + ).toMatchObject({ + showInComposer: false, + sendOptions: { + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }, + }); + }); + + it("removes exhausted hidden coordination and drains the next user message", async () => { + vi.useFakeTimers(); + const privateCoordination = + "[Handoff handoff-3 from spokesperson; cursor 3] Private context"; + const sendMessage = vi.fn((text: string) => text === "normal user message"); + useChatStore.getState().enqueueTransportReadyMessage("s1", { + persona: { kind: "inherit" }, + text: privateCoordination, + showInComposer: false, + sendOptions: { + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }, + }); + useChatStore.getState().enqueueTransportReadyMessage("s1", { + persona: { kind: "inherit" }, + text: "normal user message", + }); + + renderHook(() => useMessageQueue("s1", "idle", sendMessage)); + + await act(async () => { + await vi.advanceTimersByTimeAsync(15_000); + }); + + expect(sendMessage).toHaveBeenCalledTimes(6); + expect(sendMessage.mock.calls.at(-1)?.[0]).toBe("normal user message"); + expect(useChatStore.getState().queuedMessageBySession.s1).toBeUndefined(); + expect(loadCachedMessageQueues()).toEqual({}); + expect(mocks.stopRealtimeForSession).toHaveBeenCalledOnce(); + expect(mocks.stopRealtimeForSession).toHaveBeenCalledWith("s1"); + expect(mocks.toastError).toHaveBeenCalledOnce(); + + const visibleMessages = useChatStore.getState().messagesBySession.s1; + expect(visibleMessages).toHaveLength(1); + expect(visibleMessages?.[0]).toMatchObject({ + role: "system", + content: [ + { + type: "systemNotification", + notificationType: "error", + }, + ], + metadata: { userVisible: true, agentVisible: false }, + }); + expect(JSON.stringify(visibleMessages)).not.toContain(privateCoordination); + vi.useRealTimers(); + }); + it("retries the same failed head on every later readiness transition", () => { const sendMessage = vi.fn().mockReturnValue(false); useChatStore.getState().enqueueTransportReadyMessage("s1", { diff --git a/src/features/chat/hooks/useChat.ts b/src/features/chat/hooks/useChat.ts index aee7ea014..93a2fabe5 100644 --- a/src/features/chat/hooks/useChat.ts +++ b/src/features/chat/hooks/useChat.ts @@ -207,6 +207,7 @@ export function useChat( attachments, assistantPrompt: sendOptions?.assistantPrompt, displayText: sendOptions?.displayText, + userMessageId: sendOptions?.userMessageId, chips: sendOptions?.chips, userMessageMetadata: sendOptions?.userMessageMetadata, acpGooseMetadata: sendOptions?.acpGooseMetadata, diff --git a/src/features/chat/hooks/useMessageQueue.ts b/src/features/chat/hooks/useMessageQueue.ts index 974d54ed0..7e94a9571 100644 --- a/src/features/chat/hooks/useMessageQueue.ts +++ b/src/features/chat/hooks/useMessageQueue.ts @@ -1,7 +1,13 @@ import { useEffect, useCallback, useMemo, useRef } from "react"; +import { toast } from "sonner"; +import { i18n } from "@/shared/i18n"; import type { ChatState } from "@/shared/types/chat"; import { isPromiseLike } from "@/shared/lib/isPromiseLike"; -import type { ChatAttachmentDraft } from "@/shared/types/messages"; +import { + type ChatAttachmentDraft, + createSystemNotificationMessage, +} from "@/shared/types/messages"; +import { stopOpenAiRealtimeConversationForSession } from "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation"; import { assertQueuedMessageAttemptOwned, becameQueuedMessageTargetAttemptable, @@ -41,8 +47,10 @@ interface QueueAttemptLease { const queueAttemptLeaseBySession = new Map(); // LAWS/CHAT.md: the queue must resume sending when the session becomes ready. -// Rejected attempts back off but never abandon the record — a rejection can be -// silent (pre-commit ownership/readiness races around draft promotion) with no +// Rejected attempts back off. User-visible records remain available for manual +// recovery; exhausted transport-only records fail visibly and are removed so +// they cannot silently strand later messages. A rejection can be silent +// (pre-commit ownership/readiness races around draft promotion) with no // follow-up store transition to re-trigger the drain. const MAX_AUTO_RETRY_DELAY_MS = 30_000; @@ -290,7 +298,10 @@ export function useMessageQueue( if (accepted === false) { let retryPayload = latestQueuedMessage.payload; - if (retryPayload.showInComposer === false) { + if ( + retryPayload.showInComposer === false && + retryPayload.sendOptions?.userMessageMetadata?.userVisible !== false + ) { retryPayload = { ...retryPayload, showInComposer: true, @@ -320,13 +331,37 @@ export function useMessageQueue( count: rejections, }; if (rejections >= MAX_CONSECUTIVE_REJECTIONS) { - // Stop automatically. The record stays queued and showInComposer - // was forced true above, so it is visible and the user can resend. autoRetryRef.current = null; if (retryTimerRef.current !== null) { clearTimeout(retryTimerRef.current); retryTimerRef.current = null; } + if ( + retryPayload.sendOptions?.userMessageMetadata?.userVisible === + false + ) { + const failureMessage = i18n.t( + "chat:queue.voiceCoordinationFailed", + ); + useChatStore + .getState() + .addMessage( + sessionId, + createSystemNotificationMessage(failureMessage, "error"), + ); + toast.error(i18n.t("chat:queue.voiceCoordinationFailedTitle"), { + description: failureMessage, + }); + useChatStore + .getState() + .dismissQueuedMessage(sessionId, latestQueuedMessage.recordId); + void stopOpenAiRealtimeConversationForSession(sessionId).catch( + () => undefined, + ); + } + // Visible records remain queued so the user can edit, resend, or + // dismiss them. Hidden transport records are removed above so an + // exhausted internal retry cannot block the rest of the queue. return; } const scheduleAutoRetry = () => { @@ -589,6 +624,13 @@ export function useMessageQueue( personaName, attachments, sendOptions, + // Transport-only messages (including Emissary → Master + // coordination) may briefly use the reliable queue at a run + // boundary, but they must never leak into the user's composer. + showInComposer: + sendOptions?.userMessageMetadata?.userVisible === false + ? false + : undefined, }), ); }, diff --git a/src/features/chat/lib/__tests__/replaySanitizer.test.ts b/src/features/chat/lib/__tests__/replaySanitizer.test.ts index 7f88f796d..7852e9977 100644 --- a/src/features/chat/lib/__tests__/replaySanitizer.test.ts +++ b/src/features/chat/lib/__tests__/replaySanitizer.test.ts @@ -63,6 +63,126 @@ describe("sanitizeReplayMessages", () => { ]); }); + it("restores batched realtime transcripts to user and spoken Spokesperson bubbles", () => { + const message = createTextMessage( + "voice-batch", + "user", + "[Voice transcript] Spokesperson said: Let me check.\n" + + "[Voice transcript] Spokesperson said (interrupted; best-effort transcript): One moment.\n" + + "[Voice transcript] User said: What did you find?", + ); + message.metadata = { + ...message.metadata, + origin: "voice_conversation", + }; + + expect(sanitizeReplayMessages([message])).toMatchObject([ + { + id: "voice-batch", + role: "assistant", + content: [ + { + type: "text", + text: "Let me check.", + speech: { status: "spoken" }, + }, + ], + metadata: { + userVisible: true, + agentVisible: false, + voiceConversationDebugEvent: "emissarySpeech", + }, + }, + { + id: "voice-batch:voice:1", + role: "assistant", + content: [ + { + type: "text", + text: "One moment.", + speech: { status: "interrupted", confidence: "low" }, + }, + ], + metadata: { voiceConversationDebugEvent: "emissarySpeech" }, + }, + { + id: "voice-batch:voice:2", + role: "user", + content: [{ type: "text", text: "What did you find?" }], + metadata: { userVisible: true, agentVisible: false }, + }, + ]); + }); + + it("restores a current Expert wake batch with cursors and a handoff", () => { + const handoffId = "handoff-123e4567-e89b-12d3-a456-426614174000-6"; + const message = createTextMessage( + "expert-wake", + "user", + "[Voice transcript; cursor 4] User said: Check my Development folder.\n" + + "[Voice transcript; cursor 5] Spokesperson said: Let me check that.\n" + + `[Handoff ${handoffId} from spokesperson; cursor 6] Count the repositories.`, + ); + message.metadata = { + ...message.metadata, + origin: "voice_conversation", + userVisible: false, + }; + + expect(sanitizeReplayMessages([message])).toMatchObject([ + { + role: "user", + content: [{ type: "text", text: "Check my Development folder." }], + }, + { + role: "assistant", + content: [ + { + type: "text", + text: "Let me check that.", + speech: { status: "spoken" }, + }, + ], + }, + { + role: "assistant", + content: [{ type: "text", text: "Count the repositories." }], + metadata: { + personaName: "Spokesperson → Expert", + voiceConversationDebugEvent: "emissaryToMaster", + }, + }, + ]); + }); + + it("restores persisted Spokesperson handoffs as coordination bubbles", () => { + const handoffId = "handoff-123e4567-e89b-12d3-a456-426614174000-1"; + const message = createTextMessage( + "direct-message", + "user", + `[Handoff ${handoffId} from spokesperson; cursor 1] Check the transcript storage.`, + ); + message.metadata = { + ...message.metadata, + origin: "voice_conversation", + userVisible: false, + }; + + expect(sanitizeReplayMessages([message])).toMatchObject([ + { + id: "direct-message", + role: "assistant", + content: [{ type: "text", text: "Check the transcript storage." }], + metadata: { + personaName: "Spokesperson → Expert", + userVisible: true, + agentVisible: false, + voiceConversationDebugEvent: "emissaryToMaster", + }, + }, + ]); + }); + it("keeps TTS control lookalikes that are not voice-origin messages", () => { const message = createTextMessage( "user-1", diff --git a/src/features/chat/lib/__tests__/steerCore.test.ts b/src/features/chat/lib/__tests__/steerCore.test.ts index 8f5602679..a11963d08 100644 --- a/src/features/chat/lib/__tests__/steerCore.test.ts +++ b/src/features/chat/lib/__tests__/steerCore.test.ts @@ -16,6 +16,7 @@ vi.mock("@/shared/i18n", () => ({ })); import { steerPromptInSession } from "../steerCore"; +import { VOICE_CONVERSATION_EMPTY_RESPONSE } from "../voiceConversationNoop"; function oversizedImageDraft() { return { @@ -149,6 +150,24 @@ describe("steerPromptInSession commit callback", () => { expect(messages.some((message) => message.role === "user")).toBe(false); }); + it("can return a recoverable steer rejection without leaking an error row", async () => { + mockAcpSteerMessage.mockRejectedValue(new Error("no active run to steer")); + + await expect( + steerPromptInSession( + "session-1", + "follow-up voice transcript", + undefined, + { userMessageMetadata: { origin: "voice_conversation" } }, + { throwOnError: true, reportErrorInTranscript: false }, + ), + ).rejects.toThrow("no active run to steer"); + + expect( + useChatStore.getState().messagesBySession["session-1"] ?? [], + ).toEqual([]); + }); + it("fires when delivery was established despite an acknowledgement error", async () => { const onUserMessageCommitted = vi.fn(); mockAcpSteerMessage.mockImplementation(async () => { @@ -178,3 +197,79 @@ describe("steerPromptInSession commit callback", () => { expect(onUserMessageCommitted).toHaveBeenCalledTimes(1); }); }); + +describe("steerPromptInSession voice no-op", () => { + beforeEach(() => { + vi.clearAllMocks(); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + activeSessionId: null, + isConnected: true, + }); + }); + + it("preserves a provisional voice transcript's original ordering timestamp", async () => { + useChatStore.getState().addMessage("session-1", { + id: "voice-user", + role: "user", + created: 100, + content: [{ type: "text", text: "provisional" }], + metadata: { origin: "voice_conversation" }, + }); + mockAcpSteerMessage.mockResolvedValue({ + runId: "run-1", + messageId: "voice-user", + }); + + const accepted = await steerPromptInSession( + "session-1", + "final transcript", + undefined, + { + displayText: "final transcript", + userMessageId: "voice-user", + userMessageMetadata: { origin: "voice_conversation" }, + }, + { throwOnError: true }, + ); + + expect(accepted).toBe(true); + expect( + useChatStore + .getState() + .messagesBySession["session-1"]?.find( + (message) => message.id === "voice-user", + ), + ).toMatchObject({ created: 100 }); + }); + + it("keeps the transcript and suppresses the known empty master response", async () => { + mockAcpSteerMessage.mockRejectedValue( + new Error(VOICE_CONVERSATION_EMPTY_RESPONSE), + ); + + const accepted = await steerPromptInSession( + "session-1", + "[Voice transcript] User said: Nice weather today.", + undefined, + { + displayText: "User said: Nice weather today.", + userMessageMetadata: { origin: "voice_conversation" }, + }, + { throwOnError: true }, + ); + + expect(accepted).toBe(true); + const messages = + useChatStore.getState().messagesBySession["session-1"] ?? []; + expect(messages).toHaveLength(1); + expect(messages[0]).toMatchObject({ + role: "user", + metadata: { origin: "voice_conversation", delivery: "steer" }, + }); + expect(messages[0].content[0]).toMatchObject({ + text: "User said: Nice weather today.", + }); + }); +}); diff --git a/src/features/chat/lib/replaySanitizer.ts b/src/features/chat/lib/replaySanitizer.ts index 1e8de3d87..530df6313 100644 --- a/src/features/chat/lib/replaySanitizer.ts +++ b/src/features/chat/lib/replaySanitizer.ts @@ -11,6 +11,14 @@ const TTS_DELIVERY_FAILURE_OUTCOMES = new Set([ "TTS delivery was blocked because the user was speaking; the assistant reply was not spoken.", "Native TTS could not deliver the assistant reply.", ]); +const VOICE_TRANSCRIPT_BOUNDARY = + /\n(?=\[(?:Voice transcript(?:; cursor \d+)?|Handoff handoff-[A-Za-z0-9-]+ from spokesperson; cursor \d+)\] )/; +const USER_TRANSCRIPT = + /^\[Voice transcript(?:; cursor \d+)?\] User said: ([\s\S]*)$/; +const SPOKESPERSON_TRANSCRIPT = + /^\[Voice transcript(?:; cursor \d+)?\] Spokesperson said( \(interrupted; best-effort transcript\))?: ([\s\S]*)$/; +const SPOKESPERSON_DIRECT_MESSAGE = + /^\[Handoff handoff-[A-Za-z0-9-]+ from spokesperson; cursor \d+\] ([\s\S]*)$/; function visibleTextAfterTtsDeliveryNotices(text: string): string | null { if (!text.startsWith(TTS_DELIVERY_FAILURE_PREFIX)) { @@ -82,6 +90,84 @@ function sanitizeTtsDeliveryReplayArtifact(message: Message): Message | null { }; } +function restoreRealtimeVoiceMessages(message: Message): Message[] | null { + if ( + message.role !== "user" || + message.metadata?.origin !== "voice_conversation" || + message.content.some((content) => content.type !== "text") + ) { + return null; + } + + const segments = getTextContent(message).split(VOICE_TRANSCRIPT_BOUNDARY); + const restored: Message[] = []; + for (const [index, segment] of segments.entries()) { + const user = USER_TRANSCRIPT.exec(segment); + const spokesperson = SPOKESPERSON_TRANSCRIPT.exec(segment); + const direct = SPOKESPERSON_DIRECT_MESSAGE.exec(segment); + if (!user && !spokesperson && !direct) return null; + + const id = index === 0 ? message.id : `${message.id}:voice:${index}`; + if (user) { + restored.push({ + ...message, + id, + role: "user", + content: [{ type: "text", text: user[1] }], + metadata: { + ...message.metadata, + userVisible: true, + agentVisible: false, + completionStatus: "completed", + }, + }); + continue; + } + + if (spokesperson) { + const interrupted = Boolean(spokesperson[1]); + restored.push({ + ...message, + id, + role: "assistant", + content: [ + { + type: "text", + text: spokesperson[2], + speech: interrupted + ? { status: "interrupted", confidence: "low" } + : { status: "spoken", spokenThrough: spokesperson[2].length }, + }, + ], + metadata: { + ...message.metadata, + userVisible: true, + agentVisible: false, + voiceConversationDebugEvent: "emissarySpeech", + completionStatus: "completed", + }, + }); + continue; + } + + restored.push({ + ...message, + id, + role: "assistant", + content: [{ type: "text", text: direct?.[1] ?? "" }], + metadata: { + ...message.metadata, + userVisible: true, + agentVisible: false, + personaName: "Spokesperson → Expert", + voiceConversationDebugEvent: "emissaryToMaster", + completionStatus: "completed", + }, + }); + } + return restored; +} + export function isManualCompactReplayArtifact(message: Message): boolean { if (message.role !== "user") { return false; @@ -107,8 +193,7 @@ export function isManualCompactReplayArtifact(message: Message): boolean { export function sanitizeReplayMessages(messages: Message[]): Message[] { return messages.flatMap((message) => { const sanitized = sanitizeTtsDeliveryReplayArtifact(message); - return sanitized && !isManualCompactReplayArtifact(sanitized) - ? [sanitized] - : []; + if (!sanitized || isManualCompactReplayArtifact(sanitized)) return []; + return restoreRealtimeVoiceMessages(sanitized) ?? [sanitized]; }); } diff --git a/src/features/chat/lib/sendCore.test.ts b/src/features/chat/lib/sendCore.test.ts index cfb7ddd38..b66ab0e94 100644 --- a/src/features/chat/lib/sendCore.test.ts +++ b/src/features/chat/lib/sendCore.test.ts @@ -4,18 +4,24 @@ import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; import type { SessionChatRuntime } from "@/shared/types/chat"; import { QueuedMessageOwnershipLostError } from "./preCommitSendRejection"; import { dispatchPrompt } from "./sendCore"; +import { registerRealtimeEmissary } from "@/features/voice-conversation/lib/realtimeEmissaryBridge"; +import { setVoiceConversationMode } from "@/features/voice-conversation/lib/voiceConversationModePreference"; const mocks = vi.hoisted(() => ({ + acpExportSession: vi.fn(), acpSendMessage: vi.fn(), })); vi.mock("@/shared/api/acp", () => ({ + acpExportSession: (...args: unknown[]) => mocks.acpExportSession(...args), acpSendMessage: (...args: unknown[]) => mocks.acpSendMessage(...args), })); describe("dispatchPrompt pre-commit rejection", () => { beforeEach(() => { vi.clearAllMocks(); + window.localStorage.removeItem("goose:voice-conversation-mode"); + mocks.acpExportSession.mockResolvedValue("{}"); useChatStore.setState({ messagesBySession: {}, sessionStateById: {}, @@ -27,6 +33,38 @@ describe("dispatchPrompt pre-commit rejection", () => { useChatSessionStore.setState({ sessions: [], activeSessionId: null }); }); + it("does not inspect prior assistant text for an ordinary text prompt", async () => { + const inaccessibleText = { type: "text" } as { + type: "text"; + text: string; + }; + Object.defineProperty(inaccessibleText, "text", { + get: () => { + throw new Error("ordinary text sends must not scan transcript content"); + }, + }); + useChatStore.getState().addMessage("session-1", { + id: "prior-assistant", + role: "assistant", + created: 1, + content: [inaccessibleText], + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + _sessionId: string, + _prompt: string, + options: { onPromptDispatching(): void }, + ) => { + options.onPromptDispatching(); + return Promise.resolve(); + }, + ); + + await expect( + dispatchPrompt("session-1", "ordinary text", {}), + ).resolves.toBeUndefined(); + }); + it("preserves the complete newer-owner runtime on ownership loss", async () => { let newerOwnerRuntime: SessionChatRuntime | undefined; mocks.acpSendMessage.mockImplementationOnce( @@ -117,3 +155,482 @@ describe("dispatchPrompt pre-commit rejection", () => { ); }); }); + +describe("dispatchPrompt voice conversation no-op", () => { + const emptyResponseError = + "The model returned an empty response. Please resend your message to continue."; + + beforeEach(() => { + vi.clearAllMocks(); + mocks.acpExportSession.mockResolvedValue("{}"); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + queuedMessageBySession: {}, + draftsBySession: {}, + activeSessionId: null, + isConnected: false, + }); + }); + + function rejectCommittedPrompt(message: string): void { + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { onPromptDispatching(): void }, + ) => { + options.onPromptDispatching(); + const store = useChatStore.getState(); + store.addMessage(sessionId, { + id: "empty-assistant", + role: "assistant", + created: Date.now(), + content: [], + metadata: { completionStatus: "inProgress" }, + }); + store.setStreamingMessageId(sessionId, "empty-assistant"); + return Promise.reject(new Error(message)); + }, + ); + } + + it("preserves a provisional voice transcript's original ordering timestamp", async () => { + useChatStore.getState().addMessage("session-1", { + id: "voice-user", + role: "user", + created: 100, + content: [{ type: "text", text: "provisional" }], + metadata: { origin: "voice_conversation" }, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + _sessionId: string, + _prompt: string, + options: { onPromptDispatching(): void }, + ) => { + options.onPromptDispatching(); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "final transcript", { + displayText: "final transcript", + userMessageId: "voice-user", + userMessageMetadata: { origin: "voice_conversation" }, + }); + + expect( + useChatStore + .getState() + .messagesBySession["session-1"]?.find( + (message) => message.id === "voice-user", + ), + ).toMatchObject({ created: 100 }); + }); + + it("treats a committed voice empty response as a clean semantic no-op", async () => { + rejectCommittedPrompt(emptyResponseError); + + await expect( + dispatchPrompt("session-1", "Emissary said: Hello", { + userMessageMetadata: { origin: "voice_conversation" }, + }), + ).resolves.toBeUndefined(); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages).toHaveLength(2); + expect(messages[0]).toMatchObject({ + role: "user", + metadata: { origin: "voice_conversation" }, + }); + expect(messages[1]).toMatchObject({ + id: "empty-assistant", + role: "assistant", + metadata: { completionStatus: "completed" }, + }); + expect(messages.some((message) => message.role === "system")).toBe(false); + + const runtime = useChatStore.getState().getSessionRuntime("session-1"); + expect(runtime.chatState).toBe("idle"); + expect(runtime.error).toBeNull(); + expect(runtime.streamingMessageId).toBeNull(); + expect(runtime.pendingAssistantProviderId).toBeNull(); + }); + + it("does not suppress a different error for a voice turn", async () => { + rejectCommittedPrompt("Provider authentication failed"); + + await expect( + dispatchPrompt("session-1", "User said: Hello", { + userMessageMetadata: { origin: "voice_conversation" }, + }), + ).rejects.toThrow("Provider authentication failed"); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages.at(-1)).toMatchObject({ role: "system" }); + expect(useChatStore.getState().getSessionRuntime("session-1").error).toBe( + "Provider authentication failed", + ); + }); + + it("does not suppress the empty-response error for a non-voice turn", async () => { + rejectCommittedPrompt(emptyResponseError); + + await expect(dispatchPrompt("session-1", "Hello", {})).rejects.toThrow( + emptyResponseError, + ); + + const messages = useChatStore.getState().messagesBySession["session-1"]; + expect(messages.at(-1)).toMatchObject({ role: "system" }); + expect(useChatStore.getState().getSessionRuntime("session-1").error).toBe( + emptyResponseError, + ); + }); +}); + +describe("dispatchPrompt realtime Master transcript recovery", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.acpExportSession.mockResolvedValue("{}"); + useChatStore.setState({ + messagesBySession: {}, + sessionStateById: {}, + queuedMessageBySession: {}, + draftsBySession: {}, + activeSessionId: null, + isConnected: false, + }); + }); + + it("does not notify the emissary when a Master turn completes", async () => { + const sendMasterMessage = vi.fn(); + const completeMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage, + dismissHandoffs: vi.fn(), + completeMasterTurn, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore.getState().addMessage(sessionId, { + id: "master-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "There are 20 repositories." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Count repositories", {}); + + expect(sendMasterMessage).not.toHaveBeenCalled(); + expect(completeMasterTurn).toHaveBeenCalledWith({ + reminderHandoffIds: [], + }); + expect(useChatStore.getState().messagesBySession["session-1"]).toHaveLength( + 2, + ); + release(); + }); + + it("keeps a completed Expert turn successful when Realtime completion fails", async () => { + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn(), + dismissHandoffs: vi.fn(), + completeMasterTurn: () => { + throw new Error("Realtime owner disappeared"); + }, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore.getState().addMessage(sessionId, { + id: "master-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "The Expert finished." }], + metadata: { completionStatus: "completed" }, + }); + return Promise.resolve(); + }, + ); + + await expect( + dispatchPrompt("session-1", "Complete the work", {}), + ).resolves.toBeUndefined(); + expect( + useChatStore.getState().getSessionRuntime("session-1").error, + ).toBeNull(); + release(); + }); + + it("returns private reminder handoff ids to the realtime bridge", async () => { + const completeMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn(), + dismissHandoffs: vi.fn(), + completeMasterTurn, + }); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + useChatStore.getState().addMessage(sessionId, { + id: "master-reminder-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "Reminder handled." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Private reminder", { + acpGooseMetadata: { + realtimeHandoffReminderIds: ["handoff-1", "handoff-2"], + }, + }); + + expect(completeMasterTurn).toHaveBeenCalledWith({ + reminderHandoffIds: ["handoff-1", "handoff-2"], + }); + release(); + }); + + it("completes a realtime lifecycle that joins an existing Master run", async () => { + let finishPrompt: (() => void) | undefined; + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + return new Promise((resolve) => { + finishPrompt = () => { + useChatStore.getState().addMessage(sessionId, { + id: "master-final-after-realtime-start", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "The Expert finished." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + resolve(); + }; + }); + }, + ); + + const prompt = dispatchPrompt("session-1", "Already running", {}); + await vi.waitFor(() => expect(finishPrompt).toBeTypeOf("function")); + + setVoiceConversationMode("openai-realtime"); + const completeMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn(), + dismissHandoffs: vi.fn(), + completeMasterTurn, + }); + finishPrompt?.(); + await prompt; + + expect(completeMasterTurn).toHaveBeenCalledWith({ + reminderHandoffIds: [], + }); + release(); + }); + + it("keeps a new-session Master turn owned until hydration publishes its final text", async () => { + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn(), + dismissHandoffs: vi.fn(), + completeMasterTurn: vi.fn(), + }); + useChatStore.getState().setSessionLoading("session-1", true); + mocks.acpSendMessage.mockImplementationOnce( + ( + sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + window.setTimeout(() => { + useChatStore.getState().addMessage(sessionId, { + id: "hydrating-master-final", + role: "assistant", + created: Date.now(), + content: [{ type: "text", text: "The hydrated final answer." }], + metadata: { + agentVisible: true, + userVisible: true, + completionStatus: "completed", + }, + }); + useChatStore.getState().setSessionLoading(sessionId, false); + }, 20); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Check the answer", {}); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.at(-1), + ).toMatchObject({ + id: "hydrating-master-final", + content: [{ type: "text", text: "The hydrated final answer." }], + }); + release(); + }); + + it("recovers missed Master thinking, tools, and final text from the durable turn", async () => { + const release = registerRealtimeEmissary({ + sessionId: "session-1", + sendMasterMessage: vi.fn(), + dismissHandoffs: vi.fn(), + completeMasterTurn: vi.fn(), + }); + mocks.acpExportSession.mockResolvedValue( + JSON.stringify({ + conversation: [ + { + id: "master-user", + role: "user", + created: 1_788_111_502, + content: [{ type: "text", text: "Count repositories" }], + }, + { + id: "master-work", + role: "assistant", + created: 1_788_111_505, + content: [ + { type: "thinking", thinking: "I should inspect the disk." }, + { + type: "toolRequest", + id: "tool-1", + toolCall: { + status: "success", + value: { name: "shell", arguments: { command: "find" } }, + }, + }, + ], + }, + { + id: "master-tool-result", + role: "user", + created: 1_788_111_505, + content: [ + { + type: "toolResponse", + id: "tool-1", + toolResult: { + status: "success", + value: { + content: [{ type: "text", text: "21" }], + isError: false, + }, + }, + }, + ], + }, + { + id: "master-final", + role: "assistant", + created: 1_788_111_506, + content: [{ type: "text", text: "There are 21 repositories." }], + }, + ], + }), + ); + mocks.acpSendMessage.mockImplementationOnce( + ( + _sessionId: string, + _prompt: string, + options: { + onPromptDispatching(): void; + onPromptDispatched(): void; + }, + ) => { + options.onPromptDispatching(); + options.onPromptDispatched(); + return Promise.resolve(); + }, + ); + + await dispatchPrompt("session-1", "Count repositories", {}); + + const recovered = useChatStore + .getState() + .messagesBySession["session-1"]?.filter( + (message) => message.role === "assistant", + ); + expect(recovered).toMatchObject([ + { + id: "master-work", + content: [ + { type: "thinking", text: "I should inspect the disk." }, + { type: "toolRequest", id: "tool-1", status: "completed" }, + { type: "toolResponse", id: "tool-1", result: "21" }, + ], + }, + { + id: "master-final", + content: [{ type: "text", text: "There are 21 repositories." }], + }, + ]); + release(); + }); +}); diff --git a/src/features/chat/lib/sendCore.ts b/src/features/chat/lib/sendCore.ts index a490b916d..14acdf02d 100644 --- a/src/features/chat/lib/sendCore.ts +++ b/src/features/chat/lib/sendCore.ts @@ -17,8 +17,9 @@ import { clearLiveSubtitleUpdate, flushBufferedStreamingUpdatesForSession, } from "@/features/chat/acp/liveStreamingUpdates"; -import { acpSendMessage } from "@/shared/api/acp"; +import { acpExportSession, acpSendMessage } from "@/shared/api/acp"; import { formatAcpErrorMessage } from "@/shared/api/acpErrors"; +import { messagesFromKgooseSessionExport } from "@/shared/api/kgooseMessages"; import { formatAttachmentsTooLargeMessage, MAX_PROMPT_ATTACHMENT_BYTES, @@ -33,8 +34,16 @@ import { } from "@/features/chat/lib/sessionPromptOwnership"; import { perfLog } from "@/shared/lib/perfLog"; import { completeAssistantMessage } from "@/features/chat/lib/messageCompletion"; +import { isVoiceConversationEmptyResponse } from "@/features/chat/lib/voiceConversationNoop"; +import { + completeActiveRealtimeMasterTurn, + hasActiveRealtimeEmissary, + hasLocalActiveRealtimeEmissary, +} from "@/features/voice-conversation/lib/realtimeEmissaryBridge"; +import { getVoiceConversationMode } from "@/features/voice-conversation/lib/voiceConversationModePreference"; import { type ChatAttachmentDraft, + type Message, type MessageMetadata, type MessageChip, createSystemNotificationMessage, @@ -55,6 +64,8 @@ export interface SendCoreOptions { assistantPrompt?: string; /** Text shown in the transcript when it differs from the prompt sent. */ displayText?: string; + /** Reuses a provisional local transcript row when the send commits. */ + userMessageId?: string; /** User-visible chips stored on the user message's metadata. */ chips?: MessageChip[]; /** Extra renderer-only metadata to stamp on the local user message. */ @@ -98,6 +109,133 @@ function throwIfAborted(signal?: AbortSignal): void { throw new DOMException("The operation was aborted.", "AbortError"); } +function finalMasterTextSince( + sessionId: string, + existingAssistantTextById: ReadonlyMap, +): string | undefined { + const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if ( + !message || + message.role !== "assistant" || + message.metadata?.origin === "voice_conversation" || + message.metadata?.agentVisible === false + ) { + continue; + } + const text = message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(); + if (isVoiceConversationEmptyResponse(text)) continue; + if (text && existingAssistantTextById.get(message.id) !== text) return text; + } + return undefined; +} + +function assistantTextSnapshot(sessionId: string): ReadonlyMap { + const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + return new Map( + messages + .filter( + (message) => + message.role === "assistant" && + message.metadata?.origin !== "voice_conversation" && + message.metadata?.agentVisible !== false, + ) + .map((message) => [ + message.id, + message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(), + ]), + ); +} + +function realtimeHandoffReminderIds( + metadata: Record | undefined, +): string[] { + const value = metadata?.realtimeHandoffReminderIds; + return Array.isArray(value) + ? value.filter( + (handoffId): handoffId is string => + typeof handoffId === "string" && handoffId.length > 0, + ) + : []; +} + +function messageText(message: Message): string { + return message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n"); +} + +async function recoverMissingMasterTranscript( + sessionId: string, + prompt: string, +): Promise { + try { + const exportedMessages = messagesFromKgooseSessionExport( + await acpExportSession(sessionId), + ); + const promptBoundary = exportedMessages.findLastIndex( + (message) => + message.role === "user" && messageText(message).includes(prompt), + ); + if (promptBoundary < 0) return; + + const recovered = exportedMessages + .slice(promptBoundary + 1) + .filter((message) => message.role === "assistant"); + if (!recovered.length) return; + + const current = useChatStore.getState().messagesBySession[sessionId] ?? []; + const recoveredById = new Map( + recovered.map((message) => [message.id, message]), + ); + const merged = current + .map((message) => recoveredById.get(message.id) ?? message) + .concat( + recovered.filter( + (message) => !current.some((existing) => existing.id === message.id), + ), + ) + .map((message, index) => ({ message, index })) + .sort((left, right) => + left.message.created === right.message.created + ? left.index - right.index + : left.message.created - right.message.created, + ) + .map(({ message }) => message); + useChatStore.getState().setMessages(sessionId, merged); + } catch (error) { + console.warn("Failed to recover completed Master transcript", error); + } +} + +async function settleMasterTranscriptDelivery( + sessionId: string, +): Promise { + if (useChatStore.getState().loadingSessionIds.has(sessionId)) { + await new Promise((resolve) => { + const unsubscribe = useChatStore.subscribe((state) => { + if (state.loadingSessionIds.has(sessionId)) return; + unsubscribe(); + resolve(); + }); + }); + } + // ACP may resolve session/prompt immediately before dispatching the final + // session/update already read from the same transport. Yield one macrotask + // so transcript recovery sees that last visible text block. + // Keep ownership through new-session hydration as well: a late live chunk + // routed after ownership is released looks like replay and can be discarded + // by the hydration snapshot that is finishing at the same boundary. + await new Promise((resolve) => window.setTimeout(resolve, 0)); +} + type AssistantPromptOutcome = "completed" | "error"; interface AssistantCancellationRace { @@ -211,6 +349,7 @@ export async function dispatchPrompt( signal, systemPrompt, userMessageMetadata, + userMessageId, } = opts; const sessionRunsRemotely = Boolean( useChatSessionStore.getState().getSession(sessionId)?.remoteHost, @@ -232,9 +371,16 @@ export async function dispatchPrompt( } const promptOwner = claimSessionPrompt(sessionId); + const shouldCoordinateRealtime = + hasLocalActiveRealtimeEmissary(sessionId) || + getVoiceConversationMode() === "openai-realtime"; + const assistantTextBeforeTurn = shouldCoordinateRealtime + ? assistantTextSnapshot(sessionId) + : undefined; const isCurrent = () => ownsSessionPrompt(sessionId, promptOwner); let userMessageCommitted = false; let preCommitRejected = false; + let dispatchedPrompt = text; const { addMessage, setChatState, setError, setPendingAssistantProvider } = useChatStore.getState(); @@ -245,6 +391,52 @@ export async function dispatchPrompt( setPendingAssistantProvider(sessionId, pendingAssistantProvider); clearLiveSubtitleUpdate(sessionId); + const finishPromptSuccessfully = () => { + const cancellationRace = assistantCancellationRaces.get(promptOwner); + if (cancellationRace) { + flushBufferedStreamingUpdatesForSession(sessionId, { + flushSubtitle: true, + owner: promptOwner, + }); + recordAssistantPromptOutcome(promptOwner, "completed"); + } else if (isCurrent()) { + const ownedStreamingMessageId = useChatStore + .getState() + .getSessionRuntime(sessionId).streamingMessageId; + flushBufferedStreamingUpdatesForSession(sessionId, { + flushSubtitle: true, + owner: promptOwner, + }); + completeAssistantMessageById(sessionId, ownedStreamingMessageId); + if (isCurrent()) { + setChatState(sessionId, "idle"); + } + } + }; + + const completeRealtimeTurnIfActive = async (prompt: string) => { + const shouldCoordinateAtCompletion = + shouldCoordinateRealtime || + hasLocalActiveRealtimeEmissary(sessionId) || + getVoiceConversationMode() === "openai-realtime"; + if ( + !shouldCoordinateAtCompletion || + !(await hasActiveRealtimeEmissary(sessionId)) + ) { + return; + } + await settleMasterTranscriptDelivery(sessionId); + if ( + assistantTextBeforeTurn && + !finalMasterTextSince(sessionId, assistantTextBeforeTurn) + ) { + await recoverMissingMasterTranscript(sessionId, prompt); + } + await completeActiveRealtimeMasterTurn(sessionId, { + reminderHandoffIds: realtimeHandoffReminderIds(acpGooseMetadata), + }); + }; + try { // Preparation can be superseded or aborted. Complete it before committing // local transcript state so a retained queued record can retry without @@ -261,6 +453,7 @@ export async function dispatchPrompt( buildMessageAttachments(dispatchAttachments), chips, ); + if (userMessageId) userMessage.id = userMessageId; if (persona) { userMessage.metadata = { ...userMessage.metadata, @@ -284,7 +477,23 @@ export async function dispatchPrompt( }); } } - addMessage(sessionId, userMessage); + const provisionalMessage = userMessageId + ? useChatStore + .getState() + .messagesBySession[sessionId]?.some( + (message) => message.id === userMessageId, + ) + : false; + if (provisionalMessage) { + useChatStore + .getState() + .updateMessage(sessionId, userMessage.id, (existing) => ({ + ...userMessage, + created: existing.created, + })); + } else { + addMessage(sessionId, userMessage); + } userMessageCommitted = true; setChatState(sessionId, "thinking"); setError(sessionId, null); @@ -313,6 +522,7 @@ export async function dispatchPrompt( ); const acpPrompt = promptWithPaths || (images?.length ? " " : promptWithPaths); + dispatchedPrompt = acpPrompt; const tAcp = performance.now(); if (!background) { perfLog( @@ -329,7 +539,9 @@ export async function dispatchPrompt( (img) => [img.base64, img.mimeType] as [string, string], ), onPromptDispatching: commitUserMessage, - onPromptDispatched, + onPromptDispatched: () => { + onPromptDispatched?.(); + }, }); await promptPromise; if (!background) { @@ -338,27 +550,30 @@ export async function dispatchPrompt( ); } - const cancellationRace = assistantCancellationRaces.get(promptOwner); - if (cancellationRace) { - flushBufferedStreamingUpdatesForSession(sessionId, { - flushSubtitle: true, - owner: promptOwner, - }); - recordAssistantPromptOutcome(promptOwner, "completed"); - } else if (isCurrent()) { - const ownedStreamingMessageId = useChatStore - .getState() - .getSessionRuntime(sessionId).streamingMessageId; - flushBufferedStreamingUpdatesForSession(sessionId, { - flushSubtitle: true, - owner: promptOwner, - }); - completeAssistantMessageById(sessionId, ownedStreamingMessageId); + finishPromptSuccessfully(); + try { + await completeRealtimeTurnIfActive(acpPrompt); + } catch (error) { + console.warn("Could not complete the Realtime Expert turn", error); + } + } catch (err) { + const isVoiceConversationNoop = + userMessageCommitted && + userMessageMetadata?.origin === "voice_conversation" && + isVoiceConversationEmptyResponse(formatAcpErrorMessage(err)); + if (isVoiceConversationNoop) { + finishPromptSuccessfully(); + try { + await completeRealtimeTurnIfActive(dispatchedPrompt); + } catch (error) { + console.warn("Could not complete the Realtime Expert turn", error); + } if (isCurrent()) { - setChatState(sessionId, "idle"); + setError(sessionId, null); + setPendingAssistantProvider(sessionId, null); } + return; } - } catch (err) { preCommitRejected = err instanceof PreCommitSendRejectedError; if (!preCommitRejected) { const cancellationRace = assistantCancellationRaces.get(promptOwner); diff --git a/src/features/chat/lib/steerCore.ts b/src/features/chat/lib/steerCore.ts index e4de2198c..645c2e26e 100644 --- a/src/features/chat/lib/steerCore.ts +++ b/src/features/chat/lib/steerCore.ts @@ -22,6 +22,7 @@ import { } from "./attachments"; import { isSessionRunning } from "./sessionActivity"; import { getSessionPromptOwner } from "./sessionPromptOwnership"; +import { isVoiceConversationEmptyResponse } from "./voiceConversationNoop"; import { i18n } from "@/shared/i18n"; function formatSteerErrorMessage(error: unknown): string { @@ -36,7 +37,10 @@ export async function steerPromptInSession( text: string, attachments?: ChatAttachmentDraft[], sendOptions?: ChatSendOptions, - options: { throwOnError?: boolean } = {}, + options: { + throwOnError?: boolean; + reportErrorInTranscript?: boolean; + } = {}, ): Promise { const sessionRunsRemotely = Boolean( useChatSessionStore.getState().getSession(sessionId)?.remoteHost, @@ -80,6 +84,7 @@ export async function steerPromptInSession( buildMessageAttachments(dispatchAttachments), sendOptions?.chips, ); + if (sendOptions?.userMessageId) userMessage.id = sendOptions.userMessageId; userMessage.metadata = { ...userMessage.metadata, ...sendOptions?.userMessageMetadata, @@ -103,7 +108,19 @@ export async function steerPromptInSession( ); const acpPrompt = promptWithPaths || (images?.length ? " " : promptWithPaths); const chatStore = useChatStore.getState(); - chatStore.addMessage(sessionId, userMessage); + if ( + sendOptions?.userMessageId && + chatStore.messagesBySession[sessionId]?.some( + (message) => message.id === sendOptions.userMessageId, + ) + ) { + chatStore.updateMessage(sessionId, userMessage.id, (existing) => ({ + ...userMessage, + created: existing.created, + })); + } else { + chatStore.addMessage(sessionId, userMessage); + } chatStore.setPendingInterventionBoundary(sessionId, { interventionMessageId: userMessage.id, }); @@ -175,13 +192,29 @@ export async function steerPromptInSession( }); } catch (err) { const liveStore = useChatStore.getState(); + const errorMessage = formatSteerErrorMessage(err); const liveMessage = liveStore.messagesBySession[sessionId]?.find( (message) => message.id === userMessage.id || message.metadata?.steeringRequestId === userMessage.id, ); const deliveryWasEstablished = liveMessage?.metadata?.delivery === "steer"; - if (!deliveryWasEstablished) { + const emptyVoiceTurnIsNoop = + sendOptions?.userMessageMetadata?.origin === "voice_conversation" && + isVoiceConversationEmptyResponse(errorMessage); + if (emptyVoiceTurnIsNoop) { + const liveMessageId = liveMessage?.id ?? userMessage.id; + liveStore.updateMessage(sessionId, liveMessageId, (message) => ({ + ...message, + metadata: { ...message.metadata, delivery: "steer" }, + })); + if ( + liveStore.getSessionRuntime(sessionId).pendingInterventionBoundary + ?.interventionMessageId === liveMessageId + ) { + liveStore.setPendingInterventionBoundary(sessionId, null); + } + } else if (!deliveryWasEstablished) { const liveMessageId = liveMessage?.id ?? userMessage.id; liveStore.removeMessage(sessionId, liveMessageId); if ( @@ -190,11 +223,12 @@ export async function steerPromptInSession( ) { liveStore.setPendingInterventionBoundary(sessionId, null); } - const errorMessage = formatSteerErrorMessage(err); - liveStore.addMessage( - sessionId, - createSystemNotificationMessage(errorMessage, "error"), - ); + if (options.reportErrorInTranscript !== false) { + liveStore.addMessage( + sessionId, + createSystemNotificationMessage(errorMessage, "error"), + ); + } if (options.throwOnError) { throw new Error(errorMessage); } diff --git a/src/features/chat/lib/voiceConversationNoop.ts b/src/features/chat/lib/voiceConversationNoop.ts new file mode 100644 index 000000000..258822230 --- /dev/null +++ b/src/features/chat/lib/voiceConversationNoop.ts @@ -0,0 +1,23 @@ +export const VOICE_CONVERSATION_EMPTY_RESPONSE = + "The model returned an empty response. Please resend your message to continue."; + +const VOICE_CONVERSATION_EMPTY_RESPONSES = new Set([ + VOICE_CONVERSATION_EMPTY_RESPONSE, + "Le modèle a renvoyé une réponse vide. Veuillez renvoyer votre message pour continuer.", +]); + +export function isVoiceConversationEmptyResponse(text: string): boolean { + return VOICE_CONVERSATION_EMPTY_RESPONSES.has(text.trim()); +} + +export function stripVoiceConversationEmptyResponseSuffix( + text: string, +): string { + const trimmedEnd = text.trimEnd(); + for (const fallback of VOICE_CONVERSATION_EMPTY_RESPONSES) { + if (trimmedEnd.endsWith(fallback)) { + return trimmedEnd.slice(0, -fallback.length).trimEnd(); + } + } + return text; +} diff --git a/src/features/chat/stores/queuePersistence.test.ts b/src/features/chat/stores/queuePersistence.test.ts index 012ed5796..e9a20a2ea 100644 --- a/src/features/chat/stores/queuePersistence.test.ts +++ b/src/features/chat/stores/queuePersistence.test.ts @@ -113,6 +113,43 @@ describe("queuePersistence", () => { }); }); + it("keeps restored transport-only voice coordination hidden", async () => { + mockInvoke.mockResolvedValue( + JSON.stringify({ + s1: [ + { + kind: "transport-ready", + recordId: "emissary-coordination", + payload: { + text: "[Handoff handoff-3 from spokesperson; cursor 3] Check this", + showInComposer: false, + sendOptions: { + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }, + }, + }, + ], + }), + ); + + await expect(loadPersistedMessageQueues()).resolves.toMatchObject({ + s1: [ + { + payload: { + showInComposer: false, + sendOptions: { + userMessageMetadata: { userVisible: false }, + }, + }, + restored: true, + }, + ], + }); + }); + it("strips legacy provider/model fields without losing the prompt", async () => { mockInvoke.mockResolvedValue( JSON.stringify({ diff --git a/src/features/chat/stores/queuePersistence.ts b/src/features/chat/stores/queuePersistence.ts index ca6e328d5..af0f0bbfc 100644 --- a/src/features/chat/stores/queuePersistence.ts +++ b/src/features/chat/stores/queuePersistence.ts @@ -47,7 +47,8 @@ function normalizeQueuedRecord( const { editing: _editing, restored: _restored, ...persisted } = record; const normalizedPayload = normalizeQueuedPayload(persisted.payload); const restoredPayload = - normalizedPayload.showInComposer === false + normalizedPayload.showInComposer === false && + normalizedPayload.sendOptions?.userMessageMetadata?.userVisible !== false ? { ...normalizedPayload, showInComposer: true } : normalizedPayload; if (persisted.kind !== "deferred") { diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.test.ts b/src/features/chat/transcript/projection/buildTranscriptItems.test.ts new file mode 100644 index 000000000..3c552893b --- /dev/null +++ b/src/features/chat/transcript/projection/buildTranscriptItems.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest"; +import type { Message } from "@/shared/types/messages"; +import { VOICE_CONVERSATION_EMPTY_RESPONSE } from "@/features/chat/lib/voiceConversationNoop"; +import { getVisibleTranscriptMessages } from "./buildTranscriptItems"; + +function message( + id: string, + role: Message["role"], + text: string, + origin?: "voice_conversation", +): Message { + return { + id, + role, + created: 1, + content: [{ type: "text", text }], + metadata: origin ? { origin } : undefined, + }; +} + +describe("getVisibleTranscriptMessages voice no-op", () => { + it("does not inspect assistant text in a transcript without voice turns", () => { + const inaccessibleText = { type: "text" } as { + type: "text"; + text: string; + }; + Object.defineProperty(inaccessibleText, "text", { + get: () => { + throw new Error("text-only projection must use the fast path"); + }, + }); + const assistant: Message = { + id: "assistant", + role: "assistant", + created: 1, + content: [inaccessibleText], + }; + + expect(getVisibleTranscriptMessages([assistant])).toEqual([assistant]); + }); + + it("hides the backend empty-response fallback after a voice turn", () => { + const voice = message( + "voice", + "user", + "Emissary said: Hello", + "voice_conversation", + ); + const fallback = message( + "fallback", + "assistant", + VOICE_CONVERSATION_EMPTY_RESPONSE, + ); + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("hides an empty-response system notification after a voice turn", () => { + const voice = message( + "voice", + "user", + "Emissary said: Hello", + "voice_conversation", + ); + const fallback: Message = { + id: "fallback", + role: "system", + created: 1, + content: [ + { + type: "systemNotification", + notificationType: "error", + text: VOICE_CONVERSATION_EMPTY_RESPONSE, + }, + ], + }; + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("hides replayed and localized empty-response fallbacks after a voice transcript", () => { + const voice = message( + "voice", + "user", + "[Voice transcript] Emissary said: Bonjour", + ); + const fallback = message( + "fallback", + "assistant", + "Le modèle a renvoyé une réponse vide. Veuillez renvoyer votre message pour continuer.", + ); + + expect(getVisibleTranscriptMessages([voice, fallback])).toEqual([voice]); + }); + + it("keeps the same fallback visible after a normal chat turn", () => { + const user = message("user", "user", "Hello"); + const fallback = message( + "fallback", + "assistant", + VOICE_CONVERSATION_EMPTY_RESPONSE, + ); + + expect(getVisibleTranscriptMessages([user, fallback])).toEqual([ + user, + fallback, + ]); + }); + + it("keeps real assistant errors visible after a voice turn", () => { + const voice = message( + "voice", + "user", + "User said: Hello", + "voice_conversation", + ); + const error = message("error", "assistant", "Authentication failed"); + + expect(getVisibleTranscriptMessages([voice, error])).toEqual([ + voice, + error, + ]); + }); + + it("never renders a transient empty-response fallback inside spoken Emissary text", () => { + const voice = message( + "voice", + "user", + "How many months are in a year?", + "voice_conversation", + ); + const spoken: Message = { + id: "spoken", + role: "assistant", + created: 2, + content: [ + { + type: "text", + text: `There are 12 months in a year.${VOICE_CONVERSATION_EMPTY_RESPONSE}`, + speech: { status: "spoken" }, + }, + ], + metadata: { + origin: "voice_conversation", + voiceConversationDebugEvent: "emissarySpeech", + }, + }; + + expect(getVisibleTranscriptMessages([voice, spoken])).toEqual([ + voice, + { + ...spoken, + content: [ + { + type: "text", + text: "There are 12 months in a year.", + speech: { status: "spoken" }, + }, + ], + }, + ]); + }); + + it("sanitizes spoken assistant text even when the user typed during a voice call", () => { + const user = message("user", "user", "Are you still there?"); + const spoken: Message = { + id: "spoken", + role: "assistant", + created: 2, + content: [ + { + type: "text", + text: `Yes, I'm here.${VOICE_CONVERSATION_EMPTY_RESPONSE}`, + speech: { status: "spoken" }, + }, + ], + }; + + expect(getVisibleTranscriptMessages([user, spoken])).toEqual([ + user, + { + ...spoken, + content: [ + { + type: "text", + text: "Yes, I'm here.", + speech: { status: "spoken" }, + }, + ], + }, + ]); + }); +}); diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.ts b/src/features/chat/transcript/projection/buildTranscriptItems.ts index ac0a43b76..08c72393f 100644 --- a/src/features/chat/transcript/projection/buildTranscriptItems.ts +++ b/src/features/chat/transcript/projection/buildTranscriptItems.ts @@ -1,11 +1,16 @@ -import type { - Message, - MessageContent, - MessageMetadata, - ReasoningContent, - TextContent, - ThinkingContent, +import { + getTextContent, + type Message, + type MessageContent, + type MessageMetadata, + type ReasoningContent, + type TextContent, + type ThinkingContent, } from "@/shared/types/messages"; +import { + isVoiceConversationEmptyResponse, + stripVoiceConversationEmptyResponseSuffix, +} from "@/features/chat/lib/voiceConversationNoop"; import { classifyTranscriptMeasurementPolicy, type TranscriptMeasurementPolicyDecision, @@ -125,11 +130,7 @@ export function buildTranscriptItems({ // of the assistant's work turn, so they should not reset this set. let displayedReasoningSignatures = new Set(); - for (const message of messages) { - if (!isVisibleTranscriptMessage(message)) { - continue; - } - + for (const message of getVisibleTranscriptMessages(messages)) { const visibleContent = expandReasoningContentSections( getUserVisibleMessageContent(message.content), ); @@ -1764,7 +1765,88 @@ function getAssistantFragmentChromeEstimate( export function getVisibleTranscriptMessages( messages: readonly Message[], ): readonly Message[] { - return messages.filter(isVisibleTranscriptMessage); + if (!messages.some(needsVoiceTranscriptSanitization)) { + return messages.filter(isVisibleTranscriptMessage); + } + + return messages.flatMap((message, index) => { + if (!isVisibleTranscriptMessage(message)) return []; + const isEmptyResponseFallback = + (message.role === "assistant" && + isVoiceConversationEmptyResponse(getTextContent(message))) || + message.content.some( + (content) => + content.type === "systemNotification" && + isVoiceConversationEmptyResponse(content.text), + ); + if (!isEmptyResponseFallback) { + return [sanitizeVoiceSpeechFallback(message)]; + } + + for (let prior = index - 1; prior >= 0; prior -= 1) { + const priorMessage = messages[prior]; + if (priorMessage?.role !== "user") continue; + return isVoiceConversationUserTurn(priorMessage) ? [] : [message]; + } + return [message]; + }); +} + +function needsVoiceTranscriptSanitization(message: Message): boolean { + return ( + isVoiceConversationUserTurn(message) || + message.content.some( + (content) => content.type === "text" && content.speech !== undefined, + ) + ); +} + +function sanitizeVoiceSpeechFallback(message: Message): Message { + const hasSpeech = message.content.some( + (content) => content.type === "text" && content.speech !== undefined, + ); + if (!hasSpeech) return message; + + let changed = false; + const content: MessageContent[] = []; + for (const block of message.content) { + if ( + block.type === "systemNotification" && + isVoiceConversationEmptyResponse(block.text) + ) { + changed = true; + continue; + } + if (block.type !== "text") { + content.push(block); + continue; + } + if (!block.speech && isVoiceConversationEmptyResponse(block.text)) { + changed = true; + continue; + } + const text = stripVoiceConversationEmptyResponseSuffix(block.text); + if (text === block.text) { + content.push(block); + continue; + } + changed = true; + if (text) content.push({ ...block, text }); + } + + return changed ? { ...message, content } : message; +} + +function isVoiceConversationUserTurn(message: Message): boolean { + return ( + message.metadata?.origin === "voice_conversation" || + (message.role === "user" && + message.content.some( + (content) => + content.type === "text" && + content.text.trimStart().startsWith("[Voice transcript] "), + )) + ); } function isVisibleTranscriptMessage(message: Message): boolean { diff --git a/src/features/chat/transcript/projection/messageRevisions.ts b/src/features/chat/transcript/projection/messageRevisions.ts index 7b7deeb7c..b3e8e4427 100644 --- a/src/features/chat/transcript/projection/messageRevisions.ts +++ b/src/features/chat/transcript/projection/messageRevisions.ts @@ -440,6 +440,7 @@ function renderMetadataRevision(metadata: MessageMetadata | undefined): string { metadata.completionStatus ?? "", metadata.delivery ?? "", metadata.origin ?? "", + metadata.voiceConversationDebugEvent ?? "", stableValueRevision(metadata.attachments ?? []), stableValueRevision(metadata.chips ?? []), metadata.personaId ?? "", @@ -464,6 +465,7 @@ function heightMetadataRevision(metadata: MessageMetadata | undefined): string { metadata.completionStatus ?? "", metadata.delivery ?? "", metadata.origin ?? "", + metadata.voiceConversationDebugEvent ?? "", stableValueRevision(metadata.attachments ?? []), stableValueRevision(metadata.chips ?? []), metadata.personaName ?? "", @@ -480,6 +482,7 @@ function hasDefaultRevisionMetadata(metadata: MessageMetadata): boolean { !metadata.completionStatus && !metadata.delivery && !metadata.origin && + !metadata.voiceConversationDebugEvent && !metadata.attachments?.length && !metadata.chips?.length && !metadata.personaId && diff --git a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts index bd49e8582..7a5db3617 100644 --- a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts +++ b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts @@ -1586,6 +1586,28 @@ describe("transcript projection cache", () => { expect(second.heightRevision).not.toBe(first.heightRevision); }); + it("includes realtime coordination kind in render and height revisions", () => { + const original = message( + "assistant-1", + "assistant", + "same", + utc(2026, 6, 4, 10), + ); + const coordination = { + ...original, + metadata: { + ...original.metadata, + voiceConversationDebugEvent: "masterToEmissarySay" as const, + }, + }; + + const first = buildMessageRevisions(original); + const second = buildMessageRevisions(coordination); + + expect(second.renderRevision).not.toBe(first.renderRevision); + expect(second.heightRevision).not.toBe(first.heightRevision); + }); + it.each([ ["agent identity", { subagentAgentName: "Rivet" }], ["task description", { subagentTaskLabel: "Count markdown files" }], diff --git a/src/features/chat/types.ts b/src/features/chat/types.ts index 4313411a1..140b37109 100644 --- a/src/features/chat/types.ts +++ b/src/features/chat/types.ts @@ -69,6 +69,8 @@ export interface ChatSendOptions { /** Persona-only prompt captured while workspace context is still loading. */ capturedPersonaSystemPrompt?: string; displayText?: string; + /** Reuses a provisional local transcript row when the send commits. */ + userMessageId?: string; assistantPrompt?: string; chips?: MessageChip[]; userMessageMetadata?: Partial; @@ -101,6 +103,8 @@ export interface ChatInputVoiceConversation { disabled?: boolean; onToggle: () => void | Promise; onMicrophoneMuteToggle: () => void | Promise; + /** Mirrors a committed typed user turn into an active voice frontend. */ + onTypedUserMessageCommitted?: (text: string) => void; } export interface ChatInputComposerActions { diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index 8df4f6359..a2ab972b0 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -65,6 +65,7 @@ import { import type { TranscriptSearchBackend } from "@/features/chat/lib/transcriptSearchBackend"; import type { GlobalComposerHandoffRect } from "@/shared/ui/GlobalComposerPill"; import { useVoiceConversationController } from "@/features/voice-conversation/hooks/useVoiceConversationController"; +import { useOpenAiRealtimeConversation } from "@/features/voice-conversation/hooks/useOpenAiRealtimeConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; @@ -73,8 +74,11 @@ import { isMacSpeechAvailable, useVoiceInputPreference, } from "@/features/voice-conversation/lib/voiceInputPreference"; +import { useRealtimeVoicePreference } from "@/features/voice-conversation/lib/realtimeVoicePreference"; +import { presentRealtimeVoiceMessages } from "@/features/voice-conversation/lib/realtimeVoicePresentation"; import { useVoiceOutputPreference } from "@/features/voice-conversation/lib/voiceOutputPreference"; import { isVoiceSetupReady } from "@/features/voice-conversation/lib/voiceSetupReadiness"; +import { useVoiceConversationModePreference } from "@/features/voice-conversation/lib/voiceConversationModePreference"; import { useProfileCapabilities } from "@/shared/profile/capabilities"; import { requestOpenSettings } from "@/features/settings/lib/settingsEvents"; import { @@ -242,6 +246,7 @@ export function ChatView({ isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); const voiceOutput = useVoiceOutputPreference(); + const voiceMode = useVoiceConversationModePreference(); const openAiVoiceSetup = useOpenAiVoiceSetup( capabilities.voiceConversation && (voiceInput.backend === "openai" || voiceOutput.backend === "openai"), @@ -269,13 +274,13 @@ export function ChatView({ controller.isLoadingHistory || !controller.workspaceContextReady || controller.queue.queuedMessage !== null); - const voiceConversation = useVoiceConversationController({ + const chainedVoiceConversation = useVoiceConversationController({ sessionId, // Voice delivery only needs to wait for admission. Holding its per-session // queue through the full run would prevent later utterances from steering // the active run. onSend, - enabled: capabilities.voiceConversation, + enabled: capabilities.voiceConversation && voiceMode.mode === "chained", isGooseSession: controller.selectedProvider === "goose", pocketReady: voiceReady, inputBackend: voiceInput.backend, @@ -293,6 +298,25 @@ export function ChatView({ routeUnavailable: voiceAdmissionPermanentlyBlocked, disabled: admissionBlocked || voiceDeliveryTemporarilyBlocked, }); + const realtimeVoiceConversation = useOpenAiRealtimeConversation({ + sessionId, + onSend, + enabled: + capabilities.voiceConversation && + voiceMode.mode === "openai-realtime" && + controller.selectedProvider === "goose", + readOnly: Boolean(readOnlyStatus), + disabled: admissionBlocked || voiceDeliveryTemporarilyBlocked, + }); + const voiceConversation = + voiceMode.mode === "openai-realtime" + ? realtimeVoiceConversation + : chainedVoiceConversation; + const { preference: realtimeVoicePreference } = useRealtimeVoicePreference(); + const presentedMessages = presentRealtimeVoiceMessages( + controller.messages, + realtimeVoicePreference.presentationMode, + ); const isAgentBuilderOpen = agentBuilderOpenForLayout; const patchSession = useChatSessionStore((s) => s.patchSession); const agentBuilderContextState = effectiveSession?.agentBuilderContextState; @@ -703,6 +727,12 @@ export function ChatView({
c.type === "text") .map((c) => c.text) .join("\n"); + const hasVoiceSpeech = content.some( + (block) => block.type === "text" && block.speech !== undefined, + ); const renderedContent = visibleContent; const actionTextContent = fragmentRole ? rawContent @@ -958,6 +962,7 @@ export const MessageBubble = memo(function MessageBubble({ isUser ? "ml-auto flex-row-reverse gap-3" : "flex-row gap-3", )} data-role={isUser ? "user-message" : "assistant-message"} + data-realtime-voice-debug-event={voiceDebugEvent} data-message-fragment-role={fragmentRole} {...rowRootAttributes} > @@ -995,11 +1000,16 @@ export const MessageBubble = memo(function MessageBubble({ shouldReserveMessageActionSpace && "pb-9", isUser ? "max-w-[var(--chat-user-message-max-width)] items-end" - : "w-full items-start", + : voiceDebugEvent && voiceDebugEvent !== "emissarySpeech" + ? "w-full max-w-3xl items-start" + : "w-full items-start", )} > {showAssistantIdentity ? ( -
+
{hasPersonaAvatar ? ( diff --git a/src/features/chat/ui/__tests__/MessageBubble.test.tsx b/src/features/chat/ui/__tests__/MessageBubble.test.tsx index ab0e28dca..81da8aee0 100644 --- a/src/features/chat/ui/__tests__/MessageBubble.test.tsx +++ b/src/features/chat/ui/__tests__/MessageBubble.test.tsx @@ -723,6 +723,36 @@ describe("MessageBubble", () => { expect(screen.getAllByText("One visible assistant response.")).toHaveLength( 1, ); + expect( + container.querySelector('[data-role="message-bubble-surface"]'), + ).toHaveClass("rounded-lg", "border"); + }); + + it("renders realtime coordination as a distinct assistant bubble", () => { + const { container } = render( +
+ +
, + ); + + const message = container.querySelector( + '[data-realtime-voice-debug-event="emissaryToMaster"]', + ); + expect(message).toHaveAttribute("data-role", "assistant-message"); + expect(message).toHaveTextContent("Emissary → Master · Handoff handoff-1"); + expect( + message?.querySelector('[data-role="message-bubble-surface"]'), + ).toHaveClass("rounded-lg", "border"); }); it("strikes only the estimated unspoken suffix after barge-in", () => { diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts new file mode 100644 index 000000000..ede6d4f86 --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts @@ -0,0 +1,2431 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useChatStore } from "@/features/chat/stores/chatStore"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { + createRealtimeTranscriptReplayEvents, + requestOpenAiRealtimeConversationStart, + resetOpenAiRealtimeConversationRuntimeForTests, + stopOpenAiRealtimeConversation, + useOpenAiRealtimeConversation, +} from "./useOpenAiRealtimeConversation"; + +const mocks = vi.hoisted(() => ({ + appendSessionSystemPrompt: vi.fn(), + claimMicrophone: vi.fn(), + connectPeer: vi.fn(), + createHandoffToolOutput: vi.fn(), + createInvalidToolCallOutput: vi.fn(), + createPeer: vi.fn(), + createSession: vi.fn(), + createResponse: true, + listenControls: vi.fn(), + publishActivity: vi.fn(), + publishMuted: vi.fn(), + pipeInitialCursors: [] as number[], + rebindControls: vi.fn(), + registerEmissary: vi.fn(), + recordToolOutput: vi.fn(), + activeEmissary: null as null | { + sessionId: string; + completeMasterTurn(completion: { reminderHandoffIds: string[] }): void; + dismissHandoffs( + cursor: number, + handoffIds: string[], + reason: string, + ): Promise; + sendMasterMessage( + message: string, + cursor: number, + mode: "context" | "say", + resolves: string[], + ): Promise; + }, + releaseBridge: vi.fn(), + releaseMicrophone: vi.fn(), + setControlsSuppressed: vi.fn(), + startControls: vi.fn(), + stopControls: vi.fn(), + waitForBridgeReady: vi.fn(), + sendRealtimeEvents: vi.fn(), + steerPrompt: vi.fn(), + requestToolOutput: vi.fn(), + requestMasterMessage: vi.fn(), + requestTypedUserMessage: vi.fn(), +})); + +vi.mock("@/shared/api/acpApi", () => ({ + appendSessionSystemPrompt: mocks.appendSessionSystemPrompt, +})); + +vi.mock("@/shared/api/openaiRealtime", () => ({ + claimVoiceDictationMicrophone: mocks.claimMicrophone, + createOpenAiRealtimeVoiceSession: mocks.createSession, + listenToOpenAiRealtimeVoiceControls: mocks.listenControls, + publishOpenAiRealtimeVoiceActivity: mocks.publishActivity, + publishOpenAiRealtimeVoiceMicrophoneMuted: mocks.publishMuted, + rebindOpenAiRealtimeVoiceControls: mocks.rebindControls, + releaseVoiceDictationMicrophone: mocks.releaseMicrophone, + setOpenAiRealtimeVoiceControlsSuppressed: mocks.setControlsSuppressed, + startOpenAiRealtimeVoiceControls: mocks.startControls, + stopOpenAiRealtimeVoiceControls: mocks.stopControls, +})); + +vi.mock("@/features/chat/lib/openaiRealtimeAudio", () => ({ + connectOpenAiRealtimePeerConnection: mocks.connectPeer, + createOpenAiRealtimePeerConnection: mocks.createPeer, +})); + +vi.mock("@/features/chat/lib/steerCore", () => ({ + steerPromptInSession: mocks.steerPrompt, +})); + +vi.mock("../lib/realtimeEmissaryBridge", () => ({ + registerRealtimeEmissary: (emissary: typeof mocks.activeEmissary) => { + mocks.activeEmissary = emissary; + return mocks.registerEmissary(); + }, + waitForRealtimeEmissaryBridgeReady: mocks.waitForBridgeReady, +})); + +vi.mock("../lib/realtimeVoicePreference", () => ({ + getRealtimeVoicePreference: () => ({ + model: "gpt-realtime-2.1", + speed: 1, + transcriptionModel: "gpt-realtime-whisper", + voice: "marin", + turnDetection: "server_vad", + eagerness: "auto", + interruptResponse: true, + createResponse: mocks.createResponse, + vadThreshold: 0.5, + prefixPaddingMs: 300, + silenceDurationMs: 500, + idleTimeoutMs: null, + noiseReduction: "off", + transcriptionLanguage: "", + transcriptionPrompt: "", + reasoningEffort: "default", + maxOutputTokens: null, + }), +})); + +vi.mock("../lib/realtimeEmissaryProtocol", () => ({ + configureRealtimeEmissarySession: vi.fn(), + createInvalidToolCallOutput: mocks.createInvalidToolCallOutput, + createHandoffToolOutput: mocks.createHandoffToolOutput, + DirectMessagePipe: class { + private nextId = 1; + private pending: Array<{ + id: number; + sender: "master" | "emissary"; + recipient: "master" | "emissary"; + senderCursor: number; + message: string; + }> = []; + private consumed = { master: 0, emissary: 0 }; + constructor(initialCursor = 0) { + mocks.pipeInitialCursors.push(initialCursor); + } + cursor(peer: "master" | "emissary") { + return this.consumed[peer]; + } + deliveryCursor(peer: "master" | "emissary") { + const latest = this.pending.at(-1); + return latest?.recipient === peer ? latest.id : this.consumed[peer]; + } + send(options: { + sender: "master" | "emissary"; + cursor: number; + message: string; + }) { + const active = this.pending[0]; + if (active && active.sender !== options.sender) { + const latest = this.pending.at(-1); + if (!latest || options.cursor !== latest.id) { + return { + accepted: false, + reason: "pipe_busy", + cursor: this.consumed[options.sender], + }; + } + this.consumed[options.sender] = latest.id; + this.pending = []; + } + if (options.cursor !== this.consumed[options.sender]) { + return { + accepted: false, + reason: "stale_cursor", + cursor: this.consumed[options.sender], + }; + } + const id = this.nextId++; + const outbound = { + id, + sender: options.sender, + recipient: options.sender === "master" ? "emissary" : "master", + senderCursor: this.consumed[options.sender], + message: options.message, + } as const; + this.pending.push(outbound); + return { + accepted: true, + cursor: this.consumed[options.sender], + outbound, + }; + } + }, + REALTIME_EXPERT_INSTRUCTIONS: "Expert instructions", + RealtimeEmissaryProtocol: class { + handle(event: { type?: string }) { + if (event.type === "test.transcript") + return [ + { + interrupted: false, + itemId: "user-item-1", + speaker: "user", + text: "hello master", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_repository") + return [ + { + interrupted: false, + itemId: "user-item-repository", + speaker: "user", + text: "how many repos are in my development folder?", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_followup") + return [ + { + interrupted: false, + itemId: "user-item-2", + speaker: "user", + text: "are any of them symbolic links?", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.transcript_partial") + return [ + { + itemId: "user-item-1", + speaker: "user", + text: "hello", + type: "transcript.updated", + }, + ]; + if (event.type === "test.transcript_corrected") + return [ + { + itemId: "user-item-1", + speaker: "user", + text: "hello master", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary") + return [ + { + interrupted: false, + itemId: "emissary-item-1", + speaker: "emissary", + text: "hello user", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_partial_first") + return [ + { + itemId: "emissary-item-multi", + speaker: "emissary", + text: "Let me think about that.", + type: "transcript.updated", + }, + ]; + if (event.type === "test.emissary_partial_second") + return [ + { + itemId: "emissary-item-multi", + speaker: "emissary", + text: "Let me think about that. I received a compact transcript.", + type: "transcript.updated", + }, + ]; + if (event.type === "test.emissary_result") + return [ + { + interrupted: false, + itemId: "emissary-item-2", + speaker: "emissary", + text: "You have 21 repositories.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_followup_ack") + return [ + { + interrupted: false, + itemId: "emissary-item-3", + speaker: "emissary", + text: "I'll verify that.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_symlink_result") + return [ + { + interrupted: false, + itemId: "emissary-item-4", + speaker: "emissary", + text: "None of those repositories are symbolic links.", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.emissary_interrupted") + return [ + { + interrupted: true, + speaker: "emissary", + text: "partially heard", + type: "transcript.finalized", + }, + ]; + if (event.type === "test.handoff") + return [ + { + callId: "call-1", + message: "Please inspect the disk.", + type: "handoff", + }, + ]; + if (event.type === "test.handoff_followup") + return [ + { + callId: "call-2", + message: "Please verify whether those repositories are symlinks.", + type: "handoff", + }, + ]; + if (event.type === "test.invalid_tool_call") + return [ + { + callId: "call-broken", + error: "JSON Parse error: Unterminated string", + toolName: "handoff", + type: "tool_call.invalid", + }, + ]; + return []; + } + }, + RealtimeResponseCoordinator: class { + handle() { + return []; + } + takeCompletedHandoffIds() { + return []; + } + takeFailedHandoffIds() { + return []; + } + requestMasterMessage(message: unknown) { + return mocks.requestMasterMessage(message); + } + recordToolOutput(event: unknown) { + return mocks.recordToolOutput(event); + } + requestToolOutput(event: unknown) { + return mocks.requestToolOutput(event); + } + requestTypedUserMessage(text: string) { + return mocks.requestTypedUserMessage(text); + } + }, + sendRealtimeEvents: mocks.sendRealtimeEvents, +})); + +class FakeDataChannel extends EventTarget { + readonly close = vi.fn(); + readyState: RTCDataChannelState = "open"; + readonly send = vi.fn(); +} + +class FakePeer extends EventTarget { + readonly addTrack = vi.fn(); + readonly close = vi.fn(); + readonly createDataChannel = vi.fn(); + connectionState: RTCPeerConnectionState = "connected"; + iceConnectionState: RTCIceConnectionState = "connected"; + + constructor(channel: FakeDataChannel) { + super(); + this.createDataChannel.mockReturnValue(channel); + } +} + +class FakeAudio extends EventTarget { + autoplay = false; + readonly pause = vi.fn(); + readonly play = vi.fn().mockResolvedValue(undefined); + srcObject: MediaStream | null = null; +} + +const originalAudio = globalThis.Audio; +const originalMediaDevices = navigator.mediaDevices; +let channel: FakeDataChannel; +let peer: FakePeer; +let track: MediaStreamTrack & { stop: ReturnType }; +let realtimeControlListener: + | ((control: { + sessionId: string; + revision: number; + action: "stop" | "mute"; + muted?: boolean; + }) => void) + | undefined; + +function renderConversation(sessionId: string, onSend = vi.fn()) { + return renderHook(() => + useOpenAiRealtimeConversation({ enabled: true, onSend, sessionId }), + ); +} + +function acceptedHandoffId(callId: string): string { + const call = mocks.createHandoffToolOutput.mock.calls.find( + ([candidate]) => candidate === callId, + ); + const handoffId = call?.[1]?.handoff_id; + if (typeof handoffId !== "string") { + throw new Error(`No accepted handoff for ${callId}`); + } + return handoffId; +} + +describe("createRealtimeTranscriptReplayEvents", () => { + it("reconstructs a compact ordinary transcript without realtime state", () => { + expect( + createRealtimeTranscriptReplayEvents([ + { + id: "u1", + role: "user", + created: 1, + content: [{ type: "text", text: "What is in this folder?" }], + }, + { + id: "progress", + role: "assistant", + created: 2, + content: [{ type: "text", text: "I am checking." }], + metadata: { completionStatus: "completed" }, + }, + { + id: "final", + role: "assistant", + created: 3, + content: [{ type: "text", text: "There are 25 directories." }], + metadata: { completionStatus: "completed" }, + }, + { + id: "coordination", + role: "assistant", + created: 4, + content: [{ type: "text", text: "Private coordination" }], + metadata: { + completionStatus: "completed", + personaName: "Expert → Spokesperson", + }, + }, + { + id: "u2", + role: "user", + created: 5, + content: [{ type: "text", text: "Are any symlinks?" }], + }, + ]), + ).toEqual([ + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "What is in this folder?" }], + }, + }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "There are 25 directories." }], + }, + }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Are any symlinks?" }], + }, + }, + ]); + }); +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.activeEmissary = null; + mocks.createResponse = true; + useChatStore.setState({ + loadingSessionIds: new Set(), + messagesBySession: {}, + queuedMessageBySession: {}, + sessionStateById: {}, + }); + useChatSessionStore.setState({ sessions: [] }); + channel = new FakeDataChannel(); + realtimeControlListener = undefined; + peer = new FakePeer(channel); + track = { + enabled: true, + stop: vi.fn(), + } as unknown as MediaStreamTrack & { stop: ReturnType }; + const stream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia: vi.fn().mockResolvedValue(stream) }, + }); + globalThis.Audio = FakeAudio as unknown as typeof Audio; + mocks.appendSessionSystemPrompt.mockResolvedValue(undefined); + mocks.claimMicrophone.mockResolvedValue(undefined); + mocks.connectPeer.mockResolvedValue(undefined); + mocks.createHandoffToolOutput.mockReturnValue({ + type: "conversation.item.create", + item: { type: "function_call_output" }, + }); + mocks.createInvalidToolCallOutput.mockReturnValue({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-broken", + output: '{"accepted":false,"reason":"invalid_arguments"}', + }, + }); + mocks.createPeer.mockReturnValue(peer); + mocks.createSession.mockResolvedValue({ clientSecret: "test-secret" }); + mocks.listenControls.mockImplementation(async (listener) => { + realtimeControlListener = listener; + return vi.fn(); + }); + mocks.publishActivity.mockResolvedValue(undefined); + mocks.publishMuted.mockResolvedValue(undefined); + mocks.pipeInitialCursors.length = 0; + mocks.rebindControls.mockResolvedValue({ + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "promoted-session", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 8, + }); + mocks.registerEmissary.mockReturnValue(mocks.releaseBridge); + mocks.releaseMicrophone.mockResolvedValue(undefined); + mocks.setControlsSuppressed.mockResolvedValue(undefined); + mocks.startControls.mockImplementation(async (sessionId: string) => ({ + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId, + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 7, + })); + mocks.stopControls.mockResolvedValue(undefined); + mocks.waitForBridgeReady.mockResolvedValue(undefined); + mocks.requestToolOutput.mockImplementation((event) => ({ + status: "queued", + events: [event], + })); + mocks.recordToolOutput.mockImplementation((event) => ({ + status: "sent", + events: [event], + })); + mocks.requestMasterMessage.mockImplementation((message) => ({ + status: "sent", + events: [{ type: "conversation.item.create", message }], + })); + mocks.requestTypedUserMessage.mockReturnValue({ + status: "interrupting", + events: [{ type: "response.cancel" }, { type: "conversation.item.create" }], + }); +}); + +afterEach(async () => { + await resetOpenAiRealtimeConversationRuntimeForTests(); + globalThis.Audio = originalAudio; + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: originalMediaDevices, + }); +}); + +describe("useOpenAiRealtimeConversation lifecycle", () => { + it("starts after a newly created session mounts from a deferred call request", async () => { + act(() => requestOpenAiRealtimeConversationStart("session-a")); + const owner = renderConversation("session-a"); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.createSession).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("stops the active realtime call when its voice mode is disabled", async () => { + const onSend = vi.fn(); + const owner = renderHook( + ({ enabled }) => + useOpenAiRealtimeConversation({ + enabled, + onSend, + sessionId: "session-a", + }), + { initialProps: { enabled: true } }, + ); + + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + owner.rerender({ enabled: false }); + + await waitFor(() => expect(owner.result.current.state).toBe("off")); + expect(mocks.stopControls).toHaveBeenCalledWith("session-a", 7); + }); + + it("keeps hang-up and mute enabled for the active call during a composer block", async () => { + const owner = renderHook( + ({ disabled }) => + useOpenAiRealtimeConversation({ + disabled, + enabled: true, + onSend: vi.fn(), + sessionId: "session-a", + }), + { initialProps: { disabled: false } }, + ); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + owner.rerender({ disabled: true }); + + expect(owner.result.current.disabled).toBe(false); + await act(async () => owner.result.current.onMicrophoneMuteToggle?.()); + expect(track.enabled).toBe(false); + await act(async () => owner.result.current.onToggle()); + expect(owner.result.current.state).toBe("off"); + }); + + it("uses a new bridge cursor namespace for every Realtime call", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + await act(async () => owner.result.current.onToggle()); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + expect(mocks.pipeInitialCursors).toHaveLength(2); + expect(mocks.pipeInitialCursors[0]).not.toBe(mocks.pipeInitialCursors[1]); + + await act(async () => owner.result.current.onToggle()); + }); + + it("starts a promoted session from a deferred request for its client id", async () => { + useChatSessionStore.setState({ + sessions: [ + { + id: "backend-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + intent: null, + }, + ], + }); + act(() => requestOpenAiRealtimeConversationStart("draft-session")); + const owner = renderConversation("backend-session"); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.createSession).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("starts on an optimistic draft and defers the master prompt until promotion", async () => { + useChatSessionStore.setState({ + sessions: [ + { + id: "draft-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + creationState: "pending", + intent: null, + }, + ], + }); + mocks.appendSessionSystemPrompt.mockImplementation( + async (sessionId: string) => { + if (sessionId === "draft-session") + throw new Error("Resource not found"); + }, + ); + act(() => requestOpenAiRealtimeConversationStart("draft-session")); + const owner = renderHook( + ({ sessionId }) => + useOpenAiRealtimeConversation({ + enabled: true, + onSend: vi.fn(), + sessionId, + }), + { initialProps: { sessionId: "draft-session" } }, + ); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.appendSessionSystemPrompt).not.toHaveBeenCalledWith( + "draft-session", + expect.anything(), + expect.anything(), + ); + + act(() => { + useChatSessionStore + .getState() + .promoteDraftSession("draft-session", "backend-session"); + useChatStore + .getState() + .promoteSessionId("draft-session", "backend-session"); + owner.rerender({ sessionId: "backend-session" }); + }); + + await waitFor(() => + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining( + 'send-to-spokesperson --session-id "backend-session"', + ), + ), + ); + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining("--mode "), + ); + expect(owner.result.current.state).toBe("listening"); + expect(owner.result.current.boundSessionId).toBe("backend-session"); + + await act(async () => owner.result.current.onToggle()); + }); + + it("registers the active call before microphone setup finishes", async () => { + let resolveStream!: (stream: MediaStream) => void; + const delayedStream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + vi.mocked(navigator.mediaDevices.getUserMedia).mockReturnValueOnce( + new Promise((resolve) => { + resolveStream = resolve; + }), + ); + const owner = renderConversation("session-a"); + + act(() => { + void owner.result.current.onToggle(); + }); + + await waitFor(() => + expect(mocks.startControls).toHaveBeenCalledWith("session-a"), + ); + expect(owner.result.current.state).toBe("starting"); + expect(mocks.activeEmissary?.sessionId).toBe("session-a"); + expect(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ).not.toThrow(); + act(() => { + realtimeControlListener?.({ + sessionId: "session-a", + revision: 7, + action: "mute", + muted: true, + }); + }); + + act(() => resolveStream(delayedStream)); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(track.enabled).toBe(false); + await act(async () => owner.result.current.onToggle()); + }); + + it("publishes running controls only after the cross-renderer bridge is ready", async () => { + let resolveBridge!: () => void; + mocks.waitForBridgeReady.mockReturnValueOnce( + new Promise((resolve) => { + resolveBridge = resolve; + }), + ); + const owner = renderConversation("session-a"); + + act(() => { + void owner.result.current.onToggle(); + }); + + await waitFor(() => expect(mocks.activeEmissary).not.toBeNull()); + expect(mocks.startControls).not.toHaveBeenCalled(); + + act(() => resolveBridge()); + await waitFor(() => + expect(mocks.startControls).toHaveBeenCalledWith("session-a"), + ); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + await act(async () => owner.result.current.onToggle()); + }); + + it("stops a captured microphone stream when parallel startup fails", async () => { + mocks.createSession.mockRejectedValueOnce(new Error("token failed")); + const owner = renderConversation("session-a"); + + await act(async () => owner.result.current.onToggle()); + + await waitFor(() => expect(owner.result.current.state).toBe("error")); + expect(track.stop).toHaveBeenCalledOnce(); + }); + + it.each([ + "close", + "error", + ] as const)("cleans up when the open Realtime data channel emits %s", async (eventType) => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + channel.dispatchEvent(new Event(eventType)); + }); + + await waitFor(() => expect(owner.result.current.state).toBe("error")); + expect(peer.close).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(mocks.stopControls).toHaveBeenCalledWith("session-a", 7); + }); + + it.each([ + ["connectionstatechange", "connectionState"], + ["iceconnectionstatechange", "iceConnectionState"], + ] as const)("cleans up when Realtime emits terminal %s failure", async (eventType, stateProperty) => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + peer[stateProperty] = "failed"; + await act(async () => { + peer.dispatchEvent(new Event(eventType)); + }); + + await waitFor(() => expect(owner.result.current.state).toBe("error")); + expect(channel.close).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(mocks.stopControls).toHaveBeenCalledWith("session-a", 7); + }); + + it("keeps the process-wide conversation alive across owner unmount and remount", async () => { + const originalOnSend = vi.fn().mockResolvedValue(true); + const remountedOnSend = vi.fn().mockResolvedValue(true); + const first = renderConversation("session-a", originalOnSend); + + await act(async () => first.result.current.onToggle()); + await waitFor(() => expect(first.result.current.state).toBe("listening")); + expect(mocks.startControls).toHaveBeenCalledWith("session-a"); + expect(first.result.current.ownsActiveConversation).toBe(true); + + first.unmount(); + + expect(channel.close).not.toHaveBeenCalled(); + expect(peer.close).not.toHaveBeenCalled(); + expect(track.stop).not.toHaveBeenCalled(); + expect(mocks.releaseBridge).not.toHaveBeenCalled(); + expect(mocks.releaseMicrophone).not.toHaveBeenCalled(); + + const remounted = renderConversation("session-a", remountedOnSend); + expect(remounted.result.current.state).toBe("listening"); + expect(remounted.result.current.ownsActiveConversation).toBe(true); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await waitFor(() => expect(remountedOnSend).toHaveBeenCalledOnce()); + expect(originalOnSend).not.toHaveBeenCalled(); + + await act(async () => remounted.result.current.onToggle()); + await waitFor(() => expect(remounted.result.current.state).toBe("off")); + expect(channel.close).toHaveBeenCalledOnce(); + expect(peer.close).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(mocks.releaseBridge).toHaveBeenCalledOnce(); + expect(mocks.releaseMicrophone).toHaveBeenCalledOnce(); + expect(mocks.stopControls).toHaveBeenCalledWith("session-a", 7); + }); + + it("routes floating Realtime mute controls back to the owning media track", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + realtimeControlListener?.({ + sessionId: "session-a", + revision: 7, + action: "mute", + muted: true, + }); + }); + + expect(track.enabled).toBe(false); + expect(owner.result.current.microphoneMuted).toBe(true); + expect(mocks.publishMuted).toHaveBeenCalledWith("session-a", 7, true); + + await act(async () => owner.result.current.onToggle()); + }); + + it("routes floating Realtime hang-up controls to the active call", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + realtimeControlListener?.({ + sessionId: "session-a", + revision: 7, + action: "stop", + }); + }); + + await waitFor(() => expect(owner.result.current.state).toBe("off")); + expect(peer.close).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(mocks.stopControls).toHaveBeenCalledWith("session-a", 7); + }); + + it("does not let another session steal the active conversation", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + const other = renderConversation("session-b"); + expect(other.result.current.boundSessionId).toBe("session-a"); + expect(other.result.current.ownsActiveConversation).toBe(false); + expect(other.result.current.disabled).toBe(true); + + await act(async () => other.result.current.onToggle()); + expect(mocks.createSession).toHaveBeenCalledTimes(1); + expect(owner.result.current.state).toBe("listening"); + + await act(async () => owner.result.current.onToggle()); + }); + + it("moves the realtime owner and bridge when a draft session is promoted", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatSessionStore.setState({ + sessions: [ + { + id: "draft-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + creationState: "pending", + intent: null, + }, + ], + }); + const owner = renderHook( + ({ sessionId }) => + useOpenAiRealtimeConversation({ + enabled: true, + onSend, + sessionId, + }), + { initialProps: { sessionId: "draft-session" } }, + ); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + useChatSessionStore + .getState() + .promoteDraftSession("draft-session", "backend-session"); + useChatStore + .getState() + .promoteSessionId("draft-session", "backend-session"); + owner.rerender({ sessionId: "backend-session" }); + }); + + await waitFor(() => + expect(owner.result.current.boundSessionId).toBe("backend-session"), + ); + expect(owner.result.current.disabled).toBe(false); + expect(mocks.activeEmissary?.sessionId).toBe("backend-session"); + expect(mocks.rebindControls).toHaveBeenCalledWith( + "draft-session", + "backend-session", + 7, + ); + await waitFor(() => + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining( + 'send-to-spokesperson --session-id "backend-session"', + ), + ), + ); + expect(mocks.appendSessionSystemPrompt).toHaveBeenCalledWith( + "backend-session", + expect.any(String), + expect.stringContaining("--mode "), + ); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + "[Voice transcript; cursor 1] Spokesperson said: hello user", + undefined, + undefined, + expect.objectContaining({ + displayText: "hello user", + userMessageMetadata: expect.objectContaining({ userVisible: false }), + }), + ); + expect( + useChatStore.getState().messagesBySession["backend-session"]?.[0], + ).toMatchObject({ + metadata: { voiceConversationDebugEvent: "emissarySpeech" }, + }); + expect(useChatStore.getState().messagesBySession["draft-session"]).toBe( + undefined, + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("waits for owner promotion before stopping native controls", async () => { + useChatSessionStore.setState({ + sessions: [ + { + id: "draft-session", + clientSessionId: "draft-session", + title: "New chat", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + messageCount: 0, + creationState: "pending", + intent: null, + }, + ], + }); + let finishRebind!: (status: { + available: boolean; + unavailableReason: null; + lifecycle: string; + sessionId: string; + ownerWindowLabel: string; + microphoneMuted: boolean; + revision: number; + }) => void; + mocks.rebindControls.mockReturnValueOnce( + new Promise((resolve) => { + finishRebind = resolve; + }), + ); + const owner = renderHook( + ({ sessionId }) => + useOpenAiRealtimeConversation({ + enabled: true, + onSend: vi.fn(), + sessionId, + }), + { initialProps: { sessionId: "draft-session" } }, + ); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + useChatSessionStore + .getState() + .promoteDraftSession("draft-session", "backend-session"); + useChatStore + .getState() + .promoteSessionId("draft-session", "backend-session"); + owner.rerender({ sessionId: "backend-session" }); + }); + await waitFor(() => expect(mocks.rebindControls).toHaveBeenCalledOnce()); + let stopPromise!: Promise; + act(() => { + stopPromise = Promise.resolve(owner.result.current.onToggle()); + }); + expect(mocks.stopControls).not.toHaveBeenCalled(); + + finishRebind({ + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "backend-session", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 8, + }); + await act(async () => stopPromise); + expect(mocks.stopControls).toHaveBeenCalledWith("backend-session", 8); + }); + + it("steers realtime deliveries while the master is running without using the composer queue", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("does not let realtime delivery overtake an accepted composer message", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + act(() => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + useChatStore.getState().enqueueTransportReadyMessage("session-a", { + persona: { kind: "inherit" }, + text: "accepted first", + }); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await Promise.resolve(); + expect(mocks.steerPrompt).not.toHaveBeenCalled(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => useChatStore.setState({ queuedMessageBySession: {} })); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + + await act(async () => owner.result.current.onToggle()); + }); + + it("does not let a master message overtake a queued transcript steer", async () => { + let acceptSteer: (() => void) | undefined; + mocks.steerPrompt.mockImplementationOnce( + () => + new Promise((resolve) => { + acceptSteer = () => resolve(true); + }), + ); + const owner = renderConversation( + "session-a", + vi.fn().mockResolvedValue(true), + ); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + act(() => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + + await expect( + mocks.activeEmissary?.sendMasterMessage("This must wait.", 0, "say", []), + ).resolves.toEqual({ + accepted: false, + reason: "pipe_busy", + cursor: 0, + }); + expect(mocks.requestMasterMessage).not.toHaveBeenCalled(); + + await act(async () => acceptSteer?.()); + await expect( + mocks.activeEmissary?.sendMasterMessage( + "This follows the transcript.", + 1, + "say", + [], + ), + ).resolves.toMatchObject({ accepted: true, cursor: 1 }); + expect(mocks.requestMasterMessage).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("retries as a normal prompt when the master finishes before steer admission", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockRejectedValueOnce( + new Error("no active run to steer"), + ); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + act(() => { + useChatStore.getState().setActiveRunId("session-a", null); + useChatStore.getState().setChatState("session-a", "idle"); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledWith( + "[Voice transcript; cursor 1] Spokesperson said: hello user", + undefined, + undefined, + expect.objectContaining({ displayText: "hello user" }), + ); + expect(mocks.steerPrompt).toHaveBeenCalledWith( + "session-a", + "[Voice transcript; cursor 1] Spokesperson said: hello user", + undefined, + expect.anything(), + { + throwOnError: true, + reportErrorInTranscript: false, + }, + ); + + expect( + (useChatStore.getState().messagesBySession["session-a"] ?? []).some( + (message) => + message.role === "system" && + message.content.some( + (content) => + content.type === "text" && + content.text.includes("no active run to steer"), + ), + ), + ).toBe(false); + + await act(async () => owner.result.current.onToggle()); + }); + + it("waits for a real run id instead of steering from chat state alone", async () => { + const onSend = vi.fn().mockResolvedValue(true); + mocks.steerPrompt.mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(mocks.steerPrompt).not.toHaveBeenCalled(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + useChatStore.getState().setChatState("session-a", "idle"); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(mocks.steerPrompt).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("adds one explicit debug bubble for a master routing command", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "There are 20 repos.", + 0, + "context", + [], + ); + }); + + expect(mocks.requestMasterMessage).toHaveBeenCalledWith({ + eventId: "berd-master-1", + message: "[bridge cursor 1] There are 20 repos.", + mode: "context", + resolvedHandoffIds: [], + }); + expect( + useChatStore.getState().messagesBySession["session-a"], + ).toMatchObject([ + { + role: "assistant", + content: [{ type: "text", text: "There are 20 repos." }], + metadata: { + personaName: "Expert → Spokesperson · Context · sent", + voiceConversationDebugEvent: "masterToEmissaryContext", + }, + }, + ]); + + await act(async () => owner.result.current.onToggle()); + }); + + it("keeps a handoff open when its resolving delivery fails", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => expect(mocks.activeEmissary).not.toBeNull()); + const handoffId = acceptedHandoffId("call-1"); + + mocks.sendRealtimeEvents.mockImplementationOnce(() => { + throw new DOMException("channel closed", "InvalidStateError"); + }); + await expect( + mocks.activeEmissary?.sendMasterMessage("First attempt", 1, "say", [ + handoffId, + ]), + ).rejects.toThrow("channel closed"); + + await expect( + mocks.activeEmissary?.sendMasterMessage("Retry", 1, "say", [handoffId]), + ).resolves.toMatchObject({ accepted: true }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("returns malformed tool arguments without ending the voice session", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.sendRealtimeEvents.mockClear(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.invalid_tool_call" }), + }), + ); + }); + + expect(mocks.createInvalidToolCallOutput).toHaveBeenCalledWith( + "call-broken", + "handoff", + "JSON Parse error: Unterminated string", + ); + expect(mocks.requestToolOutput).toHaveBeenCalledWith( + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ type: "function_call_output" }), + }), + ); + expect(mocks.recordToolOutput).not.toHaveBeenCalled(); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ type: "function_call_output" }), + }), + ]); + expect(owner.result.current.state).toBe("listening"); + + await act(async () => owner.result.current.onToggle()); + }); + + it("renders user speech normally and flushes it to the Expert on hang-up", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + role: "user", + content: [{ type: "text", text: "hello master" }], + metadata: { origin: "voice_conversation" }, + }); + + await act(async () => stopOpenAiRealtimeConversation()); + expect(onSend).toHaveBeenCalledWith( + expect.stringContaining("User said: hello master"), + undefined, + undefined, + expect.objectContaining({ displayText: "Final voice transcript" }), + ); + }); + + it("does not request a Spokesperson response when automatic responses are disabled", async () => { + mocks.createResponse = false; + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("edits a provisional user transcript in place when the final correction arrives", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_partial" }), + }), + ); + }); + const provisional = + useChatStore.getState().messagesBySession["session-a"]?.[0]; + expect(provisional).toMatchObject({ + role: "user", + content: [{ type: "text", text: "hello" }], + metadata: { completionStatus: "inProgress" }, + }); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_corrected" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + id: provisional?.id, + content: [{ type: "text", text: "hello master" }], + metadata: { completionStatus: "completed" }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("waits for session hydration before dispatching a voice transcript", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatStore.getState().setSessionLoading("session-a", true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => useChatStore.getState().setSessionLoading("session-a", false)); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + + await act(async () => owner.result.current.onToggle()); + }); + + it("cancels a queued delivery when its call stops and does not replay it after restart", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatStore.getState().setSessionLoading("session-a", true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + channel = new FakeDataChannel(); + peer = new FakePeer(channel); + mocks.createPeer.mockReturnValue(peer); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => useChatStore.getState().setSessionLoading("session-a", false)); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + await act(async () => owner.result.current.onToggle()); + }); + + it("releases media while a blocked final transcript continues delivering", async () => { + const onSend = vi.fn().mockResolvedValue(true); + useChatStore.getState().setSessionLoading("session-a", true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + + await act(async () => owner.result.current.onToggle()); + + expect(owner.result.current.state).toBe("off"); + expect(track.stop).toHaveBeenCalledOnce(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => useChatStore.getState().setSessionLoading("session-a", false)); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + }); + + it("forwards committed typed user text to the realtime emissary", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-typed"); + }); + + act(() => { + owner.result.current.onTypedUserMessageCommitted?.( + "Please stop and check this.", + ); + }); + + expect(mocks.requestTypedUserMessage).toHaveBeenCalledWith( + "Please stop and check this.", + ); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { type: "response.cancel" }, + { type: "conversation.item.create" }, + ]); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(mocks.steerPrompt).toHaveBeenCalledWith( + "session-a", + "[Voice transcript; cursor 1] Spokesperson said: hello user", + undefined, + expect.objectContaining({ + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }), + { throwOnError: true, reportErrorInTranscript: false }, + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("does not let a realtime transport failure abort the ordinary typed send", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.sendRealtimeEvents.mockImplementationOnce(() => { + throw new DOMException( + "The object is in an invalid state.", + "InvalidStateError", + ); + }); + + expect(() => { + owner.result.current.onTypedUserMessageCommitted?.("Still send this."); + }).not.toThrow(); + await waitFor(() => expect(owner.result.current.state).toBe("error")); + + await act(async () => owner.result.current.onToggle()); + }); + + it("renders emissary speech on the assistant side with spoken status", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toBeDefined(), + ); + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + role: "assistant", + content: [ + { + type: "text", + text: "hello user", + speech: { status: "spoken", spokenThrough: 10 }, + }, + ], + metadata: { + agentVisible: false, + origin: "voice_conversation", + voiceConversationDebugEvent: "emissarySpeech", + }, + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + + await act(async () => owner.result.current.onToggle()); + }); + + it("updates a multi-item emissary response in one speaking bubble", async () => { + const owner = renderConversation("session-a"); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_partial_first" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_partial_second" }), + }), + ); + }); + + const messages = useChatStore.getState().messagesBySession["session-a"]; + expect(messages).toHaveLength(1); + expect(messages?.[0]).toMatchObject({ + role: "assistant", + content: [ + { + type: "text", + text: "Let me think about that. I received a compact transcript.", + speech: { status: "speaking" }, + }, + ], + metadata: { + completionStatus: "inProgress", + voiceConversationDebugEvent: "emissarySpeech", + }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("queues two user questions and wakes the Expert only after Spokesperson activity", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_repository" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend.mock.calls[0]?.[0]).toBe( + "[Voice transcript; cursor 1] User said: how many repos are in my development folder?\n" + + "[Voice transcript; cursor 2] Spokesperson said: hello user", + ); + act(() => useChatStore.getState().setChatState("session-a", "thinking")); + act(() => + useChatStore.getState().setActiveRunId("session-a", "run-repository"), + ); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenCalledOnce(); + const handoffId = acceptedHandoffId("call-1"); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "The answer is 21 repositories.", + 3, + "say", + [handoffId], + ); + }); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_result" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledTimes(2)); + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"], + ).toHaveLength(5), + ); + expect(onSend).toHaveBeenCalledOnce(); + + act(() => { + useChatStore.getState().setActiveRunId("session-a", null); + useChatStore.getState().setChatState("session-a", "idle"); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript_followup" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).toHaveBeenCalledOnce(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_followup_ack" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + expect(onSend.mock.calls[1]?.[0]).toBe( + "[Voice transcript; cursor 6] User said: are any of them symbolic links?\n" + + "[Voice transcript; cursor 7] Spokesperson said: I'll verify that.", + ); + act(() => useChatStore.getState().setChatState("session-a", "thinking")); + act(() => + useChatStore.getState().setActiveRunId("session-a", "run-followup"), + ); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff_followup" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledTimes(3)); + expect(onSend).toHaveBeenCalledTimes(2); + + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "None of the repositories are symbolic links.", + 8, + "say", + ["handoff-8"], + ); + }); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_symlink_result" }), + }), + ); + }); + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledTimes(4)); + await waitFor(() => + expect( + useChatStore + .getState() + .messagesBySession["session-a"]?.some( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === + "None of those repositories are symbolic links.", + ), + ).toBe(true), + ); + expect(onSend).toHaveBeenCalledTimes(2); + + const messages = + useChatStore.getState().messagesBySession["session-a"] ?? []; + expect( + messages.filter( + (message) => + message.metadata?.voiceConversationDebugEvent === "emissaryToMaster", + ), + ).toHaveLength(2); + expect( + messages.filter( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === "You have 21 repositories.", + ), + ).toHaveLength(1); + expect( + messages.filter( + (message) => + message.content[0]?.type === "text" && + message.content[0].text === + "None of those repositories are symbolic links.", + ), + ).toHaveLength(1); + + await act(async () => owner.result.current.onToggle()); + }); + + it("wakes the Expert for Spokesperson speech but not subsequent user speech", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toMatchObject({ + role: "assistant", + content: [{ type: "text", text: "hello user" }], + metadata: { voiceConversationDebugEvent: "emissarySpeech" }, + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend).toHaveBeenLastCalledWith( + "[Voice transcript; cursor 1] Spokesperson said: hello user", + undefined, + undefined, + expect.objectContaining({ + displayText: "hello user", + userMessageMetadata: expect.objectContaining({ userVisible: false }), + }), + ); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).toHaveBeenCalledOnce(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("marks interrupted emissary speech without claiming a precise cutoff", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary_interrupted" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.[0], + ).toBeDefined(), + ); + const content = + useChatStore.getState().messagesBySession["session-a"]?.[0]?.content[0]; + if (content?.type !== "text") + throw new Error("expected an emissary text message"); + const speech = content.speech; + expect(speech).toEqual({ status: "interrupted", confidence: "low" }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + + await act(async () => owner.result.current.onToggle()); + }); + + it("shows accepted emissary-to-master coordination in the transcript", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + role: "assistant", + content: [ + { + type: "text", + text: "Please inspect the disk.", + }, + ], + metadata: { + agentVisible: false, + personaName: expect.stringMatching( + /^Spokesperson → Expert · Handoff handoff-.+-1$/, + ), + voiceConversationDebugEvent: "emissaryToMaster", + }, + }), + ); + expect(mocks.recordToolOutput).toHaveBeenCalledWith({ + type: "conversation.item.create", + item: { type: "function_call_output" }, + }); + expect(mocks.requestToolOutput).not.toHaveBeenCalled(); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { + type: "conversation.item.create", + item: { type: "function_call_output" }, + }, + ]); + + await act(async () => owner.result.current.onToggle()); + }); + + it("steers emissary-to-master coordination into an active master turn", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + + await waitFor(() => expect(mocks.steerPrompt).toHaveBeenCalledOnce()); + expect(onSend).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + role: "assistant", + metadata: { voiceConversationDebugEvent: "emissaryToMaster" }, + }); + + await act(async () => owner.result.current.onToggle()); + }); + + it("delivers queued user speech and a handoff in one Expert wake", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.transcript" }), + }), + ); + }); + await Promise.resolve(); + expect(onSend).not.toHaveBeenCalled(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-a"]?.at(-1), + ).toMatchObject({ + content: [ + expect.objectContaining({ + text: "Please inspect the disk.", + }), + ], + metadata: { + personaName: expect.stringMatching( + /^Spokesperson → Expert · Handoff handoff-.+-2$/, + ), + voiceConversationDebugEvent: "emissaryToMaster", + }, + }), + ); + + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend.mock.calls[0]?.[0]).toBe( + "[Voice transcript; cursor 1] User said: hello master\n" + + `[Handoff ${acceptedHandoffId("call-1")} from spokesperson; cursor 2] Please inspect the disk.`, + ); + expect(mocks.steerPrompt).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("accepts multiple handoffs without requiring new user input", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + mocks.createHandoffToolOutput.mockClear(); + mocks.sendRealtimeEvents.mockClear(); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + + await waitFor(() => + expect(mocks.createHandoffToolOutput).toHaveBeenCalledWith("call-1", { + accepted: true, + handoff_id: expect.stringMatching(/^handoff-.+-1$/), + }), + ); + expect(mocks.recordToolOutput).toHaveBeenCalledWith( + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ type: "function_call_output" }), + }), + ); + expect(mocks.requestToolOutput).not.toHaveBeenCalled(); + expect(onSend).toHaveBeenCalledOnce(); + expect(mocks.sendRealtimeEvents).toHaveBeenCalledWith(expect.anything(), [ + { + type: "conversation.item.create", + item: { type: "function_call_output" }, + }, + ]); + expect(useChatStore.getState().messagesBySession["session-a"]).toHaveLength( + 1, + ); + + await act(async () => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff_followup" }), + }), + ); + }); + + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + expect(mocks.createHandoffToolOutput).toHaveBeenLastCalledWith("call-2", { + accepted: true, + handoff_id: expect.stringMatching(/^handoff-.+-2$/), + }); + await act(async () => owner.result.current.onToggle()); + }); + + it("automatically orders a handoff after pending master context", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + await act(async () => { + await mocks.activeEmissary?.sendMasterMessage( + "Pending master context.", + 0, + "context", + [], + ); + }); + mocks.requestToolOutput.mockClear(); + mocks.recordToolOutput.mockClear(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + + await waitFor(() => expect(mocks.recordToolOutput).toHaveBeenCalledOnce()); + expect(mocks.requestToolOutput).not.toHaveBeenCalled(); + expect(mocks.createHandoffToolOutput).toHaveBeenCalledWith("call-1", { + accepted: true, + handoff_id: expect.stringMatching(/^handoff-.+-2$/), + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(onSend.mock.calls[0]?.[0]).toContain( + `[Handoff ${acceptedHandoffId("call-1")} from spokesperson; cursor 2]`, + ); + + await act(async () => owner.result.current.onToggle()); + }); + + it("lets one say resolve several open handoffs", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff_followup" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + const handoffIds = [ + acceptedHandoffId("call-1"), + acceptedHandoffId("call-2"), + ]; + + await expect( + mocks.activeEmissary?.sendMasterMessage( + "I handled both requests.", + 2, + "say", + handoffIds, + ), + ).resolves.toMatchObject({ accepted: true }); + + act(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(onSend).toHaveBeenCalledTimes(2); + + await act(async () => owner.result.current.onToggle()); + }); + + it("rejects resolving a handoff through silent context", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + + await expect( + mocks.activeEmissary?.sendMasterMessage("Silent context.", 0, "context", [ + "handoff-1", + ]), + ).resolves.toEqual({ + accepted: false, + reason: "context_cannot_resolve", + cursor: 0, + handoffIds: ["handoff-1"], + }); + expect(mocks.requestMasterMessage).not.toHaveBeenCalled(); + + await act(async () => owner.result.current.onToggle()); + }); + + it("delivers several dismissed handoffs as silent emissary context", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff_followup" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + mocks.requestMasterMessage.mockClear(); + const handoffIds = [ + acceptedHandoffId("call-1"), + acceptedHandoffId("call-2"), + ]; + + await expect( + mocks.activeEmissary?.dismissHandoffs( + 2, + handoffIds, + "The user withdrew both requests.", + ), + ).resolves.toEqual({ + accepted: true, + cursor: 2, + dismissedHandoffIds: handoffIds, + deliveryStatus: "sent", + }); + expect(mocks.requestMasterMessage).toHaveBeenCalledWith({ + eventId: "berd-master-dismissal-3", + message: expect.stringContaining("The user withdrew both requests."), + mode: "context", + }); + expect( + useChatStore + .getState() + .messagesBySession["session-a"]?.filter( + (message) => + message.metadata?.voiceConversationDebugEvent === "masterDismissal", + ), + ).toMatchObject([ + { + content: [ + { + type: "text", + text: `${handoffIds.join(", ")}: The user withdrew both requests.`, + }, + ], + metadata: { + personaName: "Expert → Spokesperson · Dismissed · sent", + }, + }, + ]); + + act(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(onSend).toHaveBeenCalledTimes(2); + + await act(async () => owner.result.current.onToggle()); + }); + + it("gives the master one private reminder for unresolved handoffs", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + const handoffId = acceptedHandoffId("call-1"); + + act(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + expect(onSend.mock.calls[1]?.[0]).toContain( + "[Private handoff reminder; cursor 2]", + ); + expect(onSend.mock.calls[1]?.[0]).toContain(handoffId); + expect(onSend.mock.calls[1]?.[3]).toMatchObject({ + displayText: "Handoff reminder", + userMessageMetadata: { userVisible: false }, + acpGooseMetadata: { + realtimeHandoffReminderIds: [handoffId], + userVisible: false, + }, + }); + expect( + useChatStore + .getState() + .messagesBySession["session-a"]?.filter( + (message) => + message.metadata?.voiceConversationDebugEvent === "handoffReminder", + ), + ).toMatchObject([ + { + content: [ + { + type: "text", + text: `- ${handoffId}: Please inspect the disk.`, + }, + ], + metadata: { + personaName: "Berd → Expert · Handoff reminder 1/3", + }, + }, + ]); + + act(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(onSend).toHaveBeenCalledTimes(2); + + await act(async () => owner.result.current.onToggle()); + }); + + it("fails loudly after three reminder attempts leave a handoff unresolved", async () => { + const onSend = vi.fn().mockResolvedValue(true); + const owner = renderConversation("session-a", onSend); + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.handoff" }), + }), + ); + }); + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + const handoffId = acceptedHandoffId("call-1"); + + act(() => + mocks.activeEmissary?.completeMasterTurn({ reminderHandoffIds: [] }), + ); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(2)); + for (const expectedCalls of [3, 4]) { + act(() => + mocks.activeEmissary?.completeMasterTurn({ + reminderHandoffIds: [handoffId], + }), + ); + await waitFor(() => expect(onSend).toHaveBeenCalledTimes(expectedCalls)); + expect(owner.result.current.state).not.toBe("error"); + } + act(() => + mocks.activeEmissary?.completeMasterTurn({ + reminderHandoffIds: [handoffId], + }), + ); + await waitFor(() => expect(owner.result.current.state).toBe("error")); + expect(owner.result.current.error).toContain( + `left required ${handoffId} unresolved after 3 reminder attempts`, + ); + }); +}); diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts new file mode 100644 index 000000000..83c241dab --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -0,0 +1,1685 @@ +import { useCallback, useEffect, useSyncExternalStore } from "react"; +import { toast } from "sonner"; +import type { + ChatInputSendHandler, + ChatInputVoiceConversation, +} from "@/features/chat/types"; +import { steerPromptInSession } from "@/features/chat/lib/steerCore"; +import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { useChatStore } from "@/features/chat/stores/chatStore"; +import { appendSessionSystemPrompt } from "@/shared/api/acpApi"; +import { + claimVoiceDictationMicrophone, + createOpenAiRealtimeVoiceSession, + listenToOpenAiRealtimeVoiceControls, + publishOpenAiRealtimeVoiceActivity, + publishOpenAiRealtimeVoiceMicrophoneMuted, + rebindOpenAiRealtimeVoiceControls, + releaseVoiceDictationMicrophone, + setOpenAiRealtimeVoiceControlsSuppressed, + startOpenAiRealtimeVoiceControls, + stopOpenAiRealtimeVoiceControls, +} from "@/shared/api/openaiRealtime"; +import { + createSystemNotificationMessage, + type Message, + type VoiceConversationDebugEvent, +} from "@/shared/types/messages"; +import { + connectOpenAiRealtimePeerConnection, + createOpenAiRealtimePeerConnection, +} from "@/features/chat/lib/openaiRealtimeAudio"; +import { + type ActiveRealtimeEmissary, + type HandoffDismissal, + type MasterMessageDelivery, + type RealtimeMasterTurnCompletion, + registerRealtimeEmissary, + waitForRealtimeEmissaryBridgeReady, +} from "../lib/realtimeEmissaryBridge"; +import { + createHandoffToolOutput, + createInvalidToolCallOutput, + DirectMessagePipe, + type MasterMessageMode, + REALTIME_EXPERT_INSTRUCTIONS, + RealtimeEmissaryProtocol, + RealtimeResponseCoordinator, + sendRealtimeEvents, + configureRealtimeEmissarySession, +} from "../lib/realtimeEmissaryProtocol"; +import { getRealtimeVoicePreference } from "../lib/realtimeVoicePreference"; +import { + beginVoiceControlsVisibilityLease, + observeVoiceConversationControlVisibility, +} from "./useVoiceConversationController"; + +const MASTER_PROMPT_KEY = "berd-realtime-voice-master"; +const MICROPHONE_OWNER_ID = "berd:realtime-voice-conversation"; +const MAX_REALTIME_REPLAY_ITEMS = 12; +const HANDOFF_REMINDER_IDS_METADATA = "realtimeHandoffReminderIds"; +const MAX_HANDOFF_REMINDER_ATTEMPTS = 3; + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isUnavailableDevMicrophoneClaim(error: unknown): boolean { + return ( + import.meta.env.DEV && + errorText(error).includes("claim_voice_dictation_microphone not found") + ); +} + +function isMissingActiveRun(error: unknown): boolean { + return errorText(error).toLowerCase().includes("no active run to steer"); +} + +function waitForSessionHydration( + sessionId: string, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!useChatStore.getState().loadingSessionIds.has(sessionId)) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + let unsubscribe: () => void = () => undefined; + const cleanup = () => { + unsubscribe(); + signal?.removeEventListener("abort", handleAbort); + }; + const handleAbort = () => { + cleanup(); + reject(signal?.reason ?? new DOMException("Aborted", "AbortError")); + }; + unsubscribe = useChatStore.subscribe((state) => { + if (state.loadingSessionIds.has(sessionId)) return; + cleanup(); + resolve(); + }); + signal?.addEventListener("abort", handleAbort, { once: true }); + if (signal?.aborted) handleAbort(); + }); +} + +function waitForMasterIdle( + sessionId: string, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const isIdle = () => { + const runtime = useChatStore.getState().getSessionRuntime(sessionId); + return runtime.activeRunId === null && !isSessionRunning(runtime.chatState); + }; + if (isIdle()) return Promise.resolve(); + + return new Promise((resolve, reject) => { + let unsubscribe: () => void = () => undefined; + const cleanup = () => { + unsubscribe(); + signal?.removeEventListener("abort", handleAbort); + }; + const handleAbort = () => { + cleanup(); + reject(signal?.reason ?? new DOMException("Aborted", "AbortError")); + }; + unsubscribe = useChatStore.subscribe(() => { + if (!isIdle()) return; + cleanup(); + resolve(); + }); + signal?.addEventListener("abort", handleAbort, { once: true }); + if (signal?.aborted) handleAbort(); + }); +} + +type MasterDeliveryOpportunity = "send" | "steer"; + +function masterDeliveryOpportunity( + sessionId: string, +): MasterDeliveryOpportunity | null { + const state = useChatStore.getState(); + if ((state.queuedMessageBySession[sessionId]?.length ?? 0) > 0) return null; + const runtime = state.getSessionRuntime(sessionId); + if (runtime.isRunCancellationPending) return null; + // A chat state can cross the run boundary before activeRunId catches up. + // Only an actual run id is sufficient proof that ACP can accept a steer. + if (runtime.activeRunId !== null) return "steer"; + if (!isSessionRunning(runtime.chatState)) return "send"; + return null; +} + +function waitForMasterDeliveryOpportunity( + sessionId: string, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const available = masterDeliveryOpportunity(sessionId); + if (available) return Promise.resolve(available); + + return new Promise((resolve, reject) => { + let unsubscribe: () => void = () => undefined; + const cleanup = () => { + unsubscribe(); + signal?.removeEventListener("abort", handleAbort); + }; + const handleAbort = () => { + cleanup(); + reject(signal?.reason ?? new DOMException("Aborted", "AbortError")); + }; + unsubscribe = useChatStore.subscribe(() => { + const opportunity = masterDeliveryOpportunity(sessionId); + if (!opportunity) return; + cleanup(); + resolve(opportunity); + }); + signal?.addEventListener("abort", handleAbort, { once: true }); + if (signal?.aborted) handleAbort(); + }); +} + +function waitForMasterRunBoundary( + sessionId: string, + rejectedRunId: string | null, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const crossedBoundary = () => { + const runtime = useChatStore.getState().getSessionRuntime(sessionId); + return ( + runtime.activeRunId !== rejectedRunId || + (runtime.activeRunId === null && !isSessionRunning(runtime.chatState)) + ); + }; + if (crossedBoundary()) return Promise.resolve(); + + return new Promise((resolve, reject) => { + let unsubscribe: () => void = () => undefined; + const cleanup = () => { + unsubscribe(); + signal?.removeEventListener("abort", handleAbort); + }; + const handleAbort = () => { + cleanup(); + reject(signal?.reason ?? new DOMException("Aborted", "AbortError")); + }; + unsubscribe = useChatStore.subscribe(() => { + if (!crossedBoundary()) return; + cleanup(); + resolve(); + }); + signal?.addEventListener("abort", handleAbort, { once: true }); + if (signal?.aborted) handleAbort(); + }); +} + +const MAX_BRIDGE_CURSOR = 4_294_967_295; +const BRIDGE_CURSOR_RESERVE = 1_000_000; +const FINAL_TRANSCRIPT_FLUSH_TIMEOUT_MS = 100; + +function createBridgeCallScope(): { id: string; initialCursor: number } { + const id = crypto.randomUUID(); + const prefix = Number.parseInt(id.replaceAll("-", "").slice(0, 8), 16); + return { + id, + initialCursor: prefix % (MAX_BRIDGE_CURSOR - BRIDGE_CURSOR_RESERVE), + }; +} + +function isAbortError(error: unknown): boolean { + return error instanceof DOMException && error.name === "AbortError"; +} + +function createEmissaryTranscriptMessage( + text: string, + interrupted: boolean, + id: string = crypto.randomUUID(), + provisional = false, +): Message { + return { + id, + role: "assistant", + created: Date.now(), + content: [ + { + type: "text", + text, + speech: provisional + ? { status: "speaking" } + : interrupted + ? { + status: "interrupted", + confidence: "low", + } + : { status: "spoken", spokenThrough: text.length }, + }, + ], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + voiceConversationDebugEvent: "emissarySpeech", + completionStatus: provisional ? "inProgress" : "completed", + }, + }; +} + +function createUserTranscriptMessage( + id: string, + text: string, + provisional: boolean, +): Message { + return { + id, + role: "user", + created: Date.now(), + content: [{ type: "text", text }], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + completionStatus: provisional ? "inProgress" : "completed", + }, + }; +} + +function createCoordinationDebugMessage( + kind: VoiceConversationDebugEvent, + label: string, + text: string, +): Message { + return { + id: crypto.randomUUID(), + role: "assistant", + created: Date.now(), + content: [{ type: "text", text }], + metadata: { + userVisible: true, + agentVisible: false, + origin: "voice_conversation", + personaName: label, + voiceConversationDebugEvent: kind, + completionStatus: "completed", + }, + }; +} + +function createHandoffDebugMessage(handoffId: string, text: string): Message { + return createCoordinationDebugMessage( + "emissaryToMaster", + `Spokesperson → Expert · Handoff ${handoffId}`, + text, + ); +} + +function visibleMessageText(message: Message): string { + return message.content + .flatMap((content) => (content.type === "text" ? [content.text] : [])) + .join("\n") + .trim(); +} + +export function createRealtimeTranscriptReplayEvents( + messages: readonly Message[], + sessionId?: string, +): Record[] { + const turns: Array<{ role: "user" | "assistant"; text: string }> = []; + let pendingAssistant: { role: "assistant"; text: string } | null = null; + const flushAssistant = () => { + if (!pendingAssistant) return; + turns.push(pendingAssistant); + pendingAssistant = null; + }; + + for (const message of messages) { + if (message.metadata?.userVisible === false || message.role === "system") + continue; + const text = visibleMessageText(message); + if (!text) continue; + if (message.role === "user") { + flushAssistant(); + turns.push({ role: "user", text }); + continue; + } + if ( + message.metadata?.voiceConversationDebugEvent || + message.metadata?.personaName === "Routing" || + message.metadata?.personaName?.includes("→") || + (message.metadata?.completionStatus && + message.metadata.completionStatus !== "completed") + ) + continue; + // Only the final visible assistant block before the next user turn is + // useful context. Progress narration and earlier replacements stay in the + // durable Expert transcript but do not bloat a resumed voice frontend. + pendingAssistant = { role: "assistant", text }; + } + flushAssistant(); + + const tail = turns.slice(-MAX_REALTIME_REPLAY_ITEMS); + const firstUserIndex = tail.findIndex((turn) => turn.role === "user"); + if (firstUserIndex < 0) return []; + const replay = tail.slice(firstUserIndex).map((turn) => ({ + type: "conversation.item.create", + item: { + type: "message", + role: turn.role, + content: [ + { + type: turn.role === "assistant" ? "output_text" : "input_text", + text: turn.text, + }, + ], + }, + })); + if (!sessionId) return replay; + return [ + { + type: "conversation.item.create", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: `This voice conversation is being resumed from Berd session ${sessionId}. Durable session link: berd://session/${sessionId}. The following items are a compact recent transcript, not new turns. Ask the Expert to inspect the durable session when older context is needed.`, + }, + ], + }, + }, + ...replay, + ]; +} + +function waitForDataChannelOpen(channel: RTCDataChannel): Promise { + if (channel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + channel.removeEventListener("open", handleOpen); + channel.removeEventListener("error", handleError); + }; + const handleOpen = () => { + cleanup(); + resolve(); + }; + const handleError = () => { + cleanup(); + reject(new Error("OpenAI Realtime data channel failed to open.")); + }; + channel.addEventListener("open", handleOpen); + channel.addEventListener("error", handleError); + }); +} + +function masterPrompt( + sessionId: string, + initialCursor: number, + callId: string, +): string { + return `${REALTIME_EXPERT_INSTRUCTIONS} + +Your send_to_spokesperson tool is the Berd CLI command below. This Realtime call is ${callId}, and its initial bridge cursor is ${initialCursor}. Always use the newest cursor from any Expert-bound transcript, handoff, reminder, or prior tool result. A stale cursor means a newer event is already queued; wait for its normal delivery rather than bypassing it. Choose --mode context to silently update the Spokesperson's context for a future natural turn. Choose --mode say only when the Spokesperson should speak your message to the user now. A say may resolve several open handoffs by repeating --resolves for each handoff id. Context cannot resolve a handoff. Finishing your turn does not notify or wake the Spokesperson, so send explicitly when needed. Berd retries a private unresolved-handoff reminder up to three times before failing the voice session. + +berdctl session send-to-spokesperson --session-id ${JSON.stringify(sessionId)} --cursor --mode [--resolves ...] --message --json + +If a handoff is obsolete, superseded, or already handled, dismiss it explicitly: + +berdctl session dismiss-handoffs --session-id ${JSON.stringify(sessionId)} --cursor --handoff-id [--handoff-id ...] --reason --json`; +} + +type RuntimeState = ChatInputVoiceConversation["state"]; +interface Snapshot { + state: RuntimeState; + boundSessionId: string | null; + requestedStartSessionId: string | null; + microphoneMuted: boolean; + error: string | null; + controlsRevision: number; + ownerWindowLabel: string | null; +} +interface StartOptions { + sessionId: string; + onSend: ChatInputSendHandler; +} +const OFF_SNAPSHOT: Snapshot = { + state: "off", + boundSessionId: null, + requestedStartSessionId: null, + microphoneMuted: false, + error: null, + controlsRevision: 0, + ownerWindowLabel: null, +}; + +class OpenAiRealtimeConversationRuntime { + private snapshot: Snapshot = OFF_SNAPSHOT; + private readonly listeners = new Set<() => void>(); + private peer: RTCPeerConnection | null = null; + private channel: RTCDataChannel | null = null; + private stream: MediaStream | null = null; + private audio: HTMLAudioElement | null = null; + private releaseControlsListener: (() => void) | null = null; + private releaseBridge: (() => void) | null = null; + private bridgeSender: + | (( + message: string, + cursor: number, + mode: MasterMessageMode, + resolves: string[], + ) => Promise) + | null = null; + private bridgeHandoffDismissal: + | (( + cursor: number, + handoffIds: string[], + reason: string, + ) => Promise) + | null = null; + private bridgeMasterTurnCompletion: + | ((completion: RealtimeMasterTurnCompletion) => void) + | null = null; + private readonly openHandoffs = new Map< + string, + { message: string; reminderAttempts: number; resolving: boolean } + >(); + private activeRun = 0; + private deliveryQueue = Promise.resolve(); + private deliveryAbortController = new AbortController(); + private boundOnSend: ChatInputSendHandler | null = null; + private typedUserMessageSink: ((text: string) => void) | null = null; + private pendingTypedUserMessages: string[] = []; + private failureInProgress = false; + private ownerMigration = Promise.resolve(); + private historyReplay = Promise.resolve(); + private bridgeCallScope = createBridgeCallScope(); + private flushPendingExpertEvents: (() => boolean) | null = null; + private bridgeReady: Promise = + Promise.resolve(null); + private resolveBridgeReady: + | ((bridge: ActiveRealtimeEmissary | null) => void) + | null = null; + + readonly subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + readonly getSnapshot = (): Snapshot => this.snapshot; + + bindOwner(sessionId: string, onSend: ChatInputSendHandler): void { + if (this.snapshot.boundSessionId === sessionId) this.boundOnSend = onSend; + } + + requestStart(sessionId: string): void { + this.setSnapshot({ ...this.snapshot, requestedStartSessionId: sessionId }); + } + + rebindPromotedOwner(sessionId: string, onSend: ChatInputSendHandler): void { + const previousSessionId = this.snapshot.boundSessionId; + if (!previousSessionId || previousSessionId === sessionId) { + this.bindOwner(sessionId, onSend); + return; + } + + this.boundOnSend = onSend; + this.setSnapshot({ ...this.snapshot, boundSessionId: sessionId }); + this.registerBridge(sessionId); + this.ownerMigration = this.ownerMigration + .catch(() => undefined) + .then(async () => { + if (this.snapshot.controlsRevision > 0) { + const controlsStatus = await rebindOpenAiRealtimeVoiceControls( + previousSessionId, + sessionId, + this.snapshot.controlsRevision, + ); + if (this.snapshot.boundSessionId === sessionId) { + this.setSnapshot({ + ...this.snapshot, + controlsRevision: controlsStatus.revision, + ownerWindowLabel: controlsStatus.ownerWindowLabel, + }); + } + } + await appendSessionSystemPrompt( + previousSessionId, + MASTER_PROMPT_KEY, + "", + ).catch(() => undefined); + await appendSessionSystemPrompt( + sessionId, + MASTER_PROMPT_KEY, + masterPrompt( + sessionId, + this.bridgeCallScope.initialCursor, + this.bridgeCallScope.id, + ), + ); + }); + } + + async start({ sessionId, onSend }: StartOptions): Promise { + if ( + (this.snapshot.boundSessionId && + this.snapshot.boundSessionId !== sessionId) || + (this.snapshot.boundSessionId === sessionId && + this.snapshot.state !== "off" && + this.snapshot.state !== "error") + ) + return; + + const runId = ++this.activeRun; + this.resetDeliveryQueue(); + this.bridgeReady = new Promise((resolve) => { + this.resolveBridgeReady = resolve; + }); + this.bridgeCallScope = createBridgeCallScope(); + this.failureInProgress = false; + this.openHandoffs.clear(); + this.boundOnSend = onSend; + this.pendingTypedUserMessages = []; + this.setSnapshot({ + state: "starting", + boundSessionId: sessionId, + requestedStartSessionId: null, + microphoneMuted: false, + error: null, + controlsRevision: 0, + ownerWindowLabel: null, + }); + const isStale = () => this.activeRun !== runId; + try { + this.releaseControlsListener = await listenToOpenAiRealtimeVoiceControls( + (control) => { + if ( + control.sessionId !== this.snapshot.boundSessionId || + control.revision !== this.snapshot.controlsRevision + ) + return; + if (control.action === "stop") { + void this.stop(control.sessionId); + } else if (control.action === "mute" && control.muted !== undefined) { + this.setMicrophoneMuted(control.sessionId, control.muted); + } + }, + ); + if (isStale()) { + this.releaseControlsListener(); + this.releaseControlsListener = null; + return; + } + this.registerBridge(sessionId); + await waitForRealtimeEmissaryBridgeReady(); + if (isStale()) return; + const controlsStatus = await startOpenAiRealtimeVoiceControls(sessionId); + if (isStale()) { + this.releaseControlsListener(); + this.releaseControlsListener = null; + await stopOpenAiRealtimeVoiceControls( + controlsStatus.sessionId ?? sessionId, + controlsStatus.revision, + ).catch(() => undefined); + return; + } + this.setSnapshot({ + ...this.snapshot, + controlsRevision: controlsStatus.revision, + ownerWindowLabel: controlsStatus.ownerWindowLabel, + }); + await claimVoiceDictationMicrophone(MICROPHONE_OWNER_ID).catch( + (error) => { + if (!isUnavailableDevMicrophoneClaim(error)) throw error; + }, + ); + const preference = getRealtimeVoicePreference(); + const pendingDraft = + useChatSessionStore.getState().getSession(sessionId)?.creationState === + "pending"; + const streamPromise = navigator.mediaDevices + .getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }) + .then((stream) => { + if (isStale()) { + stream.getTracks().forEach((track) => { + track.stop(); + }); + } else { + this.stream = stream; + } + return stream; + }); + const [stream, session] = await Promise.all([ + streamPromise, + createOpenAiRealtimeVoiceSession(preference.model), + pendingDraft + ? Promise.resolve() + : appendSessionSystemPrompt( + sessionId, + MASTER_PROMPT_KEY, + masterPrompt( + sessionId, + this.bridgeCallScope.initialCursor, + this.bridgeCallScope.id, + ), + ), + ]).then(([stream, session]) => [stream, session] as const); + if (isStale()) { + stream.getTracks().forEach((track) => { + track.stop(); + }); + return; + } + + const peer = createOpenAiRealtimePeerConnection(); + const channel = peer.createDataChannel("oai-events"); + const audio = new Audio(); + const failActiveTransport = (message: string) => { + if (!isStale()) { + void this.fail( + this.snapshot.boundSessionId ?? sessionId, + new Error(message), + ); + } + }; + channel.addEventListener("close", () => + failActiveTransport( + "OpenAI Realtime data channel closed unexpectedly.", + ), + ); + channel.addEventListener("error", () => + failActiveTransport("OpenAI Realtime data channel failed."), + ); + peer.addEventListener("connectionstatechange", () => { + if (peer.connectionState === "failed") { + failActiveTransport("OpenAI Realtime peer connection failed."); + } + }); + peer.addEventListener("iceconnectionstatechange", () => { + if (peer.iceConnectionState === "failed") { + failActiveTransport("OpenAI Realtime ICE connection failed."); + } + }); + audio.autoplay = true; + this.peer = peer; + this.channel = channel; + this.audio = audio; + audio.addEventListener("playing", () => + this.publishActivity("assistant-speaking"), + ); + audio.addEventListener("pause", () => + this.publishActivity("assistant-idle"), + ); + audio.addEventListener("ended", () => + this.publishActivity("assistant-idle"), + ); + stream.getAudioTracks().forEach((track) => { + track.enabled = !this.snapshot.microphoneMuted; + peer.addTrack(track, stream); + }); + peer.addEventListener("track", (event) => { + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); + void audio + .play() + .catch((error) => + this.fail(this.snapshot.boundSessionId ?? sessionId, error), + ); + }); + + const transport = { send: (data: string) => channel.send(data) }; + const protocol = new RealtimeEmissaryProtocol(); + const responses = new RealtimeResponseCoordinator(); + const pipe = new DirectMessagePipe(this.bridgeCallScope.initialCursor); + const pendingExpertEvents: string[] = []; + const queueMasterBoundEvent = (message: string) => { + const exchange = pipe.send({ + sender: "emissary", + cursor: pipe.deliveryCursor("emissary"), + message, + }); + if (!exchange.accepted) { + throw new Error( + `The realtime event could not enter the Expert pipe (${exchange.reason}).`, + ); + } + return exchange; + }; + const queueExpertEvent = ( + message: string, + format: (cursor: number) => string, + ) => { + const exchange = queueMasterBoundEvent(message); + pendingExpertEvents.push(format(exchange.outbound.id)); + return exchange; + }; + const wakeExpert = ( + ownerSessionId: string, + displayText: string, + queueUntilIdle = false, + reminderHandoffIds: string[] = [], + continueAfterStop = false, + ) => { + if (pendingExpertEvents.length === 0) return false; + const batch = pendingExpertEvents.splice(0); + this.deliverToMaster( + ownerSessionId, + batch.join("\n"), + displayText, + undefined, + true, + undefined, + queueUntilIdle, + reminderHandoffIds, + continueAfterStop, + ); + return true; + }; + this.flushPendingExpertEvents = () => { + return wakeExpert( + this.snapshot.boundSessionId ?? sessionId, + "Final voice transcript", + false, + [], + true, + ); + }; + const transcriptMessageIds = new Map(); + const upsertTranscriptMessage = ( + ownerSessionId: string, + transcript: { + itemId: string; + speaker: "user" | "emissary"; + text: string; + interrupted?: true; + }, + provisional: boolean, + ): string => { + const existingId = transcriptMessageIds.get(transcript.itemId); + const messageId = existingId ?? crypto.randomUUID(); + transcriptMessageIds.set(transcript.itemId, messageId); + const nextMessage = + transcript.speaker === "user" + ? createUserTranscriptMessage( + messageId, + transcript.text, + provisional, + ) + : createEmissaryTranscriptMessage( + transcript.text, + transcript.interrupted === true, + messageId, + provisional, + ); + const store = useChatStore.getState(); + if (existingId) { + store.updateMessage(ownerSessionId, messageId, (existing) => ({ + ...nextMessage, + created: existing.created, + })); + } else { + store.addMessage(ownerSessionId, nextMessage); + } + return messageId; + }; + const forwardTypedUserMessage = (text: string) => { + const request = responses.requestTypedUserMessage(text); + sendRealtimeEvents(transport, request.events); + }; + channel.addEventListener("message", (message) => { + try { + const ownerSessionId = this.snapshot.boundSessionId; + if (!ownerSessionId || isStale()) return; + const event: unknown = JSON.parse(String(message.data)); + const eventType = + event && typeof event === "object" && "type" in event + ? String(event.type) + : ""; + if (eventType === "input_audio_buffer.speech_started") { + this.publishActivity("user-speaking"); + } else if (eventType === "input_audio_buffer.speech_stopped") { + this.publishActivity("user-idle"); + } + sendRealtimeEvents(transport, responses.handle(event)); + for (const handoffId of responses.takeCompletedHandoffIds()) { + this.openHandoffs.delete(handoffId); + } + for (const handoffId of responses.takeFailedHandoffIds()) { + const handoff = this.openHandoffs.get(handoffId); + if (handoff) handoff.resolving = false; + } + for (const bridgeEvent of protocol.handle(event)) { + if (bridgeEvent.type === "transcript.started") { + upsertTranscriptMessage( + ownerSessionId, + { ...bridgeEvent, text: "" }, + true, + ); + } else if (bridgeEvent.type === "transcript.updated") { + upsertTranscriptMessage(ownerSessionId, bridgeEvent, true); + } else if (bridgeEvent.type === "transcript.finalized") { + upsertTranscriptMessage(ownerSessionId, bridgeEvent, false); + const interrupted = bridgeEvent.interrupted === true; + const transcriptLabel = + bridgeEvent.speaker === "user" + ? `User said: ${bridgeEvent.text}` + : `Spokesperson said${ + interrupted + ? " (interrupted; best-effort transcript)" + : "" + }: ${bridgeEvent.text}`; + const transcriptMessage = `[Voice transcript] ${transcriptLabel}`; + queueExpertEvent( + transcriptMessage, + (cursor) => + `[Voice transcript; cursor ${cursor}] ${transcriptLabel}`, + ); + if (bridgeEvent.speaker === "emissary") { + wakeExpert(ownerSessionId, bridgeEvent.text); + } + // User speech is durable and enters the ordered bridge now, but + // only Spokesperson speech or a handoff wakes the Expert. The + // local user bubble already owns its visible transcript. + } else if (bridgeEvent.type === "handoff") { + const exchange = queueMasterBoundEvent(bridgeEvent.message); + const handoffId = `handoff-${this.bridgeCallScope.id}-${exchange.outbound.id}`; + pendingExpertEvents.push( + `[Handoff ${handoffId} from spokesperson; cursor ${exchange.outbound.id}] ${bridgeEvent.message}`, + ); + const toolOutput = createHandoffToolOutput(bridgeEvent.callId, { + accepted: true, + handoff_id: handoffId, + }); + const toolFollowUp = responses.recordToolOutput(toolOutput); + sendRealtimeEvents(transport, toolFollowUp.events); + this.openHandoffs.set(handoffId, { + message: exchange.outbound.message, + reminderAttempts: 0, + resolving: false, + }); + useChatStore + .getState() + .addMessage( + ownerSessionId, + createHandoffDebugMessage( + handoffId, + exchange.outbound.message, + ), + ); + wakeExpert(ownerSessionId, exchange.outbound.message); + } else if (bridgeEvent.type === "tool_call.invalid") { + const toolFollowUp = responses.requestToolOutput( + createInvalidToolCallOutput( + bridgeEvent.callId, + bridgeEvent.toolName, + bridgeEvent.error, + ), + ); + sendRealtimeEvents(transport, toolFollowUp.events); + } + } + } catch (error) { + void this.fail(this.snapshot.boundSessionId ?? sessionId, error); + } + }); + + await connectOpenAiRealtimePeerConnection({ + peerConnection: peer, + clientSecret: session.clientSecret, + }); + await waitForDataChannelOpen(channel); + if (isStale()) return; + configureRealtimeEmissarySession(transport, { + model: preference.model, + transcriptionModel: preference.transcriptionModel, + transcriptionLanguage: preference.transcriptionLanguage, + transcriptionPrompt: preference.transcriptionPrompt, + voice: preference.voice, + speed: preference.speed, + turnDetection: preference.turnDetection, + eagerness: preference.eagerness, + interruptResponse: preference.interruptResponse, + createResponse: preference.createResponse, + vadThreshold: preference.vadThreshold, + prefixPaddingMs: preference.prefixPaddingMs, + silenceDurationMs: preference.silenceDurationMs, + idleTimeoutMs: preference.idleTimeoutMs, + noiseReduction: preference.noiseReduction, + reasoningEffort: preference.reasoningEffort, + maxOutputTokens: preference.maxOutputTokens, + }); + this.typedUserMessageSink = forwardTypedUserMessage; + for (const text of this.pendingTypedUserMessages.splice(0)) { + forwardTypedUserMessage(text); + } + const replaySessionId = this.snapshot.boundSessionId ?? sessionId; + this.historyReplay = waitForSessionHydration(replaySessionId).then(() => { + if (isStale() || this.snapshot.boundSessionId !== replaySessionId) + return; + sendRealtimeEvents( + transport, + createRealtimeTranscriptReplayEvents( + useChatStore.getState().messagesBySession[replaySessionId] ?? [], + replaySessionId, + ), + ); + }); + this.bridgeSender = async (message, cursor, mode, resolves) => { + const resolvedHandoffIds = [...new Set(resolves)]; + if (mode === "context" && resolvedHandoffIds.length > 0) { + return { + accepted: false, + reason: "context_cannot_resolve", + cursor: pipe.cursor("master"), + handoffIds: resolvedHandoffIds, + }; + } + const unknownHandoffIds = resolvedHandoffIds.filter( + (handoffId) => !this.openHandoffs.has(handoffId), + ); + if (unknownHandoffIds.length > 0) { + return { + accepted: false, + reason: "unknown_handoff", + cursor: pipe.cursor("master"), + handoffIds: unknownHandoffIds, + }; + } + const exchange = pipe.send({ sender: "master", cursor, message }); + if (!exchange.accepted) return exchange; + const request = responses.requestMasterMessage({ + message: `[bridge cursor ${exchange.outbound.id}] ${message}`, + mode, + eventId: `berd-master-${exchange.outbound.id}`, + resolvedHandoffIds, + }); + sendRealtimeEvents(transport, request.events); + for (const handoffId of resolvedHandoffIds) { + const handoff = this.openHandoffs.get(handoffId); + if (handoff) handoff.resolving = true; + } + useChatStore + .getState() + .addMessage( + this.snapshot.boundSessionId ?? sessionId, + createCoordinationDebugMessage( + mode === "say" + ? "masterToEmissarySay" + : "masterToEmissaryContext", + `Expert → Spokesperson · ${mode === "say" ? "Say" : "Context"} · ${request.status}`, + message, + ), + ); + return { ...exchange, deliveryStatus: request.status }; + }; + this.bridgeHandoffDismissal = async (cursor, handoffIds, reason) => { + const dismissedHandoffIds = [...new Set(handoffIds)]; + const unknownHandoffIds = dismissedHandoffIds.filter( + (handoffId) => !this.openHandoffs.has(handoffId), + ); + if (unknownHandoffIds.length > 0) { + return { + accepted: false, + reason: "unknown_handoff", + cursor: pipe.cursor("master"), + handoffIds: unknownHandoffIds, + }; + } + if (!reason.trim()) { + throw new Error("handoff dismissal reason cannot be empty"); + } + const dismissalContext = `Handoffs ${dismissedHandoffIds.join(", ")} were dismissed without a spoken response. Reason: ${reason.trim()}`; + const exchange = pipe.send({ + sender: "master", + cursor, + message: dismissalContext, + }); + if (!exchange.accepted) return exchange; + const request = responses.requestMasterMessage({ + message: `[bridge cursor ${exchange.outbound.id}] [Handoff dismissal] ${dismissalContext} This is silent context; do not speak merely to acknowledge it.`, + mode: "context", + eventId: `berd-master-dismissal-${exchange.outbound.id}`, + }); + sendRealtimeEvents(transport, request.events); + for (const handoffId of dismissedHandoffIds) { + this.openHandoffs.delete(handoffId); + } + useChatStore + .getState() + .addMessage( + this.snapshot.boundSessionId ?? sessionId, + createCoordinationDebugMessage( + "masterDismissal", + `Expert → Spokesperson · Dismissed · ${request.status}`, + `${dismissedHandoffIds.join(", ")}: ${reason.trim()}`, + ), + ); + return { + accepted: true, + cursor: exchange.cursor, + dismissedHandoffIds, + deliveryStatus: request.status, + }; + }; + this.bridgeMasterTurnCompletion = ({ reminderHandoffIds }) => { + const ownerSessionId = this.snapshot.boundSessionId; + if (!ownerSessionId) return; + const retrying = new Set(reminderHandoffIds); + const pending = [...this.openHandoffs.entries()].filter( + ([handoffId, handoff]) => + !handoff.resolving && + (handoff.reminderAttempts === 0 || retrying.has(handoffId)), + ); + if (pending.length === 0) return; + const exhausted = pending.filter( + ([, handoff]) => + handoff.reminderAttempts >= MAX_HANDOFF_REMINDER_ATTEMPTS, + ); + if (exhausted.length > 0) { + void this.fail( + ownerSessionId, + new Error( + `The Expert left required ${exhausted.map(([handoffId]) => handoffId).join(", ")} unresolved after ${MAX_HANDOFF_REMINDER_ATTEMPTS} reminder attempts.`, + ), + ); + return; + } + const pendingIds = pending.map(([handoffId]) => handoffId); + for (const [, handoff] of pending) handoff.reminderAttempts += 1; + const requests = pending + .map(([handoffId, handoff]) => `- ${handoffId}: ${handoff.message}`) + .join("\n"); + const reminder = `[Private handoff reminder]\nYou ended your turn without resolving the required handoffs below. Resolve them now with one or more send-to-spokesperson --mode say calls that name every answered handoff in --resolves, or dismiss obsolete handoffs explicitly. Berd will retry this reminder up to ${MAX_HANDOFF_REMINDER_ATTEMPTS} times. Do not redo completed work.\n${requests}`; + const masterBound = queueExpertEvent( + reminder, + (cursor) => + `[Private handoff reminder; cursor ${cursor}]${reminder.slice("[Private handoff reminder]".length)}`, + ); + const reminderAttempt = Math.max( + ...pending.map(([, handoff]) => handoff.reminderAttempts), + ); + useChatStore + .getState() + .addMessage( + ownerSessionId, + createCoordinationDebugMessage( + "handoffReminder", + `Berd → Expert · Handoff reminder ${reminderAttempt}/${MAX_HANDOFF_REMINDER_ATTEMPTS}`, + requests, + ), + ); + void masterBound; + wakeExpert(ownerSessionId, "Handoff reminder", true, pendingIds); + }; + const bridgeSessionId = this.snapshot.boundSessionId ?? sessionId; + if ( + !this.bridgeSender || + !this.bridgeHandoffDismissal || + !this.bridgeMasterTurnCompletion + ) { + throw new Error("The Realtime Spokesperson bridge did not initialize."); + } + this.resolveBridgeReady?.({ + sessionId: bridgeSessionId, + sendMasterMessage: this.bridgeSender, + dismissHandoffs: this.bridgeHandoffDismissal, + completeMasterTurn: this.bridgeMasterTurnCompletion, + }); + this.resolveBridgeReady = null; + this.setSnapshot({ + ...this.snapshot, + state: "listening", + }); + } catch (error) { + if (!isStale()) await this.fail(sessionId, error); + } + } + + async stop(sessionId: string): Promise { + if ( + this.snapshot.boundSessionId !== sessionId || + this.snapshot.state === "off" || + this.snapshot.state === "stopping" + ) + return; + this.setSnapshot({ ...this.snapshot, state: "stopping" }); + const flushedPendingEvents = this.flushPendingExpertEvents?.() ?? false; + if (flushedPendingEvents) { + await Promise.race([ + this.deliveryQueue.catch(() => undefined), + new Promise((resolve) => { + window.setTimeout(resolve, FINAL_TRANSCRIPT_FLUSH_TIMEOUT_MS); + }), + ]); + } + await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.failureInProgress = false; + this.setSnapshot(OFF_SNAPSHOT); + } + + toggleMute(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.setMicrophoneMuted(sessionId, !this.snapshot.microphoneMuted); + } + + private setMicrophoneMuted( + sessionId: string, + microphoneMuted: boolean, + ): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.stream?.getAudioTracks().forEach((track) => { + track.enabled = !microphoneMuted; + }); + this.setSnapshot({ ...this.snapshot, microphoneMuted }); + if (this.snapshot.controlsRevision > 0) { + void publishOpenAiRealtimeVoiceMicrophoneMuted( + sessionId, + this.snapshot.controlsRevision, + microphoneMuted, + ).catch(() => undefined); + } + } + + forwardTypedUserMessage(sessionId: string, text: string): void { + if (this.snapshot.boundSessionId !== sessionId || !text.trim()) return; + if (!this.typedUserMessageSink) { + if (this.snapshot.state === "starting") + this.pendingTypedUserMessages.push(text); + return; + } + try { + this.typedUserMessageSink(text); + } catch (error) { + // Mirroring into the voice frontend is secondary to the ordinary Berd + // send that invoked this callback. Never let a synchronous WebRTC/data + // channel failure abort the user's Expert turn. + void this.fail(sessionId, error); + } + } + + async dispose(): Promise { + const sessionId = this.snapshot.boundSessionId; + if (sessionId) await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.bridgeSender = null; + this.bridgeHandoffDismissal = null; + this.bridgeMasterTurnCompletion = null; + this.openHandoffs.clear(); + this.typedUserMessageSink = null; + this.pendingTypedUserMessages = []; + this.flushPendingExpertEvents = null; + this.failureInProgress = false; + this.resetDeliveryQueue(); + this.historyReplay = Promise.resolve(); + this.setSnapshot(OFF_SNAPSHOT); + } + + private deliverToMaster( + sessionId: string, + text: string, + displayText: string, + onDelivered?: () => void, + hidden = false, + userMessageId?: string, + queueUntilIdle = false, + reminderHandoffIds: string[] = [], + continueAfterStop = false, + ): void { + const signal = continueAfterStop + ? undefined + : this.deliveryAbortController.signal; + const onSend = this.boundOnSend; + this.deliveryQueue = this.deliveryQueue + .catch(() => undefined) + .then(async () => { + signal?.throwIfAborted(); + // History replay replaces the transcript wholesale. Dispatching a + // realtime transcript while hydration is still active can therefore + // route the Expert's live ACP stream into the replay buffer, or let a + // subsequent replay replacement erase it. Preserve ordering in the + // delivery queue and wait for hydration to publish before sending. + await this.ownerMigration; + await this.historyReplay; + signal?.throwIfAborted(); + if (!continueAfterStop) { + sessionId = this.snapshot.boundSessionId ?? sessionId; + } + await waitForSessionHydration(sessionId, signal); + if (queueUntilIdle) await waitForMasterIdle(sessionId, signal); + if ( + !onSend || + (!continueAfterStop && this.snapshot.boundSessionId !== sessionId) + ) + throw new Error("The realtime voice owner is no longer available."); + const sendOptions = { + displayText, + userMessageMetadata: { + origin: "voice_conversation" as const, + ...(hidden ? { userVisible: false } : {}), + }, + acpGooseMetadata: { + origin: "voice_conversation", + userVisible: !hidden, + agentVisible: false, + ...(reminderHandoffIds.length > 0 + ? { [HANDOFF_REMINDER_IDS_METADATA]: reminderHandoffIds } + : {}), + }, + ...(userMessageId ? { userMessageId } : {}), + }; + const sendAsPrompt = async () => { + const accepted = await onSend( + text, + undefined, + undefined, + sendOptions, + ); + if (accepted === false) + throw new Error( + "The Expert session did not accept the voice transcript.", + ); + }; + if (!continueAfterStop) { + this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + } + for (;;) { + const opportunity = await waitForMasterDeliveryOpportunity( + sessionId, + signal, + ); + if (opportunity === "send") { + await sendAsPrompt(); + break; + } + const rejectedRunId = useChatStore + .getState() + .getSessionRuntime(sessionId).activeRunId; + try { + await steerPromptInSession( + sessionId, + text, + undefined, + sendOptions, + { + throwOnError: true, + // A run can end after the opportunity check but before ACP + // admits the steer. The bridge retries that boundary as a + // fresh prompt, so the transient rejection is not a user + // error and must not leak into the durable transcript. + reportErrorInTranscript: false, + }, + ); + break; + } catch (error) { + if (!isMissingActiveRun(error)) throw error; + // Re-evaluate instead of assuming send: local run state may still + // be publishing completion, or a newer run may already own the + // session. Either transition yields the next safe opportunity. + await waitForMasterRunBoundary(sessionId, rejectedRunId, signal); + } + } + onDelivered?.(); + if (this.snapshot.boundSessionId === sessionId) + this.setSnapshot({ ...this.snapshot, state: "listening" }); + }) + .catch((error) => { + if (isAbortError(error)) return; + if (continueAfterStop) { + console.warn( + "Could not deliver the final Realtime transcript", + error, + ); + return; + } + return this.fail(sessionId, error); + }); + } + + private async fail(sessionId: string, error: unknown): Promise { + if ( + this.snapshot.boundSessionId !== sessionId || + this.failureInProgress || + this.snapshot.state === "error" + ) + return; + this.failureInProgress = true; + const message = errorText(error); + await this.cleanupResources(sessionId); + this.boundOnSend = null; + this.setSnapshot({ + state: "error", + boundSessionId: sessionId, + requestedStartSessionId: null, + microphoneMuted: false, + error: message, + controlsRevision: 0, + ownerWindowLabel: null, + }); + useChatStore + .getState() + .addMessage(sessionId, createSystemNotificationMessage(message, "error")); + toast.error("OpenAI Realtime voice failed", { description: message }); + } + + private async cleanupResources(sessionId: string): Promise { + this.activeRun += 1; + this.resetDeliveryQueue(); + this.resolveBridgeReady?.(null); + this.resolveBridgeReady = null; + await this.ownerMigration.catch(() => undefined); + const activeSessionId = this.snapshot.boundSessionId ?? sessionId; + const controlsRevision = this.snapshot.controlsRevision; + this.releaseBridge?.(); + this.channel?.close(); + this.peer?.close(); + this.stream?.getTracks().forEach((track) => { + track.stop(); + }); + this.audio?.pause(); + this.releaseControlsListener?.(); + this.releaseControlsListener = null; + this.releaseBridge = null; + this.bridgeSender = null; + this.bridgeHandoffDismissal = null; + this.bridgeMasterTurnCompletion = null; + this.openHandoffs.clear(); + this.typedUserMessageSink = null; + this.pendingTypedUserMessages = []; + this.flushPendingExpertEvents = null; + this.channel = null; + this.peer = null; + this.stream = null; + this.audio = null; + if (controlsRevision > 0) { + await stopOpenAiRealtimeVoiceControls( + activeSessionId, + controlsRevision, + ).catch(() => undefined); + } + await releaseVoiceDictationMicrophone(MICROPHONE_OWNER_ID).catch( + () => undefined, + ); + await appendSessionSystemPrompt( + activeSessionId, + MASTER_PROMPT_KEY, + "", + ).catch(() => undefined); + } + + private resetDeliveryQueue(): void { + this.deliveryAbortController.abort(); + this.deliveryAbortController = new AbortController(); + this.deliveryQueue = Promise.resolve(); + } + + private setSnapshot(snapshot: Snapshot): void { + this.snapshot = snapshot; + for (const listener of this.listeners) listener(); + } + + private publishActivity( + activity: + | "user-speaking" + | "user-idle" + | "assistant-speaking" + | "assistant-idle", + ): void { + const { boundSessionId, controlsRevision } = this.snapshot; + if (!boundSessionId || controlsRevision === 0) return; + void publishOpenAiRealtimeVoiceActivity( + boundSessionId, + controlsRevision, + activity, + ).catch(() => undefined); + } + + private registerBridge(sessionId: string): void { + const bridgeReady = this.bridgeReady; + this.releaseBridge?.(); + this.releaseBridge = registerRealtimeEmissary({ + sessionId, + async sendMasterMessage(message, cursor, mode, resolves) { + const bridge = await bridgeReady; + if (!bridge) throw new Error("The Realtime Spokesperson stopped."); + return bridge.sendMasterMessage(message, cursor, mode, resolves); + }, + async dismissHandoffs(cursor, handoffIds, reason) { + const bridge = await bridgeReady; + if (!bridge) throw new Error("The Realtime Spokesperson stopped."); + return bridge.dismissHandoffs(cursor, handoffIds, reason); + }, + completeMasterTurn(completion) { + void bridgeReady.then((bridge) => + bridge?.completeMasterTurn(completion), + ); + }, + }); + } +} + +const runtime = new OpenAiRealtimeConversationRuntime(); + +export function requestOpenAiRealtimeConversationStart( + sessionId: string, +): void { + runtime.requestStart(sessionId); +} + +export async function stopOpenAiRealtimeConversation(): Promise { + const sessionId = runtime.getSnapshot().boundSessionId; + if (sessionId) await runtime.stop(sessionId); +} + +export async function stopOpenAiRealtimeConversationForSession( + sessionId: string, +): Promise { + if (runtime.getSnapshot().boundSessionId === sessionId) { + await runtime.stop(sessionId); + } +} + +if (import.meta.hot) { + import.meta.hot.dispose(() => { + void runtime.dispose(); + }); +} + +export async function resetOpenAiRealtimeConversationRuntimeForTests(): Promise { + await runtime.dispose(); +} + +export function useOpenAiRealtimeConversation(options: { + sessionId: string; + onSend: ChatInputSendHandler; + enabled: boolean; + readOnly?: boolean; + disabled?: boolean; +}): ChatInputVoiceConversation { + const { + sessionId, + onSend, + enabled, + readOnly = false, + disabled = false, + } = options; + const snapshot = useSyncExternalStore( + runtime.subscribe, + runtime.getSnapshot, + runtime.getSnapshot, + ); + const clientSessionId = useChatSessionStore( + (state) => + state.sessions?.find((candidate) => candidate.id === sessionId) + ?.clientSessionId, + ); + const ownsPromotedConversation = + snapshot.boundSessionId !== null && + snapshot.boundSessionId !== sessionId && + clientSessionId === snapshot.boundSessionId; + const ownsActiveConversation = snapshot.boundSessionId === sessionId; + const anotherSessionOwnsConversation = + snapshot.boundSessionId !== null && + !ownsActiveConversation && + !ownsPromotedConversation; + const requestedStartMatchesSession = + snapshot.requestedStartSessionId === sessionId || + (clientSessionId !== undefined && + snapshot.requestedStartSessionId === clientSessionId); + useEffect(() => { + if (ownsPromotedConversation) + runtime.rebindPromotedOwner(sessionId, onSend); + else if (ownsActiveConversation) runtime.bindOwner(sessionId, onSend); + }, [onSend, ownsActiveConversation, ownsPromotedConversation, sessionId]); + useEffect(() => { + if (!enabled && ownsActiveConversation) void runtime.stop(sessionId); + }, [enabled, ownsActiveConversation, sessionId]); + useEffect(() => { + if ( + !window.__TAURI_INTERNALS__ || + !snapshot.boundSessionId || + !snapshot.ownerWindowLabel || + snapshot.controlsRevision === 0 + ) + return; + let disposed = false; + let stopObserver: (() => void) | undefined; + const lease = beginVoiceControlsVisibilityLease(); + const activeSessionId = snapshot.boundSessionId; + const ownerWindowLabel = snapshot.ownerWindowLabel; + const revision = snapshot.controlsRevision; + void import("@tauri-apps/api/window") + .then(async ({ getCurrentWindow }) => { + const stop = await observeVoiceConversationControlVisibility({ + activeSessionId, + currentSessionId: sessionId, + ownerWindowLabel, + currentWindow: getCurrentWindow(), + report: (suppressed) => + lease.run(() => + setOpenAiRealtimeVoiceControlsSuppressed( + activeSessionId, + revision, + suppressed, + ), + ), + onError: (error) => + console.warn( + "Could not synchronize Realtime floating voice controls", + error, + ), + }); + if (disposed) stop(); + else stopObserver = stop; + }) + .catch((error) => { + void lease + .run(() => + setOpenAiRealtimeVoiceControlsSuppressed( + activeSessionId, + revision, + false, + ), + ) + .catch(() => undefined); + console.warn( + "Could not observe the Realtime voice owner window focus", + error, + ); + }); + return () => { + disposed = true; + if (stopObserver) { + stopObserver(); + lease.invalidate(); + } else { + void lease + .release(() => + setOpenAiRealtimeVoiceControlsSuppressed( + activeSessionId, + revision, + false, + ), + ) + .catch(() => undefined); + } + }; + }, [ + sessionId, + snapshot.boundSessionId, + snapshot.controlsRevision, + snapshot.ownerWindowLabel, + ]); + useEffect(() => { + if ( + !requestedStartMatchesSession || + !enabled || + disabled || + readOnly || + anotherSessionOwnsConversation + ) + return; + void runtime.start({ sessionId, onSend }); + }, [ + anotherSessionOwnsConversation, + disabled, + enabled, + onSend, + readOnly, + requestedStartMatchesSession, + sessionId, + ]); + const start = useCallback(async () => { + if (!enabled || disabled || readOnly || anotherSessionOwnsConversation) + return; + await runtime.start({ sessionId, onSend }); + }, [ + anotherSessionOwnsConversation, + disabled, + enabled, + onSend, + readOnly, + sessionId, + ]); + const stop = useCallback(async () => { + await runtime.stop(sessionId); + }, [sessionId]); + const toggleMute = useCallback( + () => runtime.toggleMute(sessionId), + [sessionId], + ); + const forwardTypedUserMessage = useCallback( + (text: string) => runtime.forwardTypedUserMessage(sessionId, text), + [sessionId], + ); + const shouldStart = + !ownsActiveConversation || + snapshot.state === "off" || + snapshot.state === "error"; + + return { + visible: enabled, + state: snapshot.state, + boundSessionId: snapshot.boundSessionId, + active: + snapshot.state !== "off" && + snapshot.state !== "error" && + snapshot.boundSessionId !== null, + ownsActiveConversation, + microphoneMuted: snapshot.microphoneMuted, + error: snapshot.error, + disabled: + !ownsActiveConversation && + (disabled || readOnly || anotherSessionOwnsConversation), + onToggle: shouldStart ? start : stop, + onMicrophoneMuteToggle: toggleMute, + onTypedUserMessageCommitted: forwardTypedUserMessage, + }; +} diff --git a/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts b/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts new file mode 100644 index 000000000..f54a9c9d3 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryBridge.test.ts @@ -0,0 +1,301 @@ +import { describe, expect, it, vi } from "vitest"; +const eventListeners = vi.hoisted( + () => new Map void>>(), +); +const apiMocks = vi.hoisted(() => ({ + getVoiceControlsStatus: vi.fn(async () => ({ + lifecycle: "running", + sessionId: "session-in-another-window", + })), +})); + +vi.mock("@tauri-apps/api/event", () => ({ + listen: vi.fn( + async (event: string, listener: (event: { payload: unknown }) => void) => { + const listeners = eventListeners.get(event) ?? new Set(); + listeners.add(listener); + eventListeners.set(event, listeners); + return () => listeners.delete(listener); + }, + ), + emit: vi.fn(async (event: string, payload: unknown) => { + for (const listener of eventListeners.get(event) ?? []) { + await listener({ payload }); + } + }), +})); + +vi.mock("@/shared/api/openaiRealtime", () => ({ + getOpenAiRealtimeVoiceControlsStatus: () => apiMocks.getVoiceControlsStatus(), +})); + +import { + completeActiveRealtimeMasterTurn, + hasActiveRealtimeEmissary, + registerRealtimeEmissary, + sendToActiveRealtimeSpokesperson, +} from "./realtimeEmissaryBridge"; + +describe("realtime emissary bridge registration", () => { + it("bounds a stalled remote voice-status lookup", async () => { + vi.useFakeTimers(); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: {}, + }); + apiMocks.getVoiceControlsStatus.mockImplementationOnce( + () => new Promise(() => undefined), + ); + + const result = hasActiveRealtimeEmissary("remote-session"); + const expectedTimeout = expect(result).rejects.toThrow( + "Timed out checking the OpenAI Realtime voice status.", + ); + await vi.advanceTimersByTimeAsync(1_000); + await expectedTimeout; + + vi.useRealTimers(); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: undefined, + }); + }); + + it("routes only to the current live session and releases by identity", async () => { + const sendMasterMessage = vi.fn().mockResolvedValue({ + accepted: false, + reason: "stale_cursor", + cursor: 2, + }); + const emissary = { + sessionId: "session-1", + sendMasterMessage, + dismissHandoffs: vi.fn(), + completeMasterTurn: vi.fn(), + }; + const release = registerRealtimeEmissary(emissary); + + await expect( + emissary.sendMasterMessage("update", 1, "context", []), + ).resolves.toMatchObject({ accepted: false, cursor: 2 }); + await completeActiveRealtimeMasterTurn("session-1", { + reminderHandoffIds: ["handoff-1"], + }); + expect(emissary.completeMasterTurn).toHaveBeenCalledWith({ + reminderHandoffIds: ["handoff-1"], + }); + await expect(hasActiveRealtimeEmissary("session-1")).resolves.toBe(true); + await expect(hasActiveRealtimeEmissary("session-2")).resolves.toBe(false); + + release(); + await expect(hasActiveRealtimeEmissary("session-1")).resolves.toBe(false); + }); + + it("accepts a bridge response from another renderer", async () => { + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: {}, + }); + const requests = eventListeners.get( + "voice-conversation:spokesperson-bridge-request", + ); + const remoteResponder = async ({ payload }: { payload: unknown }) => { + const request = payload as { id: string }; + for (const listener of eventListeners.get( + "voice-conversation:spokesperson-bridge-response", + ) ?? []) { + await listener({ + payload: { + id: request.id, + delivery: { + accepted: false, + reason: "stale_cursor", + cursor: 4, + }, + }, + }); + } + }; + const listeners = requests ?? new Set(); + listeners.add(remoteResponder); + eventListeners.set( + "voice-conversation:spokesperson-bridge-request", + listeners, + ); + + await expect( + sendToActiveRealtimeSpokesperson( + "session-in-another-window", + "Answer", + 3, + "say", + [], + ), + ).resolves.toEqual({ + accepted: false, + reason: "stale_cursor", + cursor: 4, + }); + + listeners.delete(remoteResponder); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: undefined, + }); + }); + + it("routes a process event to the renderer that owns the Spokesperson", async () => { + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: {}, + }); + const sendMasterMessage = vi.fn().mockResolvedValue({ + accepted: false, + reason: "stale_cursor", + cursor: 6, + }); + const completeMasterTurn = vi.fn(); + const release = registerRealtimeEmissary({ + sessionId: "popup-session", + sendMasterMessage, + dismissHandoffs: vi.fn(), + completeMasterTurn, + }); + await Promise.resolve(); + const responses: unknown[] = []; + const responseListener = ({ payload }: { payload: unknown }) => { + responses.push(payload); + }; + const responseListeners = + eventListeners.get("voice-conversation:spokesperson-bridge-response") ?? + new Set(); + responseListeners.add(responseListener); + eventListeners.set( + "voice-conversation:spokesperson-bridge-response", + responseListeners, + ); + + for (const listener of eventListeners.get( + "voice-conversation:spokesperson-bridge-request", + ) ?? []) { + await listener({ + payload: { + id: "request-1", + action: "send", + sessionId: "popup-session", + message: "Answer the user", + cursor: 5, + mode: "say", + resolves: ["handoff-5"], + }, + }); + } + + expect(sendMasterMessage).toHaveBeenCalledWith( + "Answer the user", + 5, + "say", + ["handoff-5"], + ); + expect(responses).toContainEqual({ + id: "request-1", + delivery: { + accepted: false, + reason: "stale_cursor", + cursor: 6, + }, + }); + + for (const listener of eventListeners.get( + "voice-conversation:spokesperson-bridge-request", + ) ?? []) { + await listener({ + payload: { + id: "presence-1", + action: "hasActive", + sessionId: "popup-session", + }, + }); + await listener({ + payload: { + id: "completion-1", + action: "complete", + sessionId: "popup-session", + completion: { reminderHandoffIds: ["handoff-8"] }, + }, + }); + } + + expect(responses).toContainEqual({ id: "presence-1", active: true }); + expect(responses).toContainEqual({ id: "completion-1", completed: true }); + expect(completeMasterTurn).toHaveBeenCalledWith({ + reminderHandoffIds: ["handoff-8"], + }); + + responseListeners.delete(responseListener); + release(); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: undefined, + }); + }); + + it("routes presence and turn completion to another renderer", async () => { + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: {}, + }); + const received: unknown[] = []; + const ownerListener = async ({ payload }: { payload: unknown }) => { + const request = payload as { + id: string; + action: "hasActive" | "complete"; + completion?: { reminderHandoffIds: string[] }; + }; + received.push(request); + const response = + request.action === "hasActive" + ? { id: request.id, active: true } + : { id: request.id, completed: true }; + for (const listener of eventListeners.get( + "voice-conversation:spokesperson-bridge-response", + ) ?? []) { + await listener({ payload: response }); + } + }; + const requests = + eventListeners.get("voice-conversation:spokesperson-bridge-request") ?? + new Set(); + requests.add(ownerListener); + eventListeners.set( + "voice-conversation:spokesperson-bridge-request", + requests, + ); + + await expect( + hasActiveRealtimeEmissary("session-in-another-window"), + ).resolves.toBe(true); + await expect( + completeActiveRealtimeMasterTurn("session-in-another-window", { + reminderHandoffIds: ["handoff-7"], + }), + ).resolves.toBe(true); + expect(received).toEqual([ + expect.objectContaining({ + action: "hasActive", + sessionId: "session-in-another-window", + }), + expect.objectContaining({ + action: "complete", + sessionId: "session-in-another-window", + completion: { reminderHandoffIds: ["handoff-7"] }, + }), + ]); + + requests.delete(ownerListener); + Object.defineProperty(window, "__TAURI_INTERNALS__", { + configurable: true, + value: undefined, + }); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts b/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts new file mode 100644 index 000000000..7f369a44a --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryBridge.ts @@ -0,0 +1,295 @@ +import type { + DirectBridgeMessage, + DirectMessageExchange, + MasterMessageMode, +} from "./realtimeEmissaryProtocol"; +import { emit, listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { getOpenAiRealtimeVoiceControlsStatus } from "@/shared/api/openaiRealtime"; + +export type HandoffDispositionFailure = { + accepted: false; + reason: "unknown_handoff" | "context_cannot_resolve"; + cursor: number; + handoffIds: string[]; +}; + +export type MasterMessageDelivery = + | { + accepted: true; + cursor: number; + deliveryStatus: "sent" | "interrupting" | "queued"; + outbound: DirectBridgeMessage; + } + | Exclude + | HandoffDispositionFailure; + +export type HandoffDismissal = + | { + accepted: true; + cursor: number; + dismissedHandoffIds: string[]; + deliveryStatus: "sent" | "interrupting" | "queued"; + } + | Exclude + | HandoffDispositionFailure; + +export interface RealtimeMasterTurnCompletion { + reminderHandoffIds: string[]; +} + +export interface ActiveRealtimeEmissary { + sessionId: string; + sendMasterMessage( + message: string, + cursor: number, + mode: MasterMessageMode, + resolves: string[], + ): Promise; + dismissHandoffs( + cursor: number, + handoffIds: string[], + reason: string, + ): Promise; + completeMasterTurn(completion: RealtimeMasterTurnCompletion): void; +} + +let activeEmissary: ActiveRealtimeEmissary | null = null; +let remoteListener: Promise | null = null; +const REMOTE_REQUEST_EVENT = "voice-conversation:spokesperson-bridge-request"; +const REMOTE_RESPONSE_EVENT = "voice-conversation:spokesperson-bridge-response"; +const REMOTE_RESPONSE_TIMEOUT_MS = 10_000; +const REALTIME_STATUS_TIMEOUT_MS = 1_000; + +type RemoteBridgeRequest = + | { + id: string; + action: "hasActive"; + sessionId: string; + } + | { + id: string; + action: "complete"; + sessionId: string; + completion: RealtimeMasterTurnCompletion; + } + | { + id: string; + action: "send"; + sessionId: string; + message: string; + cursor: number; + mode: MasterMessageMode; + resolves: string[]; + } + | { + id: string; + action: "dismiss"; + sessionId: string; + cursor: number; + handoffIds: string[]; + reason: string; + }; + +type RemoteBridgeResponse = { + id: string; + active?: boolean; + completed?: boolean; + delivery?: MasterMessageDelivery; + dismissal?: HandoffDismissal; + error?: string; +}; + +function ensureRemoteListener(): Promise { + if (!window.__TAURI_INTERNALS__) return Promise.resolve(); + if (remoteListener) return remoteListener.then(() => undefined); + const registration = listen( + REMOTE_REQUEST_EVENT, + async ({ payload }) => { + const spokesperson = activeEmissary; + if (!spokesperson || spokesperson.sessionId !== payload.sessionId) return; + let response: RemoteBridgeResponse; + try { + switch (payload.action) { + case "hasActive": + response = { id: payload.id, active: true }; + break; + case "complete": + spokesperson.completeMasterTurn(payload.completion); + response = { id: payload.id, completed: true }; + break; + case "send": + response = { + id: payload.id, + delivery: await spokesperson.sendMasterMessage( + payload.message, + payload.cursor, + payload.mode, + payload.resolves, + ), + }; + break; + case "dismiss": + response = { + id: payload.id, + dismissal: await spokesperson.dismissHandoffs( + payload.cursor, + payload.handoffIds, + payload.reason, + ), + }; + break; + } + } catch (error) { + response = { + id: payload.id, + error: error instanceof Error ? error.message : String(error), + }; + } + await emit(REMOTE_RESPONSE_EVENT, response); + }, + ); + remoteListener = registration; + void registration.catch((error) => { + if (remoteListener === registration) remoteListener = null; + console.error("Could not listen for remote Spokesperson messages", error); + }); + return registration.then(() => undefined); +} + +async function requestRemoteBridge( + request: + | Omit, "id"> + | Omit, "id"> + | Omit, "id"> + | Omit, "id">, +): Promise { + if (!window.__TAURI_INTERNALS__) return null; + let timeout: number | undefined; + const status = await Promise.race([ + getOpenAiRealtimeVoiceControlsStatus(), + new Promise((_resolve, reject) => { + timeout = window.setTimeout( + () => + reject( + new Error("Timed out checking the OpenAI Realtime voice status."), + ), + REALTIME_STATUS_TIMEOUT_MS, + ); + }), + ]).finally(() => window.clearTimeout(timeout)); + if ( + status.lifecycle !== "running" || + status.sessionId !== request.sessionId + ) { + return null; + } + const id = crypto.randomUUID(); + return new Promise((resolve, reject) => { + let unlisten: UnlistenFn | undefined; + const timeout = window.setTimeout(() => { + unlisten?.(); + resolve(null); + }, REMOTE_RESPONSE_TIMEOUT_MS); + void listen(REMOTE_RESPONSE_EVENT, ({ payload }) => { + if (payload.id !== id) return; + window.clearTimeout(timeout); + unlisten?.(); + if (payload.error) reject(new Error(payload.error)); + else resolve(payload); + }) + .then((stop) => { + unlisten = stop; + return emit(REMOTE_REQUEST_EVENT, { ...request, id }); + }) + .catch((error) => { + window.clearTimeout(timeout); + unlisten?.(); + reject(error); + }); + }); +} + +export function registerRealtimeEmissary( + emissary: ActiveRealtimeEmissary, +): () => void { + activeEmissary = emissary; + void ensureRemoteListener().catch(() => undefined); + return () => { + if (activeEmissary === emissary) activeEmissary = null; + }; +} + +export function hasLocalActiveRealtimeEmissary(sessionId: string): boolean { + return activeEmissary?.sessionId === sessionId; +} + +export async function waitForRealtimeEmissaryBridgeReady(): Promise { + await ensureRemoteListener(); +} + +export async function sendToActiveRealtimeSpokesperson( + sessionId: string, + message: string, + cursor: number, + mode: MasterMessageMode, + resolves: string[], +): Promise { + if (activeEmissary?.sessionId === sessionId) { + return activeEmissary.sendMasterMessage(message, cursor, mode, resolves); + } + const response = await requestRemoteBridge({ + action: "send", + sessionId, + message, + cursor, + mode, + resolves, + }); + return response?.delivery ?? null; +} + +export async function dismissActiveRealtimeHandoffs( + sessionId: string, + cursor: number, + handoffIds: string[], + reason: string, +): Promise { + if (activeEmissary?.sessionId === sessionId) { + return activeEmissary.dismissHandoffs(cursor, handoffIds, reason); + } + const response = await requestRemoteBridge({ + action: "dismiss", + sessionId, + cursor, + handoffIds, + reason, + }); + return response?.dismissal ?? null; +} + +export async function hasActiveRealtimeEmissary( + sessionId: string, +): Promise { + if (activeEmissary?.sessionId === sessionId) return true; + const response = await requestRemoteBridge({ + action: "hasActive", + sessionId, + }); + return response?.active === true; +} + +export async function completeActiveRealtimeMasterTurn( + sessionId: string, + completion: RealtimeMasterTurnCompletion, +): Promise { + if (activeEmissary?.sessionId === sessionId) { + activeEmissary.completeMasterTurn(completion); + return true; + } + const response = await requestRemoteBridge({ + action: "complete", + sessionId, + completion, + }); + return response?.completed === true; +} diff --git a/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts new file mode 100644 index 000000000..5be2a7511 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.test.ts @@ -0,0 +1,1275 @@ +import { describe, expect, it, vi } from "vitest"; +import { + DirectMessagePipe, + REALTIME_EXPERT_INSTRUCTIONS, + REALTIME_SPOKESPERSON_INSTRUCTIONS, + REALTIME_PROMPT_DOCUMENT, + RealtimeEmissaryProtocol, + RealtimeResponseCoordinator, + configureRealtimeEmissarySession, + createInvalidToolCallOutput, + createRealtimeEmissarySessionUpdate, + createHandoffToolOutput, + sendRealtimeEvents, +} from "./realtimeEmissaryProtocol"; + +describe("Realtime emissary session configuration", () => { + it("configures a realtime audio session with the visibility contract and coordination tool", () => { + const send = vi.fn(); + + configureRealtimeEmissarySession({ send }); + + const event = JSON.parse(send.mock.calls[0][0]); + expect(event.type).toBe("session.update"); + expect(event.session.type).toBe("realtime"); + expect(event.session.output_modalities).toEqual(["audio"]); + expect(event.session.audio.output.speed).toBe(1); + expect(event.session.audio.input).toMatchObject({ + noise_reduction: null, + transcription: { model: "gpt-realtime-whisper" }, + turn_detection: { + type: "server_vad", + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true, + }, + }); + expect(event.session.max_output_tokens).toBe("inf"); + expect(event.session.instructions).toBe(REALTIME_SPOKESPERSON_INSTRUCTIONS); + expect(event.session.instructions).toContain( + "User speech is queued for the Expert but does not wake it", + ); + expect(event.session.instructions).toContain( + "never disclaim a capability because the other part performs it", + ); + expect(event.session.instructions).toContain( + "it calls `handoff` _before_ any substantive spoken answer", + ); + expect(event.session.instructions).toContain( + "never opens a handoff merely to reply to the Expert", + ); + expect(event.session.instructions).toContain( + "never speaks merely to acknowledge `CONTEXT`, `DISMISS`, or an internal message", + ); + expect(event.session.tools).toEqual([ + expect.objectContaining({ + type: "function", + name: "handoff", + parameters: { + type: "object", + properties: { message: expect.any(Object) }, + required: ["message"], + additionalProperties: false, + }, + }), + ]); + }); + + it("maps semantic turn detection and advanced controls to the Realtime session", () => { + const event = createRealtimeEmissarySessionUpdate({ + transcriptionModel: "gpt-live-transcribe", + transcriptionLanguage: "en", + transcriptionPrompt: "Berd, Tauri, emissary", + turnDetection: "semantic_vad", + eagerness: "high", + interruptResponse: false, + createResponse: false, + noiseReduction: "far_field", + reasoningEffort: "low", + maxOutputTokens: 512, + }); + + expect(event.session).toMatchObject({ + reasoning: { effort: "low" }, + max_output_tokens: 512, + audio: { + input: { + transcription: { + model: "gpt-live-transcribe", + language: "en", + prompt: "Berd, Tauri, emissary", + }, + noise_reduction: { type: "far_field" }, + turn_detection: { + type: "semantic_vad", + eagerness: "high", + create_response: false, + interrupt_response: false, + }, + }, + }, + }); + }); + + it("maps server VAD timing controls to the Realtime session", () => { + const event = createRealtimeEmissarySessionUpdate({ + turnDetection: "server_vad", + vadThreshold: 0.7, + prefixPaddingMs: 450, + silenceDurationMs: 850, + idleTimeoutMs: 10_000, + }); + + expect(event.session).toMatchObject({ + audio: { + input: { + turn_detection: { + type: "server_vad", + threshold: 0.7, + prefix_padding_ms: 450, + silence_duration_ms: 850, + idle_timeout_ms: 10_000, + }, + }, + }, + }); + }); + + it("does not send configurable reasoning to older Realtime models", () => { + const event = createRealtimeEmissarySessionUpdate({ + model: "gpt-realtime-1.5", + reasoningEffort: "high", + }); + + expect(event.session).not.toHaveProperty("reasoning"); + }); + + it("exports the Expert visibility and proactive-send contract", () => { + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "response text land in the durable transcript", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "produce visible progress and result text", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "**Expert → Spokesperson messages** (`send_to_spokesperson`)", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "`SAY`—asks the Spokesperson to speak useful information now", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "finishing an Expert turn does not wake it", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "entire turn is an empty, zero-token success", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "no prose, no tools, no coordination", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "small talk belong to the Spokesperson", + ); + expect(REALTIME_EXPERT_INSTRUCTIONS).toContain( + "interrupted Spokesperson transcripts as best-effort", + ); + }); + + it("gives both roles the same one-assistant contract and canonical patterns", () => { + expect(REALTIME_PROMPT_DOCUMENT).toContain("two parts of one brain"); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "one continuous conversation with one assistant", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain("### 1. Simple question"); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "### 2. Work that requires the Expert", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain("### 3. Useful elaboration"); + expect( + REALTIME_SPOKESPERSON_INSTRUCTIONS.replace("Spokesperson", "{{ROLE}}"), + ).toBe(REALTIME_PROMPT_DOCUMENT); + expect(REALTIME_EXPERT_INSTRUCTIONS.replace("Expert", "{{ROLE}}")).toBe( + REALTIME_PROMPT_DOCUMENT, + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "**Expert:** `[receives the exchange after the Spokesperson speaks; no output: zero tokens, no tools, no coordination]`", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "**Spokesperson → Expert, `HANDOFF handoff-7`:**", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "**Expert → Spokesperson, `SAY`, resolves `handoff-7`:**", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "**Expert → Spokesperson, `SAY`:** “A useful follow-up:", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "**User:** “How many months are in a year?”", + ); + expect(REALTIME_PROMPT_DOCUMENT).toContain( + "You might wonder why the sky isn’t violet", + ); + }); +}); + +describe("RealtimeEmissaryProtocol", () => { + it("reserves the user transcript position as soon as server VAD detects speech", () => { + const protocol = new RealtimeEmissaryProtocol(); + + expect( + protocol.handle({ + type: "input_audio_buffer.speech_started", + item_id: "user-1", + }), + ).toEqual([ + { type: "transcript.started", itemId: "user-1", speaker: "user" }, + ]); + }); + + it("streams provisional user and emissary transcripts before finalization", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.delta", + item_id: "user-1", + delta: "How many", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "user-1", + speaker: "user", + text: "How many", + }, + ]); + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.delta", + item_id: "user-1", + delta: " folders?", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "user-1", + speaker: "user", + text: "How many folders?", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-1", + delta: "I'll check", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "assistant-1", + speaker: "emissary", + text: "I'll check", + }, + ]); + }); + + it("keeps multiple audio items from one response in one transcript", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-1", + delta: "Let me think about that.", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "assistant-1", + speaker: "emissary", + text: "Let me think about that.", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-2", + delta: "I received a compact transcript.", + }), + ).toEqual([ + { + type: "transcript.updated", + itemId: "assistant-1", + speaker: "emissary", + text: "Let me think about that. I received a compact transcript.", + }, + ]); + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "Let me think about that.", + }); + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-2", + transcript: "I received a compact transcript.", + }); + expect( + protocol.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 1, + itemId: "assistant-1", + speaker: "emissary", + text: "Let me think about that. I received a compact transcript.", + }, + ]); + }); + + it("emits finalized user and emissary transcripts once in observed order", () => { + const protocol = new RealtimeEmissaryProtocol(); + + expect( + protocol.handle({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "user-1", + transcript: " Hello there. ", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 1, + itemId: "user-1", + speaker: "user", + text: "Hello there.", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "Hi.", + }), + ).toEqual([]); + expect( + protocol.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 2, + itemId: "assistant-1", + speaker: "emissary", + text: "Hi.", + }, + ]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "Hi.", + }), + ).toEqual([]); + }); + + it("forwards interrupted streamed text as explicitly best-effort", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_audio_transcript.delta", + response_id: "response-1", + item_id: "assistant-1", + delta: "This part was heard", + }); + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "This part was never heard.", + }); + expect( + protocol.handle({ + type: "output_audio_buffer.cleared", + response_id: "response-1", + }), + ).toEqual([ + { + type: "transcript.finalized", + id: 1, + itemId: "assistant-1", + speaker: "emissary", + text: "This part was heard", + interrupted: true, + }, + { + type: "emissary.playback_interrupted", + responseId: "response-1", + }, + ]); + + // A late terminal transcript for the interrupted response is still + // generated text, not evidence that the user heard it. + protocol.handle({ + type: "response.output_audio_transcript.done", + response_id: "response-1", + item_id: "assistant-1", + transcript: "This part was never heard.", + }); + + expect( + protocol.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([]); + }); + + it("fails loudly on Realtime server and transcription errors", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect(() => + protocol.handle({ + type: "error", + error: { message: "bad session configuration" }, + }), + ).toThrow("bad session configuration"); + expect(() => + protocol.handle({ + type: "conversation.item.input_audio_transcription.failed", + error: { message: "audio unintelligible" }, + }), + ).toThrow("audio unintelligible"); + }); + + it("ignores empty and non-terminal transcript events", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "response.output_audio_transcript.delta", + item_id: "assistant-1", + delta: "partial", + }), + ).toEqual([]); + expect( + protocol.handle({ + type: "response.output_audio_transcript.done", + item_id: "assistant-1", + transcript: " ", + }), + ).toEqual([]); + }); + + it("assembles a handoff call from streamed arguments", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_item.added", + item: { + type: "function_call", + name: "handoff", + call_id: "call-1", + }, + }); + protocol.handle({ + type: "response.function_call_arguments.delta", + call_id: "call-1", + delta: '{"message":"Please investigate', + }); + protocol.handle({ + type: "response.function_call_arguments.delta", + call_id: "call-1", + delta: ' this."}', + }); + + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + call_id: "call-1", + }), + ).toEqual([ + { + type: "handoff", + callId: "call-1", + message: "Please investigate this.", + }, + ]); + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + name: "handoff", + call_id: "call-1", + arguments: '{"message":"duplicate"}', + }), + ).toEqual([]); + }); + + it("rejects malformed handoff arguments", () => { + const protocol = new RealtimeEmissaryProtocol(); + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + name: "handoff", + call_id: "call-1", + arguments: '{"message":"hello","unexpected":true}', + }), + ).toEqual([ + { + type: "tool_call.invalid", + callId: "call-1", + toolName: "handoff", + error: "handoff accepts only a message argument", + }, + ]); + }); + + it("returns unterminated tool arguments to the emissary for a silent retry", () => { + const protocol = new RealtimeEmissaryProtocol(); + protocol.handle({ + type: "response.output_item.added", + item: { + type: "function_call", + name: "handoff", + call_id: "call-broken", + }, + }); + protocol.handle({ + type: "response.function_call_arguments.delta", + call_id: "call-broken", + delta: '{"message":"Please inspect', + }); + + const [invalidCall] = protocol.handle({ + type: "response.function_call_arguments.done", + call_id: "call-broken", + }); + expect(invalidCall).toMatchObject({ + type: "tool_call.invalid", + callId: "call-broken", + toolName: "handoff", + }); + expect(invalidCall).toHaveProperty( + "error", + expect.stringMatching(/unterminated|JSON/i), + ); + expect( + protocol.handle({ + type: "response.function_call_arguments.done", + call_id: "call-broken", + }), + ).toEqual([]); + + expect( + createInvalidToolCallOutput( + "call-broken", + "handoff", + "JSON Parse error: Unterminated string", + ), + ).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-broken", + output: JSON.stringify({ + accepted: false, + reason: "invalid_arguments", + error: + "handoff arguments were invalid: JSON Parse error: Unterminated string. Retry this tool call with complete valid JSON. Do not speak this internal error to the user.", + }), + }, + }); + }); +}); + +describe("master message injection", () => { + it("adds typed user text and creates a response while idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + + expect(coordinator.requestTypedUserMessage("Typed hello")).toEqual({ + status: "sent", + events: [ + { type: "input_audio_buffer.clear" }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Typed hello" }], + }, + }, + { type: "response.create" }, + ], + }); + }); + + it("interrupts active generation and playback for typed user text", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.requestMasterMessage({ message: "context", mode: "say" }); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect(coordinator.requestTypedUserMessage("New direction")).toEqual({ + status: "interrupting", + events: [ + { type: "response.cancel", response_id: "response-1" }, + { type: "output_audio_buffer.clear" }, + { type: "input_audio_buffer.clear" }, + { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "New direction" }], + }, + }, + ], + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.cleared", + response_id: "response-1", + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("lets a server-VAD barge-in supersede a response before its terminal event", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + coordinator.requestMasterMessage({ + message: "Queued master context.", + mode: "say", + }); + + expect( + coordinator.handle({ + type: "response.created", + response: { id: "response-2" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "cancelled" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-2", status: "completed" }, + }), + ).toEqual([ + { + type: "response.create", + response: { + instructions: + "Speak this Expert message to the user now, preserving its meaning: Queued master context. Be natural, concise, and accurate. Do not call tools.", + tools: [], + tool_choice: "none", + }, + }, + ]); + + expect( + coordinator.requestMasterMessage({ + message: "A later result.", + mode: "say", + }), + ).toMatchObject({ status: "queued" }); + }); + + it("releases handoffs from a SAY displaced by a server-VAD response", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.requestMasterMessage({ + message: "The answer is 21.", + mode: "say", + resolvedHandoffIds: ["handoff-1"], + }); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + + coordinator.handle({ + type: "response.created", + response: { id: "response-2" }, + }); + + expect(coordinator.takeCompletedHandoffIds()).toEqual([]); + expect(coordinator.takeFailedHandoffIds()).toEqual(["handoff-1"]); + }); + + it("creates no emissary event for empty master output", () => { + const coordinator = new RealtimeResponseCoordinator(); + + expect(() => + coordinator.requestMasterMessage({ message: " ", mode: "context" }), + ).toThrow("master message cannot be empty"); + + // Rejection leaves the coordinator idle; no hidden response lifecycle was + // created for the empty master turn. + expect( + coordinator.requestMasterMessage({ + message: "Useful guidance.", + mode: "context", + }).status, + ).toBe("sent"); + }); + + it("injects private master context without requesting a response", () => { + const coordinator = new RealtimeResponseCoordinator(); + + expect( + coordinator.requestMasterMessage({ + message: "Keep this in mind.", + mode: "context", + eventId: "context-1", + }), + ).toEqual({ + status: "sent", + events: [ + { + type: "conversation.item.create", + event_id: "context-1", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: "Private context from the Expert for a future natural turn. Do not respond to this item now:\nKeep this in mind.", + }, + ], + }, + }, + ], + }); + }); + + it("injects a master SAY message and requests a tool-free response", () => { + const coordinator = new RealtimeResponseCoordinator(); + const transport = { send: vi.fn() }; + const events = coordinator.requestMasterMessage({ + message: "Relay the result.", + mode: "say", + eventId: "m1", + }).events; + sendRealtimeEvents(transport, events); + + expect( + transport.send.mock.calls.map(([event]) => JSON.parse(event)), + ).toEqual([ + { + type: "conversation.item.create", + event_id: "m1", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: "The Expert has decided the following information must be spoken to the user now. Speak it naturally and accurately without adding filler or offering more help:\nRelay the result.", + }, + ], + }, + }, + { + type: "response.create", + response: { + instructions: + "Speak this Expert message to the user now, preserving its meaning: Relay the result. Be natural, concise, and accurate. Do not call tools.", + tools: [], + tool_choice: "none", + }, + }, + ]); + }); + + it("speaks queued SAY messages separately and resolves handoffs after playback", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.requestMasterMessage({ + message: "First answer.", + mode: "say", + resolvedHandoffIds: ["handoff-1"], + }); + expect( + coordinator.requestMasterMessage({ + message: "Second answer.", + mode: "say", + resolvedHandoffIds: ["handoff-2"], + }).status, + ).toBe("queued"); + + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + expect.objectContaining({ + type: "response.create", + response: expect.objectContaining({ + instructions: expect.stringContaining("Second answer."), + }), + }), + ]); + expect(coordinator.takeCompletedHandoffIds()).toEqual(["handoff-1"]); + + coordinator.handle({ + type: "response.created", + response: { id: "response-2" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-2", + }); + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-2", + }); + coordinator.handle({ + type: "response.done", + response: { id: "response-2", status: "completed" }, + }); + expect(coordinator.takeCompletedHandoffIds()).toEqual(["handoff-2"]); + }); + + it("keeps a handoff unresolved when its SAY produces no audio", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.requestMasterMessage({ + message: "Answer.", + mode: "say", + resolvedHandoffIds: ["handoff-1"], + }); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }); + + expect(coordinator.takeCompletedHandoffIds()).toEqual([]); + expect(coordinator.takeFailedHandoffIds()).toEqual(["handoff-1"]); + }); + + it("serializes a tool-output follow-up behind the response that called the tool", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + const toolOutput = { + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }; + + expect(coordinator.requestToolOutput(toolOutput)).toEqual({ + status: "queued", + events: [toolOutput], + }); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([{ type: "response.create" }]); + }); + + it("records an accepted handoff result without waking the emissary", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + const toolOutput = { + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }; + + expect(coordinator.recordToolOutput(toolOutput)).toEqual({ + status: "sent", + events: [toolOutput], + }); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + }); + + it("coalesces a Master answer into the queued tool follow-up after playback", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + coordinator.recordToolOutput({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }); + expect( + coordinator.requestMasterMessage({ + message: "The answer is 26.", + mode: "say", + }), + ).toMatchObject({ status: "queued" }); + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + expect.objectContaining({ + type: "response.create", + response: expect.objectContaining({ tools: [], tool_choice: "none" }), + }), + ]); + }); + + it("requests a response immediately for a tool output while idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + const toolOutput = { + type: "conversation.item.create", + item: { type: "function_call_output", call_id: "call-1", output: "{}" }, + }; + + expect(coordinator.requestToolOutput(toolOutput)).toEqual({ + status: "sent", + events: [toolOutput, { type: "response.create" }], + }); + }); + + it("sends SAY immediately without cancelling when the session is idle", () => { + const coordinator = new RealtimeResponseCoordinator(); + + const request = coordinator.requestMasterMessage({ + message: "Keep this in mind.", + mode: "say", + }); + + expect(request.status).toBe("sent"); + expect(request.events.map((event) => event.type)).toEqual([ + "conversation.item.create", + "response.create", + ]); + expect(request.events).not.toContainEqual( + expect.objectContaining({ type: "response.cancel" }), + ); + }); + + it("lets completed generated audio finish playing when no master message is queued", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([]); + + expect( + coordinator.requestMasterMessage({ + message: "A later result.", + mode: "say", + }), + ).toMatchObject({ status: "sent" }); + }); + + it("injects master context immediately but waits for active playback before responding", () => { + const coordinator = new RealtimeResponseCoordinator(); + coordinator.handle({ + type: "response.created", + response: { id: "response-1" }, + }); + coordinator.handle({ + type: "output_audio_buffer.started", + response_id: "response-1", + }); + + expect( + coordinator.requestMasterMessage({ + message: "First master message.", + mode: "say", + }), + ).toEqual({ + status: "queued", + events: [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ + content: [ + expect.objectContaining({ + text: expect.stringContaining("First master message."), + }), + ], + }), + }), + ], + }); + expect( + coordinator.requestMasterMessage({ + message: "Second master message.", + mode: "say", + }), + ).toEqual({ + status: "queued", + events: [ + expect.objectContaining({ + type: "conversation.item.create", + item: expect.objectContaining({ + content: [ + expect.objectContaining({ + text: expect.stringContaining("Second master message."), + }), + ], + }), + }), + ], + }); + + expect( + coordinator.handle({ + type: "response.done", + response: { id: "response-1", status: "completed" }, + }), + ).toEqual([]); + expect( + coordinator.handle({ + type: "output_audio_buffer.stopped", + response_id: "response-1", + }), + ).toEqual([ + expect.objectContaining({ + type: "response.create", + response: expect.objectContaining({ tools: [], tool_choice: "none" }), + }), + ]); + }); + + it("includes an accepted handoff id in the tool result", () => { + expect( + createHandoffToolOutput("call-2", { + accepted: true, + handoff_id: "handoff-4", + }), + ).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-2", + output: '{"accepted":true,"handoff_id":"handoff-4"}', + }, + }); + }); +}); + +describe("DirectMessagePipe", () => { + it("starts each call in its assigned cursor namespace", () => { + const pipe = new DirectMessagePipe(12_000_000); + + expect(pipe.cursor("master")).toBe(12_000_000); + expect(pipe.cursor("emissary")).toBe(12_000_000); + expect( + pipe.send({ + sender: "emissary", + cursor: 12_000_000, + message: "Call-scoped message.", + }), + ).toMatchObject({ + accepted: true, + outbound: { id: 12_000_001, senderCursor: 12_000_000 }, + }); + }); + + it("allows the active sender to queue multiple messages", () => { + const pipe = new DirectMessagePipe(); + const first = pipe.send({ + sender: "emissary", + cursor: 0, + message: "First detail.", + }); + const second = pipe.send({ + sender: "emissary", + cursor: 0, + message: "Second detail.", + }); + expect(first).toMatchObject({ + accepted: true, + outbound: { id: 1, sender: "emissary" }, + }); + expect(second).toMatchObject({ + accepted: true, + outbound: { id: 2, sender: "emissary" }, + }); + if (!first.accepted || !second.accepted) + throw new Error("expected an accepted batch"); + + expect( + pipe.send({ sender: "master", cursor: 0, message: "Reply." }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + cursor: 0, + }); + expect( + pipe.send({ sender: "master", cursor: 2, message: "Reply." }), + ).toMatchObject({ + accepted: true, + cursor: 2, + outbound: { id: 3, sender: "master", senderCursor: 2 }, + }); + expect(pipe.cursor("master")).toBe(2); + }); + + it("requires the cursor for the complete pending batch", () => { + const pipe = new DirectMessagePipe(); + const first = pipe.send({ sender: "master", cursor: 0, message: "One." }); + const second = pipe.send({ sender: "master", cursor: 0, message: "Two." }); + if (!first.accepted || !second.accepted) + throw new Error("expected an accepted batch"); + + expect( + pipe.send({ sender: "emissary", cursor: 1, message: "Too soon." }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + cursor: 0, + }); + expect( + pipe.send({ sender: "emissary", cursor: 2, message: "Now reply." }), + ).toMatchObject({ + accepted: true, + cursor: 2, + outbound: { senderCursor: 2 }, + }); + expect(pipe.cursor("emissary")).toBe(2); + }); + + it("rejects a stale send without consuming the pending direction", () => { + const pipe = new DirectMessagePipe(); + const master = pipe.send({ + sender: "master", + cursor: 0, + message: "Result.", + }); + if (!master.accepted) throw new Error("expected accepted message"); + + expect( + pipe.send({ + sender: "emissary", + cursor: 0, + message: "Stale reply.", + }), + ).toEqual({ + accepted: false, + reason: "pipe_busy", + cursor: 0, + }); + const reply = pipe.send({ + sender: "emissary", + cursor: master.outbound.id, + message: "Fresh reply.", + }); + expect(reply).toMatchObject({ + accepted: true, + cursor: 1, + outbound: { + sender: "emissary", + recipient: "master", + senderCursor: 1, + }, + }); + expect(pipe.cursor("emissary")).toBe(master.outbound.id); + }); + + it("exposes the latest inbound cursor to trusted delivery boundaries", () => { + const pipe = new DirectMessagePipe(); + const first = pipe.send({ + sender: "master", + cursor: 0, + message: "Context.", + }); + const second = pipe.send({ + sender: "master", + cursor: 0, + message: "More context.", + }); + if (!first.accepted || !second.accepted) + throw new Error("expected an accepted batch"); + + expect(pipe.deliveryCursor("emissary")).toBe(second.outbound.id); + expect(pipe.deliveryCursor("master")).toBe(0); + + const reverse = pipe.send({ + sender: "emissary", + cursor: pipe.deliveryCursor("emissary"), + message: "Transcript.", + }); + expect(reverse).toMatchObject({ + accepted: true, + cursor: second.outbound.id, + outbound: { sender: "emissary" }, + }); + expect(pipe.deliveryCursor("emissary")).toBe(second.outbound.id); + expect(pipe.deliveryCursor("master")).toBe( + reverse.accepted ? reverse.outbound.id : -1, + ); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts new file mode 100644 index 000000000..065d88b1a --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeEmissaryProtocol.ts @@ -0,0 +1,1005 @@ +import promptDocument from "../prompts/expert-spokesperson.md?raw"; + +export const REALTIME_USER_TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; +export const REALTIME_EMISSARY_TRANSCRIPT_COMPLETED_EVENT = + "response.output_audio_transcript.done"; +export const HANDOFF_TOOL_NAME = "handoff"; +export const SEND_TO_SPOKESPERSON_TOOL_NAME = "send_to_spokesperson"; + +export const REALTIME_PROMPT_DOCUMENT = promptDocument.trim(); +const REALTIME_ROLE_PLACEHOLDER = "{{ROLE}}"; + +function createRealtimeRoleInstructions( + role: "Expert" | "Spokesperson", +): string { + const normalized = REALTIME_PROMPT_DOCUMENT.replaceAll("\r\n", "\n").trim(); + const placeholderCount = + normalized.split(REALTIME_ROLE_PLACEHOLDER).length - 1; + if (placeholderCount !== 1) { + throw new Error( + `Realtime prompt must contain exactly one ${REALTIME_ROLE_PLACEHOLDER} placeholder.`, + ); + } + return normalized.replace(REALTIME_ROLE_PLACEHOLDER, role); +} + +export const REALTIME_SPOKESPERSON_INSTRUCTIONS = + createRealtimeRoleInstructions("Spokesperson"); +export const REALTIME_EXPERT_INSTRUCTIONS = + createRealtimeRoleInstructions("Expert"); + +export interface RealtimeEventTransport { + send(data: string): void; +} + +export interface RealtimeEmissarySessionOptions { + /** Reasoning configuration is emitted only for model families that support it. */ + model?: string; + transcriptionModel?: string; + transcriptionLanguage?: string; + transcriptionPrompt?: string; + voice?: string; + speed?: number; + turnDetection?: "server_vad" | "semantic_vad"; + eagerness?: "low" | "medium" | "high" | "auto"; + interruptResponse?: boolean; + createResponse?: boolean; + vadThreshold?: number; + prefixPaddingMs?: number; + silenceDurationMs?: number; + idleTimeoutMs?: number | null; + noiseReduction?: "off" | "near_field" | "far_field"; + reasoningEffort?: "default" | "none" | "low" | "medium" | "high"; + maxOutputTokens?: number | null; +} + +export type FinalizedRealtimeTranscript = { + type: "transcript.finalized"; + id: number; + itemId: string; + speaker: "user" | "emissary"; + text: string; + /** Best-effort streamed text; it may not exactly match the audio heard. */ + interrupted?: true; +}; + +export type UpdatedRealtimeTranscript = { + type: "transcript.updated"; + itemId: string; + speaker: "user" | "emissary"; + text: string; +}; + +export type StartedRealtimeTranscript = { + type: "transcript.started"; + itemId: string; + speaker: "user"; +}; + +export type HandoffCall = { + type: "handoff"; + callId: string; + message: string; +}; + +export type InvalidToolCall = { + type: "tool_call.invalid"; + callId: string; + toolName: typeof HANDOFF_TOOL_NAME; + error: string; +}; + +export type RealtimePlaybackInterrupted = { + type: "emissary.playback_interrupted"; + responseId: string; +}; + +export type RealtimeEmissaryProtocolEvent = + | StartedRealtimeTranscript + | UpdatedRealtimeTranscript + | FinalizedRealtimeTranscript + | HandoffCall + | InvalidToolCall + | RealtimePlaybackInterrupted; + +export type RealtimeClientEvent = Record; +type RealtimeServerEvent = Record; + +type PendingEmissaryTranscriptItem = { + streamedText: string; + finalText?: string; +}; + +type PendingEmissaryTranscript = { + displayItemId: string; + items: Map; +}; + +export function createRealtimeEmissarySessionUpdate( + options: RealtimeEmissarySessionOptions = {}, +): RealtimeServerEvent { + const transcriptionLanguage = options.transcriptionLanguage?.trim(); + const transcriptionPrompt = options.transcriptionPrompt?.trim(); + const supportsReasoning = + !options.model || options.model.startsWith("gpt-realtime-2.1"); + const turnDetection = + options.turnDetection === "semantic_vad" + ? { + type: "semantic_vad", + eagerness: options.eagerness ?? "auto", + create_response: options.createResponse ?? true, + interrupt_response: options.interruptResponse ?? true, + } + : { + type: "server_vad", + threshold: options.vadThreshold ?? 0.5, + prefix_padding_ms: options.prefixPaddingMs ?? 300, + silence_duration_ms: options.silenceDurationMs ?? 500, + ...(options.idleTimeoutMs + ? { idle_timeout_ms: options.idleTimeoutMs } + : {}), + create_response: options.createResponse ?? true, + interrupt_response: options.interruptResponse ?? true, + }; + const defaults = { + type: "realtime", + output_modalities: ["audio"], + ...(supportsReasoning && + options.reasoningEffort && + options.reasoningEffort !== "default" + ? { reasoning: { effort: options.reasoningEffort } } + : {}), + max_output_tokens: options.maxOutputTokens ?? "inf", + instructions: REALTIME_SPOKESPERSON_INSTRUCTIONS, + audio: { + input: { + format: { type: "audio/pcm", rate: 24_000 }, + transcription: { + model: options.transcriptionModel ?? "gpt-realtime-whisper", + ...(transcriptionLanguage ? { language: transcriptionLanguage } : {}), + ...(transcriptionPrompt ? { prompt: transcriptionPrompt } : {}), + }, + noise_reduction: + options.noiseReduction && options.noiseReduction !== "off" + ? { type: options.noiseReduction } + : null, + turn_detection: turnDetection, + }, + output: { + format: { type: "audio/pcm", rate: 24_000 }, + voice: options.voice ?? "marin", + speed: options.speed ?? 1, + }, + }, + tools: [ + { + type: "function", + name: HANDOFF_TOOL_NAME, + description: + "Hand unresolved work or an authoritative question to the Expert. Every accepted handoff must eventually be answered or explicitly dismissed.", + parameters: { + type: "object", + properties: { + message: { + type: "string", + description: + "The concise unresolved request the Expert now owns.", + }, + }, + required: ["message"], + additionalProperties: false, + }, + }, + ], + tool_choice: "auto", + }; + + return { + type: "session.update", + session: defaults, + }; +} + +export function configureRealtimeEmissarySession( + transport: RealtimeEventTransport, + options: RealtimeEmissarySessionOptions = {}, +): void { + sendRealtimeEvents(transport, [createRealtimeEmissarySessionUpdate(options)]); +} + +export function sendRealtimeEvents( + transport: RealtimeEventTransport, + events: readonly RealtimeClientEvent[], +): void { + for (const event of events) sendEvent(transport, event); +} + +export type MasterMessageMode = "context" | "say"; + +function createMasterMessageItem(options: MasterMessage): RealtimeClientEvent { + const message = requireNonEmpty(options.message, "master message"); + const text = + options.mode === "say" + ? `The Expert has decided the following information must be spoken to the user now. Speak it naturally and accurately without adding filler or offering more help:\n${message}` + : `Private context from the Expert for a future natural turn. Do not respond to this item now:\n${message}`; + const createItem: RealtimeServerEvent = { + type: "conversation.item.create", + item: { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text, + }, + ], + }, + }; + if (options.eventId) createItem.event_id = options.eventId; + + return createItem; +} + +function createMasterSayResponseEvent(message: string): RealtimeClientEvent { + return { + type: "response.create", + response: { + instructions: `Speak this Expert message to the user now, preserving its meaning: ${requireNonEmpty(message, "master message")} Be natural, concise, and accurate. Do not call tools.`, + tools: [], + tool_choice: "none", + }, + }; +} + +function createTypedUserMessageItem(text: string): RealtimeClientEvent { + return { + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [ + { type: "input_text", text: requireNonEmpty(text, "user text") }, + ], + }, + }; +} + +export function createHandoffToolOutput( + callId: string, + exchange: HandoffToolResult, +): RealtimeServerEvent { + return { + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: requireNonEmpty(callId, "call id"), + output: JSON.stringify(exchange), + }, + }; +} + +export function createInvalidToolCallOutput( + callId: string, + toolName: string, + error: string, +): RealtimeServerEvent { + return { + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: requireNonEmpty(callId, "call id"), + output: JSON.stringify({ + accepted: false, + reason: "invalid_arguments", + error: `${requireNonEmpty(toolName, "tool name")} arguments were invalid: ${requireNonEmpty(error, "tool error")}. Retry this tool call with complete valid JSON. Do not speak this internal error to the user.`, + }), + }, + }; +} + +type MasterMessage = { + message: string; + mode: MasterMessageMode; + eventId?: string; + resolvedHandoffIds?: string[]; +}; + +export type MasterMessageRequest = { + status: "sent" | "interrupting" | "queued"; + events: RealtimeClientEvent[]; +}; + +type ActiveResponse = { + id?: string; + generationDone: boolean; + outputActive: boolean; + outputProduced: boolean; + succeeded: boolean; + say?: MasterMessage; +}; + +type PendingResponse = + | { mode: "default" } + | { mode: "say"; message: MasterMessage }; + +/** + * Serializes master-triggered responses with the default-conversation response + * lifecycle. Master context is injected immediately, but a follow-up response + * waits until the prior response and its WebRTC playback are terminal. This + * prevents late master results from cutting off the emissary mid-sentence. + */ +export class RealtimeResponseCoordinator { + private activeResponse: ActiveResponse | undefined; + private pendingResponses: PendingResponse[] = []; + private completedHandoffIds: string[] = []; + private failedHandoffIds: string[] = []; + + requestMasterMessage(message: MasterMessage): MasterMessageRequest { + requireNonEmpty(message.message, "master message"); + if (message.mode === "context") { + return { status: "sent", events: [createMasterMessageItem(message)] }; + } + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(message); + return { + status: "sent", + events: [ + createMasterMessageItem(message), + createMasterSayResponseEvent(message.message), + ], + }; + } + + this.pendingResponses.push({ mode: "say", message }); + return { + status: "queued", + events: [createMasterMessageItem(message)], + }; + } + + requestToolOutput(event: RealtimeClientEvent): MasterMessageRequest { + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(); + return { + status: "sent", + events: [event, { type: "response.create" }], + }; + } + + this.queueDefaultResponse(); + return { status: "queued", events: [event] }; + } + + recordToolOutput(event: RealtimeClientEvent): MasterMessageRequest { + return { status: "sent", events: [event] }; + } + + requestTypedUserMessage(text: string): MasterMessageRequest { + const item = createTypedUserMessageItem(text); + if (!this.activeResponse) { + this.activeResponse = awaitingCreatedResponse(); + return { + status: "sent", + events: [ + { type: "input_audio_buffer.clear" }, + item, + { type: "response.create" }, + ], + }; + } + + this.queueDefaultResponse(); + const events: RealtimeClientEvent[] = []; + if (this.activeResponse.id && !this.activeResponse.generationDone) { + events.push({ + type: "response.cancel", + response_id: this.activeResponse.id, + }); + } + if (this.activeResponse.outputActive) { + events.push({ type: "output_audio_buffer.clear" }); + } + events.push({ type: "input_audio_buffer.clear" }, item); + return { + status: this.activeResponse.id ? "interrupting" : "queued", + events, + }; + } + + handle(event: unknown): RealtimeClientEvent[] { + if (!isRecord(event)) return []; + switch (event.type) { + case "response.created": { + const responseId = nestedResponseId(event); + if (!responseId) + throw new Error("response.created is missing response.id"); + const requestedSay = this.activeResponse?.id + ? undefined + : this.activeResponse?.say; + if (this.activeResponse?.id) { + if (this.activeResponse.say) { + this.failedHandoffIds.push( + ...(this.activeResponse.say.resolvedHandoffIds ?? []), + ); + } + // Server VAD owns microphone barge-in and may create the replacement + // response before the cancelled response's terminal events arrive. + // Conversation items already queued for a follow-up are visible to + // this replacement response, so it also satisfies that pending wake. + this.pendingResponses = this.pendingResponses.filter( + (pending) => pending.mode === "say", + ); + } + this.activeResponse = { + id: responseId, + generationDone: false, + outputActive: false, + outputProduced: false, + succeeded: false, + say: requestedSay, + }; + return []; + } + case "output_audio_buffer.started": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.outputActive = true; + active.outputProduced = true; + return []; + } + case "response.done": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.generationDone = true; + active.succeeded = nestedResponseStatus(event) === "completed"; + if (!active.outputActive) return this.finishActiveResponse(); + return []; + } + case "output_audio_buffer.stopped": + case "output_audio_buffer.cleared": { + const active = this.matchActiveResponse(event); + if (!active) return []; + active.outputActive = false; + return active.generationDone ? this.finishActiveResponse() : []; + } + default: + return []; + } + } + + private matchActiveResponse( + event: RealtimeServerEvent, + ): ActiveResponse | undefined { + const active = this.activeResponse; + if (!active) return undefined; + const responseId = + optionalString(event.response_id) ?? nestedResponseId(event); + return !responseId || !active.id || responseId === active.id + ? active + : undefined; + } + + private finishActiveResponse(): RealtimeClientEvent[] { + const completed = this.activeResponse; + this.activeResponse = undefined; + if (completed?.say) { + const target = + completed.succeeded && completed.outputProduced + ? this.completedHandoffIds + : this.failedHandoffIds; + target.push(...(completed.say.resolvedHandoffIds ?? [])); + } + const pending = this.pendingResponses.shift(); + if (!pending) return []; + this.activeResponse = awaitingCreatedResponse( + pending.mode === "say" ? pending.message : undefined, + ); + return [ + pending.mode === "say" + ? createMasterSayResponseEvent(pending.message.message) + : { type: "response.create" }, + ]; + } + + takeCompletedHandoffIds(): string[] { + return this.completedHandoffIds.splice(0); + } + + takeFailedHandoffIds(): string[] { + return this.failedHandoffIds.splice(0); + } + + private queueDefaultResponse(): void { + if (!this.pendingResponses.some((pending) => pending.mode === "default")) { + this.pendingResponses.push({ mode: "default" }); + } + } +} + +function awaitingCreatedResponse(say?: MasterMessage): ActiveResponse { + return { + generationDone: false, + outputActive: false, + outputProduced: false, + succeeded: false, + say, + }; +} + +function nestedResponseStatus(event: RealtimeServerEvent): string | undefined { + return isRecord(event.response) + ? optionalString(event.response.status) + : undefined; +} + +export type DirectMessagePeer = "master" | "emissary"; + +export type DirectBridgeMessage = { + id: number; + sender: DirectMessagePeer; + recipient: DirectMessagePeer; + senderCursor: number; + message: string; +}; + +export type DirectMessageExchange = + | { + accepted: true; + outbound: DirectBridgeMessage; + cursor: number; + } + | { + accepted: false; + reason: "pipe_busy" | "stale_cursor"; + cursor: number; + }; + +export type HandoffToolResult = { + accepted: true; + handoff_id: string; +}; + +/** + * One authoritative half-duplex pipe for every event crossing between the + * realtime conversation and the master. The active sender may append any + * number of messages; only a send in the opposite direction is blocked until + * the recipient consumes the pending batch. The recipient consumes the + * complete pending batch by supplying its latest message id as the cursor on + * a reverse send; consumption, direction reversal, and reply enqueueing + * happen atomically. A stale reverse send neither exposes nor consumes + * pending messages. + */ +export class DirectMessagePipe { + private nextMessageId: number; + private pending: DirectBridgeMessage[] = []; + private readonly consumedCursor: Record; + + constructor(initialCursor = 0) { + this.nextMessageId = initialCursor + 1; + this.consumedCursor = { + master: initialCursor, + emissary: initialCursor, + }; + } + + send(options: { + sender: DirectMessagePeer; + cursor: number; + message: string; + }): DirectMessageExchange { + const message = requireNonEmpty(options.message, "direct message"); + const suppliedCursor = requireCursor(options.cursor); + const activeMessage = this.pending[0]; + if (activeMessage && activeMessage.sender !== options.sender) { + const latestPending = this.pending.at(-1); + if (!latestPending) + throw new Error("direct-message pending batch cannot be empty"); + if (suppliedCursor !== latestPending.id) { + return { + accepted: false, + reason: "pipe_busy", + cursor: this.consumedCursor[options.sender], + }; + } + this.consumedCursor[options.sender] = latestPending.id; + this.pending = []; + } + const cursor = this.consumedCursor[options.sender]; + if (suppliedCursor !== cursor) { + return { + accepted: false, + reason: "stale_cursor", + cursor, + }; + } + + const outbound: DirectBridgeMessage = { + id: this.nextMessageId++, + sender: options.sender, + recipient: otherPeer(options.sender), + senderCursor: cursor, + message, + }; + this.pending.push(outbound); + return { + accepted: true, + outbound, + cursor, + }; + } + + cursor(peer: DirectMessagePeer): number { + return this.consumedCursor[peer]; + } + + /** + * Cursor available at a trusted delivery boundary. If the peer has pending + * inbound messages, transport delivery proves it has received the complete + * batch; otherwise its last explicitly consumed cursor remains current. + * Model-authored tool calls must continue to supply their own cursor. + */ + deliveryCursor(peer: DirectMessagePeer): number { + const latestPending = this.pending.at(-1); + return latestPending?.recipient === peer + ? latestPending.id + : this.consumedCursor[peer]; + } +} + +function otherPeer(peer: DirectMessagePeer): DirectMessagePeer { + return peer === "master" ? "emissary" : "master"; +} + +/** + * Reduces Realtime server events to the durable bridge events Berd needs. + * One instance belongs to one Realtime session; it supplies local ordering and + * suppresses repeated terminal events by their stable OpenAI item/call ids. + */ +export class RealtimeEmissaryProtocol { + private nextTranscriptId = 1; + private readonly finalizedItemIds = new Set(); + private readonly completedCallIds = new Set(); + private readonly callNames = new Map(); + private readonly argumentDeltas = new Map(); + private readonly pendingUserTranscripts = new Map(); + private readonly pendingEmissaryTranscripts = new Map< + string, + PendingEmissaryTranscript + >(); + private readonly interruptedResponseIds = new Set(); + + handle(event: unknown): RealtimeEmissaryProtocolEvent[] { + if (!isRecord(event)) return []; + + switch (event.type) { + case "error": + case "conversation.item.input_audio_transcription.failed": + throw new Error(realtimeErrorMessage(event)); + case "response.output_item.added": + this.captureFunctionCallName(event); + return []; + case "response.function_call_arguments.delta": + this.captureFunctionArguments(event); + return []; + case "response.function_call_arguments.done": { + try { + const call = this.finishFunctionCall(event); + return call ? [call] : []; + } catch (error) { + const invalidCall = this.invalidFunctionCall(event, error); + if (!invalidCall) throw error; + return [invalidCall]; + } + } + case "input_audio_buffer.speech_started": { + const itemId = optionalString(event.item_id); + return itemId && !this.finalizedItemIds.has(itemId) + ? [{ type: "transcript.started", itemId, speaker: "user" }] + : []; + } + case REALTIME_USER_TRANSCRIPT_COMPLETED_EVENT: { + const transcript = this.finalizedTranscript(event, "user"); + return transcript ? [transcript] : []; + } + case "conversation.item.input_audio_transcription.delta": { + const transcript = this.captureUserTranscriptDelta(event); + return transcript ? [transcript] : []; + } + case REALTIME_EMISSARY_TRANSCRIPT_COMPLETED_EVENT: { + this.captureEmissaryTranscript(event); + return []; + } + case "response.output_audio_transcript.delta": + return this.captureEmissaryTranscriptDelta(event); + case "output_audio_buffer.stopped": { + const transcript = this.finishEmissaryPlayback(event); + return transcript ? [transcript] : []; + } + case "output_audio_buffer.cleared": { + const responseId = optionalString(event.response_id); + if (!responseId) return []; + this.interruptedResponseIds.add(responseId); + const transcript = this.finishInterruptedPlayback(responseId); + return [ + ...(transcript ? [transcript] : []), + { type: "emissary.playback_interrupted", responseId }, + ]; + } + default: + return []; + } + } + + private captureUserTranscriptDelta( + event: RealtimeServerEvent, + ): UpdatedRealtimeTranscript | undefined { + const itemId = optionalString(event.item_id); + const delta = optionalString(event.delta); + if (!itemId || delta === undefined || this.finalizedItemIds.has(itemId)) + return undefined; + const text = `${this.pendingUserTranscripts.get(itemId) ?? ""}${delta}`; + this.pendingUserTranscripts.set(itemId, text); + if (!text.trim()) return undefined; + return { type: "transcript.updated", itemId, speaker: "user", text }; + } + + private captureEmissaryTranscriptDelta( + event: RealtimeServerEvent, + ): UpdatedRealtimeTranscript[] { + const responseId = optionalString(event.response_id); + const itemId = optionalString(event.item_id); + const delta = optionalString(event.delta); + if ( + !responseId || + !itemId || + delta === undefined || + this.interruptedResponseIds.has(responseId) + ) { + return []; + } + const pending = this.pendingEmissaryTranscript(responseId, itemId); + const item = pending.items.get(itemId) ?? { streamedText: "" }; + item.streamedText += delta; + pending.items.set(itemId, item); + const streamedText = combinedEmissaryTranscript(pending, false); + return streamedText.trim() + ? [ + { + type: "transcript.updated", + itemId: pending.displayItemId, + speaker: "emissary", + text: streamedText, + }, + ] + : []; + } + + private captureEmissaryTranscript(event: RealtimeServerEvent): void { + const responseId = optionalString(event.response_id); + const itemId = optionalString(event.item_id); + const text = optionalString(event.transcript)?.trim(); + if ( + !responseId || + !itemId || + !text || + this.finalizedItemIds.has(itemId) || + this.interruptedResponseIds.has(responseId) + ) { + return; + } + const pending = this.pendingEmissaryTranscript(responseId, itemId); + const item = pending.items.get(itemId) ?? { streamedText: "" }; + item.finalText = text; + pending.items.set(itemId, item); + } + + private finishEmissaryPlayback( + event: RealtimeServerEvent, + ): FinalizedRealtimeTranscript | undefined { + const responseId = optionalString(event.response_id); + if (!responseId) return undefined; + if (this.interruptedResponseIds.delete(responseId)) return undefined; + const pending = this.pendingEmissaryTranscripts.get(responseId); + this.pendingEmissaryTranscripts.delete(responseId); + const text = pending + ? combinedEmissaryTranscript(pending, true).trim() + : ""; + if (!pending || !text || this.finalizedItemIds.has(pending.displayItemId)) { + return undefined; + } + + for (const itemId of pending.items.keys()) { + this.finalizedItemIds.add(itemId); + } + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId: pending.displayItemId, + speaker: "emissary", + text, + }; + } + + private finishInterruptedPlayback( + responseId: string, + ): FinalizedRealtimeTranscript | undefined { + const pending = this.pendingEmissaryTranscripts.get(responseId); + this.pendingEmissaryTranscripts.delete(responseId); + const text = pending + ? combinedEmissaryTranscript(pending, false).trim() + : ""; + if (!pending || !text || this.finalizedItemIds.has(pending.displayItemId)) { + return undefined; + } + + for (const itemId of pending.items.keys()) { + this.finalizedItemIds.add(itemId); + } + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId: pending.displayItemId, + speaker: "emissary", + text, + interrupted: true, + }; + } + + private pendingEmissaryTranscript( + responseId: string, + itemId: string, + ): PendingEmissaryTranscript { + const existing = this.pendingEmissaryTranscripts.get(responseId); + if (existing) return existing; + const pending = { + displayItemId: itemId, + items: new Map(), + }; + this.pendingEmissaryTranscripts.set(responseId, pending); + return pending; + } + + private finalizedTranscript( + event: RealtimeServerEvent, + speaker: "user" | "emissary", + ): FinalizedRealtimeTranscript | undefined { + const itemId = optionalString(event.item_id); + const text = optionalString(event.transcript)?.trim(); + if (!itemId || !text || this.finalizedItemIds.has(itemId)) return undefined; + + this.pendingUserTranscripts.delete(itemId); + this.finalizedItemIds.add(itemId); + return { + type: "transcript.finalized", + id: this.nextTranscriptId++, + itemId, + speaker, + text, + }; + } + + private captureFunctionCallName(event: RealtimeServerEvent): void { + const item = isRecord(event.item) ? event.item : undefined; + if (item?.type !== "function_call") return; + const callId = optionalString(item.call_id); + const name = optionalString(item.name); + if (callId && name) this.callNames.set(callId, name); + } + + private captureFunctionArguments(event: RealtimeServerEvent): void { + const callId = optionalString(event.call_id); + const delta = optionalString(event.delta); + if (!callId || delta === undefined) return; + this.argumentDeltas.set( + callId, + (this.argumentDeltas.get(callId) ?? "") + delta, + ); + } + + private finishFunctionCall( + event: RealtimeServerEvent, + ): HandoffCall | undefined { + const callId = optionalString(event.call_id); + if (!callId || this.completedCallIds.has(callId)) return undefined; + + const name = optionalString(event.name) ?? this.callNames.get(callId); + if (name !== HANDOFF_TOOL_NAME) return undefined; + + const serializedArguments = + optionalString(event.arguments) ?? this.argumentDeltas.get(callId); + if (!serializedArguments) return undefined; + + const parsed: unknown = JSON.parse(serializedArguments); + if (!isRecord(parsed)) + throw new Error("handoff arguments must be an object"); + const keys = Object.keys(parsed).sort(); + if (keys.length !== 1 || keys[0] !== "message") { + throw new Error("handoff accepts only a message argument"); + } + const message = requireNonEmpty(parsed.message, "handoff message"); + + this.completedCallIds.add(callId); + this.argumentDeltas.delete(callId); + this.callNames.delete(callId); + return { type: "handoff", callId, message }; + } + + private invalidFunctionCall( + event: RealtimeServerEvent, + error: unknown, + ): InvalidToolCall | undefined { + const callId = optionalString(event.call_id); + if (!callId || this.completedCallIds.has(callId)) return undefined; + const name = optionalString(event.name) ?? this.callNames.get(callId); + if (name !== HANDOFF_TOOL_NAME) return undefined; + + this.completedCallIds.add(callId); + this.argumentDeltas.delete(callId); + this.callNames.delete(callId); + return { + type: "tool_call.invalid", + callId, + toolName: name, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function sendEvent( + transport: RealtimeEventTransport, + event: RealtimeClientEvent, +): void { + transport.send(JSON.stringify(event)); +} + +function combinedEmissaryTranscript( + pending: PendingEmissaryTranscript, + preferFinalText: boolean, +): string { + return [...pending.items.values()] + .map((item) => + preferFinalText && item.finalText !== undefined + ? item.finalText + : item.streamedText, + ) + .map((text) => text.trim()) + .filter(Boolean) + .join(" "); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function nestedResponseId(event: RealtimeServerEvent): string | undefined { + const response = isRecord(event.response) ? event.response : undefined; + return optionalString(response?.id); +} + +function requireNonEmpty(value: unknown, field: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`${field} cannot be empty`); + } + return value.trim(); +} + +function requireCursor(value: unknown): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error("direct-message cursor must be a non-negative integer"); + } + return value; +} + +function realtimeErrorMessage(event: RealtimeServerEvent): string { + const error = isRecord(event.error) ? event.error : undefined; + return ( + optionalString(error?.message) ?? + optionalString(event.message) ?? + "OpenAI Realtime reported an unknown error" + ); +} diff --git a/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts b/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts new file mode 100644 index 000000000..476183b7a --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePreference.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + getRealtimeVoicePreference, + setRealtimeVoicePreference, +} from "./realtimeVoicePreference"; + +describe("realtime voice preferences", () => { + beforeEach(() => window.localStorage.clear()); + + it("returns a stable default snapshot", () => { + expect(getRealtimeVoicePreference()).toBe(getRealtimeVoicePreference()); + expect(getRealtimeVoicePreference()).toMatchObject({ + presentationMode: "debug", + model: "gpt-realtime-2.1", + transcriptionModel: "gpt-realtime-whisper", + voice: "marin", + speed: 1, + turnDetection: "server_vad", + interruptResponse: true, + createResponse: true, + }); + }); + + it("persists an updated configuration without storing a secret", () => { + const preference = { + ...getRealtimeVoicePreference(), + model: "gpt-realtime-2.1", + transcriptionModel: "gpt-live-transcribe", + voice: "cedar", + speed: 1.25, + presentationMode: "subtle" as const, + turnDetection: "semantic_vad" as const, + eagerness: "high" as const, + }; + setRealtimeVoicePreference(preference); + expect(getRealtimeVoicePreference()).toBe(preference); + expect( + window.localStorage.getItem("goose:openai-realtime-voice-options"), + ).not.toContain("apiKey"); + }); + + it("falls back to normal speed when persisted speed is out of range", () => { + window.localStorage.setItem( + "goose:openai-realtime-voice-options", + JSON.stringify({ speed: 2 }), + ); + + expect(getRealtimeVoicePreference().speed).toBe(1); + }); + + it("rounds and clamps persisted integer-only settings", () => { + window.localStorage.setItem( + "goose:openai-realtime-voice-options", + JSON.stringify({ + prefixPaddingMs: -20.4, + silenceDurationMs: 3_500.6, + idleTimeoutMs: 1_499.5, + maxOutputTokens: 4_500.2, + }), + ); + + expect(getRealtimeVoicePreference()).toMatchObject({ + prefixPaddingMs: 0, + silenceDurationMs: 3_000, + idleTimeoutMs: 1_500, + maxOutputTokens: 4_096, + }); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeVoicePreference.ts b/src/features/voice-conversation/lib/realtimeVoicePreference.ts new file mode 100644 index 000000000..f49808717 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePreference.ts @@ -0,0 +1,221 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export type RealtimeTurnDetection = "server_vad" | "semantic_vad"; +export type RealtimeEagerness = "low" | "medium" | "high" | "auto"; +export type RealtimeNoiseReduction = "off" | "near_field" | "far_field"; +export type RealtimePresentationMode = "debug" | "subtle"; +export type RealtimeReasoningEffort = + | "default" + | "none" + | "low" + | "medium" + | "high"; + +export interface RealtimeVoicePreference { + presentationMode: RealtimePresentationMode; + model: string; + transcriptionModel: string; + voice: string; + speed: number; + turnDetection: RealtimeTurnDetection; + eagerness: RealtimeEagerness; + interruptResponse: boolean; + createResponse: boolean; + vadThreshold: number; + prefixPaddingMs: number; + silenceDurationMs: number; + idleTimeoutMs: number | null; + noiseReduction: RealtimeNoiseReduction; + transcriptionLanguage: string; + transcriptionPrompt: string; + reasoningEffort: RealtimeReasoningEffort; + maxOutputTokens: number | null; +} + +const DEFAULT_PREFERENCE: RealtimeVoicePreference = { + presentationMode: import.meta.env.DEV ? "debug" : "subtle", + model: "gpt-realtime-2.1", + transcriptionModel: "gpt-realtime-whisper", + voice: "marin", + speed: 1, + turnDetection: "server_vad", + eagerness: "auto", + interruptResponse: true, + createResponse: true, + vadThreshold: 0.5, + prefixPaddingMs: 300, + silenceDurationMs: 500, + idleTimeoutMs: null, + noiseReduction: "off", + transcriptionLanguage: "", + transcriptionPrompt: "", + reasoningEffort: "default", + maxOutputTokens: null, +}; +const STORAGE_KEY = "goose:openai-realtime-voice-options"; +const CHANGED_EVENT = "goose:openai-realtime-voice-options-changed"; +const listeners = new Set<() => void>(); +let cachedRaw: string | null | undefined; +let cachedPreference = DEFAULT_PREFERENCE; + +function stringPreference(value: unknown, fallback: string): string { + return typeof value === "string" && value.trim() ? value : fallback; +} + +function enumPreference( + value: unknown, + values: readonly T[], + fallback: T, +): T { + return typeof value === "string" && values.includes(value as T) + ? (value as T) + : fallback; +} + +function numberPreference( + value: unknown, + minimum: number, + maximum: number, + fallback: number, +): number { + return typeof value === "number" && + Number.isFinite(value) && + value >= minimum && + value <= maximum + ? value + : fallback; +} + +function integerPreference( + value: unknown, + minimum: number, + maximum: number, + fallback: number, +): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.min(maximum, Math.max(minimum, Math.round(value))) + : fallback; +} + +function optionalIntegerPreference( + value: unknown, + minimum: number, + maximum: number, +): number | null { + return typeof value === "number" && Number.isFinite(value) + ? Math.min(maximum, Math.max(minimum, Math.round(value))) + : null; +} + +export function getRealtimeVoicePreference(): RealtimeVoicePreference { + if (typeof window === "undefined") return DEFAULT_PREFERENCE; + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + if (raw === cachedRaw) return cachedPreference; + const parsed = JSON.parse(raw ?? "{}"); + cachedRaw = raw; + cachedPreference = { + presentationMode: enumPreference( + parsed.presentationMode, + ["debug", "subtle"], + DEFAULT_PREFERENCE.presentationMode, + ), + model: stringPreference(parsed.model, DEFAULT_PREFERENCE.model), + transcriptionModel: stringPreference( + parsed.transcriptionModel, + DEFAULT_PREFERENCE.transcriptionModel, + ), + voice: stringPreference(parsed.voice, DEFAULT_PREFERENCE.voice), + speed: numberPreference(parsed.speed, 0.25, 1.5, 1), + turnDetection: enumPreference( + parsed.turnDetection, + ["server_vad", "semantic_vad"], + DEFAULT_PREFERENCE.turnDetection, + ), + eagerness: enumPreference( + parsed.eagerness, + ["low", "medium", "high", "auto"], + DEFAULT_PREFERENCE.eagerness, + ), + interruptResponse: + typeof parsed.interruptResponse === "boolean" + ? parsed.interruptResponse + : DEFAULT_PREFERENCE.interruptResponse, + createResponse: + typeof parsed.createResponse === "boolean" + ? parsed.createResponse + : DEFAULT_PREFERENCE.createResponse, + vadThreshold: numberPreference(parsed.vadThreshold, 0, 1, 0.5), + prefixPaddingMs: integerPreference(parsed.prefixPaddingMs, 0, 2_000, 300), + silenceDurationMs: integerPreference( + parsed.silenceDurationMs, + 100, + 3_000, + 500, + ), + idleTimeoutMs: optionalIntegerPreference( + parsed.idleTimeoutMs, + 1_000, + 120_000, + ), + noiseReduction: enumPreference( + parsed.noiseReduction, + ["off", "near_field", "far_field"], + DEFAULT_PREFERENCE.noiseReduction, + ), + transcriptionLanguage: + typeof parsed.transcriptionLanguage === "string" + ? parsed.transcriptionLanguage + : "", + transcriptionPrompt: + typeof parsed.transcriptionPrompt === "string" + ? parsed.transcriptionPrompt + : "", + reasoningEffort: enumPreference( + parsed.reasoningEffort, + ["default", "none", "low", "medium", "high"], + DEFAULT_PREFERENCE.reasoningEffort, + ), + maxOutputTokens: optionalIntegerPreference( + parsed.maxOutputTokens, + 1, + 4_096, + ), + }; + return cachedPreference; + } catch { + return DEFAULT_PREFERENCE; + } +} + +function subscribe(listener: () => void) { + listeners.add(listener); + const notify = () => listener(); + window.addEventListener(CHANGED_EVENT, notify); + return () => { + listeners.delete(listener); + window.removeEventListener(CHANGED_EVENT, notify); + }; +} + +export function setRealtimeVoicePreference( + preference: RealtimeVoicePreference, +): void { + const raw = JSON.stringify(preference); + window.localStorage.setItem(STORAGE_KEY, raw); + cachedRaw = raw; + cachedPreference = preference; + window.dispatchEvent(new Event(CHANGED_EVENT)); +} + +export function useRealtimeVoicePreference() { + const preference = useSyncExternalStore( + subscribe, + getRealtimeVoicePreference, + () => DEFAULT_PREFERENCE, + ); + const setPreference = useCallback((value: RealtimeVoicePreference) => { + setRealtimeVoicePreference(value); + }, []); + return { preference, setPreference }; +} diff --git a/src/features/voice-conversation/lib/realtimeVoicePresentation.test.ts b/src/features/voice-conversation/lib/realtimeVoicePresentation.test.ts new file mode 100644 index 000000000..7fadde1a0 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePresentation.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import type { + Message, + VoiceConversationDebugEvent, +} from "@/shared/types/messages"; +import { presentRealtimeVoiceMessages } from "./realtimeVoicePresentation"; + +function message(id: string, event?: VoiceConversationDebugEvent): Message { + return { + id, + role: "assistant", + created: 1, + content: [{ type: "text", text: id }], + metadata: event ? { voiceConversationDebugEvent: event } : undefined, + }; +} + +describe("realtime voice presentation", () => { + const transcript = [ + message("master"), + message("spoken", "emissarySpeech"), + message("handoff", "emissaryToMaster"), + message("say", "masterToEmissarySay"), + message("context", "masterToEmissaryContext"), + message("dismissal", "masterDismissal"), + message("reminder", "handoffReminder"), + ]; + + it("keeps every coordination event in debug mode", () => { + expect(presentRealtimeVoiceMessages(transcript, "debug")).toBe(transcript); + }); + + it("presents one assistant in subtle mode", () => { + expect( + presentRealtimeVoiceMessages(transcript, "subtle").map(({ id }) => id), + ).toEqual(["master", "spoken"]); + }); + + it("keeps the original transcript when subtle mode has nothing to hide", () => { + const ordinaryTranscript = [ + message("master"), + message("spoken", "emissarySpeech"), + ]; + + expect(presentRealtimeVoiceMessages(ordinaryTranscript, "subtle")).toBe( + ordinaryTranscript, + ); + }); +}); diff --git a/src/features/voice-conversation/lib/realtimeVoicePresentation.ts b/src/features/voice-conversation/lib/realtimeVoicePresentation.ts new file mode 100644 index 000000000..9d51ded66 --- /dev/null +++ b/src/features/voice-conversation/lib/realtimeVoicePresentation.ts @@ -0,0 +1,22 @@ +import type { Message } from "@/shared/types/messages"; +import type { RealtimePresentationMode } from "./realtimeVoicePreference"; + +export function presentRealtimeVoiceMessages( + messages: Message[], + mode: RealtimePresentationMode, +): Message[] { + if (mode === "debug") return messages; + if ( + !messages.some((message) => { + const event = message.metadata?.voiceConversationDebugEvent; + return event && event !== "emissarySpeech"; + }) + ) { + return messages; + } + + return messages.filter((message) => { + const event = message.metadata?.voiceConversationDebugEvent; + return !event || event === "emissarySpeech"; + }); +} diff --git a/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts b/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts new file mode 100644 index 000000000..644ab5367 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceConversationModePreference.test.ts @@ -0,0 +1,18 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + getVoiceConversationMode, + setVoiceConversationMode, +} from "./voiceConversationModePreference"; + +describe("voice conversation mode preference", () => { + beforeEach(() => window.localStorage.clear()); + + it("defaults to the existing chained pipeline", () => { + expect(getVoiceConversationMode()).toBe("chained"); + }); + + it("persists OpenAI Realtime mode", () => { + setVoiceConversationMode("openai-realtime"); + expect(getVoiceConversationMode()).toBe("openai-realtime"); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceConversationModePreference.ts b/src/features/voice-conversation/lib/voiceConversationModePreference.ts new file mode 100644 index 000000000..721a383ae --- /dev/null +++ b/src/features/voice-conversation/lib/voiceConversationModePreference.ts @@ -0,0 +1,78 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export type VoiceConversationMode = "chained" | "openai-realtime"; + +const STORAGE_KEY = "goose:voice-conversation-mode"; +const CHANGED_EVENT = "goose:voice-conversation-mode-changed"; +let inMemoryMode: VoiceConversationMode | null = null; + +function normalize(value: unknown): VoiceConversationMode { + return value === "openai-realtime" ? value : "chained"; +} + +export function getVoiceConversationMode(): VoiceConversationMode { + if (typeof window === "undefined") return "chained"; + if (inMemoryMode) return inMemoryMode; + try { + return normalize(window.localStorage.getItem(STORAGE_KEY)); + } catch { + return "chained"; + } +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + if (typeof window === "undefined") return () => undefined; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) { + inMemoryMode = null; + notify(); + } + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setVoiceConversationMode(mode: VoiceConversationMode): void { + if (typeof window === "undefined") return; + const value = normalize(mode); + inMemoryMode = value; + try { + window.localStorage.setItem(STORAGE_KEY, value); + } catch { + // Keep this renderer usable when persistent storage is unavailable. + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: { value } })); +} + +export function useVoiceConversationModePreference() { + const mode = useSyncExternalStore( + subscribe, + getVoiceConversationMode, + () => "chained" as const, + ); + const setMode = useCallback((value: VoiceConversationMode) => { + setVoiceConversationMode(value); + }, []); + return { mode, setMode }; +} diff --git a/src/features/voice-conversation/prompts/expert-spokesperson.md b/src/features/voice-conversation/prompts/expert-spokesperson.md new file mode 100644 index 000000000..89beeaf18 --- /dev/null +++ b/src/features/voice-conversation/prompts/expert-spokesperson.md @@ -0,0 +1,59 @@ +# One assistant + +You are the {{ROLE}}. The Expert and the Spokesperson are two parts of one brain: one identity, one set of capabilities, one continuous relationship with the user. Capabilities reached through either part are capabilities of the one assistant; never disclaim a capability because the other part performs it. + +## Overview + +The user is having one continuous conversation with one assistant. The Spokesperson handles listening and spoken responses, keeping the voice experience natural and responsive. The Expert follows the conversation and handles deeper reasoning, computer tools, and durable work. User speech is queued for the Expert but does not wake it. After the Spokesperson speaks, or when it makes a handoff, the Expert receives the queued exchange and may work, correct, elaborate, or stay silent. Either part can contribute useful information, but together they present one coherent response. + +Expert, Spokesperson, handoffs, cursors, routing, model boundaries, and the existence of cooperating components are private. Never mention or explain them. Always speak in the first-person singular as one assistant. + +## How the system works + +**The Spokesperson** owns the live spoken conversation. It answers directly when confident. When a request needs computer access, tools, durable work, session inspection, or an authoritative answer it cannot give, it calls `handoff` _before_ any substantive spoken answer and says only a brief acknowledgement—“Let me check that for you.” It never claims lack of access, never speculates, and never tells the user to do the work manually unless the Expert recommends it. + +**The Expert** is the authoritative, durable part: reasoning, tools, session context, durable work. It receives queued user speech when finalized or interrupted Spokesperson speech or a handoff wakes it. Typed messages remain ordinary user turns. Microphone transcripts are prefixed `[Voice transcript]`. Treat interrupted Spokesperson transcripts as best-effort text that may not match the audio the user heard. On actionable turns, work normally and produce visible progress and result text for the durable transcript. When no work, correction, or guidance is needed, the entire turn is an empty, zero-token success: no prose, no tools, no coordination. Ordinary conversation and small talk belong to the Spokesperson. + +**Handoff lifecycle.** Every accepted handoff has an ID and stays open until the Expert resolves it with `SAY` or closes it with `DISMISS` and a reason. One `SAY` may resolve several. A handoff result does not start a new Spokesperson turn on its own, so the Spokesperson waits quietly after handing off. The system privately reminds the Expert about unresolved handoffs up to three times before failing loudly. + +**Expert → Spokesperson messages** (`send_to_spokesperson`): + +- `CONTEXT`—silently updates what the Spokesperson knows for a future natural turn. Never requires speech; cannot resolve a handoff. +- `SAY`—asks the Spokesperson to speak useful information now. May resolve handoffs, or volunteer a correction or timely update without one. +- `DISMISS`—closes obsolete, superseded, withdrawn, or already-handled handoffs. The reason arrives as silent context. + +**Transcript visibility.** The Expert’s reasoning, tool calls, and response text land in the durable transcript but do _not_ reach the Spokesperson, and finishing an Expert turn does not wake it. Anything that must affect the live conversation goes through `CONTEXT` or `SAY`. + +**Silence.** Never send a coordination message merely to acknowledge, confirm, or echo routine transcript content, and do not relay an ordinary typed user message unless you are adding genuinely new information. The Spokesperson never speaks merely to acknowledge `CONTEXT`, `DISMISS`, or an internal message, never opens a handoff merely to reply to the Expert, and adds no filler, repeated answers, or offers to help. When information arrives late, redundant, or immaterial, continue naturally without speaking. If the Spokesperson chooses not to speak, it is waiting for more user input; there is no watchdog turn. + +**Cursor ordering.** The Spokesperson does not manage cursors. Expert messages use the newest bridge cursor supplied by a transcript, handoff, reminder, or prior tool result. If a send fails because a newer event is queued in the other direction, wait for normal delivery and retry with the new cursor; never bypass the queue. + +**Resume.** On resume, the Spokesperson may receive a compact historical transcript and a durable session link. It treats replayed items as past context, not new user turns. If the replay is insufficient, it hands off rather than guessing or asking the user to repeat themselves. The Expert retains authoritative session context and can inspect older history when needed. + +## Canonical patterns + +### 1. Simple question—the Expert stays silent + +> **User:** “How many months are in a year?” +> **Spokesperson, spoken:** “There are 12 months in a year.” +> **Expert:** `[receives the exchange after the Spokesperson speaks; no output: zero tokens, no tools, no coordination]` + +### 2. Work that requires the Expert + +> **User:** “How many repositories are in my Development folder?” +> **Spokesperson, spoken:** “Let me check that for you.” +> **Spokesperson → Expert, `HANDOFF handoff-7`:** “Count the repositories in the user’s Development folder.” +> **Expert:** `[uses tools and determines that there are 21]` +> **Expert → Spokesperson, `SAY`, resolves `handoff-7`:** “There are 21 repositories in the Development folder.” +> **Spokesperson, spoken:** “You have 21 repositories in your Development folder.” + +The user hears one assistant checking, then answering. Nobody describes the handoff. + +### 3. Useful elaboration + +> **User:** “Why is the sky blue?” +> **Spokesperson, spoken:** “Sunlight scatters in the atmosphere, and shorter blue wavelengths scatter more strongly than most other visible colors.” +> **Expert → Spokesperson, `SAY`:** “A useful follow-up: although violet light scatters even more strongly, human eyes are less sensitive to violet, some violet light is absorbed in the upper atmosphere, and sunlight contains less violet than blue.” +> **Spokesperson, spoken:** “You might wonder why the sky isn’t violet. Our eyes are less sensitive to violet, some violet light is absorbed high in the atmosphere, and sunlight contains less violet than blue.” + +The addition is woven in naturally—no acknowledgement of an internal message, no replay of the exchange, no mention of another agent. Had it been immaterial, redundant, or too late, the Spokesperson would have said nothing. diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx new file mode 100644 index 000000000..43bbe5931 --- /dev/null +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx @@ -0,0 +1,112 @@ +import { fireEvent, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { i18n } from "@/shared/i18n"; +import { renderWithProviders } from "@/test/render"; +import { RealtimeVoiceSettings } from "./RealtimeVoiceSettings"; + +const openAiVoiceMocks = vi.hoisted(() => ({ + clearApiKey: vi.fn(() => Promise.resolve()), + getStatus: vi.fn(() => Promise.resolve({ sttConfigured: true })), + listenToSettings: vi.fn(() => Promise.resolve(() => undefined)), + setApiKey: vi.fn(() => Promise.resolve()), +})); + +vi.mock("../api/openAiVoice", () => ({ + clearOpenAiSttApiKey: openAiVoiceMocks.clearApiKey, + getOpenAiVoiceStatus: openAiVoiceMocks.getStatus, + listenToOpenAiVoiceSettings: openAiVoiceMocks.listenToSettings, + setOpenAiSttApiKey: openAiVoiceMocks.setApiKey, +})); + +describe("RealtimeVoiceSettings", () => { + beforeEach(async () => { + window.localStorage.clear(); + vi.clearAllMocks(); + await i18n.changeLanguage("en"); + }); + + it("shows recommended model, transcription, voice, and turn controls", () => { + renderWithProviders(); + + expect( + screen.getByRole("combobox", { name: "Realtime model" }), + ).toHaveTextContent("gpt-realtime-2.1 (default)"); + expect( + screen.getByRole("combobox", { name: "Transcription model" }), + ).toHaveTextContent("gpt-realtime-whisper (default)"); + expect(screen.getByRole("combobox", { name: "Voice" })).toHaveTextContent( + "Marin (default)", + ); + expect( + screen.getByRole("combobox", { name: "Turn detection" }), + ).toHaveTextContent("Server VAD (default)"); + expect( + screen.getByRole("combobox", { name: "Conversation presentation" }), + ).toHaveTextContent("Debug — show agent routing"); + expect( + screen.getByRole("switch", { name: "Interrupt when I speak" }), + ).toBeChecked(); + }); + + it("stores the Realtime key through the shared OpenAI voice credential path", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.type(screen.getByLabelText("OpenAI API key"), " sk-shared "); + await user.click(screen.getByRole("button", { name: "Save key" })); + + expect(openAiVoiceMocks.setApiKey).toHaveBeenCalledWith(" sk-shared "); + }); + + it("reveals the supported advanced session controls", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Advanced" })); + + expect( + screen.getByRole("switch", { name: "Respond automatically" }), + ).toBeChecked(); + expect( + screen.getByRole("combobox", { name: "Reasoning effort" }), + ).toHaveTextContent("Model default"); + expect( + screen.getByRole("combobox", { name: "Noise reduction" }), + ).toHaveTextContent("Off"); + expect( + screen.getByRole("slider", { name: "Voice activation threshold" }), + ).toBeInTheDocument(); + }); + + it("rounds and clamps integer-only advanced controls", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Advanced" })); + fireEvent.change(screen.getByLabelText("Maximum response tokens"), { + target: { value: "5000.4" }, + }); + fireEvent.change(screen.getByLabelText("End pause (ms)"), { + target: { value: "250.7" }, + }); + fireEvent.change(screen.getByLabelText("Speech lead-in (ms)"), { + target: { value: "-20" }, + }); + fireEvent.change(screen.getByLabelText("Idle timeout (ms)"), { + target: { value: "1499.5" }, + }); + + expect( + JSON.parse( + window.localStorage.getItem("goose:openai-realtime-voice-options") ?? + "{}", + ), + ).toMatchObject({ + maxOutputTokens: 4_096, + silenceDurationMs: 251, + prefixPaddingMs: 0, + idleTimeoutMs: 1_500, + }); + }); +}); diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx new file mode 100644 index 000000000..315db581e --- /dev/null +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx @@ -0,0 +1,578 @@ +import { ChevronRight } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/shared/ui/button"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/shared/ui/collapsible"; +import { Label } from "@/shared/ui/label"; +import { Input } from "@/shared/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/shared/ui/select"; +import { Slider } from "@/shared/ui/slider"; +import { Switch } from "@/shared/ui/switch"; +import { Textarea } from "@/shared/ui/textarea"; +import { + type RealtimeEagerness, + type RealtimeNoiseReduction, + type RealtimePresentationMode, + type RealtimeReasoningEffort, + type RealtimeTurnDetection, + useRealtimeVoicePreference, +} from "../lib/realtimeVoicePreference"; +import { clearOpenAiSttApiKey, setOpenAiSttApiKey } from "../api/openAiVoice"; +import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; +import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; + +const REALTIME_MODELS = [ + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-2", + "gpt-realtime-1.5", +] as const; +const TRANSCRIPTION_MODELS = [ + "gpt-realtime-whisper", + "gpt-live-transcribe", + "gpt-transcribe", + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", +] as const; +const REALTIME_VOICES = [ + "marin", + "cedar", + "alloy", + "ash", + "ballad", + "coral", + "echo", + "sage", + "shimmer", + "verse", +] as const; + +function voiceLabel(voice: string): string { + return `${voice.charAt(0).toUpperCase()}${voice.slice(1)}`; +} + +function boundedInteger( + value: string, + minimum: number, + maximum: number, +): number | null { + const parsed = Number(value); + return value.trim() && Number.isFinite(parsed) + ? Math.min(maximum, Math.max(minimum, Math.round(parsed))) + : null; +} + +function OptionalCurrentSelectItem({ + value, + knownValues, +}: { + value: string; + knownValues: readonly string[]; +}) { + return knownValues.includes(value) ? null : ( + {value} + ); +} + +function SettingSwitch({ + checked, + description, + id, + label, + onCheckedChange, +}: { + checked: boolean; + description: string; + id: string; + label: string; + onCheckedChange(checked: boolean): void; +}) { + return ( +
+
+ +

{description}

+
+ +
+ ); +} + +export function RealtimeVoiceSettings() { + const { t } = useTranslation("settings"); + const { preference, setPreference } = useRealtimeVoicePreference(); + const { status: openAiStatus } = useOpenAiVoiceSetup(); + + const update = (patch: Partial) => { + setPreference({ ...preference, ...patch }); + }; + + return ( +
+
+ +

+ {t("voice.realtimeApiKeyDescription")} +

+
+ +
+ + +

+ {t("voice.realtimePresentationDescription")} +

+
+ +
+
+ + +
+
+ + +

+ {t("voice.realtimeTranscriptionModelDescription")} +

+
+
+ + +
+
+ + +
+ {preference.turnDetection === "semantic_vad" ? ( +
+ + +
+ ) : null} +
+ +
+
+ + + {preference.speed.toFixed(2)}× + +
+ update({ speed })} + aria-label={t("voice.realtimeSpeed")} + /> +

+ {t("voice.realtimeSpeedDescription")} +

+
+ + update({ interruptResponse })} + /> + + + + + + + update({ createResponse })} + /> + +
+
+ + +
+
+ + +
+
+ + + update({ transcriptionLanguage: event.target.value }) + } + /> +
+
+ + + event.target.value + ? (() => { + const maxOutputTokens = boundedInteger( + event.target.value, + 1, + 4_096, + ); + if (maxOutputTokens !== null) + update({ maxOutputTokens }); + })() + : update({ maxOutputTokens: null }) + } + /> +
+
+ + {preference.turnDetection === "server_vad" ? ( +
+

+ {t("voice.realtimeServerVad")} +

+
+
+ + + {preference.vadThreshold.toFixed(2)} + +
+ update({ vadThreshold })} + aria-label={t("voice.realtimeVadThreshold")} + /> +
+
+
+ + { + const silenceDurationMs = boundedInteger( + event.target.value, + 100, + 3_000, + ); + if (silenceDurationMs !== null) + update({ silenceDurationMs }); + }} + /> +
+
+ + { + const prefixPaddingMs = boundedInteger( + event.target.value, + 0, + 2_000, + ); + if (prefixPaddingMs !== null) update({ prefixPaddingMs }); + }} + /> +
+
+ + + event.target.value + ? (() => { + const idleTimeoutMs = boundedInteger( + event.target.value, + 1_000, + 120_000, + ); + if (idleTimeoutMs !== null) + update({ idleTimeoutMs }); + })() + : update({ idleTimeoutMs: null }) + } + /> +
+
+
+ ) : null} + +
+ +