diff --git a/devlog/_plan/260912_audio_apis_stack/000_plan.md b/devlog/_plan/260912_audio_apis_stack/000_plan.md index e237ea9722..b7cd772090 100644 --- a/devlog/_plan/260912_audio_apis_stack/000_plan.md +++ b/devlog/_plan/260912_audio_apis_stack/000_plan.md @@ -6,6 +6,8 @@ Expose file transcription, streaming dictation and GPT-Live to external clients Owner steering during wp1 C: no local Bun suites, product tests, typecheck, build or dependency installation. Push with --no-verify and use exact-head remote CI for remaining executable verification. This supersedes every local command example in the layer plans. Already completed checks are historical evidence only; interrupted/crashed checks are not passing evidence. All active task-owned local suites were stopped. Functional layer closure uses the completed source review and pre-restriction focused evidence; PR readiness and final completion retain the remote CI gate under wp3 publication. +Owner scope steering during wp2: finish the audio stack and record pre-existing unrelated CI failures separately. Do not extend this task into further journal-restore or CLI stale-process repairs. Audio-owned regressions and source-review blockers still require closure. The already-published prerequisite corrections remain in the bottom branch; their remote outcomes are reported honestly. Aggregate CI failures are not represented as passing checks, and PRs remain drafts where baseline failures prevent full readiness. + - Archetype: satisfy-spec, C4 API/auth and C3 dashboard integration. - Trigger: owner requested both audio capabilities, inherited subagent verification and a published dependent PR stack. - Goal: three independently reviewable ordinary PRs with protocol tests, documentation and usable client controls. diff --git a/devlog/_plan/260912_audio_apis_stack/020_streaming_voice.md b/devlog/_plan/260912_audio_apis_stack/020_streaming_voice.md index 68d6fa8392..8dceff01a8 100644 --- a/devlog/_plan/260912_audio_apis_stack/020_streaming_voice.md +++ b/devlog/_plan/260912_audio_apis_stack/020_streaming_voice.md @@ -2,19 +2,27 @@ Depends on wp1 audio upstream resolution. Preserve existing Codex transport exports and native integrations. +P revalidation at 156e28f4ba: prior D completed the functional transcription layer and retained final-head CI as a publication gate. Production interface now resolves AudioUpstream with admission/model/lease/exactAccountId/signal and owns explicit-key admission before loopback fallback. Remote-only owner policy supersedes local test commands below; implement tests but run them only in CI. + +Dependency repair: remote run34684544174 identified pre-existing Cline count/expectation/provenance gaps and a preimage-read injection fixture whose interception was not observed. The bottom branch carries narrow corrections reviewed independently, with all original preimage assertions retained and an added interception counter. This upper branch must merge the updated parent before publication. No local product checks run; the fixture hypothesis is validated only by new remote CI. + ## File changes | Operation | Path | Contract change | | --- | --- | --- | | NEW | src/server/audio-dictation.ts | exact streaming route, canonical upstream URL/protocol construction and bounded session policy | +| NEW | src/server/audio-client.ts | audio-only browser credential carrier, explicit key detection and tagged owner identity | | NEW | src/server/live-call-bindings.ts | per-server bounded expiring call ownership, keyed by opaque call id and admission owner | -| MODIFY | src/server/live.ts | bind successful call-create to its resolved upstream; join via binding; proxy Bearer credential support; default V3 model/negotiation only on standalone /live when absent | +| NEW | src/server/audio-live.ts | external keyed call-create/join, account binding, safe Location and default standalone V3 negotiation; reuse Live protocol primitives | +| MODIFY | src/server/live.ts | expose the existing multipart conversion primitive for the external audio owner; preserve native handler behavior | | MODIFY | src/server/index.ts | route dictation upgrades through existing bounded WebSocket bridge; thread lifecycle metadata and close cleanup | | MODIFY | src/server/ws-bridge.ts | add only necessary session protocol/expiry fields to WsData | | MODIFY | src/server/auth-cors.ts | advertise only HTTP call-create rows; WebSocket admission is separate audio metadata | | MODIFY | tests/server/api-key-attribution.test.ts | valid SDP multipart fixtures for HTTP call-create matrix rows; WebSocket auth is tested by real upgrades in audio/server-live tests | | NEW | tests/server/audio-dictation.test.ts | mock WebSocket upstream with real JSON audio events and close/cancel checks | | MODIFY | tests/server/server-live.test.ts | external Bearer key and call-owner/session lifecycle regressions | +| NEW | tests/server/audio-client.test.ts | browser carrier, key precedence and stable key-rotation ownership | +| NEW | tests/server/live-call-bindings.test.ts | opaque call aliases, expiry, capacity and Location parsing | | MODIFY | scripts/test-layout/layout.json | register dictation test in server domain | | MODIFY | tests/fixtures/test-layout-expected.json | matching expected test path | | MODIFY | structure/data-planes/inbound-compat.md | document streams, ownership and source/test contract | @@ -28,6 +36,14 @@ Client uses observed session.start/config, audio append and session.close shapes Observed dictation audio is JSON {type: "audio.append", audio: "BASE64_PCM16"}; mono PCM16 uses the actual sample_rate_hz supplied by the client. transcript.segment/final revisions replace prior text for the same utterance_id. Closing acknowledgment is session.updated with session.status=closed. The requested limits are client policy, not proven upstream maxima. For browser clients the downstream protocol pair is opencodex-audio plus opencodex-key.; only opencodex-audio can be selected back. Explicit HTTP admission headers retain precedence. Never accept this carrier on ordinary Responses routes. +The protocol carrier suffix is canonical base64url of the UTF-8 proxy key, not a raw key; encoding is transport syntax, not encryption. Bound header/token sizes, reject duplicates and mismatched marker/key pairs, preserve explicit HTTP-header precedence and use only opencodex-audio as the selected downstream protocol. WsData receives upstream protocols through Bun.WebSocketOptions.protocols (verified in installed bun-types), plus optional validation/lifetime callbacks. Clear retained handshake credentials after connecting. This new field is constructed at upgrade, consumed by attachLiveSidebandUpstream and cleared on close; it never enters JSON persistence. + +The per-server registry stores only tagged owner identity, provider ID, account ID/physical account identity, endpoint policy and expiry. Configured-key owner identity uses key ID (rotation preserves it); environment identity uses a digest of the verified admission token. Proxy-key-created calls reject unknown, expired, mismatched owners before selecting credentials. Resolve the recorded account freshly and verify physical identity before joining; caller-owned native context must be resupplied. Do not keep credential snapshots as a substitute for ownership. Bound capacity at 1024 and TTL at 30 minutes; pruning is demand-driven and shutdown clears the map. + +Module-responsibility refinement: external keyed live HTTP/WS orchestration belongs in audio-live.ts so the native compatibility handler retains its existing behavior. index.ts selects this owner only after audio-client resolves a verified explicit key. Shared multipart conversion, URL builders and protocol header names stay owned by live.ts. The external create path has a lease-bound operation controller, upload/overall deadlines, one final outcome, and content-free failure responses. A bound join restricts provider selection to the recorded provider as well as recorded account; a newly enabled provider cannot displace the call owner. Opaque sideband close reasons are replaced with a generic reason for external audio clients. + +A audit fold-back: transfer AudioUpstream.release and recordOutcome into a once-only WsData.liveFinish callback only after server.upgrade succeeds. Before-transfer refusal/exception releases the acquired context and lease directly. The bridge calls liveFinish exactly once from finalizeLiveSideband when the upstream is observed CLOSED (or no upstream was created), then releases the turn lease in finally. Socket construction/open/error/timeout/close paths carry a single terminal outcome; client cancellation before connection is neutral. Clear handshake/session timers and retained headers/protocols on all finalization paths. Do not release ownership merely because downstream closed while upstream remains CONNECTING/CLOSING. + Live call-create keeps SDP/multipart conversion and Location response. A successful call stores a bounded per-server binding to the resolved account/provider and caller admission identity for follow-up joins. Never retain raw client API keys in persisted state; no persistence is needed. Follow-up requests authenticate again, reject mismatched/expired owners, and cannot change the selected account. Existing native clients using the same local/session identity retain their workflow. Validate invalid Location before reporting usable creation. Bindings survive sideband disconnect for bounded reconnect; server shutdown clears them. Resolve the recorded exact account before join, preserving physical account identity across credential refresh. Tagged ownership distinguishes configured key ID, environment admission and legacy loopback native session. Unknown calls are rejected for proxy-key clients; any native externally-created-call compatibility must remain limited to explicit caller-auth and documented separately. Reuse the existing socket bridge within the composition root for this layer; extracting all legacy socket machinery is optional and requires its own regression evidence. @@ -36,6 +52,12 @@ Live resolves a presented proxy key before the global loopback shortcut, so a ke For external proxy-key call-create, return a proxy-relative Location /v1/live/ (or the matching realtime/calls form). Never return an absolute upstream Location to a proxy-key client. The upstream WebSocket destination is independently selected from trusted config, not from Location. Native legacy response compatibility remains scoped to its existing explicit caller-auth contract. A regression follows the returned relative Location with the creator key and checks the recorded account after pool rotation. +The externally returned call ID is a proxy-generated rtc_ocx_ alias mapped to the validated upstream call ID, retaining compatibility with rtc_-accepting clients. The reserved rtc_ocx_ namespace never falls through to legacy joins, including after expiry/removal. This prevents an expired external alias from being reclassified as an unowned native call. The binding also preserves the originating Frameless/realtime join style; aliases do not let the caller change the upstream protocol family. + +Keyed upstreams have no verified physical account ID, so their bindings additionally retain a digest of the upstream credential. Fresh join resolution compares that digest and refuses a changed credential instead of guessing that a replacement key owns the original call. This digest is created at call registration, copied only in the in-memory registry, consumed at join and removed at expiry/shutdown; it is not serialized or exposed. + +Source-review refinement: normal WebSocket completion is neutral for account health because transport open can be followed by a protocol rejection. Only explicit transport failures/timeouts reach the recorder. Native platform-bearer compatibility uses equality with the configured canonical OpenAI API credential, never an sk- prefix guess; unrecognized/revoked custom bearers are rejected. The existing global listener policy still governs whether a native request is admitted without a proxy key. + Standalone WS /v1/live accepts an explicit model or defaults to gpt-live-1-codex, with gpt-live-1 as documented alias if implemented. Missing V3 negotiation is added only to this Frameless path. /v1/realtime preserves its current adapter semantics. A raw live client receives delegation events; proxy does not execute tools or fabricate delegation results. WsData fields are created at server.upgrade, serialized only by Bun in process, read at open/message/close, and disposed at relay closure; no disk reviver. Call binding types are in-memory only. Public query/model and protocol inputs are validated at ingress; no secret-bearing URL query authentication is added. diff --git a/devlog/_plan/260912_audio_apis_stack/021_streaming_checks.md b/devlog/_plan/260912_audio_apis_stack/021_streaming_checks.md new file mode 100644 index 0000000000..da208293c7 --- /dev/null +++ b/devlog/_plan/260912_audio_apis_stack/021_streaming_checks.md @@ -0,0 +1,39 @@ +# Streaming outcome and next layer + +Functional layer: PASS at `011f2dff5ca3667b88f090fe711c4b2c77efd190`, PR #4392 onto +`codex/audio-transcription` (`71e22d967ffafbd3492935c299623c9127eaf17b`). +Remote run [34687731903](https://github.com/lidge-jun/opencodex/actions/runs/34687731903) +tested merge `df1c26d1`. + +- Audio client: 11 passing cases, job 103537614446. +- File transcription: 24 passing cases, job 103537614468. +- Dictation/voice ingress and lifecycle: 11 passing cases, job 103537614422. +- Call bindings: four passing cases on Linux and macOS, job 103537614428. +- Gates: typecheck, 1,979 dashboard tests, privacy, skill surface, release syntax + and CLI smoke passed, job 103537614453. Dashboard build skipped for this layer. +- Independent inherited source reviews: Ramanujan closure PASS; Feynman final + PASS after native platform-key compatibility and neutral WebSocket accounting + corrections. No unresolved blocking finding in these bounded reviews. + +All checks above ran remotely. Local product tests, typecheck, build and install +were NOT RUN for this layer, per owner instruction. Pushes used `--no-verify`. +No real upstream audio or personal recording was used. + +## Separate baseline failures + +Linux test 2/4 (103537614443) and macOS 2/2 (103537614422) still report +`tests/codex-integration/codex-journal.test.ts:170` (failed versus skipped restore) +and `:528` (routing retained after compensated failed restore). These failures +were observed before the final audio changes. They are recorded, not included +in this feature's repair scope, following the owner's explicit decision. +An earlier run also showed the stale-process status assertion at +`tests/cli/cli-status-json.test.ts:893`; do not claim it repaired without evidence. +Bun batch crashes that recovered through CI singleton retries are not runtime-fix +evidence. Whole-run green and merge readiness are not claimed. + +## Next + +Proceed to wp3: configured endpoint metadata, separate Dictation and Live Voice +controls, synthetic browser QA of the CI-built dashboard, and ordinary stacked +publication. Configuration is not entitlement or observed connectivity. Leave +all PRs open; aggregate baseline failures remain a separate publication note. diff --git a/devlog/_plan/260912_audio_apis_stack/030_connections.md b/devlog/_plan/260912_audio_apis_stack/030_connections.md index ccb9fa24ce..dd3bcac7f3 100644 --- a/devlog/_plan/260912_audio_apis_stack/030_connections.md +++ b/devlog/_plan/260912_audio_apis_stack/030_connections.md @@ -2,19 +2,31 @@ Depends on wp2 completed audio routes. The first two layers remain independently usable through external client examples. +P revalidation: wp2 D concluded functional audio tests and source reviews PASS at +011f2dff5c, with unrelated journal failures recorded separately. Follow that +direction without expanding baseline repairs. C4 because transient credentials +cross the browser/data-plane boundary. Main owns implementation; inherited +architect and independent reviewers remain read-only. No additional cost/time +budget was imposed. No paid calls, personal audio, service changes or merge. +Local product suites/typecheck/build/install are prohibited. Existing remote CI +runs the checks; browser QA reads its built artifact with synthetic routes only. + ## File changes | Operation | Path | Contract change | | --- | --- | --- | | MODIFY | src/server/management/api-access.ts | extend ApiAccessEndpoints with transcription, dictationStream, live and realtimeCalls URLs plus truthful capability metadata | -| MODIFY | src/server/management-api.ts | include audio metadata using existing management authentication | +| EXISTING | src/server/management/oauth-account-routes.ts | /api/keys already serializes ...endpoints; no new route or management authority | | MODIFY | tests/server/api-access-endpoints.test.ts | URL host/protocol and capability projection tests | | MODIFY | gui/src/pages/api-keys-utils.ts | extend endpoint type/default/derive chain for new endpoints | | MODIFY | gui/src/pages/ApiKeys.tsx | consume serialized endpoint metadata through KeysResponse, CachedKeysShape, cache validation and fetchKeys | | MODIFY | gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx | place two unframed audio sections in existing Connections/API layout | | NEW | gui/src/components/apikeys-workspace/AudioApiPanel.tsx | accessible Dictation and Live Voice controls, endpoint/model display, sample copying, transient key/file controls and result/error states | | NEW | gui/src/audio-api-client.ts | bounded cancelable upload and socket client protocol helpers; no saved secrets | -| NEW | gui/tests/audio-api-client.test.ts | request generation, cancellation and transcript assembly tests | +| NEW | gui/src/audio-api-examples.ts | executable protocol examples outside component markup; localized prompt label supplied by caller | +| MODIFY | gui/src/api.ts | narrow audio upload fetch entry bypasses management auth injection/recovery, validates exact inference path | +| NEW | gui/tests/audio-api-client.test.ts | request generation, cancellation and protocol readiness tests | +| NEW | gui/tests/audio-api-panel.test.tsx | real component upload, error, cancellation, missing metadata and deactivation flows | | MODIFY | gui/src/i18n/en.ts and every locale module | complete localized label/status/action keys | | MODIFY | gui/src/styles-apikeys-workspace.css | restrained aligned responsive audio sections using existing tokens | | MODIFY | structure/gui-and-management-api.md | metadata and control ownership/current contract | @@ -26,11 +38,23 @@ Before: ApiAccessEndpoints contains Responses/chat/messages/models only. After: Metadata fields complete chain: creation buildApiAccessEndpoints -> JSON management response -> API page validation/mapping -> ApiEndpointInfo/AudioApiPanel. Defaults cannot claim configured availability. No new provider is registered and no audio model enters a text completion test. +The optional `audio` projection has transcriptionEndpoint, dictationStreamEndpoint, +liveEndpoint, realtimeCallsEndpoint, transcriptionModel, liveModel, +transcriptionConfigured, dictationConfigured and liveConfigured. URL/model strings +and booleans are validated on both network and cached payloads; endpoint origins +and paths must match the published base after WS-to-HTTP normalization. Missing +or invalid audio metadata leaves the audio controls unavailable, while existing +key management keeps working. Configuration flags use canonical enabled provider +configuration only, never credential resolution, account reads or a network probe. +File transcription accepts canonical ChatGPT or configured OpenAI API routing; +dictation/live GPT-Live flags require canonical ChatGPT routing (an API-key-only +configuration does not prove access to the Codex live model). + Dictation section contains model and endpoint copy actions, file input, transient API key input, transcribe/cancel, text result/copy and clear error states. Stream example names extension protocol and gives start/audio/close events. Live Voice section contains actual GPT-Live model and both WS/WebRTC connection endpoints, transient client key, a connect/disconnect test with status and observed event output. Browser WebSocket auth must use a short-lived local session mechanism or supported client protocol carrier; never expose ChatGPT credentials or persist raw keys. Do not create a fake success check or billable background probe. All test actions require a deliberate user click. UI is unframed and follows existing workspace colors/type/spacing. Icons reuse gui icons, all visible text is localized. At desktop and mobile widths long endpoint text wraps or scrolls within its own element without overlapping controls. Buttons have stable dimensions and stateful controls are keyboard reachable. -Pass the existing active flag through ApiKeysWorkspace. Integrations hides panels without unmounting; requests, sockets and timers must stop on deactivation as well as unmount. Existing key rows contain only prefixes: controls use a newly generated key or an explicitly entered transient key, never pretend a key ID can authenticate. Browser voice connection uses an OpenCodex-only WebSocket protocol credential carrier accepted solely by the audio routes; exact supported carrier and precedence are documented/tested in wp2. No persistent key or query authentication. +Pass the existing active flag through ApiKeysWorkspace. Integrations hides panels without unmounting; conditionally unmount just audio controls when inactive, preserving the rest of the workspace drafts. Requests, sockets and timers stop on deactivation, origin change and unmount. Existing key rows contain only prefixes: controls use an explicitly entered transient key, never pretend a key ID can authenticate. Key edits cancel pending work. Browser voice connection uses the OpenCodex-only WebSocket protocol credential carrier accepted solely by the audio routes; exact supported carrier and precedence are documented/tested in wp2. No persistent key or query authentication. Raw upstream messages are not rendered: show localized error categories and allowlisted event types only. Socket open is not success: wait for session.started/session.updated with a nonempty session.id. Probe sends no audio and closes after a bounded interval or explicit disconnect. ## Acceptance and publication @@ -38,7 +62,33 @@ Pass the existing active flag through ApiKeysWorkspace. Integrations hides panel 2. Mocked browser flow uploads a fixture, receives text, copies it, cancels a pending call and displays a server error. No real audio/provider requests during agent QA. 3. Mocked voice flow connects, observes a protocol event, disconnects and releases callbacks/timers; API keys never enter storage, screenshots or URL queries. 4. Desktop and mobile browser screenshots are read back and corrected. Screenshot attached to UI PR with synthetic data only. -5. Run GUI focused tests, lint:i18n, lint, build and repository typecheck/full suite before review ready; per-layer CI uses exact PR head. The final PRs fill Summary, Verification and Checklist plus ordinary stack map. +5. Remote CI executes dashboard tests/lint/build and repository typecheck/suite; no local execution. Inspect exact-head logs and download its dashboard-preview artifact. The final PRs fill Summary, Verification and Checklist plus ordinary stack map. Existing unrelated CI failures stay separately documented; never attest whole-suite green. + +Conditional acceptance includes empty/malformed metadata (disabled controls, no +request), file >25,000,000 bytes (local rejection before fetch), 401/429/503 +(localized categories, no raw body), aborted upload (no stale text), WS open +without ready event (timeout), protocol error (failed, not connected), key/origin +change and inactive/unmount (all resources closed, no late callback). Copy samples +contain placeholders only, never the transient input. Metadata tests cover TLS, +wildcard, IPv6 and loopback companion URLs. No new enforcement layer is claimed; +browser guards are early UX validation, and server admission remains authoritative. + +## P/A review disposition + +Accepted architect CONN-META-01, WIRE-02, URL-03, UI-04 and LIFE-06. Folded +CONN-PROBE-05 and the independent A review's three residuals: + +- On socket open send exactly `{"type":"session.update","session":{"instructions":"","audio":{"output":{"voice":"cove"}},"delegation":{"type":"client"}}}`. + Acknowledgments with closed/error/failed session status are terminal, never + ready. An error followed by normal close remains failed. +- The API module exposes a narrow raw audio-upload entry. It validates the exact + HTTP(S) `/v1/audio/transcriptions` destination and bypasses installed management + authentication and 401 recovery. Connected-mode tests install that wrapper and + assert the typed data key survives, with no session/CSRF/machine credentials. +- Endpoint validation rejects userinfo, query, fragment and incorrect schemes as + well as wrong origins/paths. Network and cache paths use the same validator. + Only the wp2 `opencodex-audio` / `opencodex-key.` carrier is used; + observed output means allowlisted event types, not raw messages. Commands are defined by root/gui package.json. Source paths and existing stylesheet/fetch owner are revalidated at this cycle P before implementation; any renamed path is amended with exact ownership evidence. No disconnected metadata fields or fake audio model tests are acceptable. diff --git a/devlog/_plan/260912_audio_apis_stack/031_connections_checks.md b/devlog/_plan/260912_audio_apis_stack/031_connections_checks.md new file mode 100644 index 0000000000..ff74412621 --- /dev/null +++ b/devlog/_plan/260912_audio_apis_stack/031_connections_checks.md @@ -0,0 +1,78 @@ +# Connections audio verification + +## Delivered surface + +Ordinary dependency chain remains open: [#4391](https://github.com/lidge-jun/opencodex/pull/4391) +-> [#4392](https://github.com/lidge-jun/opencodex/pull/4392) +-> [#4395](https://github.com/lidge-jun/opencodex/pull/4395). +No native stack registration, merge, release, deployment or live-service restart. + +The top layer adds validated audio metadata, separate Dictation and Live Voice +blocks, temporary data-key input, cancelable upload/copy, connection-only native +session readiness, client examples and all nine dashboard locales. It does not +add audio models to text completion tests. Invalid/missing metadata leaves existing +key management usable. Configured availability is not account entitlement. + +## Executable proof + +Runtime/UI source head: `f5aefd88af3116bec4f2ddc72c0c6fa974f52a83`. +Remote [run 34690242138](https://github.com/lidge-jun/opencodex/actions/runs/34690242138), +gates job 103544165369: SUCCESS for lint, typecheck, dashboard tests, privacy, +generated skill surface, release syntax, dashboard build and CLI smoke. +Artifact 10296892796 is `dashboard-preview-bb0f30fbab4496b2b69bbc1e8148e59035ba3778`. +Its GUI tree `5ccecf06a8e144e5260f4991047db6fa36f464a1` exactly matches source. +The artifact names the tested PR merge, not a different runtime build. + +Local product tests, typecheck, build, dependency installation and suites: +**NOT RUN**, per owner restriction. Every commit/push used `--no-verify`. +The only local execution was a static Node file server for the CI-built artifact +and browser QA through the already installed Playwright dependency. No proxy +runtime, provider audio request, personal recording or microphone was used. + +## Browser matrix + +Invocation: static artifact server, then `.tmp/audio-browser-run.mjs` driving +`/#integrations/keys` with intercepted synthetic management/audio routes. +Installed agbrowse lacked its documented script command, so the existing +agbrowse Playwright dependency drove a separate CDP browser on port 9231. +No new browser dependency was installed. + +| Scenario | Observed result | +| --- | --- | +| Upload synthetic file | Expected transcript; typed data key only, no management/CSRF headers | +| Copy transcript | Actual browser clipboard contained the exact transcript | +| Live connect/disconnect | session.update, session.started with ID, session.close; no audio frames | +| HTTP 401 | Localized error, raw provider material absent | +| Cancel slow response | No late transcript published | +| Leave/re-enter API tab | Pending resources released, temporary key cleared | +| Storage inspection | No typed key in localStorage/sessionStorage | +| Keyboard | Key input -> file input follows Tab order | +| 1440/1024/768/390/320 | No audio-control overflow; settled screenshots read back | +| Korean 1440/390/320 | Labels fit; no new-section overlap or clipped Korean text | +| Runtime | No page JavaScript errors; no external network requests | + +Initial captures exposed mobile top-bar overlap and a two-line tablet section +strip. Both were fixed; final screenshots below depict the corrected source. +Mobile scroll-spy uses the same 108px reading line as section positioning. + +![Desktop audio controls](screenshots/audio-1440-light.png) +![Mobile audio controls](screenshots/audio-390-light.png) +![Narrow Korean audio controls](screenshots/audio-320-ko.png) + +`screenshots/transcription-layer-baseline.png` separately renders the bottom PR's +own CI artifact (run 34687369123, merge d9771b3133863f5be1b89213029b0a8037dbcb89, +GUI tree 999781a53536c82ccfbfd376083e945bf388c55d). That layer changes GUI test +fixtures and asset provenance only; this capture does not claim the audio UI +exists in the bottom layer. + +## Review and remaining limits + +Inherited security review closed metadata projection and post-readiness failures. +Inherited component review closed endpoint styling, replacement-race and cache +observation coverage. Subsequent xai/grok-4.6 review closed idle status and mobile +scroll-spy alignment. Source verdicts: PASS. Rendered evidence is separate from +source review; neither establishes real provider availability. + +Unrelated journal restore failures remain recorded in `021_streaming_checks.md`. +Old/superseded or cancelled whole runs are not passing final-head evidence. No +whole-suite-green or merge-readiness claim is made. PRs remain open for review. diff --git a/devlog/_plan/260912_audio_apis_stack/screenshots/audio-1440-light.png b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-1440-light.png new file mode 100644 index 0000000000..81a579bb82 Binary files /dev/null and b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-1440-light.png differ diff --git a/devlog/_plan/260912_audio_apis_stack/screenshots/audio-320-ko.png b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-320-ko.png new file mode 100644 index 0000000000..9856ac03bc Binary files /dev/null and b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-320-ko.png differ diff --git a/devlog/_plan/260912_audio_apis_stack/screenshots/audio-390-light.png b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-390-light.png new file mode 100644 index 0000000000..380b2644ce Binary files /dev/null and b/devlog/_plan/260912_audio_apis_stack/screenshots/audio-390-light.png differ diff --git a/devlog/_plan/260912_audio_apis_stack/screenshots/transcription-layer-baseline.png b/devlog/_plan/260912_audio_apis_stack/screenshots/transcription-layer-baseline.png new file mode 100644 index 0000000000..34e261a4b9 Binary files /dev/null and b/devlog/_plan/260912_audio_apis_stack/screenshots/transcription-layer-baseline.png differ diff --git a/docs-site/src/content/docs/fr/reference/proxy-formats.md b/docs-site/src/content/docs/fr/reference/proxy-formats.md index 04637428e1..ee8e9dbce4 100644 --- a/docs-site/src/content/docs/fr/reference/proxy-formats.md +++ b/docs-site/src/content/docs/fr/reference/proxy-formats.md @@ -276,6 +276,10 @@ Voir [le guide Desktop](/fr/guides/claude-code/). Relecture thinking et cache re ## `POST /v1/live` et bande latérale en temps réel +La liaison de compte ci-dessous concerne les clients Codex natifs. Pour la dictée et GPT-Live avec une clé API externe, consultez la [spécification audio en anglais](/reference/proxy-formats/#streaming-dictation). + +Connections > API keys propose deux sections, Dictée et Voix en direct. La clé de données reste uniquement en mémoire dans le formulaire. La dictée envoie le fichier choisi ; la vérification vocale attend une confirmation de session sans microphone. Une configuration présente ne garantit pas la connexion. + `POST /v1/live` accepte la surface de création d'appel ChatGPT/Codex App sans cadre. `POST /v1/realtime/calls` accepte la surface de création d'appel OpenAI Realtime. opencodex sélectionne un route OpenAI-family éligible, normalise la demande de création d'appel pour l'authentification en amont diff --git a/docs-site/src/content/docs/ja/reference/proxy-formats.md b/docs-site/src/content/docs/ja/reference/proxy-formats.md index 7f875af2f5..2220dca5fa 100644 --- a/docs-site/src/content/docs/ja/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ja/reference/proxy-formats.md @@ -202,6 +202,10 @@ API ではありません。Desktop のキー移行・復旧・切断は既存 ## `POST /v1/live` とRealtime サイドバンド +以下のアカウント連携は既存の Codex クライアント向けです。外部 API キーで利用する音声入力と GPT-Live は[英語版の音声 API 仕様](/reference/proxy-formats/#streaming-dictation)を参照してください。 + +Connections > API keys に音声入力とリアルタイム音声の項目があります。データキーは入力欄のメモリにのみ保持されます。文字起こしは選択したファイルを送信し、音声の接続確認はマイクを使わずセッション応答を待ちます。設定済みの表示は接続成功を意味しません。 + `POST /v1/live` は、ChatGPT/Codex アプリのフレームレス通話作成サーフェスを受け入れます。 `POST /v1/realtime/calls` は、OpenAI Realtime 呼び出し作成サーフェスを受け入れます。 opencodex は、適格な OpenAI ファミリ ルートを選択し、アップストリーム認証モードのコール作成リクエストを正規化し、制限付き応答を中継します。 コールの作成後、クライアントはサポートされている受信フォームを使用してサイドバンド WebSocket に参加できます。 diff --git a/docs-site/src/content/docs/ko/reference/proxy-formats.md b/docs-site/src/content/docs/ko/reference/proxy-formats.md index a537d77a73..e4fe0befb9 100644 --- a/docs-site/src/content/docs/ko/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ko/reference/proxy-formats.md @@ -249,6 +249,10 @@ HTTP 400을 반환합니다. 두 경우 모두 날짜 제거나 다른 경로로 ## `POST /v1/live`와 Realtime sideband +아래 계정 연결 설명은 기존 Codex 클라이언트 기준입니다. 외부 API 키로 쓰는 받아쓰기와 GPT-Live는 [영문 음성 API 명세](/reference/proxy-formats/#streaming-dictation)를 따릅니다. + +Connections > API keys에는 받아쓰기와 실시간 음성 블록이 있습니다. 데이터 키는 입력란에만 잠시 유지됩니다. 받아쓰기는 선택한 파일을 전송하고, 음성 연결 확인은 마이크 없이 세션 응답을 기다립니다. 설정 표시는 실제 연결 성공을 뜻하지 않습니다. + `POST /v1/live`는 ChatGPT/Codex App Frameless call-creation 표면을 받습니다. `POST /v1/realtime/calls`는 OpenAI Realtime call-creation 표면을 받습니다. opencodex는 적절한 OpenAI 계열 경로를 선택하고, 업스트림 인증 모드에 맞게 call-creation 요청을 정규화한 뒤, 제한된 응답을 릴레이합니다. diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index c5fa271bf1..4fcc72f60a 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -34,11 +34,23 @@ Credential-bearing model, image, video, and search requests do not automatically | Anthropic token count | `POST /v1/messages/count_tokens` | `{ "input_tokens": number }` | Not applicable | | Model discovery | `GET /v1/models` | Catalog or explicit Desktop snapshot | Not applicable | | File transcription | `POST /v1/audio/transcriptions` | `{ "text": string }` or plain text | Not supported on this file endpoint | +| Streaming dictation | `WS /v1/audio/transcriptions/stream` | Not applicable | Desktop dictation JSON events | | Voice and Realtime | `POST /v1/live`, `POST /v1/realtime/calls` | Relayed call-creation response | A separate sideband WebSocket relays frames in both directions | | Responses compaction | `POST /v1/responses/compact` | Replacement-history JSON | Not applicable | ## File transcription +Connections > API keys has separate **Dictation** and **Live Voice** blocks. +Enter an OpenCodex data key, not a provider or management key. Dictation uploads +the file you select and offers cancellation and transcript copying. Live Voice's +**Check connection** opens a session without microphone access or audio frames, +waits for the provider's session acknowledgment, and disconnects after one minute +or when you leave the panel. The key remains only in that panel's memory. +**Configured, not verified** describes provider configuration, not account +health or entitlement. Use the explicit action to observe a result. Examples use +key placeholders and never include the entered secret. An older server without +audio metadata leaves these controls unavailable. + `POST /v1/audio/transcriptions` accepts an OpenCodex data-plane key in `Authorization: Bearer`, `x-opencodex-api-key`, or `x-api-key`, including on a local listener. An explicitly supplied invalid key is rejected. Upload one audio file @@ -72,6 +84,40 @@ rules; Pool mode uses the selected stored account. Missing, expired or draining credentials return an error. Cancellation stops the outbound request and audio content is not written to request history. +## Streaming dictation + +`WS /v1/audio/transcriptions/stream` is an OpenCodex extension for a connected +ChatGPT account. It is separate from OpenAI's public Realtime transcription +protocol. Authenticate with the same proxy-key headers as file transcription. +Browser clients instead offer these two WebSocket subprotocols: + +```text +opencodex-audio +opencodex-key. +``` + +Only `opencodex-audio` is selected in the response. Encoding is transport syntax, +not encryption. Explicit HTTP credential headers take precedence. ChatGPT tokens +stay on the proxy; an API-key-only upstream cannot serve this dictation protocol. + +After connecting, send: + +```json +{"type":"session.start","config":{"input_audio_format":"pcm16","sample_rate_hz":48000,"num_channels":1,"max_buffer_size_bytes":4194304,"max_utterance_duration_ms":30000,"session_ttl_ms":300000,"provider_mode":"streaming_sse","transcript_delivery_mode":"segment","vad":{"type":"server_vad","threshold":0.5,"prefix_padding_ms":300,"silence_duration_ms":500}}} +``` + +Use the actual sample rate of mono PCM16 audio. Wait for `session.started`, then +send `{"type":"audio.append","audio":""}`. These are JSON text +frames, not WAV files or binary WebSocket frames. The gateway accepts sample +rates from 8,000 through 192,000 Hz; upstream support is account/service-dependent. +Client frames are limited to 64 KiB and sessions to five minutes. Unsupported or +malformed event/config fields close the stream with code 1008. + +`transcript.segment` and `transcript.final` contain `utterance_id`, `revision`, and +`text`. Replace prior text for the same utterance when its revision increases; +do not concatenate revisions. Finish with `{"type":"session.close"}` and wait +for final text and `session.updated` with `session.status="closed"`. + ## `POST /v1/responses` This is the native opencodex data-plane shape. The request body must be a JSON object with a @@ -420,6 +466,51 @@ Thinking replay and prompt-cache work remain separate in [#3719](https://github. ## `POST /v1/live` and Realtime sideband +### External API keys + +External clients use an OpenCodex key in any supported audio credential header, +or the browser subprotocol pair described above. Standalone +`WS /v1/live?model=gpt-live-1-codex` uses the Frameless protocol; an omitted model +defaults to that identifier and `gpt-live-1` is its proxy alias. This is not a +claim that every public OpenAI Realtime SDK or API key supports GPT-Live. + +For a new standalone connection, send the source-compatible initialization below +after socket open and wait for `session.started` with a nonempty `session.id`. +An updated-session event with the same shape is also accepted by the native client. + +```json +{"type":"session.update","session":{"instructions":"","audio":{"output":{"voice":"cove"}},"delegation":{"type":"client"}}} +``` + +Frameless uses `input_audio.append` and `output_audio.delta`, unlike dictation's +`audio.append`. A connection-only check needs no microphone or audio frames; send +`{"type":"session.close"}` and close the socket after readiness. Delegation +events are work requests, not readiness signals, and the external client owns +their execution and responses. + +For WebRTC, post an SDP offer to `/v1/live` as multipart `sdp` and optional JSON +`session`, JSON `{sdp, session?}`, or raw `application/sdp`. The response contains +the answer and a proxy-relative `Location` with an opaque `rtc_ocx_` call ID. Join +that location using the same proxy key. The proxy resolves the creating provider +and physical account even if Pool selection changes. Unknown, expired or +other-key aliases fail before an upstream connection. Client key rotation +preserves ownership by key ID; replacement of a keyed upstream credential +requires a new call. Existing-call sidebands do not need another session update. + +Call bindings last 30 minutes, are bounded to 1024 entries per server, and end on +server restart. Socket lifetimes are bounded independently; media travels +directly over WebRTC and is not proxied. The proxy never executes delegation +requests. OpenAI account availability is established by the actual upstream +response, not by the presence of a model name in the dashboard. + +### Native Codex compatibility + +Native API-key-mode callers on the trusted local listener may use the exact +credential configured for the canonical OpenAI API tier. Other presented bearer +values require a registered proxy key or an explicit, matching ChatGPT +token/account pair; an arbitrary key prefix is not proof of native credentials. +Credential-free trusted-local native calls retain their existing behavior. + `POST /v1/live` accepts the ChatGPT/Codex App Frameless call-creation surface. `POST /v1/realtime/calls` accepts the OpenAI Realtime call-creation surface. opencodex selects an eligible OpenAI-family route, normalizes the call-creation request for the upstream authentication diff --git a/docs-site/src/content/docs/ru/reference/proxy-formats.md b/docs-site/src/content/docs/ru/reference/proxy-formats.md index 04934d0c50..7f8b1d68ad 100644 --- a/docs-site/src/content/docs/ru/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ru/reference/proxy-formats.md @@ -260,6 +260,10 @@ HTTP 400. В обоих случаях дата не удаляется и др ## `POST /v1/live` и Realtime sideband +Привязка аккаунта ниже относится к штатным клиентам Codex. Для диктовки и GPT-Live с внешним API-ключом см. [спецификацию аудио API на английском](/reference/proxy-formats/#streaming-dictation). + +В Connections > API keys есть отдельные разделы диктовки и голоса. Ключ данных остаётся только в памяти формы. Диктовка отправляет выбранный файл, а проверка голоса ждёт подтверждения сеанса без микрофона. Наличие конфигурации не означает успешное подключение. + `POST /v1/live` принимает surface Frameless call-creation из ChatGPT/Codex App. `POST /v1/realtime/calls` принимает surface call-creation OpenAI Realtime. opencodex выбирает подходящий маршрут семейства OpenAI, нормализует запрос call-creation под нужный режим diff --git a/docs-site/src/content/docs/tr/reference/proxy-formats.md b/docs-site/src/content/docs/tr/reference/proxy-formats.md index 84ad0cf5e1..a4ce5ade7b 100644 --- a/docs-site/src/content/docs/tr/reference/proxy-formats.md +++ b/docs-site/src/content/docs/tr/reference/proxy-formats.md @@ -281,6 +281,10 @@ Thinking yeniden gönderimi ve önbellek, ayrı [#3719](https://github.com/lidge ## `POST /v1/live` ve Realtime yan bandı +Aşağıdaki hesap bağlantısı yerel Codex istemcileri içindir. Harici API anahtarıyla dikte ve GPT-Live kullanımı için [İngilizce ses API belirtimine](/reference/proxy-formats/#streaming-dictation) bakın. + +Connections > API keys altında Dikte ve Canlı Ses bölümleri bulunur. Veri anahtarı yalnızca form belleğinde tutulur. Dikte seçilen dosyayı gönderir; ses bağlantısı kontrolü mikrofon kullanmadan oturum onayını bekler. Yapılandırılmış olması bağlantının başarılı olduğu anlamına gelmez. + `POST /v1/live`, ChatGPT/Codex App Frameless çağrı oluşturma yüzeyini kabul eder. `POST /v1/realtime/calls`, OpenAI Realtime çağrı oluşturma yüzeyini kabul eder. opencodex uygun bir OpenAI ailesi rotası seçer, yukarı akış kimlik diff --git a/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md b/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md index 9f83638d3c..52e8237b48 100644 --- a/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md +++ b/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md @@ -213,6 +213,10 @@ thinking 重放与提示缓存仍由独立的 [#3719](https://github.com/lidge-j ## `POST /v1/live` 和 Realtime sideband +下文的账户绑定说明适用于原生 Codex 客户端。通过外部 API 密钥使用语音转写和 GPT-Live,请参阅[英文音频 API 规范](/reference/proxy-formats/#streaming-dictation)。 + +Connections > API keys 包含独立的听写和实时语音区域。数据密钥仅保留在表单内存中。听写会上传所选文件;语音连接检查不使用麦克风,而是等待会话确认。已配置不代表连接成功。 + `POST /v1/live` 接受 ChatGPT/Codex App 的 Frameless call-creation 表面。 `POST /v1/realtime/calls` 接受 OpenAI Realtime 的 call-creation 表面。opencodex 会选择 一个符合条件的 OpenAI 家族路由,将 call-creation 请求规范化为上游认证模式,并转发有界响应。 diff --git a/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md b/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md index 483319ac58..b53e740cd8 100644 --- a/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md +++ b/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md @@ -191,6 +191,10 @@ thinking 重播與提示快取仍由獨立的 [#3719](https://github.com/lidge-j ## `POST /v1/live` 與 Realtime sideband +下方帳戶綁定說明適用於原生 Codex 用戶端。透過外部 API 金鑰使用語音轉寫及 GPT-Live,請參閱[英文音訊 API 規格](/reference/proxy-formats/#streaming-dictation)。 + +Connections > API keys 包含獨立的聽寫與即時語音區域。資料金鑰僅保留在表單記憶體中。聽寫會上傳選取的檔案;語音連線檢查不使用麥克風,而是等待工作階段確認。已設定不代表連線成功。 + `POST /v1/live` 接受 ChatGPT/Codex App Frameless call-creation 介面。 `POST /v1/realtime/calls` 接受 OpenAI Realtime call-creation 介面。opencodex 選擇一個合格的 OpenAI 家族路由、為上游認證模式正規化 call-creation 請求,並中繼有界的回應。 diff --git a/gui/src/api.ts b/gui/src/api.ts index 020183dd47..644c5f5610 100644 --- a/gui/src/api.ts +++ b/gui/src/api.ts @@ -340,6 +340,16 @@ export function installApiAuthFetch(): void { }; } +/** Audio is data-plane traffic, even when the connected management target is a relay. */ +export function fetchAudioUpload(endpoint: string, init: RequestInit): Promise { + const url = new URL(endpoint); + if (!["http:", "https:"].includes(url.protocol) || url.pathname !== "/v1/audio/transcriptions" + || url.username || url.password || url.search || url.hash || init.method !== "POST") { + return Promise.reject(new Error("Invalid audio upload destination")); + } + return (rawFetch ?? fetch)(url.href, { ...init, credentials: "omit", redirect: "error" }); +} + export function resetApiAuthFetchForTests(adminTokenPrompt: AdminTokenPrompt = promptForAdminToken): void { installed = false; rawFetch = null; diff --git a/gui/src/audio-api-client.ts b/gui/src/audio-api-client.ts new file mode 100644 index 0000000000..17eefe2c56 --- /dev/null +++ b/gui/src/audio-api-client.ts @@ -0,0 +1,153 @@ +import { fetchAudioUpload } from "./api"; +import { createBoundedFetch } from "./bounded-fetch"; + +export type AudioErrorCode = "auth" | "unavailable" | "rateLimit" | "invalid" | "size" | "network" | "timeout" | "protocol"; +export class AudioApiError extends Error { + readonly code: AudioErrorCode; + constructor(code: AudioErrorCode) { super(code); this.code = code; } +} + +export const AUDIO_FILE_MAX_BYTES = 25_000_000; +const RESPONSE_MAX_BYTES = 2 * 1024 * 1024; + +export async function transcribeAudio(endpoint: string, model: string, key: string, file: File, signal: AbortSignal): Promise { + if (!file.size || file.size > AUDIO_FILE_MAX_BYTES) throw new AudioApiError("size"); + if (!key.trim()) throw new AudioApiError("auth"); + signal.throwIfAborted(); + const bounded = createBoundedFetch(130_000); + const abort = () => bounded.controller.abort(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + try { + const form = new FormData(); + form.set("file", file); + form.set("model", model); + form.set("response_format", "json"); + const response = await fetchAudioUpload(endpoint, { + method: "POST", headers: { "X-OpenCodex-API-Key": key.trim() }, body: form, signal: bounded.signal, + }); + if (!response.ok) { + void response.body?.cancel().catch(() => {}); + throw new AudioApiError(response.status === 401 || response.status === 403 ? "auth" + : response.status === 429 ? "rateLimit" : response.status === 413 ? "size" + : response.status >= 500 ? "unavailable" : "invalid"); + } + const reader = response.body?.getReader(); + if (!reader) throw new AudioApiError("protocol"); + const cancel = () => { void reader.cancel().catch(() => {}); }; + bounded.signal.addEventListener("abort", cancel, { once: true }); + let text = ""; + let bytes = 0; + const decoder = new TextDecoder(); + try { + bounded.signal.throwIfAborted(); + for (;;) { + const part = await reader.read(); + bounded.signal.throwIfAborted(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > RESPONSE_MAX_BYTES) throw new AudioApiError("protocol"); + text += decoder.decode(part.value, { stream: true }); + } + text += decoder.decode(); + } finally { + bounded.signal.removeEventListener("abort", cancel); + void reader.cancel().catch(() => {}); + reader.releaseLock(); + } + let data: unknown; + try { data = JSON.parse(text); } catch { throw new AudioApiError("protocol"); } + if (!data || typeof data !== "object" || typeof (data as { text?: unknown }).text !== "string") throw new AudioApiError("protocol"); + return (data as { text: string }).text; + } catch (error) { + if (signal.aborted) throw signal.reason; + if (bounded.signal.aborted) throw new AudioApiError("timeout"); + throw error instanceof AudioApiError ? error : new AudioApiError("network"); + } finally { + signal.removeEventListener("abort", abort); + bounded.clear(); + } +} + +export function audioSocketProtocols(key: string): string[] { + const bytes = new TextEncoder().encode(key.trim()); + if (!bytes.length || bytes.length > 4096) throw new AudioApiError("auth"); + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + const encoded = btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); + return ["opencodex-audio", `opencodex-key.${encoded}`]; +} + +export const LIVE_SESSION_UPDATE = { + type: "session.update", + session: { instructions: "", audio: { output: { voice: "cove" } }, delegation: { type: "client" } }, +} as const; +export type LiveAudioState = "connecting" | "connected" | "disconnected" | "failed"; +const DISPLAY_EVENTS = new Set(["session.started", "session.updated", "delegation.created", "output_audio.delta"]); + +/** Connection-only probe: no microphone, audio frame, delegation execution or reconnect. */ +export function connectLiveAudio(options: { + endpoint: string; model: string; key: string; + onState: (state: LiveAudioState, error?: AudioErrorCode) => void; + onEvent: (type: string) => void; + readyTimeoutMs?: number; + maxSessionMs?: number; +}): () => void { + const url = new URL(options.endpoint); + if (!["ws:", "wss:"].includes(url.protocol) || url.pathname !== "/v1/live" + || url.username || url.password || url.search || url.hash) throw new AudioApiError("invalid"); + url.searchParams.set("model", options.model); + const socket = new WebSocket(url.href, audioSocketProtocols(options.key)); + let finished = false; + let ready = false; + let lifetime: ReturnType | undefined; + const deadline = setTimeout(() => finish("failed", "timeout"), options.readyTimeoutMs ?? 15_000); + const dispose = () => { + clearTimeout(deadline); + clearTimeout(lifetime); + socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null; + try { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: "session.close" })); + socket.close(); + } catch { /* A connecting or already closed browser socket has nothing left to send. */ } + }; + function finish(state: LiveAudioState, error?: AudioErrorCode) { + if (finished) return; + finished = true; + dispose(); + options.onState(state, error); + } + socket.onopen = () => { + try { socket.send(JSON.stringify(LIVE_SESSION_UPDATE)); } + catch { finish("failed", "network"); } + }; + socket.onmessage = event => { + if (finished) return; + if (typeof event.data !== "string" || event.data.length > 64 * 1024) { finish("failed", "protocol"); return; } + let message: { type?: unknown; session?: { id?: unknown; status?: unknown } }; + try { message = JSON.parse(event.data); } catch { finish("failed", "protocol"); return; } + if (!message || typeof message !== "object" || typeof message.type !== "string") { finish("failed", "protocol"); return; } + if (message.type === "error" || message.type === "protocol.error") { finish("failed", "protocol"); return; } + if (message.type === "session.started" || message.type === "session.updated") { + const session = message.session; + if (session?.status === "error" || session?.status === "failed") { + finish("failed", "protocol"); + return; + } + if (session?.status === "closed") { + finish(ready ? "disconnected" : "failed", ready ? undefined : "protocol"); + return; + } + if (!ready && typeof session?.id === "string" && session.id.trim()) { + ready = true; + clearTimeout(deadline); + lifetime = setTimeout(() => finish("disconnected"), options.maxSessionMs ?? 60_000); + options.onState("connected"); + } + } + if (DISPLAY_EVENTS.has(message.type)) options.onEvent(message.type); + }; + socket.onerror = () => finish("failed", "network"); + socket.onclose = event => finish(ready && event.code === 1000 ? "disconnected" : "failed", ready && event.code === 1000 ? undefined : "network"); + options.onState("connecting"); + return () => { if (!finished) { finished = true; dispose(); } }; +} diff --git a/gui/src/audio-api-examples.ts b/gui/src/audio-api-examples.ts new file mode 100644 index 0000000000..c7580eddba --- /dev/null +++ b/gui/src/audio-api-examples.ts @@ -0,0 +1,29 @@ +import { LIVE_SESSION_UPDATE } from "./audio-api-client"; + +/** Executable protocol samples, not UI copy; credentials are supplied by the reader. */ +export function audioSocketExample(endpoint: string, live: boolean, model: string, keyLabel: string): string { + const url = new URL(endpoint); + if (live) url.searchParams.set("model", model); + const start = live ? LIVE_SESSION_UPDATE + : { type: "session.start", config: { input_audio_format: "pcm16", sample_rate_hz: 48000, num_channels: 1, max_buffer_size_bytes: 4194304, max_utterance_duration_ms: 30000, session_ttl_ms: 300000, provider_mode: "streaming_sse", transcript_delivery_mode: "segment", vad: { type: "server_vad", threshold: 0.5, prefix_padding_ms: 300, silence_duration_ms: 500 } } }; + return `const key = prompt(${JSON.stringify(keyLabel)})?.trim(); +if (!key) throw new Error("missing_data_key"); +const encoded = btoa(String.fromCharCode(...new TextEncoder().encode(key))) + .replace(/\\+/g, "-").replace(/\\//g, "_").replace(/=+$/, ""); +const ws = new WebSocket(${JSON.stringify(url.href)}, + ["opencodex-audio", "opencodex-key." + encoded]); +ws.onopen = () => ws.send(JSON.stringify(${JSON.stringify(start)})); +ws.onmessage = ({ data }) => console.log(JSON.parse(data).type); +// After session.started, send your protocol's audio frames. +// Dictation: mono PCM16 at the declared sample_rate_hz (48000 here). +// Live Voice: input_audio.append / output_audio.delta, 24 kHz mono. +const closeSession = () => ws.send(JSON.stringify({ type: "session.close" }));`; +} + +export function audioUploadExample(endpoint: string, model: string): string { + return [ + `curl ${JSON.stringify(endpoint)}`, + ' -H "X-OpenCodex-API-Key: $OPENCODEX_API_KEY"', + ` -F "file=@recording.wav" -F "model=${model}"`, + ].join(" \\\n"); +} diff --git a/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx b/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx index dba0e23591..ff885ff616 100644 --- a/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx +++ b/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx @@ -24,11 +24,13 @@ import { } from "../../pages/api-keys-panels"; import ClientConfigPanel from "./ClientConfigPanel"; import ApiKeysListPanel from "./ApiKeysListPanel"; +import { DictationPanel, LiveVoicePanel } from "./AudioApiPanel"; export interface ApiKeysWorkspaceProps { keys: ApiKeyEntry[]; /** Management API origin the client-config panel fetches from. */ apiBase: string; + active?: boolean; /** Dataset-level. Absent means nothing is attributable yet — a different * statement from a key whose counters read zero. */ attributionSince?: string; @@ -78,6 +80,7 @@ export interface ApiKeysWorkspaceProps { export default function ApiKeysWorkspace({ keys, apiBase, + active = true, attributionSince, historyTruncated, authMatrix, @@ -164,6 +167,8 @@ export default function ApiKeysWorkspace({ { id: "keys", label: t("api.section.keys"), meta: keysLoading ? undefined : String(keys.length) }, { id: "connect", label: t("api.section.connect") }, { id: "endpoints", label: t("api.section.endpoints") }, + { id: "dictation", label: t("audio.dictation") }, + { id: "live-voice", label: t("audio.liveVoice") }, { id: "models", label: t("api.section.models"), meta: String(modelCount) }, { id: "examples", label: t("api.section.examples") }, ], [t, keys.length, keysLoading, modelCount]); @@ -245,7 +250,7 @@ export default function ApiKeysWorkspace({ one, the pattern Usage / Logs / Subagents already use. A rail plus a content pane was a second vertical band competing for the same width, and at 1280px it cost the content column 252px it could not spare. */} - {!selected && } + {!selected && }
{selected ? ( @@ -484,6 +489,12 @@ export default function ApiKeysWorkspace({
+
+ {active && } +
+
+ {active && } +
void }) { + const t = useT(); + const id = useId(); + return ; +} + +function AudioError({ code }: { code: AudioErrorCode | null }) { + const t = useT(); + return code ?

{t(`audio.error.${code}`)}

: null; +} + +export function DictationPanel({ audio }: { audio?: AudioApiInfo }) { + const t = useT(); + const [key, setKey] = useState(""); + const [file, setFile] = useState(null); + const [pending, setPending] = useState(false); + const [text, setText] = useState(null); + const [error, setError] = useState(null); + const request = useRef(null); + const fileId = useId(); + const titleId = useId(); + useEffect(() => () => { request.current?.abort(); request.current = null; }, []); + const cancel = () => { + request.current?.abort(); + request.current = null; + setPending(false); + }; + const upload = async () => { + if (!audio?.transcriptionConfigured || !file || !key.trim() || request.current) return; + const controller = new AbortController(); + request.current = controller; + setPending(true); + setError(null); + setText(null); + try { + const result = await transcribeAudio(audio.transcriptionEndpoint, audio.transcriptionModel, key, file, controller.signal); + if (request.current === controller) setText(result); + } catch (failure) { + if (request.current === controller && !controller.signal.aborted) setError(failure instanceof AudioApiError ? failure.code : "network"); + } finally { + if (request.current === controller) { request.current = null; setPending(false); } + } + }; + return
+
+

{t("audio.dictation")}

+ {t(audio?.transcriptionConfigured ? "audio.configured" : audio ? "audio.notConfigured" : "audio.unknown")} +
+ {audio && <> +
{audio.transcriptionModel}
+
{ event.preventDefault(); void upload(); }}> + { cancel(); setError(null); setKey(value); }} /> + +
+ + {pending && } +
+ + + {text !== null &&

{t("audio.transcript")}

{text ? :

{t("audio.emptyTranscript")}

}
} +
{t("audio.examples")} + +

{t("audio.streaming")}

{t(audio.dictationConfigured ? "audio.configured" : "audio.notConfigured")}
+ + +
+ } +
; +} + +export function LiveVoicePanel({ audio }: { audio?: AudioApiInfo }) { + const t = useT(); + const [key, setKey] = useState(""); + const [state, setState] = useState("idle"); + const [error, setError] = useState(null); + const [events, setEvents] = useState([]); + const connection = useRef<{ dispose?: () => void } | null>(null); + const id = useId(); + useEffect(() => () => { connection.current?.dispose?.(); connection.current = null; }, []); + const disconnect = () => { + if (!connection.current) return; + connection.current?.dispose?.(); connection.current = null; + setState("disconnected"); + }; + const connect = () => { + if (!audio?.liveConfigured || !key.trim() || connection.current) return; + setError(null); setEvents([]); + const current: { dispose?: () => void } = {}; + connection.current = current; + try { + current.dispose = connectLiveAudio({ endpoint: audio.liveEndpoint, model: audio.liveModel, key, + onState: (next, code) => { + if (connection.current !== current) return; + setState(next); setError(code ?? null); + if (next === "failed" || next === "disconnected") connection.current = null; + }, + onEvent: type => { if (connection.current === current) setEvents(previous => [...previous.slice(-7), type]); }, + }); + } catch (failure) { + connection.current = null; + setState("failed"); setError(failure instanceof AudioApiError ? failure.code : "network"); + } + }; + const busy = state === "connecting" || state === "connected"; + return
+

{t("audio.liveVoice")}

{t(audio?.liveConfigured ? "audio.configured" : audio ? "audio.notConfigured" : "audio.unknown")}
+ {audio && <> +
{audio.liveModel}
+
{ event.preventDefault(); connect(); }}> + { disconnect(); setError(null); setEvents([]); setKey(value); }} /> +
+ + {busy && } + {t(`audio.state.${state}`)} +
+ + + {events.length > 0 &&
{events.join("\n")}
} +
{t("audio.examples")}
+ } +
; +} diff --git a/gui/src/components/section-tabs.tsx b/gui/src/components/section-tabs.tsx index fd5999bba4..0866ec42b6 100644 --- a/gui/src/components/section-tabs.tsx +++ b/gui/src/components/section-tabs.tsx @@ -22,11 +22,23 @@ export function SectionTabs({ scope, items, ariaLabel, + mobileReadingLine, }: { scope: string; items: SectionTabItem[]; ariaLabel: string; + /** Optional offset for a shell with a mobile top bar above its section strip. */ + mobileReadingLine?: number; }) { + const [readingLine, setReadingLine] = useState(() => mobileReadingLine && window.matchMedia("(max-width: 760px)").matches ? mobileReadingLine : 72); + useEffect(() => { + if (!mobileReadingLine) return; + const query = window.matchMedia("(max-width: 760px)"); + const update = () => setReadingLine(query.matches ? mobileReadingLine : 72); + update(); + query.addEventListener("change", update); + return () => query.removeEventListener("change", update); + }, [mobileReadingLine]); const [active, setActive] = useState(items[0]?.id ?? ""); /** While set, scroll-spy ignores intermediate sections during smooth scroll-to-click. */ const scrollLockRef = useRef(null); @@ -49,7 +61,6 @@ export function SectionTabs({ // so a destination that stopped mid-viewport still wins over an off-screen prior heading. let bestId: string | null = null; let bestDistance = Number.POSITIVE_INFINITY; - const readingLine = 72; for (const item of items) { const node = document.getElementById(sectionAnchorId(scope, item.id)); if (!node) continue; @@ -60,7 +71,7 @@ export function SectionTabs({ } } if (bestId) setActive(bestId); - }, [clearScrollLock, items, scope]); + }, [clearScrollLock, items, scope, readingLine]); useEffect(() => () => clearScrollLock(), [clearScrollLock]); @@ -92,11 +103,11 @@ export function SectionTabs({ const id = visible.target.id.slice(sectionAnchorPrefix(scope).length); setActive(current => (current === id ? current : id)); }, - { rootMargin: "-72px 0px -60% 0px", threshold: 0 }, + { rootMargin: [String(-readingLine) + "px", "0px", "-60%", "0px"].join(" "), threshold: 0 }, ); for (const node of nodes) observer.observe(node); return () => observer.disconnect(); - }, [clearScrollLock, items, scope]); + }, [clearScrollLock, items, scope, readingLine]); const go = (id: string) => { const target = document.getElementById(sectionAnchorId(scope, id)); diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index db6b8c889f..bde60bbf6c 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -5,6 +5,35 @@ import type { TKey } from "./en"; * German i18n catalog, generated from en.ts. Must match the `TKey` set (compile-checked). */ export const de: Record = { + "audio.dictation": "Diktat", + "audio.liveVoice": "Live-Sprache", + "audio.configured": "Konfiguriert, nicht geprüft", + "audio.notConfigured": "Nicht konfiguriert", + "audio.unknown": "Audio-Metadaten nicht verfügbar", + "audio.key": "OpenCodex-Datenschlüssel", + "audio.file": "Audiodatei (max. 25 MB)", + "audio.transcribe": "Transkribieren", + "audio.transcribing": "Transkription läuft...", + "audio.transcript": "Transkript", + "audio.emptyTranscript": "Keine Sprache erkannt", + "audio.examples": "API-Beispiele", + "audio.streaming": "Streaming-Diktat", + "audio.connect": "Verbindung prüfen", + "audio.disconnect": "Trennen", + "audio.events": "Sitzungsereignisse", + "audio.state.idle": "Nicht geprüft", + "audio.state.connecting": "Verbindung wird aufgebaut...", + "audio.state.connected": "Sitzung bereit", + "audio.state.disconnected": "Getrennt", + "audio.state.failed": "Verbindung fehlgeschlagen", + "audio.error.auth": "Schlüssel abgelehnt. OpenCodex-Datenschlüssel prüfen.", + "audio.error.unavailable": "Anbieter nicht verfügbar. Konto prüfen.", + "audio.error.rateLimit": "Anfragelimit erreicht. Später erneut versuchen.", + "audio.error.invalid": "Anfrage abgelehnt. Datei und Anbieter prüfen.", + "audio.error.size": "Eine nicht leere Audiodatei bis 25 MB wählen.", + "audio.error.network": "Verbindung fehlgeschlagen. Proxy-Adresse prüfen.", + "audio.error.timeout": "Zeitüberschreitung. Erneut versuchen.", + "audio.error.protocol": "Unerwartete Audioantwort. Anbieterkompatibilität prüfen.", "models.pickerOrder.label": "Modellreihenfolge", "models.pickerOrder.default": "Standard", "models.pickerOrder.alphabetical": "A–Z nach Modell", @@ -1556,7 +1585,7 @@ export const de: Record = { "api.endpointsTitle": "Gateway-Endpunkte", "api.authBaseUrlNote": "Konfiguriere Clients mit der Basis-URL und wähle dann den protokollspezifischen Endpunkt unten.", "api.authTitle": "Authentifizierung", - "api.authLoopback": "Loopback-Binds (127.0.0.1 oder ::1) umgehen die Authentifizierung. Remote-Binds benötigen einen generierten ocx_-Schlüssel oder OPENCODEX_API_AUTH_TOKEN.", + "api.authLoopback": "Loopback-Zugriff hängt von der Route ab; eigenständige Audio-Clients benötigen einen OpenCodex-Datenschlüssel. Remote-Zugriff erfordert einen Datenschlüssel oder OPENCODEX_API_AUTH_TOKEN.", "api.modelsTitle": "Externe Modelle", "api.modelsCount": "{count} aufrufbar", "api.modelsSearch": "Modelle suchen", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 74e4ded4d1..7c56acff69 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -6,6 +6,35 @@ * `{var}` are plain interpolations. */ export const en = { + "audio.dictation": "Dictation", + "audio.liveVoice": "Live Voice", + "audio.configured": "Configured, not verified", + "audio.notConfigured": "Not configured", + "audio.unknown": "Audio metadata unavailable", + "audio.key": "OpenCodex data key", + "audio.file": "Audio file (25 MB max)", + "audio.transcribe": "Transcribe", + "audio.transcribing": "Transcribing...", + "audio.transcript": "Transcript", + "audio.emptyTranscript": "No speech detected", + "audio.examples": "API examples", + "audio.streaming": "Streaming dictation", + "audio.connect": "Check connection", + "audio.disconnect": "Disconnect", + "audio.events": "Session events", + "audio.state.idle": "Not checked", + "audio.state.connecting": "Connecting...", + "audio.state.connected": "Session ready", + "audio.state.disconnected": "Disconnected", + "audio.state.failed": "Connection failed", + "audio.error.auth": "Key rejected. Check your OpenCodex data key.", + "audio.error.unavailable": "Upstream unavailable. Check the provider account.", + "audio.error.rateLimit": "Rate limit reached. Try again later.", + "audio.error.invalid": "Request rejected. Check the file and provider.", + "audio.error.size": "Choose a nonempty audio file up to 25 MB.", + "audio.error.network": "Connection failed. Check the proxy address.", + "audio.error.timeout": "The request timed out. Try again.", + "audio.error.protocol": "Unexpected audio response. Check provider compatibility.", "models.pickerOrder.label": "Picker order", "models.pickerOrder.default": "Default", "models.pickerOrder.alphabetical": "A–Z by model", @@ -2142,7 +2171,7 @@ export const en = { "api.endpointNote": "Use the base URL with OpenAI-compatible clients. Responses and Chat Completions are exposed under /v1.", "api.endpointsTitle": "Endpoints", "api.authTitle": "Authentication", - "api.authLoopback": "Loopback binds (127.0.0.1 or ::1) bypass authentication. Remote binds require a generated ocx_ key or OPENCODEX_API_AUTH_TOKEN.", + "api.authLoopback": "Loopback access is route-specific; standalone audio clients require an OpenCodex data key. Remote binds require a data key or OPENCODEX_API_AUTH_TOKEN.", "api.authBaseUrlNote": "Configure clients with the base URL, then choose the protocol-specific endpoint below.", "api.newKeyTitle": "New key created", "api.newKeyNote": "Copy this key now — it won't be shown again.", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index 2bc3d5ab51..12b4353bf5 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -4,6 +4,35 @@ import type { TKey } from "./en"; * French i18n catalog. Must match the `TKey` set. */ export const fr: Record = { + "audio.dictation": "Dictée", + "audio.liveVoice": "Voix en direct", + "audio.configured": "Configuré, non vérifié", + "audio.notConfigured": "Non configuré", + "audio.unknown": "Métadonnées audio indisponibles", + "audio.key": "Clé de données OpenCodex", + "audio.file": "Fichier audio (25 Mo max.)", + "audio.transcribe": "Transcrire", + "audio.transcribing": "Transcription en cours...", + "audio.transcript": "Transcription", + "audio.emptyTranscript": "Aucune parole détectée", + "audio.examples": "Exemples d’API", + "audio.streaming": "Dictée en continu", + "audio.connect": "Vérifier la connexion", + "audio.disconnect": "Déconnecter", + "audio.events": "Événements de session", + "audio.state.idle": "Non vérifié", + "audio.state.connecting": "Connexion en cours...", + "audio.state.connected": "Session prête", + "audio.state.disconnected": "Déconnecté", + "audio.state.failed": "Échec de connexion", + "audio.error.auth": "Clé refusée. Vérifiez votre clé de données OpenCodex.", + "audio.error.unavailable": "Fournisseur indisponible. Vérifiez le compte.", + "audio.error.rateLimit": "Limite de requêtes atteinte. Réessayez plus tard.", + "audio.error.invalid": "Requête refusée. Vérifiez le fichier et le fournisseur.", + "audio.error.size": "Choisissez un fichier audio non vide de 25 Mo maximum.", + "audio.error.network": "Connexion impossible. Vérifiez l’adresse du proxy.", + "audio.error.timeout": "Délai dépassé. Réessayez.", + "audio.error.protocol": "Réponse audio inattendue. Vérifiez la compatibilité du fournisseur.", "models.pickerOrder.label": "Ordre des modèles", "models.pickerOrder.default": "Par défaut", "models.pickerOrder.alphabetical": "A–Z par modèle", @@ -2062,7 +2091,7 @@ export const fr: Record = { "api.endpointNote": "Utilisez l’URL de base avec les clients compatibles avec OpenAI. Responses et Chat Completions sont accessibles sous /v1.", "api.endpointsTitle": "Points de terminaison", "api.authTitle": "Authentification", - "api.authLoopback": "Les écoutes en boucle locale (127.0.0.1 ou ::1) contournent l’authentification. Les écoutes distantes nécessitent une clé ocx_ générée ou OPENCODEX_API_AUTH_TOKEN.", + "api.authLoopback": "L’accès local dépend de la route ; les clients audio autonomes nécessitent une clé de données OpenCodex. L’accès distant nécessite une clé de données ou OPENCODEX_API_AUTH_TOKEN.", "api.authBaseUrlNote": "Configurez les clients avec l’URL de base, puis choisissez ci-dessous le point de terminaison propre au protocole.", "api.newKeyTitle": "Nouvelle clé créée", "api.newKeyNote": "Copiez cette clé maintenant — elle ne sera plus affichée.", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index a787ab735b..b8119052e1 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -4,6 +4,35 @@ import type { TKey } from "./en"; * Japanese i18n catalog; must match the `TKey` set (compile-checked). */ export const ja: Record = { + "audio.dictation": "音声入力", + "audio.liveVoice": "リアルタイム音声", + "audio.configured": "設定済み・接続未確認", + "audio.notConfigured": "未設定", + "audio.unknown": "音声API情報を取得できません", + "audio.key": "OpenCodexデータキー", + "audio.file": "音声ファイル(最大25 MB)", + "audio.transcribe": "文字起こし", + "audio.transcribing": "変換中...", + "audio.transcript": "文字起こし結果", + "audio.emptyTranscript": "音声が検出されませんでした", + "audio.examples": "APIの例", + "audio.streaming": "ストリーミング音声入力", + "audio.connect": "接続を確認", + "audio.disconnect": "切断", + "audio.events": "セッションイベント", + "audio.state.idle": "未確認", + "audio.state.connecting": "接続中...", + "audio.state.connected": "セッション準備完了", + "audio.state.disconnected": "切断済み", + "audio.state.failed": "接続失敗", + "audio.error.auth": "キーが拒否されました。OpenCodexデータキーを確認してください。", + "audio.error.unavailable": "プロバイダーに接続できません。アカウントを確認してください。", + "audio.error.rateLimit": "リクエスト上限に達しました。後でもう一度お試しください。", + "audio.error.invalid": "リクエストが拒否されました。ファイルとプロバイダーを確認してください。", + "audio.error.size": "空でない25 MB以下の音声ファイルを選択してください。", + "audio.error.network": "接続に失敗しました。プロキシのアドレスを確認してください。", + "audio.error.timeout": "応答がタイムアウトしました。もう一度お試しください。", + "audio.error.protocol": "音声の応答形式が異なります。プロバイダーの互換性を確認してください。", "models.pickerOrder.label": "モデル選択順", "models.pickerOrder.default": "デフォルト", "models.pickerOrder.alphabetical": "モデル名のA–Z順", @@ -1994,7 +2023,7 @@ export const ja: Record = { "api.modelsEndpoint": "Models API", "api.authTitle": "認証", "api.authBaseUrlNote": "クライアントにはベース URL を設定し、下のプロトコル別エンドポイントを選んでください。", - "api.authLoopback": "ループバック (127.0.0.1 または ::1) は認証を省略します。リモートでは生成した ocx_ キーまたは OPENCODEX_API_AUTH_TOKEN が必要です。", + "api.authLoopback": "ループバックの認証は経路によって異なります。独立した音声クライアントにはOpenCodexデータキーが必要です。リモート接続にはデータキーまたはOPENCODEX_API_AUTH_TOKENが必要です。", "api.modelsTitle": "外部モデルカタログ", "api.modelsCount": "{count} 件が利用可能", "api.modelsSearch": "モデルを検索", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index ccd2b06735..5a031a9698 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -4,6 +4,35 @@ import type { TKey } from "./en"; * Korean i18n catalog; must match the `TKey` set (compile-checked). */ export const ko: Record = { + "audio.dictation": "받아쓰기", + "audio.liveVoice": "실시간 음성", + "audio.configured": "설정됨 · 연결 미확인", + "audio.notConfigured": "설정 안 됨", + "audio.unknown": "음성 API 정보를 불러올 수 없음", + "audio.key": "OpenCodex 데이터 키", + "audio.file": "음성 파일 (최대 25 MB)", + "audio.transcribe": "받아쓰기", + "audio.transcribing": "변환 중...", + "audio.transcript": "변환 결과", + "audio.emptyTranscript": "감지된 음성 없음", + "audio.examples": "API 예제", + "audio.streaming": "스트리밍 받아쓰기", + "audio.connect": "연결 확인", + "audio.disconnect": "연결 해제", + "audio.events": "세션 이벤트", + "audio.state.idle": "확인 전", + "audio.state.connecting": "연결 중...", + "audio.state.connected": "세션 준비됨", + "audio.state.disconnected": "연결 해제됨", + "audio.state.failed": "연결 실패", + "audio.error.auth": "키가 거부됐습니다. OpenCodex 데이터 키를 확인하세요.", + "audio.error.unavailable": "제공자에 연결할 수 없습니다. 계정을 확인하세요.", + "audio.error.rateLimit": "요청 한도에 도달했습니다. 잠시 후 다시 시도하세요.", + "audio.error.invalid": "요청이 거부됐습니다. 파일과 제공자를 확인하세요.", + "audio.error.size": "내용이 있는 25 MB 이하 음성 파일을 선택하세요.", + "audio.error.network": "연결에 실패했습니다. 프록시 주소를 확인하세요.", + "audio.error.timeout": "응답 시간이 초과됐습니다. 다시 시도하세요.", + "audio.error.protocol": "음성 응답 형식이 맞지 않습니다. 제공자 호환성을 확인하세요.", "models.pickerOrder.label": "모델 선택 순서", "models.pickerOrder.default": "기본값", "models.pickerOrder.alphabetical": "모델 이름순", @@ -1595,7 +1624,7 @@ export const ko: Record = { "api.endpointsTitle": "게이트웨이 엔드포인트", "api.authBaseUrlNote": "클라이언트에는 기본 URL을 설정한 뒤 아래에서 프로토콜별 엔드포인트를 선택하세요.", "api.authTitle": "인증", - "api.authLoopback": "루프백 바인드(127.0.0.1 또는 ::1)는 인증을 건너뜁니다. 원격 바인드는 생성된 ocx_ 키 또는 OPENCODEX_API_AUTH_TOKEN이 필요합니다.", + "api.authLoopback": "루프백 인증은 경로마다 다릅니다. 독립 음성 API에는 OpenCodex 데이터 키가 필요합니다. 원격 연결에는 데이터 키 또는 OPENCODEX_API_AUTH_TOKEN이 필요합니다.", "api.modelsTitle": "외부 모델 카탈로그", "api.modelsCount": "{count}개 호출 가능", "api.modelsSearch": "모델 검색", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index ceb0c18a92..609fe25daf 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -4,6 +4,35 @@ import type { TKey } from "./en"; * Russian i18n catalog; must match the `TKey` set (compile-checked). */ export const ru: Record = { + "audio.dictation": "Диктовка", + "audio.liveVoice": "Голос в реальном времени", + "audio.configured": "Настроено, не проверено", + "audio.notConfigured": "Не настроено", + "audio.unknown": "Метаданные аудио недоступны", + "audio.key": "Ключ данных OpenCodex", + "audio.file": "Аудиофайл (до 25 МБ)", + "audio.transcribe": "Расшифровать", + "audio.transcribing": "Расшифровка...", + "audio.transcript": "Расшифровка", + "audio.emptyTranscript": "Речь не обнаружена", + "audio.examples": "Примеры API", + "audio.streaming": "Потоковая диктовка", + "audio.connect": "Проверить соединение", + "audio.disconnect": "Отключить", + "audio.events": "События сеанса", + "audio.state.idle": "Не проверено", + "audio.state.connecting": "Подключение...", + "audio.state.connected": "Сеанс готов", + "audio.state.disconnected": "Отключено", + "audio.state.failed": "Ошибка подключения", + "audio.error.auth": "Ключ отклонён. Проверьте ключ данных OpenCodex.", + "audio.error.unavailable": "Провайдер недоступен. Проверьте аккаунт.", + "audio.error.rateLimit": "Достигнут лимит запросов. Повторите позже.", + "audio.error.invalid": "Запрос отклонён. Проверьте файл и провайдера.", + "audio.error.size": "Выберите непустой аудиофайл размером до 25 МБ.", + "audio.error.network": "Ошибка подключения. Проверьте адрес прокси.", + "audio.error.timeout": "Время ожидания истекло. Повторите попытку.", + "audio.error.protocol": "Неожиданный аудиоответ. Проверьте совместимость провайдера.", "models.pickerOrder.label": "Порядок моделей", "models.pickerOrder.default": "По умолчанию", "models.pickerOrder.alphabetical": "По имени A–Z", @@ -2064,7 +2093,7 @@ export const ru: Record = { "api.endpointsTitle": "Конечные точки", "api.authTitle": "Аутентификация", "api.authBaseUrlNote": "Настройте клиентов с базовым URL, затем выберите нужный протокольный endpoint ниже.", - "api.authLoopback": "Loopback-привязки (127.0.0.1 или ::1) обходят аутентификацию. Для удалённых привязок нужен сгенерированный ocx_-ключ или OPENCODEX_API_AUTH_TOKEN.", + "api.authLoopback": "Доступ через loopback зависит от маршрута; отдельным аудиоклиентам нужен ключ данных OpenCodex. Для удалённого доступа нужен ключ данных или OPENCODEX_API_AUTH_TOKEN.", "api.modelsTitle": "Каталог внешних моделей", "api.modelsCount": "{count} доступно", "api.modelsSearch": "Поиск моделей", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 26a8f93d09..755aa8b48c 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -5,6 +5,35 @@ import type { TKey } from "./en"; * Turkish i18n catalog. Must match the `TKey` set (compile-checked). */ export const tr: Record = { + "audio.dictation": "Dikte", + "audio.liveVoice": "Canlı Ses", + "audio.configured": "Yapılandırıldı, doğrulanmadı", + "audio.notConfigured": "Yapılandırılmadı", + "audio.unknown": "Ses API bilgileri kullanılamıyor", + "audio.key": "OpenCodex veri anahtarı", + "audio.file": "Ses dosyası (en fazla 25 MB)", + "audio.transcribe": "Metne çevir", + "audio.transcribing": "Metne çevriliyor...", + "audio.transcript": "Döküm", + "audio.emptyTranscript": "Konuşma algılanmadı", + "audio.examples": "API örnekleri", + "audio.streaming": "Akışlı dikte", + "audio.connect": "Bağlantıyı kontrol et", + "audio.disconnect": "Bağlantıyı kes", + "audio.events": "Oturum olayları", + "audio.state.idle": "Kontrol edilmedi", + "audio.state.connecting": "Bağlanıyor...", + "audio.state.connected": "Oturum hazır", + "audio.state.disconnected": "Bağlantı kesildi", + "audio.state.failed": "Bağlantı başarısız", + "audio.error.auth": "Anahtar reddedildi. OpenCodex veri anahtarını kontrol edin.", + "audio.error.unavailable": "Sağlayıcı kullanılamıyor. Hesabı kontrol edin.", + "audio.error.rateLimit": "İstek sınırına ulaşıldı. Daha sonra tekrar deneyin.", + "audio.error.invalid": "İstek reddedildi. Dosyayı ve sağlayıcıyı kontrol edin.", + "audio.error.size": "En fazla 25 MB boyutunda, boş olmayan bir ses dosyası seçin.", + "audio.error.network": "Bağlantı başarısız. Proxy adresini kontrol edin.", + "audio.error.timeout": "İstek zaman aşımına uğradı. Tekrar deneyin.", + "audio.error.protocol": "Beklenmeyen ses yanıtı. Sağlayıcı uyumluluğunu kontrol edin.", "models.pickerOrder.label": "Model sırası", "models.pickerOrder.default": "Varsayılan", "models.pickerOrder.alphabetical": "Model adına göre A–Z", @@ -2083,7 +2112,7 @@ export const tr: Record = { "api.endpointNote": "OpenAI uyumlu istemcilerle taban URL'yi kullanın.", "api.endpointsTitle": "Uç noktalar", "api.authTitle": "Kimlik Doğrulama", - "api.authLoopback": "Geri döngü (loopback) bağlantıları (127.0.0.1 / ::1) kimlik doğrulamasını atlar. Harici/ağ istemcileri x-opencodex-api-key veya Authorization başlığında bir ocx_ API anahtarı ya da OPENCODEX_API_AUTH_TOKEN göndermelidir.", + "api.authLoopback": "Geri döngü erişimi yola bağlıdır; bağımsız ses istemcileri OpenCodex veri anahtarı gerektirir. Uzak bağlantılar veri anahtarı veya OPENCODEX_API_AUTH_TOKEN gerektirir.", "api.authBaseUrlNote": "İstemcileri taban URL ile yapılandırın.", "api.newKeyTitle": "Yeni anahtar oluşturuldu", "api.newKeyNote": "Bu anahtarı şimdi kopyalayın — tekrar gösterilmeyecektir.", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index 1fb8387f4d..17b483dcd3 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -2,6 +2,35 @@ import type { TKey } from "./en"; /** Traditional Chinese (Taiwan) UI strings — keys must match `en.ts` 1:1. */ export const zhTW: Record = { + "audio.dictation": "聽寫", + "audio.liveVoice": "即時語音", + "audio.configured": "已設定,未驗證", + "audio.notConfigured": "未設定", + "audio.unknown": "無法取得音訊 API 資訊", + "audio.key": "OpenCodex 資料金鑰", + "audio.file": "音訊檔案(最大 25 MB)", + "audio.transcribe": "轉寫", + "audio.transcribing": "正在轉寫...", + "audio.transcript": "轉寫結果", + "audio.emptyTranscript": "未偵測到語音", + "audio.examples": "API 範例", + "audio.streaming": "串流聽寫", + "audio.connect": "檢查連線", + "audio.disconnect": "中斷連線", + "audio.events": "工作階段事件", + "audio.state.idle": "未檢查", + "audio.state.connecting": "正在連線...", + "audio.state.connected": "工作階段已就緒", + "audio.state.disconnected": "已中斷", + "audio.state.failed": "連線失敗", + "audio.error.auth": "金鑰遭拒。請檢查 OpenCodex 資料金鑰。", + "audio.error.unavailable": "上游無法使用。請檢查供應商帳戶。", + "audio.error.rateLimit": "已達請求上限。請稍後再試。", + "audio.error.invalid": "請求遭拒。請檢查檔案與供應商。", + "audio.error.size": "請選擇不超過 25 MB 的非空音訊檔案。", + "audio.error.network": "連線失敗。請檢查代理位址。", + "audio.error.timeout": "請求逾時。請再試一次。", + "audio.error.protocol": "音訊回應格式異常。請檢查供應商相容性。", "models.pickerOrder.label": "模型選擇順序", "models.pickerOrder.default": "預設", "models.pickerOrder.alphabetical": "依模型名稱 A–Z", @@ -1601,7 +1630,7 @@ export const zhTW: Record = { "api.endpointNote": "請將基礎 URL 用於 OpenAI 相容客戶端。Responses 與 Chat Completions 在 /v1 下提供。", "api.endpointsTitle": "閘道器端點", "api.authTitle": "身份驗證", - "api.authLoopback": "迴環繫結(127.0.0.1 或 ::1)會跳過身份驗證。遠端繫結需要生成的 ocx_ 金鑰或 OPENCODEX_API_AUTH_TOKEN。", + "api.authLoopback": "迴環存取的驗證要求依路徑而異;獨立音訊用戶端需要 OpenCodex 資料金鑰。遠端連線需要資料金鑰或 OPENCODEX_API_AUTH_TOKEN。", "api.authBaseUrlNote": "客戶端應使用基礎 URL,然後選擇下面的協議端點。", "api.newKeyTitle": "已建立新金鑰", "api.newKeyNote": "請立即複製此金鑰,它不會再次顯示。", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 4e2cd831cc..af1acbbdab 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -4,6 +4,35 @@ import type { TKey } from "./en"; * Chinese i18n catalog; must match the `TKey` set (compile-checked). */ export const zh: Record = { + "audio.dictation": "听写", + "audio.liveVoice": "实时语音", + "audio.configured": "已配置,未验证", + "audio.notConfigured": "未配置", + "audio.unknown": "无法获取音频 API 信息", + "audio.key": "OpenCodex 数据密钥", + "audio.file": "音频文件(最大 25 MB)", + "audio.transcribe": "转写", + "audio.transcribing": "正在转写...", + "audio.transcript": "转写结果", + "audio.emptyTranscript": "未检测到语音", + "audio.examples": "API 示例", + "audio.streaming": "流式听写", + "audio.connect": "检查连接", + "audio.disconnect": "断开连接", + "audio.events": "会话事件", + "audio.state.idle": "未检查", + "audio.state.connecting": "正在连接...", + "audio.state.connected": "会话已就绪", + "audio.state.disconnected": "已断开", + "audio.state.failed": "连接失败", + "audio.error.auth": "密钥被拒绝。请检查 OpenCodex 数据密钥。", + "audio.error.unavailable": "上游不可用。请检查提供商账户。", + "audio.error.rateLimit": "已达到请求上限。请稍后重试。", + "audio.error.invalid": "请求被拒绝。请检查文件和提供商。", + "audio.error.size": "请选择不超过 25 MB 的非空音频文件。", + "audio.error.network": "连接失败。请检查代理地址。", + "audio.error.timeout": "请求超时。请重试。", + "audio.error.protocol": "音频响应格式异常。请检查提供商兼容性。", "models.pickerOrder.label": "模型选择顺序", "models.pickerOrder.default": "默认", "models.pickerOrder.alphabetical": "按模型名 A–Z", @@ -1576,7 +1605,7 @@ export const zh: Record = { "api.endpointsTitle": "网关端点", "api.authBaseUrlNote": "客户端应使用基础 URL,然后选择下面的协议端点。", "api.authTitle": "身份验证", - "api.authLoopback": "回环绑定(127.0.0.1 或 ::1)会跳过身份验证。远程绑定需要生成的 ocx_ 密钥或 OPENCODEX_API_AUTH_TOKEN。", + "api.authLoopback": "回环访问的认证要求因路径而异;独立音频客户端需要 OpenCodex 数据密钥。远程连接需要数据密钥或 OPENCODEX_API_AUTH_TOKEN。", "api.modelsTitle": "外部模型目录", "api.modelsCount": "{count} 个可调用", "api.modelsSearch": "搜索模型", diff --git a/gui/src/pages/ApiKeys.tsx b/gui/src/pages/ApiKeys.tsx index d10ff3c33d..ca45d8b940 100644 --- a/gui/src/pages/ApiKeys.tsx +++ b/gui/src/pages/ApiKeys.tsx @@ -20,6 +20,7 @@ import { deriveApiEndpoints, isApiAuthMatrix, isApiKeyUsage, + isAudioApiInfo, type ApiEndpointInfo, type ApiAuthMatrixRow, type ApiKeyEntry, @@ -43,6 +44,7 @@ interface KeysResponse { messagesEndpoint?: string; modelsEndpoint?: string; claudeCodeEnabled?: boolean; + audio?: unknown; } interface CreateKeyResponse { @@ -88,6 +90,10 @@ function seedEndpointsFromApiBase(apiBase: string): ApiEndpointInfo { function validCachedKeys(cached: CachedKeysShape | null): CachedKeysShape | null { if (!cached || !isApiAuthMatrix(cached.authMatrix)) return null; if (!Array.isArray(cached.keys) || cached.keys.some(key => !key || !isApiKeyUsage(key.usage) || !validPendingRotation(key.pendingRotation))) return null; + if (cached.endpoints?.audio !== undefined && !isAudioApiInfo(cached.endpoints.audio, cached.endpoints.baseUrl)) { + const { audio: _audio, ...endpoints } = cached.endpoints; + return { ...cached, endpoints }; + } return cached; } @@ -155,6 +161,7 @@ export default function ApiKeys({ apiBase, active = true }: { apiBase: string; a chatCompletions: data.chatCompletionsEndpoint ?? derived.chatCompletions, messages: data.messagesEndpoint ?? derived.messages, models: data.modelsEndpoint ?? derived.models, + ...(isAudioApiInfo(data.audio, data.baseUrl ?? derived.baseUrl) ? { audio: data.audio } : {}), }, claudeCodeEnabled: data.claudeCodeEnabled !== false, ...(data.attributionSince ? { attributionSince: data.attributionSince } : {}), @@ -496,6 +503,7 @@ export default function ApiKeys({ apiBase, active = true }: { apiBase: string; a ) : ( <> ; + const fields = new Set(["transcriptionEndpoint", "dictationStreamEndpoint", "liveEndpoint", "realtimeCallsEndpoint", "transcriptionModel", "liveModel", "transcriptionConfigured", "dictationConfigured", "liveConfigured"]); + if (Object.keys(audio).some(field => !fields.has(field))) return false; + try { + const base = new URL(baseUrl); + if (!["http:", "https:"].includes(base.protocol) || base.username || base.password || base.search || base.hash) return false; + const paths = { + transcriptionEndpoint: ["/audio/transcriptions", false], + dictationStreamEndpoint: ["/audio/transcriptions/stream", true], + liveEndpoint: ["/live", true], + realtimeCallsEndpoint: ["/realtime/calls", false], + } as const; + for (const [field, [suffix, socket]] of Object.entries(paths)) { + if (typeof audio[field] !== "string") return false; + const url = new URL(audio[field]); + const protocol = socket ? (base.protocol === "https:" ? "wss:" : "ws:") : base.protocol; + if (url.protocol !== protocol || url.username || url.password || url.search || url.hash) return false; + url.protocol = base.protocol; + if (url.origin !== base.origin || url.pathname !== `${base.pathname.replace(/\/$/, "")}${suffix}`) return false; + } + return audio.transcriptionModel === "gpt-4o-transcribe" && audio.liveModel === "gpt-live-1-codex" + && [audio.transcriptionConfigured, audio.dictationConfigured, audio.liveConfigured].every(flag => typeof flag === "boolean"); + } catch { return false; } } export type ModelTestState = "idle" | "testing" | "ok" | "error"; diff --git a/gui/src/styles-apikeys-workspace.css b/gui/src/styles-apikeys-workspace.css index ad027753eb..ce9fa08654 100644 --- a/gui/src/styles-apikeys-workspace.css +++ b/gui/src/styles-apikeys-workspace.css @@ -69,6 +69,85 @@ min-width: 0; } +.audio-api-section { + min-width: 0; + display: grid; + gap: var(--space-3); + padding: 18px 0; + border-top: 1px solid var(--border); + letter-spacing: 0; +} + +.audio-api-head, .audio-api-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: var(--space-2); + min-width: 0; +} + +.audio-api-head { justify-content: space-between; } +.audio-api-head h3, .audio-api-section h4 { + margin: 0; + font-size: var(--text-control); + line-height: var(--leading-ui); + text-wrap: balance; + overflow-wrap: anywhere; +} +.audio-api-endpoints, .audio-api-field, .audio-api-form, .audio-api-result { + display: grid; + gap: var(--space-2); + min-width: 0; +} +.audio-api-form { grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); } +.audio-api-field { font-size: var(--text-label); align-content: start; } +.audio-api-field .input { width: 100%; min-width: 0; box-sizing: border-box; } +.audio-api-field input[type="file"] { padding: 7px; } +.audio-api-actions { grid-column: 1 / -1; min-height: 36px; } +.audio-api-actions .btn { min-height: 36px; white-space: nowrap; } +.audio-api-status { font-size: var(--text-label); } +.audio-api-error { margin: 0; color: var(--danger); font-size: var(--text-label); } +.audio-api-section .api-endpoint-url-btn { + display: block; + width: 100%; + max-width: 100%; + min-width: 0; + text-align: left; + appearance: none; + border: 0; + background: transparent; + padding: 0; + margin: 0; + color: inherit; + font: inherit; + cursor: pointer; +} +.audio-api-section .api-endpoint-url { + display: block; + box-sizing: border-box; + width: 100%; + max-width: 100%; + white-space: normal; + overflow-wrap: anywhere; +} +.audio-api-examples { min-width: 0; } +.audio-api-examples summary { cursor: pointer; font-size: var(--text-label); padding: 6px 0; } +.audio-api-examples > :not(summary) { margin-top: var(--space-2); } +.audio-api-events { + margin: 0; + padding: var(--space-2) 0; + white-space: pre-wrap; + overflow-wrap: anywhere; + font-size: var(--text-label); + color: var(--muted); +} +.audio-api-result .api-example-pre { white-space: pre-wrap; overflow-wrap: anywhere; } +@container apikeys-workspace (max-width: 560px) { + .audio-api-form { grid-template-columns: minmax(0, 1fr); } + .audio-api-actions .btn { min-height: 44px; } + .audio-api-field .input { min-height: 44px; } +} + .awi-keylist-panel .awi-keylist-name { appearance: none; background: none; @@ -736,3 +815,17 @@ padding: var(--space-2) 0 0; } } + +.apikeys-workspace-shell .section-tabs { + flex-wrap: nowrap; + overflow-x: auto; + overflow-y: hidden; + padding-top: 0; + min-height: 44px; +} + +/* The shell's mobile top bar is 44px controls + 8px padding + 1px border. */ +@media (max-width: 760px) { + .apikeys-workspace-shell .section-tabs { top: 53px; } + .apikeys-workspace-shell .awi-section-anchor { scroll-margin-top: 108px; } +} diff --git a/gui/tests/api-auth-memory.test.ts b/gui/tests/api-auth-memory.test.ts index 825df3806d..96c618f6bd 100644 --- a/gui/tests/api-auth-memory.test.ts +++ b/gui/tests/api-auth-memory.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { Window } from "happy-dom"; -import { configureApiTargets, installApiAuthFetch, installApiSessionFromHtml, resetApiAuthFetchForTests } from "../src/api"; +import { configureApiTargets, fetchAudioUpload, installApiAuthFetch, installApiSessionFromHtml, resetApiAuthFetchForTests } from "../src/api"; import { targetsFromMachineStatus, type MachineStatusV1 } from "../src/api-targets"; const LEGACY_TOKEN_KEY = "opencodex-api-token"; @@ -86,6 +86,29 @@ test("installApiAuthFetch deletes legacy sessionStorage token without reading it } }); +test("audio uploads bypass connected management interception and 401 recovery", async () => { + injectSessionMeta("ocx_session_audio_machine", "audio-csrf", "http://localhost"); + const seen: Array<{ url: string; headers: Headers }> = []; + const mockFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(input), headers: new Headers(init?.headers) }); + return new Response("rejected", { status: 401 }); + }) as typeof fetch; + await installMockAuthFetch(mockFetch); + configureApiTargets({ + connected: true, + machine: { id: "machine", baseUrl: "http://localhost", serverOrigin: "http://localhost", bootstrapPath: "/opencodex-session", transport: "same-origin" }, + shared: { id: "shared", baseUrl: "https://hub.example.test", serverOrigin: "https://hub.example.test", bootstrapPath: "https://hub.example.test/opencodex-session", transport: "relay" }, + }); + const key = "ocx_data_audio_wrapper_fixture"; + const response = await fetchAudioUpload("https://hub.example.test/v1/audio/transcriptions", { method: "POST", headers: { "X-OpenCodex-API-Key": key }, body: new FormData() }); + expect(response.status).toBe(401); + expect(seen).toHaveLength(1); + expect([...seen[0]!.headers]).toEqual([["x-opencodex-api-key", key]]); + expect(sessionStorage.length).toBe(0); + await expect(fetchAudioUpload("https://hub.example.test/api/config", { method: "POST" })).rejects.toThrow(); + expect(seen).toHaveLength(1); +}); + test("prompted API tokens stay memory-only and are not written to sessionStorage", async () => { declareManagementAuthRequired(); sessionStorage.setItem(LEGACY_TOKEN_KEY, "legacy-secret"); diff --git a/gui/tests/audio-api-client.test.ts b/gui/tests/audio-api-client.test.ts new file mode 100644 index 0000000000..43ebfa727d --- /dev/null +++ b/gui/tests/audio-api-client.test.ts @@ -0,0 +1,173 @@ +import { afterEach, expect, test } from "bun:test"; +import { AUDIO_FILE_MAX_BYTES, AudioApiError, LIVE_SESSION_UPDATE, audioSocketProtocols, connectLiveAudio, transcribeAudio } from "../src/audio-api-client"; +import { resetApiAuthFetchForTests } from "../src/api"; +import { isAudioApiInfo, type AudioApiInfo } from "../src/pages/api-keys-utils"; +import { audioSocketExample, audioUploadExample } from "../src/audio-api-examples"; + +const originalFetch = globalThis.fetch; +const originalSocket = globalThis.WebSocket; +const KEY = "ocx_data_audio_client_fixture"; +const ENDPOINT = "https://gateway.example/v1/audio/transcriptions"; +afterEach(() => { globalThis.fetch = originalFetch; globalThis.WebSocket = originalSocket; resetApiAuthFetchForTests(); }); + +test("upload sends multipart with only the typed data key and no cookies or redirects", async () => { + resetApiAuthFetchForTests(); + let init!: RequestInit; + globalThis.fetch = (async (_url, options) => { init = options!; return Response.json({ text: "fixture transcript" }); }) as typeof fetch; + const text = await transcribeAudio(ENDPOINT, "gpt-4o-transcribe", KEY, new File(["synthetic"], "fixture.wav"), new AbortController().signal); + expect(text).toBe("fixture transcript"); + expect([...new Headers(init.headers)]).toEqual([["x-opencodex-api-key", KEY]]); + expect(init.credentials).toBe("omit"); expect(init.redirect).toBe("error"); + const body = init.body as FormData; + expect(body.get("model")).toBe("gpt-4o-transcribe"); + expect(body.get("response_format")).toBe("json"); + expect((body.get("file") as File).name).toBe("fixture.wav"); +}); + +test("invalid files stop before fetch and upstream errors never expose their body", async () => { + resetApiAuthFetchForTests(); + let calls = 0; + globalThis.fetch = (async () => { calls++; return new Response("private upstream detail", { status: 401 }); }) as typeof fetch; + const file = new File(["fixture"], "fixture.wav"); + for (const size of [0, AUDIO_FILE_MAX_BYTES + 1]) { + Object.defineProperty(file, "size", { configurable: true, value: size }); + await expect(transcribeAudio(ENDPOINT, "gpt-4o-transcribe", KEY, file, new AbortController().signal)).rejects.toMatchObject({ code: "size" }); + } + expect(calls).toBe(0); + Object.defineProperty(file, "size", { configurable: true, value: 7 }); + await expect(transcribeAudio(ENDPOINT, "gpt-4o-transcribe", KEY, file, new AbortController().signal)).rejects.toEqual(new AudioApiError("auth")); +}); + +test("upload cancellation aborts fetch and a locked response body", async () => { + resetApiAuthFetchForTests(); + let responseCancelled = false; + let uploadSignal: AbortSignal | undefined; + let bodyStarted!: () => void; + const started = new Promise(resolve => { bodyStarted = resolve; }); + globalThis.fetch = (async (_url, init) => { + uploadSignal = init!.signal!; + return new Response(new ReadableStream({ start() { bodyStarted(); }, cancel() { responseCancelled = true; } })); + }) as typeof fetch; + const controller = new AbortController(); + const pending = transcribeAudio(ENDPOINT, "gpt-4o-transcribe", KEY, new File(["x"], "x.wav"), controller.signal); + await started; + await Promise.resolve(); + controller.abort(); + await expect(pending).rejects.toBeDefined(); + expect(uploadSignal!.aborted).toBe(true); + expect(responseCancelled).toBe(true); +}); + +test("audio metadata accepts the exact scheme/origin/path projection only", () => { + const base = "https://[2001:db8::1]:8443/v1"; + const audio: AudioApiInfo = { + transcriptionEndpoint: `${base}/audio/transcriptions`, realtimeCallsEndpoint: `${base}/realtime/calls`, + liveEndpoint: "wss://[2001:db8::1]:8443/v1/live", dictationStreamEndpoint: "wss://[2001:db8::1]:8443/v1/audio/transcriptions/stream", + transcriptionModel: "gpt-4o-transcribe", liveModel: "gpt-live-1-codex", + transcriptionConfigured: true, dictationConfigured: true, liveConfigured: true, + }; + expect(isAudioApiInfo(audio, base)).toBe(true); + for (const endpoint of ["ws://[2001:db8::1]:8443/v1/live", `${audio.liveEndpoint}?key=value`, `${audio.liveEndpoint}#fragment`, "wss://user:pass@[2001:db8::1]:8443/v1/live", "wss://other.example/v1/live", "wss://[2001:db8::1]:8443/v1/responses"]) { + expect(isAudioApiInfo({ ...audio, liveEndpoint: endpoint }, base)).toBe(false); + } + expect(isAudioApiInfo(undefined, base)).toBe(false); + expect(isAudioApiInfo({ ...audio, liveConfigured: "true" }, base)).toBe(false); + expect(isAudioApiInfo({ ...audio, apiKey: "sensitive-marker" }, base)).toBe(false); +}); + +class FakeSocket { + static OPEN = 1; + static latest: FakeSocket; + readyState = 0; + closed = false; + sent: string[] = []; + onopen: (() => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onclose: ((event: { code: number }) => void) | null = null; + onerror: (() => void) | null = null; + constructor(readonly url: string, readonly protocols: string[]) { FakeSocket.latest = this; } + send(value: string) { this.sent.push(value); } + close() { this.closed = true; this.readyState = 3; } + open() { this.readyState = 1; this.onopen?.(); } + message(value: unknown) { this.onmessage?.({ data: JSON.stringify(value) }); } +} + +function probe(options: { readyTimeoutMs?: number; maxSessionMs?: number } = {}) { + globalThis.WebSocket = FakeSocket as unknown as typeof WebSocket; + const states: string[] = []; + const events: string[] = []; + const dispose = connectLiveAudio({ endpoint: "wss://gateway.example/v1/live", model: "gpt-live-1-codex", key: KEY, + onState: (state, code) => states.push(code ? `${state}:${code}` : state), onEvent: event => events.push(event), ...options }); + return { socket: FakeSocket.latest, states, events, dispose }; +} + +test("voice waits for a nonterminal native session ID, then closes without audio", () => { + const { socket, states, events, dispose } = probe(); + try { + expect(socket.url).not.toContain(KEY); + expect(socket.protocols).toEqual(audioSocketProtocols(KEY)); + socket.open(); + expect(JSON.parse(socket.sent[0]!)).toEqual(LIVE_SESSION_UPDATE); + expect(states).toEqual(["connecting"]); + socket.message({ type: "session.started", session: { session_id: "dictation-not-live" } }); + expect(states).toEqual(["connecting"]); + socket.message({ type: "session.started", session: { id: "fixture" }, secret: "never display" }); + expect(states).toEqual(["connecting", "connected"]); + expect(events).toEqual(["session.started", "session.started"]); + } finally { dispose(); } + expect(socket.closed).toBe(true); + expect(socket.onmessage).toBeNull(); + expect(socket.sent.map(value => JSON.parse(value).type)).toEqual(["session.update", "session.close"]); +}); + +test("terminal acknowledgments and protocol errors stay failed after a normal close", () => { + for (const event of [{ type: "session.updated", session: { id: "fixture", status: "closed" } }, { type: "protocol.error", error: "sensitive" }]) { + const { socket, states, dispose } = probe(); + socket.open(); + const oldClose = socket.onclose; + socket.message(event); + oldClose?.({ code: 1000 }); + expect(states).toEqual(["connecting", "failed:protocol"]); + expect(socket.closed).toBe(true); + dispose(); + } +}); + +test("voice readiness and established-session timers release all handlers", async () => { + const pending = probe({ readyTimeoutMs: 5 }); + pending.socket.open(); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(pending.states).toEqual(["connecting", "failed:timeout"]); + expect(pending.socket.onmessage).toBeNull(); + const ready = probe({ maxSessionMs: 5 }); + ready.socket.open(); ready.socket.message({ type: "session.started", session: { id: "fixture" } }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(ready.states).toEqual(["connecting", "connected", "disconnected"]); + expect(ready.socket.closed).toBe(true); +}); + +test("failed session updates remain failures even after readiness", () => { + for (const status of ["error", "failed"]) { + const { socket, states, dispose } = probe(); + socket.open(); socket.message({ type: "session.started", session: { id: "fixture" } }); + const oldClose = socket.onclose; + socket.message({ type: "session.updated", session: { id: "fixture", status } }); + oldClose?.({ code: 1000 }); + expect(states).toEqual(["connecting", "connected", "failed:protocol"]); + expect(socket.closed).toBe(true); + dispose(); + } +}); + +test("copied socket examples are executable protocol code with a localized key prompt", () => { + const example = audioSocketExample("wss://gateway.example/v1/live", true, "gpt-live-1-codex", "데이터 키"); + let label = ""; + const run = new Function("WebSocket", "prompt", "btoa", "TextEncoder", example + "; return ws;"); + const socket = run(FakeSocket, (value: string) => { label = value; return KEY; }, btoa, TextEncoder) as FakeSocket; + expect(label).toBe("데이터 키"); + expect(socket.protocols).toEqual(audioSocketProtocols(KEY)); + socket.open(); + expect(JSON.parse(socket.sent[0]!)).toEqual(LIVE_SESSION_UPDATE); + expect(audioUploadExample(ENDPOINT, "gpt-4o-transcribe")).toContain("$OPENCODEX_API_KEY"); + expect(audioUploadExample(ENDPOINT, "gpt-4o-transcribe")).not.toContain("\n+"); +}); diff --git a/gui/tests/audio-api-panel.test.tsx b/gui/tests/audio-api-panel.test.tsx new file mode 100644 index 0000000000..bbffa6a45b --- /dev/null +++ b/gui/tests/audio-api-panel.test.tsx @@ -0,0 +1,229 @@ +/** @jsxImportSource react */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { Window } from "happy-dom"; +import { act } from "react"; +import type { Root } from "react-dom/client"; +import { LanguageProvider } from "../src/i18n/provider"; +import { clearClientResourceStoresForTests } from "../src/client-resource"; +import { resetApiAuthFetchForTests } from "../src/api"; +import ApiKeys from "../src/pages/ApiKeys"; + +const KEY = "ocx_data_audio_panel_fixture"; +const BASE = "http://localhost/v1"; +const AUDIO = { + transcriptionEndpoint: `${BASE}/audio/transcriptions`, dictationStreamEndpoint: "ws://localhost/v1/audio/transcriptions/stream", + liveEndpoint: "ws://localhost/v1/live", realtimeCallsEndpoint: `${BASE}/realtime/calls`, + transcriptionModel: "gpt-4o-transcribe", liveModel: "gpt-live-1-codex", + transcriptionConfigured: true, dictationConfigured: true, liveConfigured: true, +}; +const globals = ["document", "window", "navigator", "localStorage", "sessionStorage", "fetch", "WebSocket", "HTMLElement", "IS_REACT_ACT_ENVIRONMENT"] as const; +let previous: Record; +let win: Window; +let root: Root | undefined; +let container: HTMLDivElement; +let audio: unknown; +let sent: RequestInit[]; +let responder: (init: RequestInit) => Promise; +let holdKeys: Promise | null; + +class Socket { + static OPEN = 1; + static latest: Socket | undefined; + readyState = 0; + closed = false; + onopen: (() => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: (() => void) | null = null; + onclose: ((event: { code: number }) => void) | null = null; + sent: string[] = []; + constructor(readonly url: string, readonly protocols: string[]) { Socket.latest = this; } + send(value: string) { this.sent.push(value); } + close() { this.closed = true; } +} + +beforeEach(() => { + previous = Object.fromEntries(globals.map(key => [key, Object.getOwnPropertyDescriptor(globalThis, key)])); + win = new Window({ url: "http://localhost/" }); + for (const key of ["document", "window", "navigator", "localStorage", "sessionStorage", "HTMLElement"] as const) { + Object.defineProperty(globalThis, key, { configurable: true, value: key === "window" ? win : win[key] }); + } + Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", { configurable: true, value: true }); + Object.defineProperty(globalThis, "WebSocket", { configurable: true, writable: true, value: Socket }); + win.localStorage.setItem("ocx-lang", "en"); + audio = AUDIO; sent = []; Socket.latest = undefined; holdKeys = null; + responder = async () => Response.json({ text: "Synthetic transcript" }); + Object.defineProperty(globalThis, "fetch", { configurable: true, writable: true, value: async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/api/keys")) { + if (holdKeys) await holdKeys; + return Response.json({ keys: [], baseUrl: BASE, endpoint: `${BASE}/responses`, authMatrix: [{ endpoint: "/v1/models", bearer: "accepted", dedicated: "accepted", xApiKey: "accepted" }], ...(audio === undefined ? {} : { audio }) }); + } + if (url.endsWith("/v1/models")) return Response.json({ data: [] }); + if (url.endsWith("/v1/audio/transcriptions")) { sent.push(init!); return responder(init!); } + return new Response(null, { status: 404 }); + } }); + resetApiAuthFetchForTests(); clearClientResourceStoresForTests(); +}); + +afterEach(async () => { + if (root) await act(async () => { root!.unmount(); }); + root = undefined; + clearClientResourceStoresForTests(); resetApiAuthFetchForTests(); + win.close(); + for (const key of globals) { + if (previous[key]) Object.defineProperty(globalThis, key, previous[key]!); + else delete (globalThis as Record)[key]; + } +}); + +async function flush() { await act(async () => { await new Promise(resolve => setTimeout(resolve, 0)); }); } +async function render(active = true, apiBase = "http://localhost") { + if (!root) { + container = win.document.createElement("div") as unknown as HTMLDivElement; + win.document.body.appendChild(container); + root = (await import("react-dom/client")).createRoot(container); + } + await act(async () => { root!.render(); }); + await flush(); +} +async function typeKey(section: string, value = KEY) { + const input = container.querySelector(`${section} input[type=password]`)!; + await act(async () => { + Object.getOwnPropertyDescriptor(win.HTMLInputElement.prototype, "value")!.set!.call(input, value); + input.dispatchEvent(new win.Event("input", { bubbles: true }) as unknown as Event); + }); +} +async function selectFile() { + const input = container.querySelector('input[type="file"]')!; + Object.defineProperty(input, "files", { configurable: true, value: [new File(["synthetic"], "fixture.wav")] }); + await act(async () => { input.dispatchEvent(new win.Event("change", { bubbles: true }) as unknown as Event); }); +} +async function submit(section: string) { + await act(async () => { container.querySelector(`${section} button[type=submit]`)!.click(); }); + await flush(); +} +const DICTATION = "#api-section-dictation"; +const LIVE = "#api-section-live-voice"; + +test("real API page uploads, copies transcript, and keeps keys out of caches and examples", async () => { + await render(); + expect(sent).toHaveLength(0); + await typeKey(DICTATION); await selectFile(); await submit(DICTATION); + expect(sent).toHaveLength(1); + expect(new Headers(sent[0]!.headers).get("x-opencodex-api-key")).toBe(KEY); + expect(container.querySelector(".audio-api-result")?.textContent).toContain("Synthetic transcript"); + let copied = ""; + Object.defineProperty(navigator, "clipboard", { configurable: true, value: { writeText: async (text: string) => { copied = text; } } }); + await act(async () => { container.querySelector(".audio-api-result button")!.click(); }); + expect(copied).toBe("Synthetic transcript"); + expect(container.querySelector(".audio-api-examples")?.textContent).not.toContain(KEY); + for (const storage of [win.localStorage, win.sessionStorage]) { + for (let i = 0; i < storage.length; i++) expect(storage.getItem(storage.key(i)!)).not.toContain(KEY); + } +}); + +test("upload errors are localized, cancellation and inactive panels cannot publish late text", async () => { + await render(); await typeKey(DICTATION); await selectFile(); + responder = async () => new Response("private account details", { status: 401 }); + await submit(DICTATION); + expect(container.querySelector(`${DICTATION} [role=alert]`)?.textContent).toContain("Key rejected"); + expect(container.textContent).not.toContain("private account details"); + let resolveResponse!: (response: Response) => void; + responder = () => new Promise(resolve => { resolveResponse = resolve; }); + await submit(DICTATION); + const signal = sent.at(-1)!.signal!; + await render(false); + expect(signal.aborted).toBe(true); + await act(async () => { resolveResponse(Response.json({ text: "late transcript" })); }); + await render(true); + expect(container.textContent).not.toContain("late transcript"); + expect(container.querySelector(`${DICTATION} input[type=password]`)!.value).toBe(""); +}); + +test("voice reports real readiness, filters event payloads and releases on deactivation", async () => { + await render(); + expect(Socket.latest).toBeUndefined(); + await typeKey(LIVE); + expect(container.querySelector(`${LIVE} [role=status]`)?.textContent).toBe("Not checked"); + await submit(LIVE); + const socket = Socket.latest!; + expect(socket).toBeDefined(); + await act(async () => { socket.readyState = 1; socket.onopen?.(); }); + expect(container.querySelector(`${LIVE} [role=status]`)?.textContent).toBe("Connecting..."); + await act(async () => { socket.onmessage?.({ data: JSON.stringify({ type: "session.started", session: { id: "fixture" }, token: "never-render" }) }); }); + expect(container.querySelector(`${LIVE} [role=status]`)?.textContent).toBe("Session ready"); + expect(container.textContent).not.toContain("never-render"); + await render(false); + expect(socket.closed).toBe(true); expect(socket.onmessage).toBeNull(); +}); + +test("missing or malformed audio metadata leaves existing key management usable", async () => { + audio = { ...AUDIO, liveEndpoint: "wss://unexpected.example/v1/live" }; + await render(); + expect(container.querySelector(DICTATION)?.textContent).toContain("Audio metadata unavailable"); + expect(container.querySelector(`${DICTATION} input`)).toBeNull(); + expect(container.textContent).toContain("Generate"); + expect(sent).toHaveLength(0); +}); + +test("Cancel, key replacement and duplicate clicks cannot publish a superseded upload", async () => { + const replies: Array<(response: Response) => void> = []; + responder = () => new Promise(resolve => { replies.push(resolve); }); + await render(); await typeKey(DICTATION); await selectFile(); + await act(async () => { + const button = container.querySelector(`${DICTATION} button[type=submit]`)!; + button.click(); button.click(); + }); + await flush(); + expect(sent).toHaveLength(1); + await act(async () => { container.querySelector(`${DICTATION} .audio-api-actions button[type=button]`)!.click(); }); + expect(sent[0]!.signal!.aborted).toBe(true); + await submit(DICTATION); + expect(sent).toHaveLength(2); + await typeKey(DICTATION, KEY + "-replacement"); + expect(sent[1]!.signal!.aborted).toBe(true); + await submit(DICTATION); + expect(sent).toHaveLength(3); + await act(async () => { replies[0]!(Response.json({ text: "stale A" })); replies[1]!(Response.json({ text: "stale B" })); }); + expect(container.textContent).not.toContain("stale A"); expect(container.textContent).not.toContain("stale B"); + expect(container.querySelector(`${DICTATION} button[type=submit]`)!.disabled).toBe(true); + expect(container.querySelector(`${DICTATION} .audio-api-actions button[type=button]`)?.textContent).toContain("Cancel"); + await submit(DICTATION); + expect(sent).toHaveLength(3); + await act(async () => { replies[2]!(Response.json({ text: "Current C" })); }); + expect(container.querySelector(".audio-api-result")?.textContent).toContain("Current C"); +}); + +test("origin changes close the prior socket and clear the transient key", async () => { + await render(); await typeKey(LIVE); await submit(LIVE); + const socket = Socket.latest!; + await render(true, "http://127.0.0.1"); + expect(socket.closed).toBe(true); + expect(container.querySelector(`${LIVE} input[type=password]`)!.value).toBe(""); +}); + +test("unknown fields cannot write secrets to the list cache", async () => { + audio = { ...AUDIO, apiKey: "sensitive-extra-marker" }; + await render(); + expect(container.querySelector(`${DICTATION} input`)).toBeNull(); + expect(win.sessionStorage.getItem("ocx.apikeys.list.v2:http://localhost")).not.toContain("sensitive-extra-marker"); +}); + +test("old server and malformed cache audio never invent configured support", async () => { + audio = undefined; + let release!: () => void; + holdKeys = new Promise(resolve => { release = resolve; }); + win.sessionStorage.setItem("ocx.apikeys.list.v2:http://localhost", JSON.stringify({ + keys: [], claudeCodeEnabled: true, + authMatrix: [{ endpoint: "/v1/models", bearer: "accepted", dedicated: "accepted", xApiKey: "accepted" }], + endpoints: { baseUrl: BASE, responses: `${BASE}/responses`, chatCompletions: `${BASE}/chat/completions`, messages: `${BASE}/messages`, models: `${BASE}/models`, audio: { ...AUDIO, liveConfigured: "yes" } }, + })); + await render(); + expect(container.querySelector(DICTATION)?.textContent).toContain("Audio metadata unavailable"); + expect(container.querySelector(`${LIVE} input`)).toBeNull(); + expect(container.textContent).toContain("Generate"); + release(); + await flush(); + expect(container.querySelector(`${LIVE} input`)).toBeNull(); + expect(sent).toHaveLength(0); +}); diff --git a/gui/tests/section-tabs-scroll-lock.test.tsx b/gui/tests/section-tabs-scroll-lock.test.tsx index 6f54bb818a..26bbdcfcc4 100644 --- a/gui/tests/section-tabs-scroll-lock.test.tsx +++ b/gui/tests/section-tabs-scroll-lock.test.tsx @@ -9,7 +9,7 @@ const globals = ["document", "window", "navigator", "IS_REACT_ACT_ENVIRONMENT"] let previousGlobals: Record<(typeof globals)[number], unknown>; let testWindow: Window; -const observers: Array<{ callback: IntersectionObserverCallback; nodes: Element[] }> = []; +const observers: Array<{ callback: IntersectionObserverCallback; nodes: Element[]; rootMargin?: string }> = []; const OriginalIntersectionObserver = globalThis.IntersectionObserver; function emitIntersecting(id: string) { @@ -42,9 +42,9 @@ beforeEach(() => { #callback: IntersectionObserverCallback; #nodes: Element[] = []; - constructor(callback: IntersectionObserverCallback) { + constructor(callback: IntersectionObserverCallback, options?: IntersectionObserverInit) { this.#callback = callback; - observers.push({ callback: this.#callback.bind(this), nodes: this.#nodes }); + observers.push({ callback: this.#callback.bind(this), nodes: this.#nodes, rootMargin: options?.rootMargin }); } observe(node: Element) { @@ -70,6 +70,26 @@ beforeEach(() => { }); }); +test("a mobile shell offset updates the observer without changing default section behavior", async () => { + const original = window.matchMedia; + let listener: (() => void) | undefined; + const query = { matches: true, addEventListener: (_name: string, callback: () => void) => { listener = callback; }, removeEventListener: () => { listener = undefined; } }; + window.matchMedia = (() => query) as unknown as typeof window.matchMedia; + const container = document.createElement("div"); document.body.append(container); + const { createRoot } = await import("react-dom/client"); + const root = createRoot(container); + try { + await act(async () => root.render(<>
)); + expect(observers.at(-1)?.rootMargin).toBe("-108px 0px -60% 0px"); + await act(async () => { query.matches = false; listener?.(); }); + expect(observers.at(-1)?.rootMargin).toBe("-72px 0px -60% 0px"); + } finally { + await act(async () => root.unmount()); + expect(listener).toBeUndefined(); + window.matchMedia = original; container.remove(); + } +}); + afterEach(() => { Object.defineProperty(globalThis, "IntersectionObserver", { configurable: true, diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c4363a7305..0b75e34eda 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -246,6 +246,8 @@ "aside-profiles-routes.test.ts": "server", "aside-profiles.test.ts": "clients", "assert-mergeable-review.test.ts": "ci-workflows", + "audio-client.test.ts": "server", + "audio-dictation.test.ts": "server", "audio-transcriptions.test.ts": "server", "auto-compact-budget.test.ts": "providers", "autostart-health.test.ts": "service", @@ -824,6 +826,7 @@ "lab-read-surfaces.test.ts": "lab", "launchd-repair.test.ts": "service", "legacy-shell-compat.test.ts": "responses", + "live-call-bindings.test.ts": "server", "live-service-manager-guard.test.ts": "service", "local-destinations.test.ts": "lib", "local-management-attestation.test.ts": "server", diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index 2b0b41d26d..1021e6a5c6 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -130,6 +130,7 @@ export async function resolveFirstUsableOpenAiSidecar( admission?: Pick; codexAuthPolicy?: CodexAuthPolicyConfig; beginCodexAccountSelection?: () => CodexAccountSelectionAdmission | undefined; + signal?: AbortSignal; } = {}, ): Promise { const { exactAccount } = options; @@ -152,9 +153,11 @@ export async function resolveFirstUsableOpenAiSidecar( modelId: exactAccount.modelId, admission: options.admission, beginCodexAccountSelection: options.beginCodexAccountSelection, + signal: options.signal, }); let selectedHeaders: Headers; try { + options.signal?.throwIfAborted(); selectedHeaders = headersForCodexAuthContext(incomingHeaders, authContext, policy, exactAccount.modelId, options.admission); } catch (error) { releaseCodexAuthContextProbeLease(authContext); @@ -202,9 +205,11 @@ export async function resolveFirstUsableOpenAiSidecar( codexAuthPolicy: policy, admission: options.admission, beginCodexAccountSelection: options.beginCodexAccountSelection, + signal: options.signal, }); let selectedHeaders: Headers; try { + options.signal?.throwIfAborted(); selectedHeaders = headersForCodexAuthContext(incomingHeaders, authContext, policy, undefined, options.admission); } catch (error) { releaseCodexAuthContextProbeLease(authContext); diff --git a/src/server/audio-client.ts b/src/server/audio-client.ts new file mode 100644 index 0000000000..0e089c1aad --- /dev/null +++ b/src/server/audio-client.ts @@ -0,0 +1,64 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { formatErrorResponse } from "../bridge"; +import type { OcxConfig } from "../types"; +import { captureExplicitOpenAiCallerAuth, selectOpenAiImagesProvider } from "../providers/openai-sidecar"; +import { isProxyAdmissionSecret, type DataPlaneAdmission } from "./auth-cors"; +import { resolveAudioAdmission } from "./audio-upstream"; + +export const AUDIO_WEBSOCKET_PROTOCOL = "opencodex-audio"; +const KEY_PROTOCOL_PREFIX = "opencodex-key."; + +export interface AudioClient { + admission: DataPlaneAdmission; + headers: Headers; + owner: string; + protocol?: string; +} + +/** Browser protocols are an audio-only credential carrier; only the public marker is echoed. */ +export function resolveAudioClient(req: Request, config: OcxConfig, required = false): AudioClient | Response | null { + const headers = new Headers(req.headers); + let protocol: string | undefined; + let carrier = false; + if (headers.get("upgrade")?.toLowerCase() === "websocket") { + const raw = headers.get("sec-websocket-protocol") ?? ""; + if (raw.length > 8192) return formatErrorResponse(400, "invalid_request_error", "Audio protocol header too large"); + const protocols = raw.split(",").map(value => value.trim()).filter(Boolean); + const keys = protocols.filter(value => value.startsWith(KEY_PROTOCOL_PREFIX)); + const markers = protocols.filter(value => value === AUDIO_WEBSOCKET_PROTOCOL); + carrier = keys.length > 0 || markers.length > 0; + if (carrier) { + if (keys.length !== 1 || markers.length !== 1 || protocols.length !== 2) { + return formatErrorResponse(400, "invalid_request_error", "Expected one audio protocol and one encoded client key"); + } + const encoded = keys[0]!.slice(KEY_PROTOCOL_PREFIX.length); + const bytes = Buffer.from(encoded, "base64url"); + const key = bytes.toString("utf8"); + if (!encoded || bytes.toString("base64url") !== encoded || Buffer.from(key).toString("base64url") !== encoded || !key.trim()) { + return formatErrorResponse(400, "invalid_request_error", "Invalid audio client key encoding"); + } + protocol = AUDIO_WEBSOCKET_PROTOCOL; + if (!["authorization", "x-opencodex-api-key", "x-api-key"].some(name => headers.has(name))) { + headers.set("x-opencodex-api-key", key); + } + } + } + const admission = resolveAudioAdmission(headers, config); + if (!admission) { + const bearer = /^Bearer\s+([^\s,]+)$/i.exec(headers.get("authorization") ?? "")?.[1]; + const platformKey = selectOpenAiImagesProvider(config).keyed?.apiKey; + const knownPlatformBearer = !!bearer && !!platformKey && !isProxyAdmissionSecret(bearer, config) + && timingSafeEqual(createHash("sha256").update(bearer).digest(), createHash("sha256").update(platformKey).digest()); + const explicit = headers.has("x-opencodex-api-key") || headers.has("x-api-key") + || (headers.has("authorization") && !knownPlatformBearer && !captureExplicitOpenAiCallerAuth(headers, config)); + return required || carrier || explicit + ? formatErrorResponse(401, "authentication_error", "opencodex API key required") + : null; + } + const credential = admission.source === "dedicated" ? headers.get("x-opencodex-api-key") + : admission.source === "x-api-key" ? headers.get("x-api-key") : headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + const owner = admission.kind === "configured" + ? JSON.stringify(["configured", admission.keyId]) + : JSON.stringify(["environment", createHash("sha256").update(credential?.trim() ?? "").digest("hex")]); + return { admission, headers, owner, ...(protocol ? { protocol } : {}) }; +} diff --git a/src/server/audio-dictation.ts b/src/server/audio-dictation.ts new file mode 100644 index 0000000000..87c2fe55d1 --- /dev/null +++ b/src/server/audio-dictation.ts @@ -0,0 +1,91 @@ +import { formatErrorResponse } from "../bridge"; +import type { AdmissionLease } from "../lib/admission"; +import type { OcxConfig } from "../types"; +import type { AudioClient } from "./audio-client"; +import { resolveAudioUpstream, TRANSCRIPTION_MODEL, type AudioUpstream } from "./audio-upstream"; +import type { RequestLogContext } from "./request-log"; + +export const DICTATION_SESSION_MAX_MS = 300_000; +const DICTATION_FRAME_MAX_BYTES = 64 * 1024; +type JsonObject = Record; +function object(value: unknown): value is JsonObject { return !!value && typeof value === "object" && !Array.isArray(value); } +function integer(value: unknown, min: number, max: number): boolean { + return typeof value === "number" && Number.isInteger(value) && value >= min && value <= max; +} + +export function createDictationFrameValidator(): (frame: string | Buffer) => boolean { + let started = false; + let closed = false; + return frame => { + if (closed || typeof frame !== "string" || Buffer.byteLength(frame) > DICTATION_FRAME_MAX_BYTES) return false; + let event: unknown; + try { event = JSON.parse(frame); } catch { return false; } + if (!object(event)) return false; + if (event.type === "session.start" && !started) { + const config = event.config; + if (!object(config) || Object.keys(event).some(key => key !== "type" && key !== "config")) return false; + const fields = new Set(["input_audio_format", "sample_rate_hz", "num_channels", "max_buffer_size_bytes", "max_utterance_duration_ms", "session_ttl_ms", "provider_mode", "transcript_delivery_mode", "vad"]); + if (Object.keys(config).some(key => !fields.has(key)) || config.input_audio_format !== "pcm16" || config.num_channels !== 1 + || !integer(config.sample_rate_hz, 8000, 192000) || !integer(config.max_buffer_size_bytes, 1, 4 * 1024 * 1024) + || !integer(config.max_utterance_duration_ms, 1, 30000) || !integer(config.session_ttl_ms, 1, DICTATION_SESSION_MAX_MS) + || !["buffered", "streaming_sse"].includes(String(config.provider_mode)) + || !["final_only", "segment", "delta"].includes(String(config.transcript_delivery_mode))) return false; + const vad = config.vad; + if (!object(vad) || Object.keys(vad).some(key => !["type", "threshold", "prefix_padding_ms", "silence_duration_ms"].includes(key)) + || vad.type !== "server_vad" || typeof vad.threshold !== "number" || !Number.isFinite(vad.threshold) || vad.threshold < 0 || vad.threshold > 1 + || !integer(vad.prefix_padding_ms, 0, 1000) || !integer(vad.silence_duration_ms, 0, 5000)) return false; + started = true; + return true; + } + if (!started) return false; + if (event.type === "session.close" && Object.keys(event).length === 1) { closed = true; return true; } + if (event.type !== "audio.append" || Object.keys(event).some(key => key !== "type" && key !== "audio") || typeof event.audio !== "string" || !event.audio) return false; + const bytes = Buffer.from(event.audio, "base64"); + return bytes.length > 0 && bytes.length % 2 === 0 && bytes.toString("base64") === event.audio; + }; +} + +export interface AudioSocketTarget { + headers: Record; + upstreamWsUrl: string; + protocols?: string[]; + validateFrame?: (frame: string | Buffer) => boolean; + maxSessionMs: number; + finish: (outcome?: number | "timeout" | "connect_error") => void; +} + +export function finishAudioUpstream(relay: AudioUpstream): AudioSocketTarget["finish"] { + let finished = false; + return outcome => { + if (finished) return; + finished = true; + try { if (outcome !== undefined) relay.recordOutcome?.(outcome); } + finally { relay.release(); } + }; +} + +export async function resolveDictationSocket( + client: AudioClient, config: OcxConfig, log: RequestLogContext, lease: AdmissionLease, signal?: AbortSignal, +): Promise { + const relay = await resolveAudioUpstream(client.headers, config, log, { admission: client.admission, model: TRANSCRIPTION_MODEL, lease, signal }); + if (relay instanceof Response) return relay; + if (relay.keyed) { + relay.release(); + return formatErrorResponse(400, "invalid_request_error", "Streaming dictation requires a connected ChatGPT account"); + } + const headers = new Headers(relay.headers); + const token = headers.get("authorization")?.replace(/^Bearer\s+/i, "").trim(); + if (!token || /[\s,]/.test(token)) { + relay.release(); + return formatErrorResponse(401, "authentication_error", "Dictation account authentication unavailable"); + } + headers.delete("authorization"); + return { + upstreamWsUrl: "wss://chatgpt.com/backend-api/dictation/stream", + headers: Object.fromEntries(headers), + protocols: ["chatgpt-dictation", `openai-bearer.${token}`, "codex-desktop"], + validateFrame: createDictationFrameValidator(), + maxSessionMs: DICTATION_SESSION_MAX_MS, + finish: finishAudioUpstream(relay), + }; +} diff --git a/src/server/audio-live.ts b/src/server/audio-live.ts new file mode 100644 index 0000000000..b90921a98e --- /dev/null +++ b/src/server/audio-live.ts @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; +import { formatErrorResponse } from "../bridge"; +import { MAIN_CODEX_ACCOUNT_ID } from "../codex/account-id"; +import { cancelBodyOnAbort, clearableDeadline } from "../lib/abort"; +import type { AdmissionLease } from "../lib/admission"; +import { captureExplicitOpenAiCallerAuth } from "../providers/openai-sidecar"; +import type { OcxConfig } from "../types"; +import type { AudioClient } from "./audio-client"; +import { finishAudioUpstream, type AudioSocketTarget } from "./audio-dictation"; +import { LIVE_AUDIO_MODEL, resolveAudioUpstream, type AudioUpstream } from "./audio-upstream"; +import { registerTurn, unregisterTurn } from "./lifecycle"; +import { + backendJsonBodyFromApiMultipart, buildLiveSidebandUpstreamWsUrl, forwardLiveUrl, keyedLiveUrl, + LIVE_CLIENT_PROTOCOL_HEADERS, LIVE_REQUEST_MAX_BYTES, LIVE_RESPONSE_MAX_BYTES, readBodyCapped, + type LiveSidebandTarget, +} from "./live"; +import { LIVE_CALL_TTL_MS, LiveCallBindings, upstreamLiveCallId } from "./live-call-bindings"; +import type { RequestLogContext } from "./request-log"; + +function protocolHeaders(client: AudioClient, relay: AudioUpstream, frameless: boolean): Headers { + const headers = new Headers(); + for (const name of LIVE_CLIENT_PROTOCOL_HEADERS) { + const value = client.headers.get(name); + if (value) headers.set(name, value); + } + for (const [name, value] of Object.entries(relay.headers)) headers.set(name, value); + if (frameless && !headers.has("openai-alpha")) headers.set("openai-alpha", "quicksilver=v2"); + return headers; +} + +async function parseExternalOffer(req: Request, signal: AbortSignal): Promise<{ sdp: string; session?: Record } | Response> { + const upload = clearableDeadline(30_000, signal); + try { + const body = await readBodyCapped(req.body, LIVE_REQUEST_MAX_BYTES, () => "Live offer too large", upload.signal); + if (body instanceof Response) return formatErrorResponse(413, "invalid_request_error", "Live offer exceeds 16 MiB"); + const type = req.headers.get("content-type") ?? ""; + let payload: unknown; + if (type.toLowerCase().includes("multipart/form-data")) { + const converted = await backendJsonBodyFromApiMultipart(body, type); + if (converted instanceof Response) return converted; + payload = JSON.parse(new TextDecoder().decode(converted.body)); + } else if (type.toLowerCase().includes("application/json")) { + payload = JSON.parse(new TextDecoder().decode(body)); + } else if (type.toLowerCase().includes("application/sdp")) { + payload = { sdp: new TextDecoder().decode(body) }; + } + if (!payload || typeof payload !== "object" || !("sdp" in payload) || typeof payload.sdp !== "string" || !payload.sdp.trim()) { + return formatErrorResponse(400, "invalid_request_error", "Live call requires a nonempty SDP offer"); + } + const session = "session" in payload ? payload.session : undefined; + if (session !== undefined && (!session || typeof session !== "object" || Array.isArray(session))) { + return formatErrorResponse(400, "invalid_request_error", "Live session must be an object"); + } + return { sdp: payload.sdp, ...(session ? { session: session as Record } : {}) }; + } catch { + if (signal.aborted) throw signal.reason; + return formatErrorResponse(upload.didExpire() ? 408 : 400, "invalid_request_error", upload.didExpire() ? "Live offer upload timed out" : "Malformed live offer"); + } finally { upload.clear(); } +} + +export async function handleExternalLive( + req: Request, config: OcxConfig, log: RequestLogContext, + options: { client: AudioClient; lease: AdmissionLease; bindings: LiveCallBindings }, +): Promise { + const controller = new AbortController(); + registerTurn(controller, options.lease); + const deadline = clearableDeadline(120_000, AbortSignal.any([req.signal, controller.signal])); + let relay: AudioUpstream | undefined; + let outcome: number | "timeout" | "connect_error" | undefined; + try { + const offer = await parseExternalOffer(req, deadline.signal); + if (offer instanceof Response) return offer; + if (!options.bindings.hasCapacity()) return formatErrorResponse(503, "server_busy", "Live call capacity reached"); + const frameless = new URL(req.url).pathname === "/v1/live"; + const session = offer.session ? { ...offer.session } : frameless ? { + model: LIVE_AUDIO_MODEL, instructions: "", audio: { output: { voice: "cove" } }, delegation: { type: "client" }, + } : undefined; + if (session?.model === "gpt-live-1") session.model = LIVE_AUDIO_MODEL; + const model = typeof session?.model === "string" ? session.model : LIVE_AUDIO_MODEL; + const resolved = await resolveAudioUpstream(options.client.headers, config, log, { + admission: options.client.admission, model, lease: options.lease, signal: deadline.signal, + }); + if (!(resolved instanceof Response)) relay = resolved; + deadline.signal.throwIfAborted(); + if (resolved instanceof Response) return resolved; + relay = resolved; + const headers = protocolHeaders(options.client, relay, frameless); + let body: BodyInit; + if (relay.keyed) { + const form = new FormData(); + form.set("sdp", offer.sdp); + if (session) form.set("session", JSON.stringify(session)); + headers.delete("content-type"); + body = form; + } else { + headers.set("content-type", "application/json"); + body = JSON.stringify({ sdp: offer.sdp, ...(session ? { session } : {}) }); + } + const url = relay.keyed + ? frameless ? forwardLiveUrl(relay.providerBaseUrl, false) : keyedLiveUrl(relay.providerBaseUrl) + : forwardLiveUrl(relay.providerBaseUrl, true); + const upstream = await fetch(url, { method: "POST", headers, body, signal: deadline.signal, redirect: "manual" }); + outcome = upstream.ok ? 502 : upstream.status; + const detach = cancelBodyOnAbort(upstream.body, deadline.signal); + let responseBody: ArrayBuffer | Response; + try { responseBody = await readBodyCapped(upstream.body, LIVE_RESPONSE_MAX_BYTES, () => "Live answer too large", deadline.signal); } + finally { detach(); } + if (responseBody instanceof Response) return responseBody; + if (!upstream.ok) return formatErrorResponse(upstream.status >= 400 ? upstream.status : 502, "upstream_error", `Live upstream returned HTTP ${upstream.status}`); + const callId = upstreamLiveCallId(upstream.headers.get("location")); + if (!callId || responseBody.byteLength === 0) return formatErrorResponse(502, "upstream_error", "Live upstream returned an invalid call answer"); + const context = relay.authContext; + const callerOwned = context?.kind === "main" && captureExplicitOpenAiCallerAuth(options.client.headers, config) !== null; + const alias = options.bindings.create({ + owner: options.client.owner, upstreamCallId: callId, + joinStyle: frameless ? "frameless-path" : "realtime-query", + providerName: relay.providerName, + accountId: context ? context.kind === "main" ? callerOwned ? undefined : MAIN_CODEX_ACCOUNT_ID : context.accountId : undefined, + chatgptAccountId: new Headers(relay.headers).get("chatgpt-account-id") ?? undefined, + keyedCredentialDigest: relay.keyed ? createHash("sha256").update(new Headers(relay.headers).get("authorization") ?? "").digest("hex") : undefined, + callerOwned, + sidebandBaseUrl: config.experimentalRealtimeWsBaseUrl, + }); + if (!alias) return formatErrorResponse(503, "server_busy", "Live call could not be registered"); + outcome = upstream.status; + return new Response(responseBody, { status: upstream.status, headers: { + "content-type": upstream.headers.get("content-type") ?? "application/sdp", + location: `/v1/${frameless ? "live" : "realtime/calls"}/${alias}`, + } }); + } catch { + if (req.signal.aborted || controller.signal.aborted) { + outcome = undefined; + return formatErrorResponse(req.signal.aborted ? 499 : 503, "client_closed_request", "Live call canceled"); + } + outcome = deadline.didExpire() ? "timeout" : "connect_error"; + return formatErrorResponse(deadline.didExpire() ? 504 : 502, "upstream_error", deadline.didExpire() ? "Live call timed out" : "Live upstream connection failed"); + } finally { + try { if (outcome !== undefined) relay?.recordOutcome?.(outcome); } + finally { relay?.release(); deadline.clear(); unregisterTurn(controller); } + } +} + +export async function resolveExternalLiveSocket( + client: AudioClient, config: OcxConfig, log: RequestLogContext, target: LiveSidebandTarget, + options: { lease: AdmissionLease; bindings: LiveCallBindings; signal?: AbortSignal }, +): Promise { + const binding = "callId" in target ? options.bindings.get(target.callId, client.owner) : undefined; + if ("callId" in target && !binding) return formatErrorResponse(404, "not_found", "Live call is unavailable for this key"); + if (binding?.callerOwned && !captureExplicitOpenAiCallerAuth(client.headers, config)) { + return formatErrorResponse(401, "authentication_error", "Live call requires its original caller account"); + } + let upstreamTarget = binding ? { style: binding.joinStyle, callId: binding.upstreamCallId } as LiveSidebandTarget : target; + const frameless = upstreamTarget.style === "frameless-path" || upstreamTarget.style === "frameless-standalone"; + let model = LIVE_AUDIO_MODEL; + if (upstreamTarget.style === "frameless-standalone") { + const query = new URLSearchParams(upstreamTarget.query); + if (!query.has("model") || query.get("model") === "gpt-live-1") query.set("model", LIVE_AUDIO_MODEL); + model = query.get("model") ?? LIVE_AUDIO_MODEL; + upstreamTarget = { ...upstreamTarget, query: query.toString() }; + } + const relay = await resolveAudioUpstream(client.headers, config, log, { + admission: client.admission, model, lease: options.lease, + signal: options.signal, + ...(binding ? { exactAccountId: binding.accountId, providerName: binding.providerName } : {}), + }); + if (relay instanceof Response) return relay; + try { + if (binding && (relay.providerName !== binding.providerName + || (new Headers(relay.headers).get("chatgpt-account-id") ?? undefined) !== binding.chatgptAccountId + || (binding.keyedCredentialDigest !== undefined + && createHash("sha256").update(new Headers(relay.headers).get("authorization") ?? "").digest("hex") !== binding.keyedCredentialDigest))) { + relay.release(); + return formatErrorResponse(409, "authentication_error", "Live call account is no longer available"); + } + return { + headers: Object.fromEntries(protocolHeaders(client, relay, frameless)), + upstreamWsUrl: buildLiveSidebandUpstreamWsUrl(upstreamTarget, binding ? binding.sidebandBaseUrl : config.experimentalRealtimeWsBaseUrl), + maxSessionMs: LIVE_CALL_TTL_MS, + finish: finishAudioUpstream(relay), + }; + } catch { + relay.release(); + return formatErrorResponse(400, "invalid_request_error", "Invalid live endpoint configuration"); + } +} diff --git a/src/server/audio-upstream.ts b/src/server/audio-upstream.ts index a59b7aef95..fa8bc4e9e7 100644 --- a/src/server/audio-upstream.ts +++ b/src/server/audio-upstream.ts @@ -54,6 +54,7 @@ export interface AudioUpstreamOptions { model: string; lease?: AdmissionLease; exactAccountId?: string; + providerName?: string; signal?: AbortSignal; } @@ -65,6 +66,10 @@ export async function resolveAudioUpstream( options: AudioUpstreamOptions, ): Promise { const candidates = selectOpenAiImagesProvider(config); + if (options.providerName) { + candidates.forwardCandidates = candidates.forwardCandidates.filter(candidate => candidate.providerName === options.providerName); + if (candidates.keyed?.providerName !== options.providerName) delete candidates.keyed; + } const headers = new Headers(incoming); const bearer = headers.get("authorization")?.replace(/^Bearer\s+/i, "").trim(); if (bearer && isProxyAdmissionSecret(bearer, config)) { @@ -90,6 +95,7 @@ export async function resolveAudioUpstream( beginCodexAccountSelection, signal: options.signal, }); + options.signal?.throwIfAborted(); selected = materializeCodexUpstreamAuth(headers, context, { config, admission: options.admission, @@ -99,6 +105,7 @@ export async function resolveAudioUpstream( const resolved = await resolveFirstUsableOpenAiSidecar(candidates.forwardCandidates, headers, config, { admission: options.admission, beginCodexAccountSelection, + signal: options.signal, ...(options.exactAccountId ? { exactAccount: { accountId: options.exactAccountId, modelId: options.model } } : {}), }); if (!resolved) return formatErrorResponse(401, "authentication_error", "Connect a ChatGPT account to use audio"); diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index 37ce4b1127..373aec18db 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -483,6 +483,8 @@ export interface ApiAuthMatrixRow { */ export const AUTH_MATRIX: readonly ApiAuthMatrixRow[] = [ { endpoint: "/v1/audio/transcriptions", bearer: "accepted", dedicated: "accepted", xApiKey: "accepted" }, + { endpoint: "/v1/live", bearer: "accepted", dedicated: "accepted", xApiKey: "accepted" }, + { endpoint: "/v1/realtime/calls", bearer: "accepted", dedicated: "accepted", xApiKey: "accepted" }, // #1686: a bearer that is one of OUR admission secrets is now accepted here. It is safe // because materializeCodexUpstreamAuth substitutes the stored main credential rather than // forwarding it; a bearer that is NOT our secret stays unadmitted and remains Codex Direct diff --git a/src/server/index.ts b/src/server/index.ts index 5460f443e9..b1a3ff7163 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -206,6 +206,11 @@ import { handleImages } from "./images"; import { handleLive, logLiveSidebandFrame, parseLiveSidebandTarget, resolveLiveSidebandUpgrade } from "./live"; import { handleAudioTranscriptions } from "./audio-transcriptions"; import { resolveAudioAdmission, TRANSCRIPTION_MODEL } from "./audio-upstream"; +import { resolveAudioClient } from "./audio-client"; +import { resolveDictationSocket } from "./audio-dictation"; +import { handleExternalLive, resolveExternalLiveSocket } from "./audio-live"; +import { EXTERNAL_CALL_PREFIX, LiveCallBindings } from "./live-call-bindings"; +import { clearableDeadline } from "../lib/abort"; import { handleSearch } from "./search"; import { handleContextHistory } from "./context-history"; import { codexCompatibleUrl, contextEndpoint, contextRelayActivated } from "../codex/context-compat"; @@ -357,6 +362,7 @@ export function enqueueLiveSidebandPendingFrame( type LiveSidebandWebSocketFactory = ( url: string, headers: Record, + protocols?: string[], ) => WebSocket; function releaseLiveSidebandAdmission(ws: ServerWebSocket): void { @@ -389,8 +395,22 @@ function finalizeLiveSideband(ws: ServerWebSocket, upstream?: WebSocket) ws.data.liveUpstream = undefined; ws.data.livePending = undefined; ws.data.livePendingBytes = undefined; + if (ws.data.liveConnectTimer !== undefined) clearTimeout(ws.data.liveConnectTimer); + if (ws.data.liveSessionTimer !== undefined) clearTimeout(ws.data.liveSessionTimer); + ws.data.liveConnectTimer = undefined; + ws.data.liveSessionTimer = undefined; + ws.data.liveUpstreamHeaders = undefined; + ws.data.liveUpstreamProtocols = undefined; + ws.data.liveValidateFrame = undefined; + if (ws.data.liveAbortListener) ws.data.liveAbortSignal?.removeEventListener("abort", ws.data.liveAbortListener); + ws.data.liveAbortSignal = undefined; + ws.data.liveAbortListener = undefined; ws.data.cancel = undefined; - releaseLiveSidebandAdmission(ws); + const finish = ws.data.liveFinish; + ws.data.liveFinish = undefined; + try { finish?.(ws.data.liveOutcome); } + catch { console.warn("[audio] upstream accounting failed during close"); } + finally { releaseLiveSidebandAdmission(ws); } } function armLiveSidebandCloseFallback(ws: ServerWebSocket, upstream: WebSocket): void { @@ -423,6 +443,10 @@ function armLiveSidebandCloseFallback(ws: ServerWebSocket, upstream: Web function closeLiveSideband(ws: ServerWebSocket, code = 1000, reason = ""): void { if (ws.data.liveClosing) return; ws.data.liveClosing = true; + if (ws.data.liveConnectTimer !== undefined) clearTimeout(ws.data.liveConnectTimer); + if (ws.data.liveSessionTimer !== undefined) clearTimeout(ws.data.liveSessionTimer); + ws.data.liveConnectTimer = undefined; + ws.data.liveSessionTimer = undefined; ws.data.livePending = undefined; ws.data.livePendingBytes = undefined; ws.data.cancel = undefined; @@ -454,10 +478,14 @@ function closeLiveSideband(ws: ServerWebSocket, code = 1000, reason = "" function attachLiveSidebandUpstream( ws: ServerWebSocket, - createWebSocket: LiveSidebandWebSocketFactory = (url, headers) => ( - new WebSocket(url, { headers } as unknown as string[]) + createWebSocket: LiveSidebandWebSocketFactory = (url, headers, protocols) => ( + new WebSocket(url, { headers, protocols } as unknown as string[]) ), ): void { + if (ws.data.liveAbortSignal?.aborted) { + closeLiveSideband(ws, 1000, "audio connection canceled"); + return; + } const url = ws.data.liveUpstreamUrl; if (!url) { closeLiveSideband(ws, 1011, "missing upstream"); @@ -466,18 +494,31 @@ function attachLiveSidebandUpstream( let upstream: WebSocket; try { // Bun accepts per-handshake headers; the DOM lib types only list protocol arrays. - upstream = createWebSocket(url, ws.data.liveUpstreamHeaders ?? {}); + upstream = createWebSocket(url, ws.data.liveUpstreamHeaders ?? {}, ws.data.liveUpstreamProtocols); } catch { closeLiveSideband(ws, 1011, "upstream connect failed"); return; } ws.data.liveUpstream = upstream; + ws.data.liveUpstreamHeaders = undefined; + ws.data.liveUpstreamProtocols = undefined; ws.data.liveClosing = false; ws.data.cancel = () => closeLiveSideband(ws, 1000, "client closed"); + if (ws.data.liveMaxSessionMs !== undefined) { + ws.data.liveConnectTimer = setTimeout(() => { + ws.data.liveOutcome = "timeout"; + closeLiveSideband(ws, 1011, "audio connection timed out"); + }, 10_000); + ws.data.liveSessionTimer = setTimeout(() => closeLiveSideband(ws, 1000, "audio session expired"), ws.data.liveMaxSessionMs); + } upstream.addEventListener("open", () => { if (ws.data.liveUpstream !== upstream || ws.data.liveClosing) return; ws.data.liveOpened = true; + if (ws.data.liveConnectTimer !== undefined) clearTimeout(ws.data.liveConnectTimer); + ws.data.liveConnectTimer = undefined; + // An accepted transport alone does not prove inference/quota recovery. + // Keep healthy closes neutral; explicit transport failures are recorded below. const pending = ws.data.livePending ?? []; ws.data.livePending = undefined; ws.data.livePendingBytes = undefined; @@ -498,29 +539,43 @@ function attachLiveSidebandUpstream( return; } logLiveSidebandFrame("u2c", event.data); - if (typeof event.data === "string") ws.send(event.data); - else if (event.data instanceof ArrayBuffer) ws.send(event.data); + let sent: number; + if (typeof event.data === "string") sent = ws.send(event.data); + else if (event.data instanceof ArrayBuffer) sent = ws.send(event.data); else if (ArrayBuffer.isView(event.data)) { - ws.send(event.data.buffer.slice(event.data.byteOffset, event.data.byteOffset + event.data.byteLength)); - } else ws.send(event.data as Buffer); + sent = ws.send(event.data.buffer.slice(event.data.byteOffset, event.data.byteOffset + event.data.byteLength)); + } else sent = ws.send(event.data as Buffer); + if (ws.data.liveMaxSessionMs !== undefined && (sent === 0 || ws.getBufferedAmount() > MAX_WS_FRAME_BYTES)) { + closeLiveSideband(ws, 1013, "audio client backpressure"); + } } catch { closeLiveSideband(ws, 1011, "client send failed"); } }); upstream.addEventListener("close", (event) => { if (ws.data.liveUpstream !== upstream) return; + if (ws.data.liveFinish && !ws.data.liveClosing && event.code !== 1000) ws.data.liveOutcome = "connect_error"; ws.data.liveClosing = true; finalizeLiveSideband(ws, upstream); try { - ws.close(event.code || 1000, event.reason || ""); + const external = ws.data.liveMaxSessionMs !== undefined; + const validCode = event.code === 1000 || (event.code >= 1001 && event.code <= 1014 && ![1004, 1005, 1006].includes(event.code)) + || (event.code >= 3000 && event.code <= 4999); + ws.close(external && !validCode ? 1011 : event.code || 1000, external ? "audio upstream closed" : event.reason || ""); } catch { /* ignore */ } }); upstream.addEventListener("error", () => { if (ws.data.liveUpstream !== upstream) return; + if (ws.data.liveFinish && !ws.data.liveClosing) ws.data.liveOutcome = "connect_error"; closeLiveSideband(ws, 1011, "upstream error"); }); + if (ws.data.liveAbortSignal) { + ws.data.liveAbortListener = () => closeLiveSideband(ws, 1000, "audio connection canceled"); + ws.data.liveAbortSignal.addEventListener("abort", ws.data.liveAbortListener, { once: true }); + if (ws.data.liveAbortSignal.aborted) closeLiveSideband(ws, 1000, "audio connection canceled"); + } } // GUI static serving extracted to ./server/gui-static. Re-exported below to keep the @@ -680,6 +735,7 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server { - const response = await handleLive(req, config, logCtx, turnAdmissionLease); + const response = audioClient + ? await handleExternalLive(req, config, logCtx, { client: audioClient, lease: turnAdmissionLease, bindings: liveCallBindings }) + : await handleLive(req, config, logCtx, turnAdmissionLease); addFinalRequestLog( requestId, start, @@ -2207,11 +2268,19 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server { + acquisition?.clear(); + if (audioController) unregisterTurn(audioController); + else turnAdmissionLease.release(); + }; let resolved; try { - resolved = await resolveLiveSidebandUpgrade(req, config, logCtx, liveSidebandTarget, turnAdmissionLease); + resolved = dictationSocket && audioClient + ? await resolveDictationSocket(audioClient, config, logCtx, turnAdmissionLease, acquisition?.signal) + : liveSidebandTarget && audioClient + ? await resolveExternalLiveSocket(audioClient, config, logCtx, liveSidebandTarget, { lease: turnAdmissionLease, bindings: liveCallBindings, signal: acquisition?.signal }) + : liveSidebandTarget + ? await resolveLiveSidebandUpgrade(req, config, logCtx, liveSidebandTarget, turnAdmissionLease) + : formatErrorResponse(401, "authentication_error", "opencodex API key required"); } catch (error) { - turnAdmissionLease.release(); + releaseAcquisition(); throw error; } + if (acquisition?.signal.aborted) { + try { if (!(resolved instanceof Response) && "finish" in resolved) resolved.finish(); } + finally { releaseAcquisition(); } + return withCors(formatErrorResponse(req.signal.aborted ? 499 : acquisition.didExpire() ? 504 : 503, + "upstream_error", acquisition.didExpire() ? "Audio connection timed out" : "Audio connection canceled"), req, policy); + } if (resolved instanceof Response) { - turnAdmissionLease.release(); + releaseAcquisition(); addFinalRequestLog(requestId, start, logCtx, resolved.status); return withCors(resolved, req, policy); } - addFinalRequestLog(requestId, start, logCtx, 101); - if (requestServer.upgrade(req, { + const audio = "finish" in resolved ? resolved : undefined; + const finish = audio ? (outcome?: number | "timeout" | "connect_error") => { + try { audio.finish(outcome); } + finally { releaseAcquisition(); } + } : undefined; + const discardUpgrade = () => { + if (finish) finish(); + else releaseAcquisition(); + }; + if (req.signal.aborted) { + discardUpgrade(); + return withCors(formatErrorResponse(499, "client_closed_request", "Audio connection canceled"), req, policy); + } + let upgraded = false; + try { + upgraded = requestServer.upgrade(req, { + ...(audioClient?.protocol ? { headers: { "sec-websocket-protocol": audioClient.protocol } } : {}), data: { kind: "live-sideband", liveUpstreamUrl: resolved.upstreamWsUrl, liveUpstreamHeaders: resolved.headers, + admission, + liveUpstreamProtocols: audio?.protocols, + liveValidateFrame: audio?.validateFrame, + liveMaxSessionMs: audio?.maxSessionMs, + liveFinish: finish, + liveAbortSignal: audioController?.signal, livePending: [], livePendingBytes: 0, liveOpened: false, liveTurnAdmissionLease: turnAdmissionLease, } satisfies WsData, - })) return undefined as unknown as Response; - turnAdmissionLease.release(); + }); + } catch { + discardUpgrade(); + return withCors(formatErrorResponse(502, "upstream_error", "Audio WebSocket upgrade failed"), req, policy); + } + if (upgraded) { + acquisition?.clear(); + addFinalRequestLog(requestId, start, logCtx, 101); + return undefined as unknown as Response; + } + discardUpgrade(); return withCors(formatErrorResponse(426, "upgrade_required", "WebSocket upgrade failed"), req, policy); } @@ -2366,6 +2486,10 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server, raw: string | Buffer) { if (ws.data.kind === "live-sideband") { if (ws.data.liveClosing) return; + if (ws.data.liveValidateFrame && !ws.data.liveValidateFrame(raw)) { + closeLiveSideband(ws, 1008, "invalid audio event"); + return; + } const rawBytes = webSocketFrameBytes(raw); if (exceedsLiveSidebandFrameByteLimit(rawBytes)) { closeLiveSideband(ws, 1009, "message too large"); @@ -2391,6 +2515,9 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server MAX_WS_FRAME_BYTES) { + closeLiveSideband(ws, 1013, "audio upstream backpressure"); + } } catch { closeLiveSideband(ws, 1011, "upstream send failed"); } @@ -2612,6 +2739,7 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server => { + liveCallBindings.clear(); // The orchestration lives in `runListenerShutdown` so its two competing properties — // cleanup completes, failure propagates — are testable without a live socket. await runListenerShutdown( diff --git a/src/server/live-call-bindings.ts b/src/server/live-call-bindings.ts new file mode 100644 index 0000000000..dd90e8aa43 --- /dev/null +++ b/src/server/live-call-bindings.ts @@ -0,0 +1,60 @@ +import { randomUUID } from "node:crypto"; + +export const EXTERNAL_CALL_PREFIX = "rtc_ocx_"; +export const LIVE_CALL_TTL_MS = 30 * 60_000; +const MAX_CALL_BINDINGS = 1024; + +export interface LiveCallBinding { + owner: string; + upstreamCallId: string; + joinStyle: "frameless-path" | "realtime-query"; + providerName: string; + accountId?: string; + chatgptAccountId?: string; + keyedCredentialDigest?: string; + callerOwned: boolean; + sidebandBaseUrl?: string; +} + +/** Caller-visible IDs are opaque aliases; expired aliases never become legacy upstream IDs. */ +export class LiveCallBindings { + private entries = new Map(); + + constructor(private readonly now: () => number = Date.now) {} + + private prune(): void { + const now = this.now(); + for (const [id, value] of this.entries) if (value.expiresAt <= now) this.entries.delete(id); + } + + hasCapacity(): boolean { + this.prune(); + return this.entries.size < MAX_CALL_BINDINGS; + } + + create(binding: LiveCallBinding): string | null { + if (!this.hasCapacity()) return null; + const id = EXTERNAL_CALL_PREFIX + randomUUID().replaceAll("-", ""); + this.entries.set(id, { binding: { ...binding }, expiresAt: this.now() + LIVE_CALL_TTL_MS }); + return id; + } + + get(id: string, owner: string): LiveCallBinding | undefined { + this.prune(); + const value = this.entries.get(id); + return value?.binding.owner === owner ? { ...value.binding } : undefined; + } + + clear(): void { this.entries.clear(); } +} + +export function upstreamLiveCallId(location: string | null): string | null { + if (!location || location.length > 4096) return null; + try { + const url = new URL(location, "https://unused.invalid"); + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) return null; + const segment = url.pathname.replace(/\/+$/, "").split("/").at(-1) ?? ""; + const id = decodeURIComponent(segment); + return /^(?:rtc_[A-Za-z0-9_-]+|[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})$/.test(id) && id.length <= 128 ? id : null; + } catch { return null; } +} diff --git a/src/server/live.ts b/src/server/live.ts index 7d6b5c7357..15322f225f 100644 --- a/src/server/live.ts +++ b/src/server/live.ts @@ -371,7 +371,7 @@ export function buildLiveSidebandUpstreamWsUrl( ); } -async function backendJsonBodyFromApiMultipart( +export async function backendJsonBodyFromApiMultipart( body: ArrayBuffer, contentType: string, ): Promise<{ body: Uint8Array; contentType: string } | Response> { diff --git a/src/server/management/api-access.ts b/src/server/management/api-access.ts index 44336919ca..0421824e5a 100644 --- a/src/server/management/api-access.ts +++ b/src/server/management/api-access.ts @@ -2,6 +2,21 @@ import type { OcxConfig } from "../../types"; import { isWildcardHostname } from "../../codex/loopback-target"; import { localInferenceDestination } from "../../lib/local-destinations"; import { probeHostname } from "../proxy-liveness"; +import { isCanonicalOpenAiForwardProvider, OPENAI_API_PROVIDER_ID, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers-destination"; +import { LIVE_AUDIO_MODEL, TRANSCRIPTION_MODEL } from "../audio-upstream"; + +export interface AudioApiAccess { + transcriptionEndpoint: string; + dictationStreamEndpoint: string; + liveEndpoint: string; + realtimeCallsEndpoint: string; + transcriptionModel: string; + liveModel: string; + /** Configuration only, not account health, entitlement or observed connectivity. */ + transcriptionConfigured: boolean; + dictationConfigured: boolean; + liveConfigured: boolean; +} export interface ApiAccessEndpoints { baseUrl: string; @@ -10,6 +25,7 @@ export interface ApiAccessEndpoints { messagesEndpoint: string; modelsEndpoint: string; claudeCodeEnabled: boolean; + audio: AudioApiAccess; /** Back-compat alias for older GUI clients. */ endpoint: string; } @@ -140,6 +156,16 @@ export function buildApiAccessEndpoints( ): ApiAccessEndpoints { const baseUrl = resolveApiAccessBaseUrl(config, opts); const responsesEndpoint = `${baseUrl}/responses`; + const socketBase = new URL(baseUrl); + socketBase.protocol = socketBase.protocol === "https:" ? "wss:" : "ws:"; + const forward = config.providers?.[OPENAI_CODEX_PROVIDER_ID]; + const chatgptConfigured = !!forward && forward.disabled !== true + && isCanonicalOpenAiForwardProvider({ ...forward, authMode: forward.authMode ?? "forward" }); + const keyed = config.providers?.[OPENAI_API_PROVIDER_ID]; + // Do not resolve key references or inspect accounts on a management metadata read. + const apiConfigured = !!keyed && keyed.disabled !== true && keyed.adapter === "openai-responses" + && keyed.authMode !== "forward" && keyed.baseUrl.replace(/\/+$/, "") === "https://api.openai.com/v1" + && typeof keyed.apiKey === "string" && !!keyed.apiKey.trim(); return { baseUrl, responsesEndpoint, @@ -147,6 +173,17 @@ export function buildApiAccessEndpoints( messagesEndpoint: `${baseUrl}/messages`, modelsEndpoint: `${baseUrl}/models`, claudeCodeEnabled: config.claudeCode?.enabled !== false, + audio: { + transcriptionEndpoint: `${baseUrl}/audio/transcriptions`, + dictationStreamEndpoint: `${socketBase.href}/audio/transcriptions/stream`, + liveEndpoint: `${socketBase.href}/live`, + realtimeCallsEndpoint: `${baseUrl}/realtime/calls`, + transcriptionModel: TRANSCRIPTION_MODEL, + liveModel: LIVE_AUDIO_MODEL, + transcriptionConfigured: chatgptConfigured || apiConfigured, + dictationConfigured: chatgptConfigured, + liveConfigured: chatgptConfigured, + }, endpoint: responsesEndpoint, }; } diff --git a/src/server/ws-bridge.ts b/src/server/ws-bridge.ts index 5777b45a10..1c8513de3d 100644 --- a/src/server/ws-bridge.ts +++ b/src/server/ws-bridge.ts @@ -8,6 +8,7 @@ import type { DataPlaneAdmission } from "./auth-cors"; import type { AdmissionLease, AdmissionReservation } from "../lib/admission"; import { BoundedSseFrameBuffer } from "./sse-frame-buffer"; import { safeResponseHeaders } from "./safe-response-headers"; +import type { AudioSocketTarget } from "./audio-dictation"; export { safeResponseHeaders } from "./safe-response-headers"; @@ -35,6 +36,15 @@ export interface WsData { liveUpstream?: WebSocket; liveUpstreamUrl?: string; liveUpstreamHeaders?: Record; + liveUpstreamProtocols?: string[]; + liveValidateFrame?: AudioSocketTarget["validateFrame"]; + liveFinish?: AudioSocketTarget["finish"]; + liveOutcome?: number | "timeout" | "connect_error"; + liveMaxSessionMs?: number; + liveConnectTimer?: ReturnType; + liveSessionTimer?: ReturnType; + liveAbortSignal?: AbortSignal; + liveAbortListener?: () => void; livePending?: Array; /** Total encoded bytes retained in livePending while the upstream connects. */ livePendingBytes?: number; diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index 01b3db72a8..5499b05944 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -17,6 +17,28 @@ keep credentials and audio content out of redirects, request logs and durable st `tests/server/audio-transcriptions.test.ts` exercises the real ingress and synthetic upstream; `tests/server/api-key-attribution.test.ts` uses multipart fixtures for the HTTP auth matrix. +## Streaming audio + +`src/server/audio-client.ts` recognizes explicit audio keys before local legacy admission. +Browser sockets offer opencodex-audio and opencodex-key.; only the public +marker is selected downstream. Invalid presented keys cannot become credential-free native calls. +`src/server/audio-dictation.ts` maps the validated desktop session.start/audio.append/session.close +protocol to ChatGPT dictation with server-owned credentials and a five-minute lifetime. +The `src/server/index.ts` byte relay retains bounded queues, handshake/session deadlines and +the account/turn lifecycle until its upstream closes. `src/server/ws-bridge.ts` carries the +in-memory callbacks and signal; handshake credentials are cleared after socket construction. + +`src/server/audio-live.ts` owns external keyed GPT-Live creation and joins while the original +`src/server/live.ts` keeps the native compatibility path. `src/server/live-call-bindings.ts` +maps opaque rtc_ocx_ aliases to the creating key, provider, physical account and protocol. +Expired aliases never fall through to native joins. Reconnect resolves the recorded account +freshly; keyed provider replacement fails unless its credential digest still matches. +The registry is per server, holds at most 1024 entries for 30 minutes and is cleared on shutdown. +It does not proxy WebRTC media or execute delegation requests. Standalone Frameless defaults +to gpt-live-1-codex; gpt-live-1 is an explicit alias. Dictation and Frameless event formats remain +separate. Coverage lives in `tests/server/audio-client.test.ts`, +`tests/server/audio-dictation.test.ts` and `tests/server/live-call-bindings.test.ts`. + ## Chat Completions inbound native path `POST /v1/chat/completions` sends eligible `openai-chat` routes directly to the provider's Chat diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 9fde77fc9e..3e637703d9 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -290,6 +290,29 @@ hard-lock evidence and reset-notification history; their retention rules are doc ## Dashboard surfaces +`src/server/management/api-access.ts` publishes an `audio` projection through the +existing `/api/keys` response in `src/server/management/oauth-account-routes.ts`. +URLs derive from the same advertised inference base as text APIs, with HTTP(S) +mapped to WS(S). Configuration flags inspect enabled canonical providers only; +they do not read credentials, inspect account health or prove entitlement. +`gui/src/pages/api-keys-utils.ts` validates the same projection on network and +cache reads. Missing or malformed audio metadata disables only audio controls. + +Connections/API keys has separate Dictation and Live Voice sections in +`gui/src/components/apikeys-workspace/AudioApiPanel.tsx`. Transient data keys never +enter caches or generated samples. `gui/src/audio-api-client.ts` owns bounded +uploads and a connection-only native voice probe; `gui/src/api.ts` sends uploads +without management auth injection or 401 recovery. Voice readiness requires a +nonterminal session acknowledgment with `session.id`, not merely socket open. +Changing keys, inference metadata, API origin or leaving the active panel releases +requests/sockets. Only allowlisted event types and localized error categories are +displayed. Tests live in `gui/tests/audio-api-client.test.ts`, +`gui/tests/audio-api-panel.test.tsx`, `gui/tests/api-auth-memory.test.ts` and +`tests/server/api-access-endpoints.test.ts`. +The API workspace gives `gui/src/components/section-tabs.tsx` its mobile reading +line so scroll-spy and the top-bar offset agree; other consumers keep their +existing reading line. The section strip stays one row at every width. + Provider Overview consumes the existing shared `add-provider-presets` resource for sponsor presentation. `matchingWorkspacePreset` requires the configured id, adapter and normalized endpoint to match; a custom endpoint or absent sponsor metadata suppresses the introduction. diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 8080c8bf04..4cae9c16da 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -411,6 +411,12 @@ materialization or usability failure before transferring a resolved context to i Audio reports one terminal upstream outcome after validating the response body; redirects remain neutral and client/shutdown cancellation does not manufacture an account failure. +External voice reconnects restrict provider selection as well as exact account selection to the +original call binding. Credential acquisition accepts a cancellation signal; post-resolution +materialization checks cancellation before returning ownership. Connectivity-only WebSocket +completion is neutral: HTTP 101 does not prove inference or quota recovery, and a normal close +may follow a protocol error. Explicit transport errors/timeouts settle once during cleanup. + The dashboard presents one OpenAI Codex card with accessible Pool/Direct controls and a separate, unchanged API-key card. `PATCH /api/providers?name=openai` persists exactly one `codexAccountMode`, clears affinity/quota cache, primes only when entering Pool, and does not refresh diff --git a/structure/runtime.md b/structure/runtime.md index 94541ef247..50077f7070 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -10,6 +10,7 @@ | `src/server/index.ts` | Bun server entrypoint: `startServer`, `/v1/responses` HTTP + WebSocket routing (compact handled before generic Responses), exact `POST /v1/images/generations` and `POST /v1/images/edits` routing, `/v1/models`, the Anthropic-shaped `/v1/messages` and OpenAI-shaped `/v1/chat/completions` compatibility surfaces, the Live/Realtime surface, the hosted-search relay, artifact serving, `/healthz`, the `/api/*` auth gate, the `/v1/*` JSON 404 guard, GUI fallback, the opt-in loopback-only hub-management listener, and facade re-exports for split server modules. | | `src/server/images.ts` | Standalone Images data plane: default OpenAI or explicit custom-provider selection, Codex account affinity, bounded opaque request relay, single-attempt upstream fetch, pool health recording, and safe response/cancellation relay. | | `src/server/audio-transcriptions.ts` | Standalone multipart transcription; audio-specific key admission, bounded upload/response, stored OpenAI credential resolution and lease-bound cancellation. See [audio contracts](data-planes/inbound-compat.md#standalone-file-transcription). | +| `src/server/audio-live.ts`, `src/server/audio-dictation.ts` | External voice/dictation orchestration using the existing bounded socket relay, server-owned credentials, cancellation and opaque call ownership. See [streaming audio](data-planes/inbound-compat.md#streaming-audio). | | `src/config.ts` | Persisted `~/.opencodex/config.json` schema, defaults, migrations, transactions, and compatibility re-exports for split config modules. | | `src/config/paths.ts` | Resolves `OPENCODEX_HOME`, `config.json`, and owner-only directory hardening. | | `src/config/atomic-write.ts` | Shared synchronous/asynchronous temp-harden-rename writer and residual-temp failure contract. | @@ -62,7 +63,7 @@ listener, the optional unauthenticated data-loopback listener, and the optional listener. The data-loopback socket serves a fixed data-plane allowlist: Responses and its compact sibling, -the native search relay, the standalone Images POSTs, keyed file transcription, `GET /v1/models`, the realtime voice shapes, +the native search relay, the standalone Images POSTs, keyed file/stream transcription, `GET /v1/models`, the realtime voice shapes, and the Anthropic and OpenAI chat wires the host's own local clients speak — `POST /v1/messages`, `POST /v1/messages/count_tokens`, and `POST /v1/chat/completions`. It never serves `/api/*`, `/healthz`, `/readyz`, or GUI routes, so local management discovery has to use an authenticated diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 63d04e6aa3..79c645414c 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -52,6 +52,9 @@ "antigravity-static-catalog.test.ts": "adapters/google", "api-access-endpoints.test.ts": "server", "audio-transcriptions.test.ts": "server", + "audio-client.test.ts": "server", + "audio-dictation.test.ts": "server", + "live-call-bindings.test.ts": "server", "api-catalog-route.test.ts": "server", "api-codex-log-guard-compact.test.ts": "server", "api-codex-log-guard-protection.test.ts": "server", diff --git a/tests/server/api-access-endpoints.test.ts b/tests/server/api-access-endpoints.test.ts index f1c6f7fa6d..ce2a55fdbf 100644 --- a/tests/server/api-access-endpoints.test.ts +++ b/tests/server/api-access-endpoints.test.ts @@ -7,7 +7,7 @@ import { describe("buildApiAccessEndpoints", () => { test("builds the external gateway URLs from hostname and port", () => { - expect(buildApiAccessEndpoints({ hostname: "127.0.0.1", port: 10100 })).toEqual({ + expect(buildApiAccessEndpoints({ hostname: "127.0.0.1", port: 10100 })).toMatchObject({ baseUrl: "http://127.0.0.1:10100/v1", endpoint: "http://127.0.0.1:10100/v1/responses", responsesEndpoint: "http://127.0.0.1:10100/v1/responses", @@ -19,7 +19,7 @@ describe("buildApiAccessEndpoints", () => { }); test("falls back to the default bind when config fields are missing", () => { - expect(buildApiAccessEndpoints({})).toEqual({ + expect(buildApiAccessEndpoints({})).toMatchObject({ baseUrl: "http://127.0.0.1:10100/v1", endpoint: "http://127.0.0.1:10100/v1/responses", responsesEndpoint: "http://127.0.0.1:10100/v1/responses", @@ -31,7 +31,7 @@ describe("buildApiAccessEndpoints", () => { }); test("brackets IPv6 hostnames for URL display", () => { - expect(buildApiAccessEndpoints({ hostname: "::1", port: 10100 })).toEqual({ + expect(buildApiAccessEndpoints({ hostname: "::1", port: 10100 })).toMatchObject({ baseUrl: "http://[::1]:10100/v1", endpoint: "http://[::1]:10100/v1/responses", responsesEndpoint: "http://[::1]:10100/v1/responses", @@ -106,6 +106,36 @@ describe("buildApiAccessEndpoints", () => { test("reflects disabled Claude inbound in API access metadata", () => { expect(buildApiAccessEndpoints({ claudeCode: { enabled: false } }).claudeCodeEnabled).toBe(false); }); + + test("audio metadata derives TLS and IPv6 URLs without claiming connectivity", () => { + const result = buildApiAccessEndpoints({ hostname: "::", port: 10100 }, { requestOrigin: "https://[2001:db8::1]:8443" }); + expect(result.audio).toEqual({ + transcriptionEndpoint: "https://[2001:db8::1]:8443/v1/audio/transcriptions", + dictationStreamEndpoint: "wss://[2001:db8::1]:8443/v1/audio/transcriptions/stream", + liveEndpoint: "wss://[2001:db8::1]:8443/v1/live", + realtimeCallsEndpoint: "https://[2001:db8::1]:8443/v1/realtime/calls", + transcriptionModel: "gpt-4o-transcribe", liveModel: "gpt-live-1-codex", + transcriptionConfigured: false, dictationConfigured: false, liveConfigured: false, + }); + const companion = buildApiAccessEndpoints({ hostname: "0.0.0.0", port: 10100, unauthenticatedLoopbackListener: { enabled: true, port: 10104 } }); + expect(companion.audio.liveEndpoint).toBe("ws://127.0.0.1:10104/v1/live"); + }); + + test("audio configuration distinguishes subscription, API key and noncanonical destinations", () => { + const forward = { adapter: "openai-responses" as const, baseUrl: "https://chatgpt.com/backend-api/codex" }; + const configured = buildApiAccessEndpoints({ providers: { openai: forward } }).audio; + expect(configured.transcriptionConfigured).toBe(true); + expect(configured.dictationConfigured).toBe(true); + expect(configured.liveConfigured).toBe(true); + for (const provider of [{ ...forward, disabled: true }, { ...forward, baseUrl: "https://example.test/v1" }, { ...forward, authMode: "key" as const }]) { + expect(buildApiAccessEndpoints({ providers: { openai: provider } }).audio.liveConfigured).toBe(false); + } + const api = buildApiAccessEndpoints({ providers: { "openai-apikey": { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1", apiKey: "env:AUDIO_METADATA_FIXTURE" } } }).audio; + expect(api.transcriptionConfigured).toBe(true); + expect(api.dictationConfigured).toBe(false); + expect(api.liveConfigured).toBe(false); + expect(JSON.stringify(api)).not.toContain("AUDIO_METADATA_FIXTURE"); + }); }); describe("formatAuthorityHost", () => { diff --git a/tests/server/api-key-attribution.test.ts b/tests/server/api-key-attribution.test.ts index 6bc42acf62..6962be9ff4 100644 --- a/tests/server/api-key-attribution.test.ts +++ b/tests/server/api-key-attribution.test.ts @@ -634,10 +634,14 @@ describe("AUTH_MATRIX is true of the running server", () => { // than admission. /v1/catalog joined this set in #809. const isGet = row.endpoint === "/v1/models" || row.endpoint === "/v1/catalog" || row.endpoint === "/v1/hub-state"; - const audio = row.endpoint === "/v1/audio/transcriptions" ? new FormData() : null; + const live = row.endpoint === "/v1/live" || row.endpoint === "/v1/realtime/calls"; + const audio = row.endpoint === "/v1/audio/transcriptions" || live ? new FormData() : null; if (audio) { - audio.append("model", "gpt-4o-transcribe"); - audio.append("file", new File([new Uint8Array([0, 0])], "sample.wav", { type: "audio/wav" })); + if (live) audio.append("sdp", "v=0\r\n"); + else { + audio.append("model", "gpt-4o-transcribe"); + audio.append("file", new File([new Uint8Array([0, 0])], "sample.wav", { type: "audio/wav" })); + } } const res = await fetch(new URL(row.endpoint, server.url), { method: isGet ? "GET" : "POST", diff --git a/tests/server/audio-client.test.ts b/tests/server/audio-client.test.ts new file mode 100644 index 0000000000..39765498bb --- /dev/null +++ b/tests/server/audio-client.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { resolveAudioClient, type AudioClient } from "../../src/server/audio-client"; +import type { OcxConfig } from "../../src/types"; + +const KEY = "ocx_data_audio_client_fixture"; +const ROTATED_KEY = "ocx_data_rotated_fixture"; +const originalToken = process.env.OPENCODEX_API_AUTH_TOKEN; +const config = { providers: {}, defaultProvider: "none", apiKeys: [{ id: "one", name: "one", key: KEY, createdAt: "2026-09-12T00:00:00Z" }] } as OcxConfig; +beforeEach(() => { delete process.env.OPENCODEX_API_AUTH_TOKEN; }); +afterEach(() => { + if (originalToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; + else process.env.OPENCODEX_API_AUTH_TOKEN = originalToken; +}); + +function request(headers: Record): Request { + return new Request("http://localhost/v1/live", { headers: { upgrade: "websocket", ...headers } }); +} +function carrier(key = KEY): string { return `opencodex-audio, opencodex-key.${Buffer.from(key).toString("base64url")}`; } +function client(result: ReturnType): AudioClient { + if (!result || result instanceof Response) throw new Error("Expected admitted audio client"); + return result; +} + +describe("audio-only WebSocket admission", () => { + test("browser carrier resolves a configured owner and selects only the public protocol", () => { + const result = client(resolveAudioClient(request({ "sec-websocket-protocol": carrier() }), config, true)); + expect(result.admission).toMatchObject({ kind: "configured", keyId: "one", source: "dedicated" }); + expect(result.admission).toHaveProperty("contextPrincipalId"); + expect(result.protocol).toBe("opencodex-audio"); + expect(result.owner).toBe('["configured","one"]'); + expect(result.owner).not.toContain(KEY); + }); + test("bad explicit headers win over a valid carrier", () => { + const result = resolveAudioClient(request({ "sec-websocket-protocol": carrier(), "x-opencodex-api-key": "wrong" }), config, true); + expect(result).toBeInstanceOf(Response); + expect((result as Response).status).toBe(401); + }); + test.each([ + "opencodex-audio", `opencodex-key.${Buffer.from(KEY).toString("base64url")}`, + carrier() + ", opencodex-audio", carrier() + ", other", + "opencodex-audio, opencodex-key.bm9uY2Fub25pY2Fs=", + ])("rejects malformed protocol pair %s", value => { + expect((resolveAudioClient(request({ "sec-websocket-protocol": value }), config, true) as Response).status).toBe(400); + }); + test("same environment key has one owner across header carriers", () => { + process.env.OPENCODEX_API_AUTH_TOKEN = "fixture-env-audio"; + const bearer = client(resolveAudioClient(request({ authorization: "Bearer fixture-env-audio" }), config)); + const dedicated = client(resolveAudioClient(request({ "x-opencodex-api-key": "fixture-env-audio" }), config)); + expect(bearer.owner).toBe(dedicated.owner); + expect(bearer.owner).not.toContain("fixture-env-audio"); + }); + test("required audio never falls through to unauthenticated loopback", () => { + expect((resolveAudioClient(request({}), config, true) as Response).status).toBe(401); + expect(resolveAudioClient(request({}), config)).toBeNull(); + expect((resolveAudioClient(request({ authorization: "Bearer custom-revoked-key" }), config) as Response).status).toBe(401); + }); + test("pending rotation key keeps configured call ownership", () => { + const rotated = { ...config, apiKeys: [{ ...config.apiKeys![0]!, pendingRotation: { id: "rotation", key: ROTATED_KEY, createdAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString() } }] }; + const old = client(resolveAudioClient(request({ authorization: `Bearer ${KEY}` }), rotated)); + const next = client(resolveAudioClient(request({ authorization: `Bearer ${ROTATED_KEY}` }), rotated)); + expect(next.owner).toBe(old.owner); + }); + test("known native platform bearer retains legacy handling without guessing by prefix", () => { + const keyed: OcxConfig = { ...config, providers: { "openai-apikey": { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1", apiKey: "sk-fixture-native", authMode: "key" } } }; + expect(resolveAudioClient(request({ authorization: "Bearer sk-fixture-native" }), keyed)).toBeNull(); + expect((resolveAudioClient(request({ authorization: "Bearer sk-other-revoked" }), keyed) as Response).status).toBe(401); + }); +}); diff --git a/tests/server/audio-dictation.test.ts b/tests/server/audio-dictation.test.ts new file mode 100644 index 0000000000..1c9020e270 --- /dev/null +++ b/tests/server/audio-dictation.test.ts @@ -0,0 +1,279 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearAccountQuota } from "../../src/codex/auth-api"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; +import * as routing from "../../src/codex/routing"; +import { saveConfig } from "../../src/config"; +import { startServer } from "../../src/server"; +import { createDictationFrameValidator } from "../../src/server/audio-dictation"; +import { abortAndReleaseAllTurns, resetLifecycleDrainStateForTests } from "../../src/server/lifecycle"; +import type { OcxConfig } from "../../src/types"; +import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const KEY = "ocx_data_audio_stream_fixture"; +const OTHER_KEY = "ocx_data_audio_other_fixture"; +const fetchOriginal = globalThis.fetch; +const previousHome = process.env.OPENCODEX_HOME; +const previousToken = process.env.OPENCODEX_API_AUTH_TOKEN; +let home: string; +let codex: IsolatedCodexHome; +let fixture: ReturnType | undefined; +const clients = new Set(); + +const startEvent = { type: "session.start", config: { + input_audio_format: "pcm16", sample_rate_hz: 48000, num_channels: 1, + max_buffer_size_bytes: 4194304, max_utterance_duration_ms: 30000, session_ttl_ms: 300000, + provider_mode: "streaming_sse", transcript_delivery_mode: "segment", + vad: { type: "server_vad", threshold: 0.5, prefix_padding_ms: 300, silence_duration_ms: 500 }, +} }; + +function createFixture(options: { failDictation?: boolean } = {}) { + const creates: Headers[] = []; + const handshakes: Array<{ url: string; headers: Headers; protocols?: string[] }> = []; + const frames: string[] = []; + const upstreamClosed = Promise.withResolvers(); + const upstream = Bun.serve({ + port: 0, + fetch(req, server) { + if (req.headers.get("upgrade")?.toLowerCase() === "websocket") { + const protocol = req.headers.get("sec-websocket-protocol")?.includes("chatgpt-dictation") ? "chatgpt-dictation" : undefined; + if (server.upgrade(req, { data: {}, ...(protocol ? { headers: { "sec-websocket-protocol": protocol } } : {}) })) return; + } + return new Response("not found", { status: 404 }); + }, + websocket: { + message(ws, raw) { + const message = String(raw); + frames.push(message); + let event: { type?: string }; + try { event = JSON.parse(message); } catch { ws.send(message); return; } + if (options.failDictation && event.type === "session.start") { + ws.send(JSON.stringify({ type: "session.error", sequence_no: 1, fatal: true, error: { code: "fixture_error", message: "fixture rejection", retryable: false } })); + ws.close(1000); + return; + } + if (event.type === "session.start") ws.send(JSON.stringify({ type: "session.started", sequence_no: 1, session: { session_id: "fixture", status: "active", config: { provider_mode: "streaming_sse", transcript_delivery_mode: "segment" } } })); + else if (event.type === "audio.append") ws.send(JSON.stringify({ type: "transcript.final", sequence_no: 2, utterance_id: "u1", revision: 1, text: "fixture transcript" })); + else if (event.type === "session.close") { + ws.send(JSON.stringify({ type: "session.updated", sequence_no: 3, session: { session_id: "fixture", status: "closed", config: { provider_mode: "streaming_sse", transcript_delivery_mode: "segment" } } })); + ws.close(1000); + } else ws.send(message); + }, + }, + }); + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const req = new Request(input, init); + if (["chatgpt.com", "api.openai.com"].includes(new URL(req.url).hostname)) { + if (new URL(req.url).pathname.endsWith("/realtime/calls") || new URL(req.url).pathname === "/v1/live") { + creates.push(new Headers(req.headers)); + return new Response("v=0\r\n", { status: 201, headers: { "content-type": "application/sdp", location: `https://api.openai.com/v1/live/rtc_upstream_${creates.length}` } }); + } + return Response.json({}); + } + return fetchOriginal(input, init); + }) as typeof fetch; + const server = startServer(0, { + liveSidebandWebSocketFactory(url, headers, protocols) { + handshakes.push({ url, headers: new Headers(headers), protocols }); + const local = new URL("/socket", upstream.url); local.protocol = "ws:"; + const socket = new WebSocket(local, { headers, protocols } as unknown as string[]); + socket.addEventListener("close", () => upstreamClosed.resolve(), { once: true }); + return socket; + }, + }); + return { server, upstream, creates, handshakes, frames, upstreamClosed: upstreamClosed.promise }; +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-stream-")); + process.env.OPENCODEX_HOME = home; + delete process.env.OPENCODEX_API_AUTH_TOKEN; + codex = installIsolatedCodexHome("ocx-stream-codex-"); + resetLifecycleDrainStateForTests(); clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "openai", openaiProviderTierVersion: 2, accountPoolStrategy: "round-robin", + providers: { openai: { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", authMode: "forward", codexAccountMode: "pool" } }, + codexAccounts: [ + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "acct-a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "acct-b" }, + ], + apiKeys: [ + { id: "one", name: "one", key: KEY, createdAt: "2026-09-12T00:00:00Z" }, + { id: "two", name: "two", key: OTHER_KEY, createdAt: "2026-09-12T00:00:00Z" }, + ], + }; + for (const [id, account] of [["pool-a", "acct-a"], ["pool-b", "acct-b"]] as const) { + saveCodexAccountCredential(id, { accessToken: fakeChatGptJwt({ chatgpt_account_id: account }), refreshToken: "fixture-refresh", expiresAt: Date.now() + 3600000, chatgptAccountId: account }); + } + saveConfig(config); +}); + +afterEach(async () => { + for (const ws of clients) ws.close(); + clients.clear(); + if (fixture) await Promise.all([fixture.server.stop(true), fixture.upstream.stop(true)]); + fixture = undefined; + globalThis.fetch = fetchOriginal; + resetLifecycleDrainStateForTests(); clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); + codex.restore(); removeTreeWithRetry(home); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; else process.env.OPENCODEX_API_AUTH_TOKEN = previousToken; +}); + +function socket(path: string, browser = false): WebSocket { + const url = new URL(path, fixture!.server.url); url.protocol = "ws:"; + const ws = browser + ? new WebSocket(url, ["opencodex-audio", `opencodex-key.${Buffer.from(KEY).toString("base64url")}`]) + : new WebSocket(url, { headers: { authorization: `Bearer ${KEY}` } } as unknown as string[]); + clients.add(ws); + return ws; +} + +function receive(ws: WebSocket, send: unknown, expected: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { ws.close(); reject(new Error("Fixture socket timed out")); }, 10000); + ws.addEventListener("message", event => { + const text = String(event.data); + if (text.includes(expected)) { clearTimeout(timer); resolve(text); } + }); + ws.addEventListener("error", () => { clearTimeout(timer); reject(new Error("Fixture socket failed")); }, { once: true }); + const transmit = () => ws.send(typeof send === "string" ? send : JSON.stringify(send)); + if (ws.readyState === WebSocket.OPEN) transmit(); else ws.addEventListener("open", transmit, { once: true }); + }); +} + +async function createCall(): Promise { + const response = await fetchOriginal(new URL("/v1/live", fixture!.server.url), { + method: "POST", headers: { authorization: `Bearer ${KEY}`, "content-type": "application/json" }, body: JSON.stringify({ sdp: "v=0\r\n" }), + }); + expect(response.status).toBe(201); + await response.text(); + return response.headers.get("location")!; +} + +describe("dictation protocol validation", () => { + test("PCM frames require a valid start, canonical base64 and an open session", () => { + const validate = createDictationFrameValidator(); + expect(validate(JSON.stringify({ type: "audio.append", audio: "AAA=" }))).toBe(false); + expect(validate(JSON.stringify(startEvent))).toBe(true); + expect(validate(JSON.stringify(startEvent))).toBe(false); + expect(validate(JSON.stringify({ type: "audio.append", audio: "AA==" }))).toBe(false); + expect(validate(JSON.stringify({ type: "audio.append", audio: "AAA=" }))).toBe(true); + expect(validate(JSON.stringify({ type: "session.close" }))).toBe(true); + expect(validate(JSON.stringify({ type: "audio.append", audio: "AAA=" }))).toBe(false); + }); + test("bad formats and oversized configurations do not start sessions", () => { + for (const change of [{ num_channels: 2 }, { sample_rate_hz: 0 }, { session_ttl_ms: 300001 }, { max_buffer_size_bytes: 4194305 }]) { + expect(createDictationFrameValidator()(JSON.stringify({ ...startEvent, config: { ...startEvent.config, ...change } }))).toBe(false); + } + }); +}); + +describe("external audio sockets", () => { + test("native platform bearer keeps HTTP creation and WebSocket relay on its configured tier", async () => { + saveConfig({ port: 0, hostname: "127.0.0.1", defaultProvider: "openai-apikey", openaiProviderTierVersion: 2, + providers: { "openai-apikey": { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1", apiKey: "sk-fixture-native", authMode: "key" } }, + apiKeys: [{ id: "one", name: "one", key: KEY, createdAt: "2026-09-12T00:00:00Z" }], + }); + fixture = createFixture(); + const form = new FormData(); form.set("sdp", "v=0\r\n"); + const response = await fetchOriginal(new URL("/v1/live", fixture.server.url), { + method: "POST", body: form, headers: { authorization: "Bearer sk-fixture-native" }, + }); + expect(response.status).toBe(201); + await response.text(); + expect(response.headers.get("location")).toBe("https://api.openai.com/v1/live/rtc_upstream_1"); + const url = new URL("/v1/realtime?model=gpt-realtime-1.5", fixture.server.url); url.protocol = "ws:"; + const ws = new WebSocket(url, { headers: { authorization: "Bearer sk-fixture-native" } } as unknown as string[]); + clients.add(ws); + await receive(ws, "native-echo", "native-echo"); + expect(fixture.handshakes[0]!.headers.get("authorization")).toBe("Bearer sk-fixture-native"); + }); + test("browser key carrier relays the actual dictation protocol without exposing upstream credentials", async () => { + fixture = createFixture(); + const ws = socket("/v1/audio/transcriptions/stream", true); + await receive(ws, startEvent, "session.started"); + expect(ws.protocol).toBe("opencodex-audio"); + const transcript = await receive(ws, { type: "audio.append", audio: "AAA=" }, "transcript.final"); + expect(JSON.parse(transcript).text).toBe("fixture transcript"); + await receive(ws, { type: "session.close" }, "session.updated"); + expect(fixture.handshakes[0]!.url).toBe("wss://chatgpt.com/backend-api/dictation/stream"); + expect(fixture.handshakes[0]!.protocols?.[0]).toBe("chatgpt-dictation"); + expect(fixture.handshakes[0]!.protocols?.[1]).toStartWith("openai-bearer."); + expect(fixture.handshakes[0]!.protocols?.join(",")).not.toContain(KEY); + }); + test("standalone live defaults model and negotiation only for external clients", async () => { + fixture = createFixture(); + const ws = socket("/v1/live"); + expect(await receive(ws, "fixture-echo", "fixture-echo")).toBe("fixture-echo"); + expect(fixture.handshakes[0]!.url).toBe("wss://api.openai.com/v1/live?model=gpt-live-1-codex"); + expect(fixture.handshakes[0]!.headers.get("openai-alpha")).toBe("quicksilver=v2"); + }); + test("connectivity-only completion does not claim inference recovery", async () => { + fixture = createFixture(); + const outcomes = spyOn(routing, "recordCodexUpstreamOutcome"); + try { + const ws = socket("/v1/live"); + await receive(ws, "healthy", "healthy"); + ws.close(); + await fixture.upstreamClosed; + expect(outcomes.mock.calls).toEqual([]); + } finally { outcomes.mockRestore(); } + }); + test("protocol failure followed by a normal close never records success", async () => { + fixture = createFixture({ failDictation: true }); + const before = routing.getCodexUpstreamHealth("pool-a"); + const outcomes = spyOn(routing, "recordCodexUpstreamOutcome"); + try { + const ws = socket("/v1/audio/transcriptions/stream", true); + await receive(ws, startEvent, "session.error"); + await fixture.upstreamClosed; + expect(outcomes.mock.calls).toEqual([]); + expect(routing.getCodexUpstreamHealth("pool-a")).toEqual(before); + } finally { outcomes.mockRestore(); } + }); + test("shutdown cancellation reaches the authenticated upstream socket", async () => { + fixture = createFixture(); + const ws = socket("/v1/live"); + await receive(ws, "before-shutdown", "before-shutdown"); + const closed = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Shutdown left the client open")), 10000); + ws.addEventListener("close", () => { clearTimeout(timer); resolve(); }, { once: true }); + }); + abortAndReleaseAllTurns(); + await Promise.all([closed, fixture.upstreamClosed]); + expect(ws.readyState).toBe(WebSocket.CLOSED); + }, { timeout: 15000 }); + test("a returned call alias keeps the creating account after pool rotation", async () => { + fixture = createFixture(); + const location = await createCall(); + expect(location).toStartWith("/v1/live/rtc_ocx_"); + await createCall(); + expect(fixture.creates[1]!.get("chatgpt-account-id")).not.toBe(fixture.creates[0]!.get("chatgpt-account-id")); + const ws = socket(location); + await receive(ws, "bound-echo", "bound-echo"); + expect(fixture.handshakes[0]!.url).toBe("wss://api.openai.com/v1/live/rtc_upstream_1"); + expect(fixture.handshakes[0]!.headers.get("chatgpt-account-id")).toBe(fixture.creates[0]!.get("chatgpt-account-id")); + }); + test("wrong key and unkeyed callers cannot join an external alias", async () => { + fixture = createFixture(); + const location = await createCall(); + for (const [key, expected] of [[OTHER_KEY, 404], ["", 401]] as const) { + const response = await fetchOriginal(new URL(location, fixture.server.url), { headers: { upgrade: "websocket", connection: "upgrade", "sec-websocket-key": "MDEyMzQ1Njc4OWFiY2RlZg==", "sec-websocket-version": "13", ...(key ? { authorization: `Bearer ${key}` } : {}) } }); + expect(response.status).toBe(expected); + await response.text(); + } + expect(fixture.handshakes).toHaveLength(0); + }); + test("missing reserved aliases never become legacy native joins", async () => { + fixture = createFixture(); + const response = await fetchOriginal(new URL("/v1/live/rtc_ocx_expired", fixture.server.url), { headers: { upgrade: "websocket" } }); + expect(response.status).toBe(401); + expect(fixture.handshakes).toHaveLength(0); + }); +}); diff --git a/tests/server/live-call-bindings.test.ts b/tests/server/live-call-bindings.test.ts new file mode 100644 index 0000000000..161c68f1fa --- /dev/null +++ b/tests/server/live-call-bindings.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from "bun:test"; +import { EXTERNAL_CALL_PREFIX, LIVE_CALL_TTL_MS, LiveCallBindings, upstreamLiveCallId, type LiveCallBinding } from "../../src/server/live-call-bindings"; + +const binding: LiveCallBinding = { owner: "key-one", upstreamCallId: "rtc_upstream", joinStyle: "frameless-path", providerName: "openai", accountId: "account-a", chatgptAccountId: "workspace-a", callerOwned: false }; + +describe("external live call ownership", () => { + test("opaque aliases isolate owners and do not expose upstream call IDs", () => { + const registry = new LiveCallBindings(); + const id = registry.create(binding)!; + expect(id.startsWith(EXTERNAL_CALL_PREFIX)).toBe(true); + expect(id).not.toContain("upstream"); + expect(registry.get(id, "key-one")).toEqual(binding); + expect(registry.get(id, "key-two")).toBeUndefined(); + const copy = registry.get(id, "key-one")!; + copy.accountId = "other"; + expect(registry.get(id, "key-one")!.accountId).toBe("account-a"); + }); + test("expiry and clear retire aliases independently of socket reconnection", () => { + let now = 0; + const registry = new LiveCallBindings(() => now); + const id = registry.create(binding)!; + now = LIVE_CALL_TTL_MS - 1; + expect(registry.get(id, "key-one")).toBeDefined(); + now += 1; + expect(registry.get(id, "key-one")).toBeUndefined(); + const next = registry.create(binding)!; + registry.clear(); + expect(registry.get(next, "key-one")).toBeUndefined(); + }); + test("capacity is bounded and expired entries reclaim capacity", () => { + let now = 0; + const registry = new LiveCallBindings(() => now); + for (let i = 0; i < 1024; i++) expect(registry.create(binding)).not.toBeNull(); + expect(registry.create(binding)).toBeNull(); + now += LIVE_CALL_TTL_MS; + expect(registry.create(binding)).not.toBeNull(); + }); + test("Location parsing extracts only a bounded rtc or UUID identifier", () => { + expect(upstreamLiveCallId("https://api.openai.com/v1/live/rtc_upstream?private=context")).toBe("rtc_upstream"); + expect(upstreamLiveCallId("/v1/live/01234567-89ab-cdef-0123-456789abcdef")).toBe("01234567-89ab-cdef-0123-456789abcdef"); + for (const location of [null, "javascript:rtc_upstream", "/v1/live/%2fsecret", "/v1/live/%ZZ", "/v1/live/unknown", "/v1/live/rtc_" + "x".repeat(128)]) { + expect(upstreamLiveCallId(location)).toBeNull(); + } + }); +});