From 131b21e597ae29224479d914ff2879697e990f48 Mon Sep 17 00:00:00 2001 From: yongzhao chen Date: Sat, 12 Sep 2026 22:31:49 +0200 Subject: [PATCH 01/13] fix(chat): preserve OCG DeepSeek timeline system instructions (cherry picked from commit e7bfb08b2823647b09d501692f759193a19d2299) --- .../src/content/docs/guides/claude-code.md | 8 ++ src/adapters/openai-chat.ts | 11 ++- structure/adapters/registry.md | 3 + structure/data-planes/inbound-compat.md | 5 + structure/providers/chat-compat.md | 13 +++ structure/providers/cursor.md | 3 + structure/runtime.md | 4 + structure/transports/inventory.md | 3 + structure/transports/responses.md | 4 + .../openai/openai-chat-system-order.test.ts | 92 +++++++++++++++++++ 10 files changed, 143 insertions(+), 3 deletions(-) diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 9be9d644df..67305832d1 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -681,3 +681,11 @@ directives, not the Agent tool's `model` argument. Make sure the directive match route. Pass `"haiku"` as the model placeholder. Set `claudeCode.stabilizePromptCache` to `true` in `config.json` to relocate supported trailing Claude harness notices from system instructions to a trailing user message on translated routes. The default is `false`. Enable it only when this role change is appropriate for your clients. It preserves fenced examples and unmatched text; native Anthropic passthrough is unchanged. The metadata-less prompt-cache key then follows stabilized instructions. This does not create conversation identity or guarantee upstream cache hits. + +On OpenCode Go's `deepseek-v4.1-flash` Chat route, translated timeline system +reminders automatically retain their position and system role, after any pending +tool results. This prevents newly appended reminders from rewriting the leading +system prompt. It applies with or without `stabilizePromptCache`; other models +and destinations keep their existing conversion. Upstream cache availability, +changes to earlier instructions or tools, and conversation compaction can still +affect cache hits. diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 7a8fac03ab..e183e0e60a 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -3,6 +3,7 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "./base"; import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTextContent, OcxThinkingContent, OcxToolCall, OcxUsage } from "../types"; import { isAllowedToolChoice, modelInList, namespacedToolName, resolveToolChoiceWireName, toolChoiceToolPredicate } from "../types"; import { mapReasoningEffort, modelRecordValue } from "../reasoning-effort"; +import { registryEntryForProviderDestination } from "../providers/registry"; import { debugProviderDiagnostic } from "../lib/debug"; import { sseFieldValue } from "../lib/sse-decoder"; import { isDebugEnabled } from "../lib/debug-settings"; @@ -738,10 +739,14 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon }; const nativeOpenAI = isNativeOpenAIChatTarget(provider); + // Hoisting a newly appended reminder rewrites the reusable prompt prefix. + // Keep this compatibility exception on the destination/model tested with OCG. + const chronologicalSystem = parsed.modelId === "deepseek-v4.1-flash" + && registryEntryForProviderDestination(provider)?.id === "opencode-go"; const toolCatalogNudge = shouldInjectNonOpenAIToolCatalogNudge(provider) ? buildNonOpenAIToolCatalogNudgeForTools(context.tools, options.toolChoice) : undefined; - const developerSystemParts = nativeOpenAI + const developerSystemParts = nativeOpenAI || chronologicalSystem ? [] : context.messages .map(developerSystemText) @@ -767,11 +772,11 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const hasImages = parts?.some(p => p.type === "image") ?? false; let chatMsg: Record; if (msg.role === "developer" && !hasImages) { - if (!nativeOpenAI) break; + if (!nativeOpenAI && !chronologicalSystem) break; const text = typeof msg.content === "string" ? msg.content : parts!.map(p => (p as OcxTextContent).text).join(""); - chatMsg = { role: "developer", content: text }; + chatMsg = { role: nativeOpenAI ? "developer" : "system", content: text }; } else if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; } else if (!hasImages) { diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 8c23e38eef..1b2ba2d4d0 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -7,6 +7,9 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Runtime adapter construction has one authority: `src/adapters/registry.ts`. +The OpenCode Go [chronological instruction exception](../providers/chat-compat.md#opencode-go-chronological-instructions) +uses the provider registry's destination identity inside the Chat adapter; it adds no adapter factory. + `src/server/adapter-resolve.ts` may resolve a provider/model onto an adapter id, but it does not maintain a second adapter factory inventory. The selected persisted/configured adapter id remains an untrusted string until the registry lookup succeeds. Unknown ids fail with the existing `Unknown adapter: ` error instead of widening configuration types around a closed compile-time union. ## Semantic inheritance is not constructor inheritance diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index 7794603072..e2b317248a 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -42,6 +42,11 @@ to gpt-live-1-codex; gpt-live-1 is an explicit alias. Dictation and Frameless ev separate. Coverage lives in `tests/server/audio-client.test.ts`, `tests/server/audio-dictation.test.ts` and `tests/server/live-call-bindings.test.ts`. +Translated Claude timeline reminders use the Chat adapter's +[OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions) +on its exact supported route. This is separate from trailing-notice stabilization +and from native Chat message passthrough. + ## Chat Completions inbound native path `POST /v1/chat/completions` sends eligible `openai-chat` routes directly to the provider's Chat diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index 2a849648de..cf830a8f1d 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -7,6 +7,19 @@ Native Codex Spark-specific request exceptions are absent. General Lite and name remain shared [Responses compatibility](../transports/responses.md#responses-httpsse), including other providers whose models happen to share a name fragment. +## OpenCode Go chronological instructions + +For the registry-recognized OpenCode Go Chat destination and exact model +`deepseek-v4.1-flash`, `src/adapters/openai-chat.ts` keeps text-only timeline +developer messages in place as system messages. Appending a reminder therefore +does not hoist new text into the leading system prompt and rewrite the existing +serialized message prefix. Pending tool results still precede deferred reminders. +The base system prompt, vision conversion and native OpenAI developer roles retain +their existing behavior; other Chat destinations and models retain leading-system +folding. This is independent of the Claude trailing-notice stabilization option +and does not guarantee upstream cache hits. Regression coverage is in +`tests/adapters/openai/openai-chat-system-order.test.ts`. + ## Reasoning and tool-result compatibility Kiro groups only consecutive original-message tool results whose raw call ID exactly matches diff --git a/structure/providers/cursor.md b/structure/providers/cursor.md index ec0c306868..ab030d8ae7 100644 --- a/structure/providers/cursor.md +++ b/structure/providers/cursor.md @@ -7,6 +7,9 @@ Codex-native retirement does not retire a Cursor-owned model name. Cursor transp namespace handling retain their provider contract; the bounded native scope lives in [the shared catalog](../catalog.md#shared-catalog). +Cursor's direct adapter does not enter the OpenAI Chat serializer's +[OpenCode Go instruction ordering](chat-compat.md#opencode-go-chronological-instructions). + ## Cursor Native Exec Cursor's experimental live transport can receive server-driven local read/write/delete/ls/grep, diff --git a/structure/runtime.md b/structure/runtime.md index 35b1e1f31a..36f20b9d67 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -3,6 +3,10 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +Chat request serialization owns the destination-scoped +[OpenCode Go instruction ordering](providers/chat-compat.md#opencode-go-chronological-instructions); +it requires no runtime lifecycle change or new configuration option. + ## Entrypoints | Path | Responsibility | diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index b2ba773ba1..ea6c440f17 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -3,6 +3,9 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +The Chat adapter's [OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions) +changes translated message placement only; endpoint selection and transport stay with their existing owners. + ## Transport inventory The sections above cover the transports with load-bearing invariants. The rest of the transport diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 96ef06d3c0..081554646f 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -5,6 +5,10 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Plaintext collaboration restoration treats a null namespace as absent, rejects non-string namespace types, and restores the native namespace/name pair before HTTP/WS delivery and continuation publication. +When internal Responses messages are translated to Chat, the adapter applies +[OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions). +Native Responses transport does not enter that conversion. + ## Responses HTTP/SSE `/v1/responses` is the main Codex-facing endpoint. The server parses Responses input, routes to a diff --git a/tests/adapters/openai/openai-chat-system-order.test.ts b/tests/adapters/openai/openai-chat-system-order.test.ts index d01ad96810..3c64414b85 100644 --- a/tests/adapters/openai/openai-chat-system-order.test.ts +++ b/tests/adapters/openai/openai-chat-system-order.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createOpenAIChatAdapter } from "../../../src/adapters/openai-chat"; +import { anthropicToResponsesBody } from "../../../src/claude/inbound"; +import { parseRequest } from "../../../src/responses/parser"; import type { OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; const provider: OcxProviderConfig = { @@ -101,3 +103,93 @@ describe("openai-chat system message ordering", () => { }); }); }); + +describe("OpenCode Go DeepSeek chronological system messages", () => { + const model = "deepseek-v4.1-flash"; + const ocg: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", + preserveReasoningContentModels: [model], + }; + const history = [ + { role: "user", content: "Inspect the synthetic project." }, + { role: "assistant", content: "First result." }, + { role: "system", content: "Synthetic reminder A." }, + ]; + function build(messages: unknown[], target = ocg, modelId = model, stabilize = false) { + const parsed = parseRequest(anthropicToResponsesBody({ + model: modelId, + system: "Stable project instructions.", + max_tokens: 100, + stream: true, + messages, + tools: [{ + name: "read_file", + description: "Read a synthetic file.", + input_schema: { type: "object", properties: { path: { type: "string" } } }, + }], + }, { stabilizePromptCache: stabilize })); + return JSON.parse(createOpenAIChatAdapter(target).buildRequest(parsed).body); + } + + test.each([false, true])("appending a reminder preserves the serialized history prefix (stabilize=%s)", stabilize => { + const first = build(history, ocg, model, stabilize); + const next = build([ + ...history, + { role: "assistant", content: "Second result." }, + { role: "user", content: "Continue." }, + { role: "system", content: "Synthetic reminder B." }, + ], ocg, model, stabilize); + expect(JSON.stringify(next.messages.slice(0, first.messages.length))).toBe(JSON.stringify(first.messages)); + expect(first.messages.map((message: { role: string }) => message.role)).toEqual(["system", "user", "assistant", "system"]); + expect(first.messages[0].content).not.toContain("Synthetic reminder A."); + expect(first.messages.at(-1)).toEqual({ role: "system", content: "Synthetic reminder A." }); + expect(next.messages.at(-1)).toEqual({ role: "system", content: "Synthetic reminder B." }); + expect(next.tools).toEqual(first.tools); + expect(next.model).toBe(model); + expect(next.stream).toBe(true); + }); + + test("defers reminders until pending tool results have arrived without losing reasoning", () => { + const body = build([ + { role: "user", content: "Read the fixture." }, + { role: "assistant", content: [{ type: "tool_use", id: "call_fixture", name: "read_file", input: {} }] }, + { role: "system", content: "Reminder during pending tool." }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "call_fixture", content: "Fixture result." }] }, + ]); + const callIndex = body.messages.findIndex((message: { tool_calls?: unknown }) => message.tool_calls); + expect(callIndex).toBeGreaterThan(0); + expect(body.messages[callIndex].reasoning_content).toBe(" "); + expect(body.messages[callIndex + 1]).toMatchObject({ role: "tool", tool_call_id: "call_fixture", content: "Fixture result." }); + expect(body.messages[callIndex + 2]).toEqual({ role: "system", content: "Reminder during pending tool." }); + }); + + test.each([ + "https://opencode.ai/zen/go/v1/", + "https://opencode.ai:443/zen/go/v1", + ])("matches the canonical destination %s", baseUrl => { + expect(build(history, { ...ocg, baseUrl }).messages.at(-1).role).toBe("system"); + }); + + test.each([ + "https://opencode.ai.example.invalid/zen/go/v1", + "https://opencode.ai/zen/v1", + "https://opencode.ai:444/zen/go/v1", + "http://opencode.ai/zen/go/v1", + "http://localhost:1234/v1", + ])("retains generic hoisting for other destinations: %s", baseUrl => { + const messages = build(history, { ...ocg, baseUrl }).messages; + expect(messages[0].content).toContain("Synthetic reminder A."); + expect(messages.map((message: { role: string }) => message.role)).toEqual(["system", "user", "assistant"]); + }); + + test("retains generic hoisting for other OCG models", () => { + expect(build(history, ocg, "kimi-k3").messages[0].content).toContain("Synthetic reminder A."); + }); + + test("retains native OpenAI developer roles", () => { + const messages = build(history, { ...ocg, baseUrl: "https://api.openai.com/v1" }).messages; + expect(messages[0].content).not.toContain("Synthetic reminder A."); + expect(messages.at(-1)).toEqual({ role: "developer", content: "Synthetic reminder A." }); + }); +}); From 7dc6c001b222a5c5d66d0c2b6188f7000f903d86 Mon Sep 17 00:00:00 2001 From: yongzhao chen Date: Sun, 13 Sep 2026 00:45:15 +0200 Subject: [PATCH 02/13] docs(claude): sync timeline cache behavior across locales (cherry picked from commit 9ebbcad263527b2cba8e629fe0e596cdf2968596) --- docs-site/src/content/docs/fr/guides/claude-code.md | 2 ++ docs-site/src/content/docs/guides/claude-code.md | 8 +++++--- docs-site/src/content/docs/ja/guides/claude-code.md | 2 ++ docs-site/src/content/docs/ko/guides/claude-code.md | 2 ++ docs-site/src/content/docs/ru/guides/claude-code.md | 2 ++ docs-site/src/content/docs/tr/guides/claude-code.md | 2 ++ docs-site/src/content/docs/zh-cn/guides/claude-code.md | 2 ++ docs-site/src/content/docs/zh-tw/guides/claude-code.md | 2 ++ 8 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs-site/src/content/docs/fr/guides/claude-code.md b/docs-site/src/content/docs/fr/guides/claude-code.md index 65d1a39a45..de614880e7 100644 --- a/docs-site/src/content/docs/fr/guides/claude-code.md +++ b/docs-site/src/content/docs/fr/guides/claude-code.md @@ -625,3 +625,5 @@ par défaut par un contenu minimal (`blockedSkills: ["claude-api"]`). Utilisez `"haiku"` comme valeur de remplacement pour le modèle. Dans `config.json`, `claudeCode.stabilizePromptCache: true` déplace les notices Claude reconnues en fin des instructions système vers un dernier message utilisateur sur les routes traduites. La valeur par défaut est `false`. Activez cette option seulement si ce changement de rôle convient à vos clients. Les exemples dans des blocs de code et le texte non reconnu sont conservés ; le transfert Anthropic natif reste inchangé. Sans métadonnées, la clé de cache suit les instructions stabilisées. Cette option ne crée pas une identité de conversation et ne garantit aucun succès du cache amont. + +Sur la route Chat d’OpenCode Go pour `deepseek-v4.1-flash`, les rappels système traduits dans l’historique conservent automatiquement leur position et leur rôle system, après les résultats d’outils encore attendus. Ainsi, l’ajout de rappels ne réécrit pas le prompt système initial. Ce comportement s’applique avec ou sans `stabilizePromptCache` ; la conversion des autres modèles et destinations, ainsi que le transfert Anthropic natif, restent inchangés. La réutilisation du cache exige toujours une identité de session stable et un cache disponible en amont. Les changements des instructions ou outils antérieurs et la compaction de la conversation peuvent aussi affecter les succès du cache ; préserver l’ordre des rappels ne suffit pas à garantir sa réutilisation. diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 67305832d1..a44867bd09 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -686,6 +686,8 @@ On OpenCode Go's `deepseek-v4.1-flash` Chat route, translated timeline system reminders automatically retain their position and system role, after any pending tool results. This prevents newly appended reminders from rewriting the leading system prompt. It applies with or without `stabilizePromptCache`; other models -and destinations keep their existing conversion. Upstream cache availability, -changes to earlier instructions or tools, and conversation compaction can still -affect cache hits. +and destinations keep their existing conversion; native Anthropic passthrough +is unchanged. Cache reuse still requires stable session identity and upstream +cache availability. Changes to earlier instructions or tools, and conversation +compaction, can still affect cache hits; preserving reminder order alone does +not guarantee reuse. diff --git a/docs-site/src/content/docs/ja/guides/claude-code.md b/docs-site/src/content/docs/ja/guides/claude-code.md index 1d3c2da548..d6b3cf30dc 100644 --- a/docs-site/src/content/docs/ja/guides/claude-code.md +++ b/docs-site/src/content/docs/ja/guides/claude-code.md @@ -496,3 +496,5 @@ Anthropic バックエンドを明示すると意図的に失敗後停止しま モデルプレースホルダとして `"haiku"` を渡してください。 `config.json` の `claudeCode.stabilizePromptCache` を `true` にすると、変換ルートのシステム指示末尾にある対応済み Claude 通知を最後のユーザーメッセージへ移します。既定値は `false` です。このロール変更が適切なクライアントでのみ有効にしてください。コードフェンス内の例と一致しない本文は保持され、Anthropic のネイティブ転送は変わりません。メタデータがない場合のキャッシュキーは安定化した指示から計算されます。会話 ID の生成やキャッシュヒットの保証は行いません。 + +OpenCode Go の `deepseek-v4.1-flash` Chat ルートでは、変換されたタイムライン上のシステムリマインダーは、保留中のツール結果の後で位置と system ロールを自動的に維持します。これにより、新しいリマインダーを追加しても先頭のシステムプロンプトが書き換わりません。`stabilizePromptCache` の設定にかかわらず適用され、他のモデルや接続先の変換、および Anthropic のネイティブ転送は変わりません。キャッシュの再利用には、安定したセッション ID と上流キャッシュの利用可能性が引き続き必要です。過去の指示やツールの変更、会話の圧縮もキャッシュヒットに影響します。リマインダーの順序を保つだけで再利用が保証されるわけではありません。 diff --git a/docs-site/src/content/docs/ko/guides/claude-code.md b/docs-site/src/content/docs/ko/guides/claude-code.md index ca3f5dfb33..b5151033f0 100644 --- a/docs-site/src/content/docs/ko/guides/claude-code.md +++ b/docs-site/src/content/docs/ko/guides/claude-code.md @@ -561,3 +561,5 @@ Anthropic 백엔드를 명시하면 의도적으로 실패 후 중단해요. 확인하고, 모델 자리 표시자로 `"haiku"`를 전달하세요. `config.json`에서 `claudeCode.stabilizePromptCache`를 `true`로 설정하면 번역 경로의 시스템 지시 끝에 붙은 지원 대상 Claude 알림을 마지막 사용자 메시지로 옮깁니다. 기본값은 `false`입니다. 사용하는 클라이언트에서 이 역할 변경을 허용할 때만 켜세요. 코드 펜스 안의 예제와 일치하지 않는 원문은 보존하며, Anthropic 원본 전달 경로는 바꾸지 않습니다. 메타데이터가 없는 요청의 캐시 키는 정리된 지시문을 기준으로 계산합니다. 대화 식별자를 만들거나 상위 서비스의 캐시 적중을 보장하는 기능은 아닙니다. + +OpenCode Go의 `deepseek-v4.1-flash` Chat 경로에서는 변환된 타임라인 시스템 알림이 대기 중인 도구 결과 뒤에서 원래 위치와 system 역할을 자동으로 유지합니다. 따라서 새 알림을 추가해도 맨 앞의 시스템 프롬프트를 다시 쓰지 않습니다. `stabilizePromptCache` 설정과 관계없이 적용되며, 다른 모델과 대상의 변환 및 Anthropic 네이티브 전달은 기존 동작을 유지합니다. 캐시 재사용에는 안정적인 세션 식별자와 사용 가능한 상위 서비스 캐시가 여전히 필요합니다. 이전 지시나 도구의 변경, 대화 압축도 캐시 적중에 영향을 줄 수 있으며, 알림 순서를 유지하는 것만으로 재사용을 보장하지는 않습니다. diff --git a/docs-site/src/content/docs/ru/guides/claude-code.md b/docs-site/src/content/docs/ru/guides/claude-code.md index 41cefcc951..93f926aae3 100644 --- a/docs-site/src/content/docs/ru/guides/claude-code.md +++ b/docs-site/src/content/docs/ru/guides/claude-code.md @@ -527,3 +527,5 @@ Responses `web_search_call` в парные блоки Anthropic `server_tool_us соответствует нужному маршруту. В качестве плейсхолдера модели передавайте `"haiku"`. Параметр `claudeCode.stabilizePromptCache: true` в `config.json` переносит поддерживаемые уведомления Claude в конце системных инструкций в последнее пользовательское сообщение на маршрутах с преобразованием. По умолчанию он выключен (`false`). Включайте его только когда такое изменение роли допустимо для ваших клиентов. Примеры в блоках кода и нераспознанный текст сохраняются; нативная передача Anthropic не меняется. Без метаданных ключ кэша рассчитывается по стабилизированным инструкциям. Идентификатор разговора не создаётся, попадания в кэш не гарантируются. + +На Chat-маршруте OpenCode Go для `deepseek-v4.1-flash` преобразованные системные напоминания в истории автоматически сохраняют свою позицию и роль system после ожидаемых результатов инструментов. Поэтому добавление новых напоминаний не переписывает начальный системный промпт. Это работает независимо от `stabilizePromptCache`; преобразование для других моделей и адресатов, а также нативная передача Anthropic остаются прежними. Для повторного использования кэша по-прежнему нужны стабильный идентификатор сессии и доступный кэш провайдера. Изменения прежних инструкций или инструментов и сжатие разговора также могут влиять на попадания в кэш; само сохранение порядка напоминаний не гарантирует повторного использования. diff --git a/docs-site/src/content/docs/tr/guides/claude-code.md b/docs-site/src/content/docs/tr/guides/claude-code.md index f4846148f3..510a7818a7 100644 --- a/docs-site/src/content/docs/tr/guides/claude-code.md +++ b/docs-site/src/content/docs/tr/guides/claude-code.md @@ -734,3 +734,5 @@ kullanır. Yönergenin hedeflenen rotayla eşleştiğinden emin olun. Model yer tutucusu olarak `"haiku"` iletin. `config.json` içindeki `claudeCode.stabilizePromptCache: true`, dönüştürülen rotalarda sistem talimatlarının sonundaki desteklenen Claude bildirimlerini son kullanıcı mesajına taşır. Varsayılan değer `false` olur. Yalnızca bu rol değişikliği istemcileriniz için uygunsa etkinleştirin. Kod bloklarındaki örnekler ve eşleşmeyen metin korunur; yerel Anthropic aktarımı değişmez. Meta veri yoksa önbellek anahtarı kararlı talimatlardan hesaplanır. Bu seçenek konuşma kimliği oluşturmaz veya üst hizmette önbellek isabeti garanti etmez. + +OpenCode Go’nun `deepseek-v4.1-flash` Chat rotasında, dönüştürülen zaman çizelgesi sistem hatırlatmaları bekleyen araç sonuçlarından sonra konumlarını ve system rolünü otomatik olarak korur. Böylece yeni hatırlatmalar eklenmesi, baştaki sistem istemini yeniden yazmaz. Bu davranış `stabilizePromptCache` açık veya kapalıyken geçerlidir; diğer modellerin ve hedeflerin dönüşümü ile yerel Anthropic aktarımı değişmez. Önbelleğin yeniden kullanımı için kararlı bir oturum kimliği ve kullanılabilir üst hizmet önbelleği hâlâ gereklidir. Önceki talimatların veya araçların değişmesi ve konuşmanın sıkıştırılması da önbellek isabetini etkileyebilir; hatırlatma sırasını korumak tek başına yeniden kullanımı garanti etmez. diff --git a/docs-site/src/content/docs/zh-cn/guides/claude-code.md b/docs-site/src/content/docs/zh-cn/guides/claude-code.md index 4c4982908f..75d7ab422e 100644 --- a/docs-site/src/content/docs/zh-cn/guides/claude-code.md +++ b/docs-site/src/content/docs/zh-cn/guides/claude-code.md @@ -465,3 +465,5 @@ Claude 模型时自动加载。对于原生透传,这是正常现象;对于 而不是 Agent 工具的 `model` 参数。请确保指令与预期路由一致。传入 `"haiku"` 作为模型占位符。 在 `config.json` 中设置 `claudeCode.stabilizePromptCache: true`,可在转换路由上将系统指令末尾受支持的 Claude 提示移到最后一条用户消息。默认值为 `false`。仅在客户端允许这种角色变化时启用。代码围栏内的示例和不匹配的文本会保留,Anthropic 原生透传不变。没有元数据时,缓存键按稳定后的指令计算。该选项不会生成会话标识,也不保证上游缓存命中。 + +在 OpenCode Go 的 `deepseek-v4.1-flash` Chat 路由上,转换后的时间线系统提醒会自动保留原有位置和 system 角色,并排在尚待返回的工具结果之后。因此,追加提醒不会重写开头的系统提示。无论 `stabilizePromptCache` 是否启用,该行为都会生效;其他模型、目标地址的转换方式以及 Anthropic 原生透传保持不变。缓存复用仍需要稳定的会话标识和可用的上游缓存。修改较早的指令或工具、压缩对话也可能影响缓存命中;仅保留提醒顺序并不保证缓存复用。 diff --git a/docs-site/src/content/docs/zh-tw/guides/claude-code.md b/docs-site/src/content/docs/zh-tw/guides/claude-code.md index 6df147ab6a..2c1a995cc3 100644 --- a/docs-site/src/content/docs/zh-tw/guides/claude-code.md +++ b/docs-site/src/content/docs/zh-tw/guides/claude-code.md @@ -535,3 +535,5 @@ Claude 模型時自動載入。對於原生透傳,這是正常現象;對於 而不是 Agent 工具的 `model` 引數。請確保指令與預期路由一致。傳入 `"haiku"` 作為模型佔位符。 在 `config.json` 中設定 `claudeCode.stabilizePromptCache: true`,可在轉換路由上將系統指令末尾支援的 Claude 提示移到最後一則使用者訊息。預設值為 `false`。僅在用戶端允許這種角色變更時啟用。程式碼圍欄中的範例和不符合的文字會保留,Anthropic 原生轉送不變。沒有中繼資料時,快取鍵依穩定後的指令計算。此選項不會產生對話識別碼,也不保證上游快取命中。 + +在 OpenCode Go 的 `deepseek-v4.1-flash` Chat 路由上,轉換後的時間線系統提醒會自動保留原有位置和 system 角色,並排在尚待傳回的工具結果之後。因此,新增提醒不會重寫開頭的系統提示。無論 `stabilizePromptCache` 是否啟用,此行為都會生效;其他模型、目標位址的轉換方式以及 Anthropic 原生轉送維持不變。快取重用仍需要穩定的工作階段識別碼和可用的上游快取。修改較早的指令或工具、壓縮對話也可能影響快取命中;僅保留提醒順序並不保證快取重用。 From 6645ccb0607aba28159b5ae71188aae7d996ad49 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 13 Sep 2026 15:55:07 +0900 Subject: [PATCH 03/13] fix(chat): keep non-text timeline messages out of the OCG system role [skip ci] Carry of #4438 by Yongzhaooo, with the open CodeRabbit finding on src/adapters/openai-chat.ts folded in. A timeline developer message whose only part is non-text (a video part, for example) serializes to an empty string here. The generic Chat path drops such a message through the existing break, but the new chronological exception turned it into { role: "system", content: "" }, which some upstreams reject. Skip it on the non-native path so the OCG route matches the generic path instead of inventing a content-free system message. Native OpenAI developer behavior is unchanged. The finding also asked for video parts to be mapped to a Chat video_url part. That is declined here: the Chat serializer has never emitted video for any destination or role, including ordinary user messages on current dev, so it is a pre-existing gap across every Chat provider rather than something this change introduces, and no upstream in this repository is known to accept that part type. Landing it inside a destination-scoped ordering fix would change every Chat destination on unvalidated wire format. structure/transports/responses.md is at its 600-line budget on dev with no headroom, so its four-line cross-reference is dropped rather than adding the repository's first grace.oversizeDocs entry for a cross-link. The owning description stays in structure/providers/chat-compat.md and the cross-references in runtime.md, transports/inventory.md, providers/cursor.md and data-planes/inbound-compat.md are unchanged. Co-authored-by: Yongzhao <133014490+Yongzhaooo@users.noreply.github.com> --- src/adapters/openai-chat.ts | 4 ++++ structure/transports/responses.md | 4 ---- .../openai/openai-chat-system-order.test.ts | 23 +++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index e183e0e60a..c1cb0f418d 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -776,6 +776,10 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const text = typeof msg.content === "string" ? msg.content : parts!.map(p => (p as OcxTextContent).text).join(""); + // A non-text timeline part (video, for example) serializes to nothing here. + // The generic path drops such a message; the chronological exception must not + // turn it into an empty system message that some upstreams reject. + if (!nativeOpenAI && text.length === 0) break; chatMsg = { role: nativeOpenAI ? "developer" : "system", content: text }; } else if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 081554646f..96ef06d3c0 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -5,10 +5,6 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Plaintext collaboration restoration treats a null namespace as absent, rejects non-string namespace types, and restores the native namespace/name pair before HTTP/WS delivery and continuation publication. -When internal Responses messages are translated to Chat, the adapter applies -[OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions). -Native Responses transport does not enter that conversion. - ## Responses HTTP/SSE `/v1/responses` is the main Codex-facing endpoint. The server parses Responses input, routes to a diff --git a/tests/adapters/openai/openai-chat-system-order.test.ts b/tests/adapters/openai/openai-chat-system-order.test.ts index 3c64414b85..def3179d81 100644 --- a/tests/adapters/openai/openai-chat-system-order.test.ts +++ b/tests/adapters/openai/openai-chat-system-order.test.ts @@ -192,4 +192,27 @@ describe("OpenCode Go DeepSeek chronological system messages", () => { expect(messages[0].content).not.toContain("Synthetic reminder A."); expect(messages.at(-1)).toEqual({ role: "developer", content: "Synthetic reminder A." }); }); + + test("drops a non-text timeline message instead of emitting an empty system message", () => { + const context = { + messages: [ + { role: "user", content: "Inspect the synthetic project.", timestamp: 0 }, + { role: "developer", content: [{ type: "video", videoUrl: "data:video/mp4;base64,AA==" }], timestamp: 0 }, + ], + } as unknown as OcxParsedRequest["context"]; + const request = (target: OcxProviderConfig) => JSON.parse(createOpenAIChatAdapter(target).buildRequest({ + modelId: model, + context, + stream: false, + options: {}, + } as unknown as Parameters["buildRequest"]>[0]).body) as { + messages: Array>; + }; + + // The generic serializer drops this message, so the chronological exception + // must not introduce a content-free system message on the OCG route. + expect(request(ocg).messages).toEqual([{ role: "user", content: "Inspect the synthetic project." }]); + expect(request({ ...ocg, baseUrl: "http://localhost:1234/v1" }).messages) + .toEqual([{ role: "user", content: "Inspect the synthetic project." }]); + }); }); From c6f7bc963a1c48f45eb5ce548f73ba2762be55cc Mon Sep 17 00:00:00 2001 From: olddonkey Date: Fri, 11 Sep 2026 21:10:03 -0700 Subject: [PATCH 04/13] perf(server): avoid request-sized UTF-8 accounting buffers (cherry picked from commit 324008b27fd7ca05f7101b6f91e76ce6ff42e0dc) --- src/server/request-decompress.ts | 9 ++- structure/adapters/registry.md | 2 + structure/catalog.md | 2 + structure/clients/claude-desktop.md | 2 + structure/data-planes/images.md | 2 + structure/data-planes/inbound-compat.md | 2 + structure/gui-and-management-api.md | 2 + structure/ops/service-and-sidecars.md | 2 + structure/providers/xai-grok.md | 2 + structure/runtime.md | 2 + structure/subagents.md | 2 + structure/transports/inventory.md | 2 + structure/transports/responses.md | 10 ++++ structure/transports/streaming-health.md | 2 + tests/usage/request-decompress.test.ts | 74 +++++++++++++++++++++++- 15 files changed, 113 insertions(+), 4 deletions(-) diff --git a/src/server/request-decompress.ts b/src/server/request-decompress.ts index 1939479429..f2d2d71ff7 100644 --- a/src/server/request-decompress.ts +++ b/src/server/request-decompress.ts @@ -1,3 +1,4 @@ +import { Buffer } from "node:buffer"; import { gunzipSync, inflateRawSync, inflateSync, zstdDecompressSync } from "node:zlib"; import type { TranslatorBudget } from "../lib/translator-budget"; @@ -29,7 +30,7 @@ export const MAX_DECOMPRESSED_BODY_BYTES = 256 * 1024 * 1024; * shrink and is stuck. An UNBOUNDED inbound cap is not an acceptable answer: this admission * limit is the only thing standing between one request and the process heap, and * `readBoundedJsonRequestBody` materializes the body several times over (retained wire bytes, - * decoded bytes, the decoded string, the re-encoded measurement copies, and the parsed object + * decoded bytes, the decoded string, the serialized measurement string, and the parsed object * graph), so peak RSS is a MULTIPLE of whatever is admitted here. 512 MiB is the largest value * that keeps that multiple survivable on an ordinary machine, and it is what #3573 asked for. */ @@ -324,12 +325,14 @@ export async function readBoundedJsonRequestBody( const decoded = decodeRequestBody(raw, encoding, maxBytes); releaseDecoded = decoded === raw ? undefined : budget?.observeAcceptedRequestCopy(decoded.byteLength); const text = new TextDecoder().decode(decoded); - releaseText = budget?.observeAcceptedRequestCopy(new TextEncoder().encode(text).byteLength); + // Count UTF-8 without allocating another request-sized byte array for diagnostics. + releaseText = budget?.observeAcceptedRequestCopy(Buffer.byteLength(text, "utf8")); if (options && "emptyBodyFallback" in options && text.trim() === "") { return options.emptyBodyFallback; } const parsed = JSON.parse(text); - budget?.observeAcceptedRequestCopy(new TextEncoder().encode(JSON.stringify(parsed)).byteLength); + // Keep the serialized-size contract: normalization can expand numeric literals. + budget?.observeAcceptedRequestCopy(Buffer.byteLength(JSON.stringify(parsed), "utf8")); return parsed; } finally { releaseText?.(); diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 1b2ba2d4d0..56a9666ef1 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -3,6 +3,8 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## Decision Runtime adapter construction has one authority: `src/adapters/registry.ts`. diff --git a/structure/catalog.md b/structure/catalog.md index 23846a6a50..fc8adf41ba 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -3,6 +3,8 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). + ## Shared catalog `src/codex/catalog.ts` builds a shared Codex-shaped catalog for CLI, TUI, App, and SDK. It: diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index 1412a498d1..31edca54d5 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -6,6 +6,8 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Codex-native model discovery follows the [shared retirement policy](../catalog.md#shared-catalog). That projection does not migrate existing user-selected Desktop configuration or usage history. +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## Connected Claude Desktop profiles Connected `ocx claude desktop apply` reads the hub's Desktop snapshot and writes the hub origin diff --git a/structure/data-planes/images.md b/structure/data-planes/images.md index d36a1f713e..314fafbceb 100644 --- a/structure/data-planes/images.md +++ b/structure/data-planes/images.md @@ -7,6 +7,8 @@ Hosted Responses image-tool eligibility uses the shared compatibility policy wit Codex Spark exception; standalone Images retain the separate relay contract below. See [Responses transport](../transports/responses.md#responses-httpsse). +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## Standalone Images Codex's local `image_gen.imagegen` tool makes a second Images request after the model calls it: diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index e2b317248a..1aef6459ef 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -47,6 +47,8 @@ Translated Claude timeline reminders use the Chat adapter's on its exact supported route. This is separate from trailing-notice stabilization and from native Chat message passthrough. +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## Chat Completions inbound native path `POST /v1/chat/completions` sends eligible `openai-chat` routes directly to the provider's Chat diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index bf7e70cd8b..9c8578c2f2 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -3,6 +3,8 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). + ## Dashboard serving The bundled React dashboard is built into `gui/dist` and served by the same Bun proxy. `ocx gui` diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index c99164a8cf..f8ea6c7c0a 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -6,6 +6,8 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Service startup and restore use the [catalog retirement policy](../catalog.md#shared-catalog); retirement does not itself change service registration or user-selected model configuration. +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## Background service command selection A bare `ocx service` is an idempotent install-or-repair command. Argument validation happens before diff --git a/structure/providers/xai-grok.md b/structure/providers/xai-grok.md index ada68474a8..e01d62a1bd 100644 --- a/structure/providers/xai-grok.md +++ b/structure/providers/xai-grok.md @@ -7,6 +7,8 @@ Codex-native retirement is scoped to OpenAI catalog/quota evidence. Shared Respo retains xAI provider behavior; see [the catalog boundary](../catalog.md#shared-catalog). +For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). + ## xAI Grok hardening (official Grok Build contract parity) Grounded in the open-sourced official client (xai-org/grok-build); unit + evidence: diff --git a/structure/runtime.md b/structure/runtime.md index 36f20b9d67..33238c301f 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -7,6 +7,8 @@ Chat request serialization owns the destination-scoped [OpenCode Go instruction ordering](providers/chat-compat.md#opencode-go-chronological-instructions); it requires no runtime lifecycle change or new configuration option. +For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). + ## Entrypoints | Path | Responsibility | diff --git a/structure/subagents.md b/structure/subagents.md index 5c9e486b45..0270523ef6 100644 --- a/structure/subagents.md +++ b/structure/subagents.md @@ -25,6 +25,8 @@ Codex treats qualified names literally and defaults absent namespaces to functio declarations inherit their restored namespace container; the compiler never invents an empty encryption marker when the upstream omitted it or returned a nonempty marker. +For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). + ## Multi-agent surface mode (3-state) `OcxConfig.multiAgentMode` controls the `multi_agent_version` field stamped on catalog entries: diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index ea6c440f17..9f13441275 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -6,6 +6,8 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior The Chat adapter's [OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions) changes translated message placement only; endpoint selection and transport stay with their existing owners. +For shared JSON request-body parsing, see [request-copy accounting](responses.md#request-copy-accounting). + ## Transport inventory The sections above cover the transports with load-bearing invariants. The rest of the transport diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 96ef06d3c0..1a5b1e40d1 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -15,6 +15,16 @@ Retired Codex Spark has no model-specific tool or Responses Lite override; gener namespace scrubbing remain shared compatibility behavior. Codex quota/reset evidence follows the [shared/Reserve policy](../providers/openai-tiers.md#public-provider-contract), including suppression of retired model-derived evidence before shared recovery. +### Request-copy accounting + +`src/server/request-decompress.ts` observes the UTF-8 sizes of decoded text and reserialized JSON +without allocating encoded byte arrays solely to count them. Parsed-body accounting still uses +`JSON.stringify(parsed)`: numeric normalization can make it larger than the input text. These +observations retain the existing ownership and release lifecycle and do not consume the translator's +hard byte cap. Admission limits, parsing, compression, and error envelopes are unchanged. +`tests/usage/request-decompress.test.ts` covers exact accounting across codecs and Unicode/numeric +normalization, UTF-8 counting without encoded copies, and release after malformed or optional empty input. + ### Credential-bearing HTTP redirects Credential/body-bearing HTTP sends use `redirect: "manual"` at the final executor boundary, diff --git a/structure/transports/streaming-health.md b/structure/transports/streaming-health.md index 7040009835..1d9741111c 100644 --- a/structure/transports/streaming-health.md +++ b/structure/transports/streaming-health.md @@ -7,6 +7,8 @@ Codex WebSocket quota-family normalization remains generic; retired-model eviden by the [OpenAI quota owner](../providers/openai-tiers.md#public-provider-contract), not by removing support for non-default WebSocket quota families. +For shared JSON request-body parsing, see [request-copy accounting](responses.md#request-copy-accounting). + ## Heartbeat and stall deadline The HTTP/SSE bridge emits an SSE comment-line keep-alive (`: opencodex heartbeat`) during upstream diff --git a/tests/usage/request-decompress.test.ts b/tests/usage/request-decompress.test.ts index cb88dd29e1..e6c0fcfc56 100644 --- a/tests/usage/request-decompress.test.ts +++ b/tests/usage/request-decompress.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { deflateRawSync, deflateSync } from "node:zlib"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; import { DecompressedBodyTooLargeError, decodeRequestBody, @@ -315,6 +316,77 @@ describe("configurable inbound body limit (Issue #3573)", () => { }); describe("readJsonRequestBody", () => { + const accountingBodies = [ + ["Unicode and escaped surrogates", new TextEncoder().encode('{"text":"中文😀é","escaped":"\\ud800\\udc00\\ud800x\\udc00"}')], + ["numeric normalization and duplicate keys", new TextEncoder().encode(' { "n": 1e20, "small": 1e-7, "zero": -0, "dup": 1, "dup": 2 } ')], + ["replacement decoding and BOM", Uint8Array.from([0xef, 0xbb, 0xbf, 0x22, 0xff, 0x22])], + ] as const; + + for (const encoding of ["identity", "zstd", "gzip", "deflate"] as const) { + for (const [label, decoded] of accountingBodies) { + test(`keeps exact request-copy accounting for ${encoding}: ${label}`, async () => { + const wire = encoding === "identity" ? decoded + : encoding === "zstd" ? Bun.zstdCompressSync(decoded) + : encoding === "gzip" ? Bun.gzipSync(decoded) + : deflateSync(decoded); + const text = new TextDecoder().decode(decoded); + const expected = JSON.parse(text); + const textBytes = new TextEncoder().encode(text).byteLength; + const parsedBytes = new TextEncoder().encode(JSON.stringify(expected)).byteLength; + // Accepted request copies remain observed even when they exceed the translator cap. + const budget = createTranslatorBudget({ maxTurnBytes: 1 }); + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-encoding": encoding, "content-length": String(wire.byteLength) }, + body: wire, + }); + expect(await readJsonRequestBody(request, budget)).toEqual(expected); + expect(budget.snapshot()).toMatchObject({ + currentBytes: parsedBytes, + highWaterBytes: wire.byteLength + (encoding === "identity" ? 0 : decoded.byteLength) + + textBytes + parsedBytes, + overflows: 0, + }); + } finally { + budget.dispose(); + } + expect(budget.snapshot().currentBytes).toBe(0); + }); + } + } + + test("request-copy accounting avoids allocating UTF-8 copies of the body", async () => { + const text = JSON.stringify({ input: "x".repeat(256 * 1024) }); + const wire = new TextEncoder().encode(text); + const request = new Request("http://localhost/v1/responses", { method: "POST", body: wire }); + const budget = createTranslatorBudget(); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + expect(await readJsonRequestBody(request, budget)).toEqual(JSON.parse(text)); + expect(budget.snapshot().currentBytes).toBe(wire.byteLength); + expect(encode).not.toHaveBeenCalled(); + } finally { + encode.mockRestore(); + budget.dispose(); + } + }); + + test("releases observed request copies after malformed JSON and empty-body fallback", async () => { + for (const text of ['{"input":', " \n"]) { + const budget = createTranslatorBudget(); + const request = new Request("http://localhost/v1/responses", { method: "POST", body: text }); + try { + const pending = readBoundedJsonRequestBody(request, 1024, budget, { emptyBodyFallback: null }); + if (text.trim() === "") expect(await pending).toBeNull(); + else await expect(pending).rejects.toBeInstanceOf(SyntaxError); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + budget.dispose(); + } + } + }); + test("reports a compressed declaration without reading or echoing request metadata", async () => { const { body, stats } = trackedBodyStream([Bun.gzipSync(PAYLOAD_BYTES)]); const req = new Request("http://localhost/v1/responses/compact?private-query", { From f51a59bcf108bf0e0deeba531869c7b2dacf3bba Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 00:16:34 -0700 Subject: [PATCH 05/13] perf(runtime): avoid UTF-8 accounting copies (cherry picked from commit 4eafb3ff60a4e2e288200cf39cd66561d9fe4370) --- src/adapters/anthropic.ts | 2 +- src/adapters/google.ts | 4 +- src/adapters/openai-chat.ts | 6 +- src/lib/admission.ts | 18 ++++-- src/lib/translator-budget.ts | 7 ++- src/server/chat-completions.ts | 2 +- tests/adapters/translator-budget.test.ts | 78 +++++++++++++++++++++++- tests/lib/debug.test.ts | 69 ++++++++++++++++++++- 8 files changed, 168 insertions(+), 18 deletions(-) diff --git a/src/adapters/anthropic.ts b/src/adapters/anthropic.ts index cc3dfebf3f..e85cf64418 100644 --- a/src/adapters/anthropic.ts +++ b/src/adapters/anthropic.ts @@ -1320,7 +1320,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti }]; } const json = parsed; - const responseBytes = new TextEncoder().encode(JSON.stringify(json)).byteLength; + const responseBytes = Buffer.byteLength(JSON.stringify(json), "utf8"); budget.chargeRetained(responseBytes, { kind: "retained_collectors" }); try { const events: AdapterEvent[] = []; diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 518aca3903..53206045b0 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -1325,7 +1325,7 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte bytesReservation.commitRetained(); budget.releaseRetained(total, { kind: "retained_collectors" }); rawText = new TextDecoder().decode(bytes); - rawTextBytes = new TextEncoder().encode(rawText).byteLength; + rawTextBytes = Buffer.byteLength(rawText, "utf8"); const textReservation = budget.reserveTransient(rawTextBytes, { kind: "retained_collectors" }); textReservation.commitRetained(); budget.releaseRetained(total, { kind: "retained_collectors" }); @@ -1347,7 +1347,7 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte return [{ type: "error", message: `google response was not a JSON object (${valueType})` }]; } raw = parsedRaw; - rawBytes = new TextEncoder().encode(JSON.stringify(raw)).byteLength; + rawBytes = Buffer.byteLength(JSON.stringify(raw), "utf8"); const rawReservation = budget.reserveTransient(rawBytes, { kind: "retained_collectors" }); rawReservation.commitRetained(); budget.releaseRetained(rawTextBytes, { kind: "retained_collectors" }); diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index c1cb0f418d..d32c5615d3 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -210,7 +210,7 @@ export function buildOpenAIChatPassthroughRequest( messageCount: Array.isArray(body.messages) ? body.messages.length : 0, toolCount: Array.isArray(body.tools) ? body.tools.length : 0, hasCredential, - bodyBytes: new TextEncoder().encode(bodyJson).length, + bodyBytes: Buffer.byteLength(bodyJson, "utf8"), }); } @@ -1665,7 +1665,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd messageCount: Array.isArray(messages) ? messages.length : 0, toolCount: Array.isArray(tools) ? tools.length : 0, hasCredential, - bodyBytes: new TextEncoder().encode(bodyJson).length, + bodyBytes: Buffer.byteLength(bodyJson, "utf8"), }); } @@ -2100,7 +2100,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (Object.hasOwn(json, "service_tier")) { tierMetadata?.observeResponseServiceTier(json.service_tier); } - const responseBytes = new TextEncoder().encode(JSON.stringify(json)).byteLength; + const responseBytes = Buffer.byteLength(JSON.stringify(json), "utf8"); budget.chargeRetained(responseBytes, { kind: "retained_collectors" }); try { const payload = unwrapChatCompletionPayload(json); diff --git a/src/lib/admission.ts b/src/lib/admission.ts index 7160603b6c..9965b781ea 100644 --- a/src/lib/admission.ts +++ b/src/lib/admission.ts @@ -58,20 +58,26 @@ export function createAdmissionGate(name: string, limit: number): { } export function retainedUtf8Bytes(value: string): number { - return new TextEncoder().encode(value).byteLength; + // Keep TextEncoder's runtime coercion for legacy callers outside the string-typed contract. + // Template coercion rejects Symbols; String(value) would silently accept them. + return Buffer.byteLength(typeof value === "string" ? value : value === undefined ? "" : `${value}`, "utf8"); } function utf8Prefix(value: string, maxBytes: number): string { if (maxBytes <= 0) return ""; let bytes = 0; - let result = ""; - for (const character of value) { - const size = retainedUtf8Bytes(character); + let end = 0; + while (end < value.length) { + const code = value.charCodeAt(end); + const next = value.charCodeAt(end + 1); + const pair = code >= 0xd800 && code <= 0xdbff && next >= 0xdc00 && next <= 0xdfff; + // An unpaired surrogate encodes as a three-byte replacement, like TextEncoder. + const size = code <= 0x7f ? 1 : code <= 0x7ff ? 2 : pair ? 4 : 3; if (bytes + size > maxBytes) break; - result += character; bytes += size; + end += pair ? 2 : 1; } - return result; + return value.slice(0, end); } export function truncateRetainedUtf8(value: string, maxBytes: number): string { diff --git a/src/lib/translator-budget.ts b/src/lib/translator-budget.ts index 18400eebf7..d1914894f7 100644 --- a/src/lib/translator-budget.ts +++ b/src/lib/translator-budget.ts @@ -112,14 +112,15 @@ export function retainTranslatedEvent( */ export function retainTranslatedEventBatch(events: T[], budget: TranslatorBudget): void { if (events.length === 0) return; - const serialized = events.map(event => JSON.stringify(event)); - const totalBytes = Buffer.byteLength(`[${serialized.join(",")}]`); + // Preserve atomic batch admission without retaining serialized strings or joining a second copy. + const eventBytes = events.map(event => Buffer.byteLength(JSON.stringify(event))); + const totalBytes = eventBytes.reduce((total, bytes) => total + bytes, events.length + 1); budget.chargeRetained(totalBytes, { kind: "retained_collectors" }); for (let index = 0; index < events.length; index++) { const delimiterBytes = index === events.length - 1 ? 2 : 1; retainedEventOwnership.set(events[index]!, { budget, - bytes: Buffer.byteLength(serialized[index]!) + delimiterBytes, + bytes: eventBytes[index]! + delimiterBytes, }); } } diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index 3234dfed00..b50af22cb5 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -288,7 +288,7 @@ async function handleChatCompletionsWithBudget( try { internalBodyJson = JSON.stringify(internalBody); translatorBudget.chargeRetained( - new TextEncoder().encode(internalBodyJson).byteLength, + Buffer.byteLength(internalBodyJson, "utf8"), { kind: "request_copies" }, ); } catch (err) { diff --git a/tests/adapters/translator-budget.test.ts b/tests/adapters/translator-budget.test.ts index 8e9800899c..a15103c4bc 100644 --- a/tests/adapters/translator-budget.test.ts +++ b/tests/adapters/translator-budget.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { bridgeToResponsesSSE, buildResponseJSON } from "../../src/bridge"; import { createAnthropicAdapter } from "../../src/adapters/anthropic"; import { createGoogleAdapter } from "../../src/adapters/google"; @@ -9,6 +9,7 @@ import { createTranslatorBudget, releaseTranslatedEvent, retainTranslatedEvent, + retainTranslatedEventBatch, translatorObservedBufferSnapshot, } from "../../src/lib/translator-budget"; import type { AdapterEvent } from "../../src/types"; @@ -22,6 +23,81 @@ async function textWithin(stream: ReadableStream, timeoutMs = 2_000) } describe("translator budget", () => { + for (const kind of ["anthropic", "google", "openai-chat"] as const) { + test(`${kind} buffered response sizing avoids encoded measurement copies`, async () => { + const text = "中文😀\ud800".repeat(1024); + const provider = { adapter: kind, apiKey: "fixture", baseUrl: "https://example.test/v1" }; + const adapter = kind === "anthropic" ? createAnthropicAdapter(provider) + : kind === "google" ? createGoogleAdapter(provider) + : createOpenAIChatAdapter(provider); + const payload = kind === "anthropic" + ? { content: [{ type: "text", text }], stop_reason: "end_turn" } + : kind === "google" + ? { candidates: [{ content: { parts: [{ text }] }, finishReason: "STOP" }] } + : { choices: [{ message: { content: text }, finish_reason: "stop" }] }; + const response = new Response(Buffer.from(JSON.stringify(payload))); + const budget = createTranslatorBudget(); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const events = await adapter.parseResponse(response, budget); + expect(events).toContainEqual({ type: "text_delta", text }); + expect(encode).not.toHaveBeenCalled(); + for (const event of events) releaseTranslatedEvent(event, budget); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + encode.mockRestore(); + budget.dispose(); + } + }); + } + + test("batch retention counts each event once without constructing a serialized batch", () => { + const events = [ + { type: "text_delta", text: "中文😀\ud800".repeat(1024) }, + { type: "done", usage: { inputTokens: 1e20, outputTokens: -0 } }, + ]; + const eventBytes = events.map(event => Buffer.byteLength(JSON.stringify(event))); + const total = Buffer.byteLength(JSON.stringify(events)); + const budget = createTranslatorBudget({ maxTurnBytes: total }); + const count = spyOn(Buffer, "byteLength"); + try { + retainTranslatedEventBatch(events, budget); + expect(count).toHaveBeenCalledTimes(events.length); + expect(budget.snapshot()).toMatchObject({ currentBytes: total, highWaterBytes: total, overflows: 0 }); + releaseTranslatedEvent(events[0]!, budget); + expect(budget.snapshot().currentBytes).toBe(eventBytes[1]! + 2); + releaseTranslatedEvent(events[1]!, budget); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + count.mockRestore(); + budget.dispose(); + } + }); + + test("batch overflow and serialization failure acquire no partial event ownership", () => { + const events = [{ type: "text_delta", text: "first" }, { type: "done" }]; + const bytes = Buffer.byteLength(JSON.stringify(events)); + const budget = createTranslatorBudget({ maxTurnBytes: bytes - 1 }); + try { + expect(() => retainTranslatedEventBatch(events, budget)).toThrow(/translator/); + expect(budget.snapshot().currentBytes).toBe(0); + for (const event of events) releaseTranslatedEvent(event, budget); + expect(budget.snapshot().currentBytes).toBe(0); + retainTranslatedEvent(events[0]!, budget); + releaseTranslatedEvent(events[0]!, budget); + expect(budget.snapshot().currentBytes).toBe(0); + + const invalid = { toJSON() { throw new Error("invalid event"); } }; + expect(() => retainTranslatedEventBatch([events[0]!, invalid], budget)).toThrow("invalid event"); + expect(budget.snapshot().currentBytes).toBe(0); + retainTranslatedEvent(events[0]!, budget); + releaseTranslatedEvent(events[0]!, budget); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + budget.dispose(); + } + }); + test("incremental event retention transfers array-tail ownership during in-order release", () => { const budget = createTranslatorBudget({ maxTurnBytes: 4_096 }); const first = { type: "text_delta", text: "first" }; diff --git a/tests/lib/debug.test.ts b/tests/lib/debug.test.ts index 6ce7359d2d..b80f7db41f 100644 --- a/tests/lib/debug.test.ts +++ b/tests/lib/debug.test.ts @@ -1,11 +1,78 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { appendDebugLogLine, debugBufferMetrics, getDebugLogEntries, resetDebugLogBufferForTests, subscribeDebugLogEntries } from "../../src/lib/debug-log-buffer"; -import { ResourceAdmissionError, RETAINED_TRUNCATION_MARKER, retainedUtf8Bytes } from "../../src/lib/admission"; +import { ResourceAdmissionError, RETAINED_TRUNCATION_MARKER, retainedUtf8Bytes, truncateRetainedUtf8 } from "../../src/lib/admission"; import { getInjectionDebugLogEntries, injectionDebugLog, resetInjectionDebugLogBufferForTests } from "../../src/lib/injection-debug-log"; import { markActivity, activityBreadcrumb } from "../../src/lib/sidecar-tracker"; import { debugDroppedFrame, debugProviderDiagnostic } from "../../src/lib/debug"; import { resetDebugSettingsForTests, setDebugSettings } from "../../src/lib/debug-settings"; +describe("retained UTF-8 sizing", () => { + test("preserves TextEncoder coercion for non-string runtime inputs", () => { + const encoder = new TextEncoder(); + const inputs: unknown[] = [ + undefined, null, true, 0, -0, 1e20, NaN, Infinity, 42n, + {}, ["中文", "\ud800"], new String("😀\udc00"), + new Uint8Array([1, 2]), Buffer.from([0xff]), + { [Symbol.toPrimitive](hint: string) { return hint === "string" ? "中\ud800" : 7; } }, + ]; + for (const value of inputs) { + const expected = Reflect.apply(encoder.encode, encoder, [value]).byteLength; + expect(retainedUtf8Bytes(value as string)).toBe(expected); + } + }); + + test("preserves TextEncoder rejection of Symbols and failed string coercion", () => { + const encoder = new TextEncoder(); + for (const value of [Symbol("input"), Object(Symbol("input")), Object.create(null)]) { + expect(() => Reflect.apply(encoder.encode, encoder, [value])).toThrow(TypeError); + expect(() => retainedUtf8Bytes(value as string)).toThrow(TypeError); + } + const failure = new Error("string conversion failed"); + const value = { toString() { throw failure; } }; + expect(() => Reflect.apply(encoder.encode, encoder, [value])).toThrow(failure); + expect(() => retainedUtf8Bytes(value as unknown as string)).toThrow(failure); + }); + + test("keeps UTF-8 and truncation boundaries for multibyte and unpaired surrogate text", () => { + const encoder = new TextEncoder(); + const markerBytes = encoder.encode(RETAINED_TRUNCATION_MARKER).byteLength; + const samples = ["", "plain", "é中😀", "\ud800x\udc00", "😀\ud800中éx".repeat(12)]; + for (const value of samples) { + const bytes = encoder.encode(value).byteLength; + expect(retainedUtf8Bytes(value)).toBe(bytes); + for (const cap of [0, 1, 2, 3, 4, markerBytes - 1, markerBytes, markerBytes + 1, markerBytes + 4, markerBytes + 7, bytes]) { + const prefix = (text: string, limit: number) => { + const points = Array.from(text); + let end = 0; + let size = 0; + while (end < points.length && size + encoder.encode(points[end]!).byteLength <= limit) { + size += encoder.encode(points[end]!).byteLength; + end += 1; + } + return points.slice(0, end).join(""); + }; + const expected = bytes <= cap ? value + : cap < markerBytes ? prefix(RETAINED_TRUNCATION_MARKER, cap) + : prefix(value, cap - markerBytes) + RETAINED_TRUNCATION_MARKER; + expect(truncateRetainedUtf8(value, cap)).toBe(expected); + } + } + }); + + test("truncates large diagnostics without per-character encoded arrays", () => { + const value = "x".repeat(1024 * 1024); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const result = truncateRetainedUtf8(value, 16 * 1024); + expect(result.endsWith(RETAINED_TRUNCATION_MARKER)).toBe(true); + expect(Buffer.byteLength(result)).toBe(16 * 1024); + expect(encode).not.toHaveBeenCalled(); + } finally { + encode.mockRestore(); + } + }); +}); + describe("debug frame logging", () => { const previous = process.env.OCX_DEBUG; From a8bd2e68de5044954dc9aa64335da1d23e7da5b7 Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 00:16:41 -0700 Subject: [PATCH 06/13] perf(responses): count compaction fragments incrementally (cherry picked from commit 930c916ad6bd07a8bea827c8496f7dd61374e6a4) --- src/adapters/openai-responses.ts | 66 +++--- .../openai-responses-passthrough.test.ts | 194 +++++++++++++++++- 2 files changed, 234 insertions(+), 26 deletions(-) diff --git a/src/adapters/openai-responses.ts b/src/adapters/openai-responses.ts index 79b8597df3..e2c7cf2a14 100644 --- a/src/adapters/openai-responses.ts +++ b/src/adapters/openai-responses.ts @@ -3,6 +3,7 @@ import { stripBracketedModelSuffix } from "./openai-chat"; import { normalizeOpenCodeGoAdditionalTools } from "./opencode-go-additional-tools"; import { isXaiResponsesDestination } from "../providers/xai-transport"; import { createHash } from "node:crypto"; +import { Buffer } from "node:buffer"; import type { IncomingMeta, ProviderAdapter } from "./base"; import { namespacedToolName, type AdapterEvent, type OcxParsedRequest, type OcxProviderConfig, type OcxUsage, type TierDecision } from "../types"; import { catalogModelSupportsReasoningSummaries } from "../codex/catalog"; @@ -2145,6 +2146,15 @@ function responsesErrorMessage(payload: unknown): string { return "upstream compaction failed"; } +/** Count an append without rescanning accumulated text, including split surrogate pairs. */ +function appendedUtf8Bytes(previousBytes: number, lastCodeUnit: number, fragment: string): number { + const first = fragment.charCodeAt(0); + // Separate lone surrogates each count as a three-byte replacement character; together + // they encode as one four-byte scalar. Empty fragments produce NaN and never pair. + const joinsSurrogatePair = lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff && first >= 0xdc00 && first <= 0xdfff; + return previousBytes + Buffer.byteLength(fragment, "utf8") - (joinsSurrogatePair ? 2 : 0); +} + export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): ProviderAdapter & { passthrough: true } { return { name: "openai-responses", @@ -2406,7 +2416,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): const body = JSON.stringify(finalBody); const releaseBodyObservation = translatorBudget.observeExternallyCapped( "passthrough_serialization", - new TextEncoder().encode(body).byteLength, + Buffer.byteLength(body, "utf8"), ); return { url, @@ -2433,12 +2443,18 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): yield { type: "error", message: "passthrough adapter received no response body" }; return; } - const budgetEncoder = new TextEncoder(); let deltas = ""; + let deltasBytes = 0; + let deltasLastCodeUnit = 0; let doneText = ""; + let doneTextBytes = 0; + let doneTextLastCodeUnit = 0; let snapshot = ""; + let snapshotBytes = 0; let usage: OcxUsage | undefined; + let usageRawBytes = 0; let compactionEncryptedContent: string | undefined; + let compactionEncryptedContentBytes = 0; let completedSeen = false; for await (const event of decodeServerSentEvents(response.body, { translatorBudget: budget })) { let payload: unknown; @@ -2448,21 +2464,25 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): case "response.output_text.delta": if (typeof payload.delta === "string") { const next = deltas + payload.delta; - const previousBytes = budgetEncoder.encode(deltas).byteLength; - const reservation = budget.reserveTransient(budgetEncoder.encode(next).byteLength, { kind: "retained_collectors" }); + const nextBytes = appendedUtf8Bytes(deltasBytes, deltasLastCodeUnit, payload.delta); + const reservation = budget.reserveTransient(nextBytes, { kind: "retained_collectors" }); deltas = next; reservation.commitRetained(); - budget.releaseRetained(previousBytes, { kind: "retained_collectors" }); + budget.releaseRetained(deltasBytes, { kind: "retained_collectors" }); + deltasBytes = nextBytes; + if (payload.delta.length > 0) deltasLastCodeUnit = payload.delta.charCodeAt(payload.delta.length - 1); } break; case "response.output_text.done": if (typeof payload.text === "string") { const next = doneText + payload.text; - const previousBytes = budgetEncoder.encode(doneText).byteLength; - const reservation = budget.reserveTransient(budgetEncoder.encode(next).byteLength, { kind: "retained_collectors" }); + const nextBytes = appendedUtf8Bytes(doneTextBytes, doneTextLastCodeUnit, payload.text); + const reservation = budget.reserveTransient(nextBytes, { kind: "retained_collectors" }); doneText = next; reservation.commitRetained(); - budget.releaseRetained(previousBytes, { kind: "retained_collectors" }); + budget.releaseRetained(doneTextBytes, { kind: "retained_collectors" }); + doneTextBytes = nextBytes; + if (payload.text.length > 0) doneTextLastCodeUnit = payload.text.charCodeAt(payload.text.length - 1); } break; case "response.failed": @@ -2480,28 +2500,28 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): const compaction = output.find(item => isPlainObject(item) && item.type === "compaction"); if (isPlainObject(compaction) && typeof compaction.encrypted_content === "string") { const nextEncryptedContent = compaction.encrypted_content; - const previousBytes = budgetEncoder.encode(compactionEncryptedContent ?? "").byteLength; - const reservation = budget.reserveTransient(budgetEncoder.encode(nextEncryptedContent).byteLength, { kind: "retained_collectors" }); + const nextEncryptedContentBytes = Buffer.byteLength(nextEncryptedContent, "utf8"); + const reservation = budget.reserveTransient(nextEncryptedContentBytes, { kind: "retained_collectors" }); compactionEncryptedContent = nextEncryptedContent; reservation.commitRetained(); - budget.releaseRetained(previousBytes, { kind: "retained_collectors" }); + budget.releaseRetained(compactionEncryptedContentBytes, { kind: "retained_collectors" }); + compactionEncryptedContentBytes = nextEncryptedContentBytes; } const next = responsesPayloadText(payload.response); - const previousBytes = budgetEncoder.encode(snapshot).byteLength; - const reservation = budget.reserveTransient(budgetEncoder.encode(next).byteLength, { kind: "retained_collectors" }); + const nextBytes = Buffer.byteLength(next, "utf8"); + const reservation = budget.reserveTransient(nextBytes, { kind: "retained_collectors" }); snapshot = next; reservation.commitRetained(); - budget.releaseRetained(previousBytes, { kind: "retained_collectors" }); + budget.releaseRetained(snapshotBytes, { kind: "retained_collectors" }); + snapshotBytes = nextBytes; } { const nextUsage = usageFromResponsesPayload(payload.response); // The attached raw usage object can be event-sized (unknown keys carry arbitrary // values); it stays reachable until the terminal yields, so charge it like the // adjacent retained collectors or it would defeat the per-request memory cap. - const previousRawBytes = usage?.rawUsage === undefined ? 0 - : budgetEncoder.encode(JSON.stringify(usage.rawUsage)).byteLength; const nextRawBytes = nextUsage?.rawUsage === undefined ? 0 - : budgetEncoder.encode(JSON.stringify(nextUsage.rawUsage)).byteLength; + : Buffer.byteLength(JSON.stringify(nextUsage.rawUsage), "utf8"); if (nextRawBytes > 0) { const reservation = budget.reserveTransient(nextRawBytes, { kind: "retained_collectors" }); usage = nextUsage; @@ -2509,9 +2529,10 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): } else { usage = nextUsage; } - if (previousRawBytes > 0) { - budget.releaseRetained(previousRawBytes, { kind: "retained_collectors" }); + if (usageRawBytes > 0) { + budget.releaseRetained(usageRawBytes, { kind: "retained_collectors" }); } + usageRawBytes = nextRawBytes; } break; } @@ -2533,10 +2554,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): const text = snapshot || doneText || deltas; if (text) yield { type: "text_delta", text }; budget.releaseRetained( - budgetEncoder.encode(deltas).byteLength - + budgetEncoder.encode(doneText).byteLength - + budgetEncoder.encode(snapshot).byteLength - + (usage?.rawUsage === undefined ? 0 : budgetEncoder.encode(JSON.stringify(usage.rawUsage)).byteLength), + deltasBytes + doneTextBytes + snapshotBytes + usageRawBytes, { kind: "retained_collectors" }, ); yield { @@ -2551,7 +2569,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): try { payload = await response.json(); } catch { return [{ type: "error", message: "malformed upstream compaction response" }]; } - budget.chargeRetained(new TextEncoder().encode(JSON.stringify(payload)).byteLength, { kind: "retained_collectors" }); + budget.chargeRetained(Buffer.byteLength(JSON.stringify(payload), "utf8"), { kind: "retained_collectors" }); if (!isPlainObject(payload)) { return [{ type: "error", message: "malformed upstream compaction response" }]; } diff --git a/tests/responses/openai-responses-passthrough.test.ts b/tests/responses/openai-responses-passthrough.test.ts index 1e7eb66986..42ace1b77a 100644 --- a/tests/responses/openai-responses-passthrough.test.ts +++ b/tests/responses/openai-responses-passthrough.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; +import { Buffer } from "node:buffer"; import { createOpenAIChatAdapter } from "../../src/adapters/openai-chat"; import { createResponsesPassthroughAdapter as createResponsesPassthroughAdapterProduction } from "../../src/adapters/openai-responses"; import { openaiResponsesUrl } from "../../src/adapters/openai-responses-url"; @@ -19,7 +20,7 @@ import { SUMMARY_PREFIX, } from "../../src/responses/compaction"; import { createTranslatorBudget } from "../../src/lib/translator-budget"; -import type { OcxConfig } from "../../src/types"; +import type { AdapterEvent, OcxConfig } from "../../src/types"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; import { restoreRoutedNamespaceCalls } from "../../src/responses/namespace-tool-compat"; import { restoreRoutedCustomCalls } from "../../src/responses/custom-tool-compat"; @@ -33,6 +34,195 @@ const provider = { authMode: "forward" as const, }; +describe("Responses request and compaction byte accounting", () => { + const encoder = new TextEncoder(); + const frame = (payload: unknown) => `data: ${JSON.stringify(payload)}\n\n`; + const adapter = () => createResponsesPassthroughAdapterProduction(provider); + + test("outbound accounting measures serialized UTF-8 without an encoded copy", () => { + const budget = createTranslatorBudget({ maxTurnBytes: 1 }); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const request = adapter().buildRequest({ + modelId: "example-model", context: { messages: [] }, stream: true, options: {}, + _rawBody: { model: "example-model", input: "中文😀é\ud800x\udc00", temperature: 1e20 }, + }, { headers: new Headers(), translatorBudget: budget }); + expect(encode).not.toHaveBeenCalled(); + encode.mockRestore(); + expect(budget.snapshot()).toMatchObject({ + currentBytes: encoder.encode(request.body).byteLength, overflows: 0, + }); + request.releaseBodyObservation?.(); + request.releaseBodyObservation?.(); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { encode.mockRestore(); budget.dispose(); } + }); + + test("buffered compaction preserves the serialized payload cap without an encoded copy", async () => { + const payload = { + output: [{ type: "message", content: [{ type: "output_text", text: "中文😀\ud800" }] }], + usage: { input_tokens: 1e20, output_tokens: 1, metadata: "\udc00" }, + }; + const wire = encoder.encode(JSON.stringify(payload)); + const budget = createTranslatorBudget({ maxTurnBytes: wire.byteLength }); + const response = new Response(wire); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const events = await adapter().parseResponse!(response, budget); + expect(events[0]).toEqual({ type: "text_delta", text: "中文😀\ud800" }); + expect(encode).not.toHaveBeenCalled(); + expect(budget.snapshot()).toMatchObject({ currentBytes: wire.byteLength, overflows: 0 }); + } finally { encode.mockRestore(); budget.dispose(); } + const limited = createTranslatorBudget({ maxTurnBytes: wire.byteLength - 1 }); + try { + await expect(adapter().parseResponse!(new Response(wire), limited)).rejects.toMatchObject({ + code: "translation_buffer_limit", + }); + } finally { limited.dispose(); } + }); + + test("compaction counts only new fragments without request-sized encoded arrays", async () => { + const count = 256; + const fragment = "x".repeat(1024); + const wire = encoder.encode(frame({ type: "response.output_text.delta", delta: fragment }).repeat(count) + + frame({ type: "response.completed", response: { output: [] } })); + const response = new Response(wire); + const budget = createTranslatorBudget(); + const originalByteLength = Buffer.byteLength; + let countedCodeUnits = 0; + const byteLength = spyOn(Buffer, "byteLength").mockImplementation((value, encoding) => { + if (typeof value === "string") countedCodeUnits += value.length; + return originalByteLength(value, encoding); + }); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const events: AdapterEvent[] = []; + for await (const event of adapter().parseStream(response, budget)) events.push(event); + expect(events.filter(event => event.type === "heartbeat")).toHaveLength(count); + expect(events.at(-2)).toEqual({ type: "text_delta", text: fragment.repeat(count) }); + expect(events.at(-1)).toEqual({ type: "done" }); + expect(budget.snapshot()).toMatchObject({ currentBytes: 0, overflows: 0 }); + // Replacing encode with byteLength alone still recounts all preceding deltas. + expect(countedCodeUnits).toBeLessThanOrEqual(count * fragment.length + 1024); + expect(encode).not.toHaveBeenCalled(); + } finally { byteLength.mockRestore(); encode.mockRestore(); budget.dispose(); } + }); + + test.each(["response.output_text.delta", "response.output_text.done"])( + "%s counts surrogate pairs joined across fragments exactly", async type => { + const fragments = ["中\ud83d", "", "\ude00", "\ud800", "x\udc00", "é"]; + let combined = ""; + const expectedBytes = fragments.map(fragment => encoder.encode(combined += fragment).byteLength); + const wire = encoder.encode(fragments.map(fragment => frame({ + type, [type.endsWith("delta") ? "delta" : "text"]: fragment, + })).join("")); + const budget = createTranslatorBudget(); + const reserve = spyOn(budget, "reserveTransient"); + try { + const events: AdapterEvent[] = []; + for await (const event of adapter().parseStream(new Response(wire), budget)) events.push(event); + expect(reserve.mock.calls.filter(([, scope]) => scope.kind === "retained_collectors") + .map(([bytes]) => bytes)).toEqual(expectedBytes); + expect(events.at(-2)).toEqual({ type: "text_delta", text: combined }); + expect(events.at(-1)).toEqual({ type: "done" }); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { reserve.mockRestore(); budget.dispose(); } + }, + ); + + test("compaction replacement still admits the full old and new text overlap", async () => { + const fragment = "x".repeat(10_000); + const bytes = encoder.encode(frame({ type: "response.output_text.delta", delta: fragment })); + const budget = createTranslatorBudget({ maxTurnBytes: 35_000 }); + let sent = 0; + let cancelled = false; + const response = new Response(new ReadableStream({ + pull(controller) { + if (sent++ < 2) controller.enqueue(bytes); + else controller.close(); + }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 })); + const iterator = adapter().parseStream(response, budget); + try { + expect(await iterator.next()).toEqual({ done: false, value: { type: "heartbeat" } }); + await expect(iterator.next()).rejects.toMatchObject({ code: "translation_buffer_limit" }); + expect(cancelled).toBe(true); + expect(budget.snapshot()).toMatchObject({ currentBytes: fragment.length, overflows: 1 }); + } finally { await iterator.return(undefined); budget.dispose(); } + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test("terminal replacement releases old snapshot and usage while retaining the current ciphertext", async () => { + const oldUsage = { input_tokens: 1, output_tokens: 2, metadata: "中文\ud800" }; + const newUsage = { input_tokens: 3, output_tokens: 4, metadata: "😀\udc00" }; + const oldCiphertext = "old-中文"; + const ciphertext = "new-😀"; + const terminal = (text: string, encrypted: string, usage?: unknown) => ({ + type: "response.completed", response: { + output: [ + { type: "message", content: [{ type: "output_text", text }] }, + { type: "compaction", encrypted_content: encrypted }, + ], ...(usage === undefined ? {} : { usage }), + }, + }); + const wire = encoder.encode(frame({ type: "response.output_text.delta", delta: "partial" }) + + frame({ type: "response.output_text.done", text: "fallback" }) + + frame(terminal("old snapshot", oldCiphertext, oldUsage)) + + frame(terminal("new snapshot", ciphertext, newUsage)) + + frame(terminal("", ciphertext))); + const budget = createTranslatorBudget(); + const reserve = spyOn(budget, "reserveTransient"); + try { + const events: AdapterEvent[] = []; + for await (const event of adapter().parseStream(new Response(wire), budget)) events.push(event); + expect(events).toEqual([ + { type: "heartbeat" }, { type: "text_delta", text: "fallback" }, + { type: "done", compactionEncryptedContent: ciphertext }, + ]); + const expected = [ + "partial", "fallback", oldCiphertext, "old snapshot", JSON.stringify(oldUsage), + ciphertext, "new snapshot", JSON.stringify(newUsage), ciphertext, "", + ].map(text => encoder.encode(text).byteLength); + expect(reserve.mock.calls.filter(([, scope]) => scope.kind === "retained_collectors") + .map(([bytes]) => bytes)).toEqual(expected); + expect(budget.snapshot().currentBytes).toBe(encoder.encode(ciphertext).byteLength); + } finally { reserve.mockRestore(); budget.dispose(); } + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test.each(["response.failed", "response.incomplete", "consumer return"])( + "%s leaves collector cleanup with the owning budget and cancels upstream", async ending => { + const partial = "中\ud800"; + const endingPayload = ending === "response.failed" + ? { type: ending, response: { error: { message: "stopped" } } } + : { type: ending, response: { incomplete_details: { reason: "stopped" } } }; + const bytes = encoder.encode(frame({ type: "response.output_text.delta", delta: partial }) + + (ending === "consumer return" ? "" : frame(endingPayload))); + const budget = createTranslatorBudget(); + let cancelled = false; + const response = new Response(new ReadableStream({ + start(controller) { controller.enqueue(bytes); }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 })); + const iterator = adapter().parseStream(response, budget); + try { + expect(await iterator.next()).toEqual({ done: false, value: { type: "heartbeat" } }); + if (ending !== "consumer return") { + expect(await iterator.next()).toEqual({ done: false, value: ending === "response.failed" + ? { type: "error", message: "stopped" } : { type: "incomplete", reason: "stopped" } }); + expect(await iterator.next()).toEqual({ done: true, value: undefined }); + } else { + await iterator.return(undefined); + } + expect(cancelled).toBe(true); + expect(budget.snapshot().currentBytes).toBe(encoder.encode(partial).byteLength); + } finally { await iterator.return(undefined); budget.dispose(); } + expect(budget.snapshot().currentBytes).toBe(0); + }, + ); +}); + describe("native routed code-mode result visibility", () => { const routed = { adapter: "openai-responses", baseUrl: "https://api.x.ai/v1", authMode: "key" as const }; const exec = { type: "custom", name: "exec", description: "Run JavaScript in a V8 isolate." }; From 4974740538fcb439aef39db2c6025f36579eb98a Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 00:16:41 -0700 Subject: [PATCH 07/13] perf(sse): drain buffers without recounting suffixes (cherry picked from commit b7d1330d0ef10a6d5e813c7fa9e60efc88e606f2) --- src/server/sse-payload-rewrite.ts | 187 +++++++++---- tests/responses/sse-payload-rewrite.test.ts | 276 +++++++++++++++++++- 2 files changed, 411 insertions(+), 52 deletions(-) diff --git a/src/server/sse-payload-rewrite.ts b/src/server/sse-payload-rewrite.ts index f9fb62065d..1c2f38b510 100644 --- a/src/server/sse-payload-rewrite.ts +++ b/src/server/sse-payload-rewrite.ts @@ -1,4 +1,5 @@ import type { TranslatorBudget } from "../lib/translator-budget"; +import { Buffer } from "node:buffer"; /** * Shared client-facing SSE payload rewrite shell. @@ -37,18 +38,21 @@ export function payloadRewriteAsBlockRewrite(rewrite: SsePayloadRewrite): SseBlo export function composeSseBlockRewrites(...rewrites: SseBlockRewrite[]): SseBlockRewrite { const active = rewrites.filter(Boolean); if (active.length === 0) return Object.assign((block: string) => [block], {}); + let disposed = false; const composed: SseBlockRewrite = (block: string) => { let blocks: readonly string[] = [block]; for (const rewrite of active) { const next: string[] = []; - for (const current of blocks) next.push(...rewrite(current)); + for (const current of blocks) { + if (disposed) return []; + next.push(...rewrite(current)); + } blocks = next; } return blocks; }; // Child disposal is part of the contract: one idempotent disposer for the // whole chain, so relay teardown never leaks a nested collector. - let disposed = false; composed.dispose = () => { if (disposed) return; disposed = true; @@ -70,10 +74,116 @@ export function nextSseBlock(buffer: string): { block: string; delimiter: string }; } +/** + * Incremental form of nextSseBlock for bounded relays. The scan cursor visits only new text and + * consuming a block subtracts its byte length instead of recounting the remaining suffix. + * Old/new buffer overlap still requires admission before either append or consumption commits. + * Call compact before yielding to stop retaining an already-consumed prefix across pulls. + */ +export function createSseBlockBuffer( + budget: TranslatorBudget, + assertAppendSize?: (bytes: number) => void, +): { + append(fragment: string): void; + next(): { block: string; delimiter: string } | null; + tail(): string; + compact(): void; + clear(): void; +} { + const scope = { kind: "live_transient" as const }; + let buffer = ""; + let offset = 0; + let scanOffset = 0; + let bufferBytes = 0; + + const compact = (): void => { + if (offset === 0) return; + buffer = buffer.slice(offset); + scanOffset -= offset; + offset = 0; + }; + + return { + append(fragment) { + if (!fragment) return; + let fragmentBytes = Buffer.byteLength(fragment, "utf8"); + // Decoder output never splits a surrogate pair, but keep the helper exact for string callers. + const last = buffer.charCodeAt(buffer.length - 1); + const first = fragment.charCodeAt(0); + if (offset < buffer.length && last >= 0xd800 && last <= 0xdbff && first >= 0xdc00 && first <= 0xdfff) { + fragmentBytes -= 2; + } + const nextBytes = bufferBytes + fragmentBytes; + assertAppendSize?.(nextBytes); + const reservation = budget.reserveTransient(nextBytes, scope); + try { + compact(); + buffer += fragment; + reservation.commitRetained(); + budget.releaseRetained(bufferBytes, scope); + bufferBytes = nextBytes; + } catch (error) { + reservation.release(); + throw error; + } + }, + next() { + for (;;) { + const newline = buffer.indexOf("\n", scanOffset); + if (newline < 0) { + scanOffset = buffer.length; + return null; + } + let end = newline + 1; + if (buffer[end] === "\r") end += 1; + if (end === buffer.length) { + // Keep the candidate first newline until its possible blank-line delimiter arrives. + scanOffset = newline; + return null; + } + if (buffer[end] !== "\n") { + scanOffset = newline + 1; + continue; + } + end += 1; + const start = newline > offset && buffer[newline - 1] === "\r" ? newline - 1 : newline; + const block = buffer.slice(offset, start); + const delimiter = buffer.slice(start, end); + const nextBytes = bufferBytes - Buffer.byteLength(block, "utf8") - delimiter.length; + const reservation = budget.reserveTransient(nextBytes, scope); + reservation.commitRetained(); + budget.releaseRetained(bufferBytes, scope); + bufferBytes = nextBytes; + offset = end; + scanOffset = end; + if (offset === buffer.length) { + buffer = ""; + offset = 0; + scanOffset = 0; + } + return { block, delimiter }; + } + }, + tail: () => buffer.slice(offset), + compact, + clear() { + budget.releaseRetained(bufferBytes, scope); + buffer = ""; + offset = 0; + scanOffset = 0; + bufferBytes = 0; + }, + }; +} + /** Join all data lines from one SSE event according to the event-stream field rules. */ export function sseDataPayload(block: string): string | null { const data: string[] = []; for (const line of block.split(/\r?\n/)) { + if (line === "data") { + data.push(""); + continue; + } if (!line.startsWith("data:")) continue; const value = line.slice(5); data.push(value.startsWith(" ") ? value.slice(1) : value); @@ -88,7 +198,7 @@ export function replaceSseDataPayload(block: string, payload: string): string { const rewritten: string[] = []; let replaced = false; for (const line of lines) { - if (!line.startsWith("data:")) { + if (line !== "data" && !line.startsWith("data:")) { rewritten.push(line); continue; } @@ -137,8 +247,7 @@ export function relaySseWithBlockRewrite( const reader = body.getReader(); const decoder = new TextDecoder(); const encoder = new TextEncoder(); - let buffer = ""; - let bufferBytes = 0; + const buffer = createSseBlockBuffer(translatorBudget); // Relays have several independent teardown paths; disposal is exactly once. let disposed = false; let cancelled = false; @@ -148,40 +257,16 @@ export function relaySseWithBlockRewrite( try { rewrite.dispose?.(); } catch { /* teardown must not throw */ } }; - const appendBuffer = (fragment: string): void => { - if (!fragment) return; - const nextBytes = bufferBytes + encoder.encode(fragment).byteLength; - const reservation = translatorBudget.reserveTransient(nextBytes, { kind: "live_transient" }); - try { - buffer += fragment; - reservation.commitRetained(); - translatorBudget.releaseRetained(bufferBytes, { kind: "live_transient" }); - bufferBytes = nextBytes; - } catch (error) { - reservation.release(); - throw error; - } - }; - - const replaceBuffer = (next: string): void => { - const nextBytes = encoder.encode(next).byteLength; - const reservation = translatorBudget.reserveTransient(nextBytes, { kind: "live_transient" }); - reservation.commitRetained(); - buffer = next; - translatorBudget.releaseRetained(bufferBytes, { kind: "live_transient" }); - bufferBytes = nextBytes; - }; - const enqueueText = ( controller: ReadableStreamDefaultController, text: string, ): void => { - const bytes = encoder.encode(text).byteLength; + const bytes = Buffer.byteLength(text, "utf8"); const reservation = translatorBudget.reserveTransient(bytes, { kind: "live_transient" }); try { const encoded = encoder.encode(text); - reservation.commitRetained(); controller.enqueue(encoded); + reservation.commitRetained(); translatorBudget.releaseRetained(bytes, { kind: "live_transient" }); } catch (error) { reservation.release(); @@ -189,35 +274,33 @@ export function relaySseWithBlockRewrite( } }; - const releaseBuffer = (): void => { - translatorBudget.releaseRetained(bufferBytes, { kind: "live_transient" }); - buffer = ""; - bufferBytes = 0; - }; - const emitProcessedBlocks = ( controller: ReadableStreamDefaultController, flushFinal = false, ): number => { let emitted = 0; - let next: { block: string; delimiter: string; rest: string } | null; - while ((next = nextSseBlock(buffer))) { - replaceBuffer(next.rest); - for (const outBlock of rewrite(next.block)) { + let next: { block: string; delimiter: string } | null; + while (!cancelled && (next = buffer.next())) { + const outBlocks = rewrite(next.block); + if (cancelled) return emitted; + for (const outBlock of outBlocks) { enqueueText(controller, outBlock + next.delimiter); emitted += 1; } } - if (flushFinal && buffer.length > 0) { - const tailBlocks = rewrite(buffer); + buffer.compact(); + const tail = flushFinal ? buffer.tail() : ""; + if (tail.length > 0) { + const tailBlocks = rewrite(tail); + if (cancelled) return emitted; // A trailing fragment has no delimiter of its own; multiple emitted // blocks must still be framed as separate events (#893 review). - const tailDelimiter = buffer.includes("\r\n") ? "\r\n\r\n" : "\n\n"; + const tailDelimiter = tail.includes("\r\n") ? "\r\n\r\n" : "\n\n"; for (let i = 0; i < tailBlocks.length; i++) { enqueueText(controller, tailBlocks[i]! + (i < tailBlocks.length - 1 ? tailDelimiter : "")); emitted += 1; } - releaseBuffer(); + buffer.clear(); } return emitted; }; @@ -236,18 +319,20 @@ export function relaySseWithBlockRewrite( // after its disposal (#893 review). if (cancelled) return; if (done) { - appendBuffer(decoder.decode()); + buffer.append(decoder.decode()); emitProcessedBlocks(controller, true); - releaseBuffer(); + if (cancelled) return; + buffer.clear(); disposeRewrite(); controller.close(); return; } - appendBuffer(decoder.decode(value, { stream: true })); - if (emitProcessedBlocks(controller) > 0) return; + buffer.append(decoder.decode(value, { stream: true })); + const emitted = emitProcessedBlocks(controller); + if (cancelled || emitted > 0) return; } } catch (error) { - releaseBuffer(); + buffer.clear(); disposeRewrite(); // Cancelling one tee branch waits for its sibling. Surface the failure // now so downstream can abort upstream and release the inspection branch. @@ -257,7 +342,7 @@ export function relaySseWithBlockRewrite( }, cancel(reason) { cancelled = true; - releaseBuffer(); + buffer.clear(); disposeRewrite(); reader.cancel(reason).catch(() => {}); }, diff --git a/tests/responses/sse-payload-rewrite.test.ts b/tests/responses/sse-payload-rewrite.test.ts index 773665a054..8fbfa0e957 100644 --- a/tests/responses/sse-payload-rewrite.test.ts +++ b/tests/responses/sse-payload-rewrite.test.ts @@ -1,13 +1,19 @@ /** * Single-pass composition of client-facing SSE payload rewrites (#588 follow-up). */ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; +import { Buffer } from "node:buffer"; import { createImageGenCallRestoreRewrite } from "../../src/server/responses-image-gen-repair"; import { createResponsesItemIdPayloadRewrite } from "../../src/server/responses-item-id-repair"; import { composeSsePayloadRewrites, + composeSseBlockRewrites, + createSseBlockBuffer, + nextSseBlock, + replaceSseDataPayload, relaySseWithBlockRewrite, relaySseWithPayloadRewrite, + sseDataPayload, } from "../../src/server/sse-payload-rewrite"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; import { relaySseWithFailedTail } from "../../src/server/relay"; @@ -55,6 +61,274 @@ async function readAll(stream: ReadableStream): Promise { } describe("SSE payload rewrite composition", () => { + test.each(["\n", "\r\n"])("reads and replaces colonless data fields with %j lines", newline => { + const block = ["event: update", "data", 'data: {"text":"hello"}', "data", "database: unchanged"].join(newline); + expect(sseDataPayload(block)).toBe('\n{"text":"hello"}\n'); + expect(replaceSseDataPayload(block, '{"text":"changed"}')).toBe( + ["event: update", 'data: {"text":"changed"}', "database: unchanged"].join(newline), + ); + expect(sseDataPayload("data")).toBe(""); + expect(sseDataPayload("database")).toBeNull(); + }); + + test("encodes only delivered blocks and counts coalesced input in linear space", async () => { + const text = Array.from({ length: 256 }, (_, index) => `data: ${index} 中文😀${"x".repeat(128)}\n\n`).join(""); + const upstream = streamFromText(text); + const budget = createTestTranslatorBudget(); + const originalEncode = TextEncoder.prototype.encode; + const originalByteLength = Buffer.byteLength; + let encodedBytes = 0; + let countedCodeUnits = 0; + const encode = spyOn(TextEncoder.prototype, "encode").mockImplementation(function (this: TextEncoder, input) { + const encoded = originalEncode.call(this, input); + encodedBytes += encoded.byteLength; + return encoded; + }); + const byteLength = spyOn(Buffer, "byteLength").mockImplementation((value, encoding) => { + if (typeof value === "string") countedCodeUnits += value.length; + return originalByteLength(value, encoding); + }); + try { + expect(await readAll(relaySseWithBlockRewrite(upstream, block => [block], budget))).toBe(text); + expect(encodedBytes).toBe(originalByteLength(text, "utf8")); + expect(countedCodeUnits).toBeLessThanOrEqual(text.length * 3); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + encode.mockRestore(); + byteLength.mockRestore(); + } + }); + + test("does not rescan an unterminated prefix after each transport fragment", async () => { + const fragments = Array.from({ length: 256 }, () => "x".repeat(128)); + fragments[0] = "data: " + fragments[0]; + fragments.push("\r\n\r\n"); + const originalMatch = String.prototype.match; + const originalIndexOf = String.prototype.indexOf; + let scannedCodeUnits = 0; + const match = spyOn(String.prototype, "match").mockImplementation(function (this: string, regexp) { + if (regexp instanceof RegExp && regexp.source === "\\r?\\n\\r?\\n") scannedCodeUnits += this.length; + return originalMatch.call(this, regexp); + }); + const indexOf = spyOn(String.prototype, "indexOf").mockImplementation(function (this: string, search, position) { + const found = originalIndexOf.call(this, search, position); + if (search === "\n") scannedCodeUnits += (found < 0 ? this.length : found + 1) - (position ?? 0); + return found; + }); + try { + const text = fragments.join(""); + expect(await readAll(relaySseWithBlockRewrite(streamFromTexts(fragments), block => [block], createTestTranslatorBudget()))).toBe(text); + expect(scannedCodeUnits).toBeLessThanOrEqual(text.length * 2); + } finally { + match.mockRestore(); + indexOf.mockRestore(); + } + }); + + test("rejects an oversized replacement before encoding or delivering it", async () => { + const upstream = streamFromText("data: small\n\n"); + const budget = createTestTranslatorBudget({ maxTurnBytes: 64 }); + const encode = spyOn(TextEncoder.prototype, "encode"); + let disposeCalls = 0; + const rewrite = Object.assign(() => [`data: ${"界".repeat(32)}`], { + dispose() { disposeCalls += 1; }, + }); + try { + await expect(readAll(relaySseWithBlockRewrite(upstream, rewrite, budget))).rejects.toMatchObject({ code: "translation_buffer_limit" }); + expect(encode).not.toHaveBeenCalled(); + expect(disposeCalls).toBe(1); + expect(budget.snapshot()).toMatchObject({ currentBytes: 0, overflows: 1 }); + } finally { + encode.mockRestore(); + } + }); + + test.each(["\n\n", "\r\n\r\n", "\r\n\n", "\n\r\n"])( + "preserves delimiter %j through byte-split Unicode, drops, injection, and EOF", + async delimiter => { + const first = "event: keep\r\ndata: 中文😀"; + const tail = "event: tail\r\ndata: é"; + const text = `${first}${delimiter}data: drop${delimiter}data: inject${delimiter}${tail}`; + const bytes = new TextEncoder().encode(text); + let index = 0; + let disposeCalls = 0; + const source = new ReadableStream({ + pull(controller) { + if (index === bytes.length) controller.close(); + else controller.enqueue(bytes.subarray(index, ++index)); + }, + }); + const rewrite = Object.assign((block: string) => { + if (block === "data: drop") return []; + if (block === "data: inject") return ["data: one", "data: two"]; + if (block === tail) return [tail, "data: end"]; + return [block]; + }, { dispose() { disposeCalls += 1; } }); + const budget = createTestTranslatorBudget(); + expect(await readAll(relaySseWithBlockRewrite(source, rewrite, budget))).toBe( + `${first}${delimiter}data: one${delimiter}data: two${delimiter}${tail}\r\n\r\ndata: end`, + ); + expect(budget.snapshot().currentBytes).toBe(0); + expect(disposeCalls).toBe(1); + }, + ); + + test("offset framing preserves exact suffix and overlap accounting through compaction", () => { + const budget = createTestTranslatorBudget(); + const buffer = createSseBlockBuffer(budget); + let reference = ""; + let peak = 0; + for (const fragment of ["data: 中文😀\r\n\r\ndata: é\n\npartial", "界\r", "\n", "\r", "\nlast\n\n", "\ud800", "\udc00\n\n"]) { + const previousBytes = Buffer.byteLength(reference, "utf8"); + reference += fragment; + peak = Math.max(peak, previousBytes + Buffer.byteLength(reference, "utf8")); + buffer.append(fragment); + for (;;) { + const expected = nextSseBlock(reference); + const beforeBytes = Buffer.byteLength(reference, "utf8"); + const actual = buffer.next(); + if (!expected) { + expect(actual).toBeNull(); + break; + } + expect(actual).toEqual({ block: expected.block, delimiter: expected.delimiter }); + reference = expected.rest; + peak = Math.max(peak, beforeBytes + Buffer.byteLength(reference, "utf8")); + // Native Chat yields after one event; HTTP rewriting drains the whole batch. + buffer.compact(); + expect(buffer.tail()).toBe(reference); + expect(budget.snapshot().currentBytes).toBe(Buffer.byteLength(reference, "utf8")); + } + expect(buffer.tail()).toBe(reference); + expect(budget.snapshot()).toMatchObject({ + currentBytes: Buffer.byteLength(reference, "utf8"), + highWaterBytes: peak, + overflows: 0, + }); + } + buffer.clear(); + buffer.clear(); + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test.each(["append", "consume"] as const)("keeps old/new %s overlap admission atomic", operation => { + const initial = operation === "append" ? "data: partial" : "data: first\n\ndata: second\n\n"; + const budget = createTestTranslatorBudget({ maxTurnBytes: Buffer.byteLength(initial) + 1 }); + const buffer = createSseBlockBuffer(budget); + buffer.append(initial); + expect(() => operation === "append" ? buffer.append("x") : buffer.next()).toThrow("buffer exceeded"); + expect(buffer.tail()).toBe(initial); + expect(budget.snapshot()).toMatchObject({ currentBytes: Buffer.byteLength(initial), overflows: 1 }); + buffer.clear(); + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test("cancel during a pending read releases partial input and disposes once", async () => { + const waiting = Promise.withResolvers(); + const bytes = new TextEncoder().encode("data: partial 中文"); + let sent = false; + let cancelCalls = 0; + let disposeCalls = 0; + let rewriteCalls = 0; + const source = new ReadableStream({ + pull(controller) { + if (!sent) { + sent = true; + controller.enqueue(bytes); + } else waiting.resolve(); + }, + cancel() { cancelCalls += 1; }, + }); + const rewrite = Object.assign((block: string) => { + rewriteCalls += 1; + return [block]; + }, { dispose() { disposeCalls += 1; } }); + const budget = createTestTranslatorBudget(); + const reader = relaySseWithBlockRewrite(source, rewrite, budget).getReader(); + const pending = reader.read(); + await waiting.promise; + await reader.cancel("client left"); + expect(await pending).toMatchObject({ done: true }); + expect(rewriteCalls).toBe(0); + expect(cancelCalls).toBe(1); + expect(disposeCalls).toBe(1); + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test.each([false, true])("cancel inside rewrite stops queued blocks and EOF rewrites (tail: %s)", async tail => { + const text = tail ? "data: unfinished" : "data: first\n\ndata: second\n\n"; + let disposeCalls = 0; + let rewriteCalls = 0; + let cancellation: Promise | undefined; + let reader: ReadableStreamDefaultReader; + const rewrite = Object.assign((block: string) => { + rewriteCalls += 1; + cancellation = reader.cancel("cancel during rewrite"); + return [block, "data: injected"]; + }, { dispose() { disposeCalls += 1; } }); + const budget = createTestTranslatorBudget(); + const encode = spyOn(TextEncoder.prototype, "encode"); + const upstream = streamFromText(text); + encode.mockClear(); + try { + reader = relaySseWithBlockRewrite(upstream, rewrite, budget).getReader(); + expect(await reader.read()).toMatchObject({ done: true }); + await cancellation; + expect(encode).not.toHaveBeenCalled(); + expect(rewriteCalls).toBe(1); + expect(disposeCalls).toBe(1); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + encode.mockRestore(); + } + }); + + test("releases the output reservation when enqueue fails after allocation", async () => { + const upstream = streamFromText("data: first\n\n"); + const budget = createTestTranslatorBudget(); + const originalEncode = TextEncoder.prototype.encode; + let reader: ReadableStreamDefaultReader; + let cancellation: Promise | undefined; + const encode = spyOn(TextEncoder.prototype, "encode").mockImplementation(function (this: TextEncoder, text) { + const encoded = originalEncode.call(this, text); + // Close the downstream immediately before enqueue to exercise reservation cleanup. + cancellation = reader.cancel("closed before enqueue"); + return encoded; + }); + try { + reader = relaySseWithBlockRewrite(upstream, block => [block], budget).getReader(); + expect(await reader.read()).toMatchObject({ done: true }); + await cancellation; + expect(budget.snapshot().currentBytes).toBe(0); + } finally { + encode.mockRestore(); + } + }); + + test("cancellation in one composed stage never invokes a disposed later stage", async () => { + let reader: ReadableStreamDefaultReader; + let cancellation: Promise | undefined; + let laterCalls = 0; + let disposeCalls = 0; + const rewrite = composeSseBlockRewrites( + block => { + cancellation = reader.cancel("cancel during first stage"); + return [block]; + }, + Object.assign((block: string) => { + laterCalls += 1; + return [block]; + }, { dispose() { disposeCalls += 1; } }), + ); + const budget = createTestTranslatorBudget(); + reader = relaySseWithBlockRewrite(streamFromText("data: event\n\n"), rewrite, budget).getReader(); + expect(await reader.read()).toMatchObject({ done: true }); + await cancellation; + expect(laterCalls).toBe(0); + expect(disposeCalls).toBe(1); + expect(budget.snapshot().currentBytes).toBe(0); + }); + test("applies image-gen restore and item-id repair in one relay pass", async () => { const upstream = [ 'event: response.output_item.added\ndata: {"type":"response.output_item.added","output_index":0,"item":{"type":"message","id":"msg_0","role":"assistant"}}\n\n', From 8fe32c55a8aebca3f04a9b8e7df7e576921cd867 Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 00:16:41 -0700 Subject: [PATCH 08/13] fix(chat): bound native stream stalls and preserve completion state (cherry picked from commit 6e162748c01aea9515306a33b67dafc1372b400e) --- src/chat/outbound.ts | 206 ++++++++-------- src/server/chat-native-sse.ts | 132 ++++++---- src/server/chat-native.ts | 32 ++- .../chat-completions-endpoint.test.ts | 232 +++++++++++++++++- 4 files changed, 441 insertions(+), 161 deletions(-) diff --git a/src/chat/outbound.ts b/src/chat/outbound.ts index e4ed29512a..8197739c07 100644 --- a/src/chat/outbound.ts +++ b/src/chat/outbound.ts @@ -7,7 +7,7 @@ */ type Rec = Record; -import { decodeServerSentEvents, sseFieldValue } from "../lib/sse-decoder"; +import { decodeServerSentEvents } from "../lib/sse-decoder"; import { isTranslatorBudgetExceededError, type TranslatorBudget, @@ -21,6 +21,7 @@ import { isCyberPolicyMessage, } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; +import { createSseBlockBuffer, sseDataPayload } from "../server/sse-payload-rewrite"; function isRec(v: unknown): v is Rec { return !!v && typeof v === "object" && !Array.isArray(v); @@ -915,11 +916,13 @@ export async function collectChatCompletion( model: string, translatorBudget: TranslatorBudget, ): Promise { + const reader = stream.getReader(); const decoder = new TextDecoder(); - let buffer = ""; + const buffer = createSseBlockBuffer(translatorBudget); let content = ""; let refusal: string | null = null; let reasoning = ""; + const retainedBytes = { content: 0, refusal: 0, reasoning: 0 }; const toolCalls = new Map(); // Per-call budget scopes (2 MiB/call enforced by the budget): the map key is the // wire index, which is stable across deltas and present before the call id. @@ -927,119 +930,111 @@ export async function collectChatCompletion( let finishReason = "stop"; let usage: unknown; let serviceTier: unknown; - const replaceRetained = (previous: string, next: string, kind: "live_transient" | "retained_collectors") => { - const reservation = translatorBudget.reserveTransient(Buffer.byteLength(next), { kind }); - reservation.commitRetained(); - translatorBudget.releaseRetained(Buffer.byteLength(previous), { kind }); - return next; + const appendRetained = (key: keyof typeof retainedBytes, previous: string, fragment: string): string => { + const previousBytes = retainedBytes[key]; + const nextBytes = appendedUtf8Bytes(previous, previousBytes, fragment); + const reservation = translatorBudget.reserveTransient(nextBytes, { kind: "retained_collectors" }); + try { + const next = previous + fragment; + reservation.commitRetained(); + translatorBudget.releaseRetained(previousBytes, { kind: "retained_collectors" }); + retainedBytes[key] = nextBytes; + return next; + } catch (error) { + reservation.release(); + throw error; + } + }; + const releaseCollectors = () => { + translatorBudget.releaseRetained(retainedBytes.content + retainedBytes.refusal + retainedBytes.reasoning, + { kind: "retained_collectors" }); }; - const reader = stream.getReader(); try { + // Share native relay framing, while retaining the collector's existing admission + // order: release each consumed input frame before accumulating its output fields. + // CRLF and multiline data obey the same contract as the streaming response. for (;;) { - let done = false; - let value: Uint8Array | undefined; - try { - ({ done, value } = await reader.read()); - } catch (err) { - if (isChatCompletionsStreamError(err)) throw err; - if (isTranslatorBudgetExceededError(err)) { - // Provider-controlled overflow is an upstream failure, not a client - // request error: match the adapter/bridge contract (502 upstream_error). - throw new ChatCompletionsStreamError(err.message, { - status: 502, - type: "upstream_error", - code: err.code, - }); - } - throw new ChatCompletionsStreamError(err instanceof Error ? err.message : String(err)); - } + const { done, value } = await reader.read(); if (done) break; if (!value) continue; - buffer = replaceRetained(buffer, buffer + decoder.decode(value, { stream: true }), "live_transient"); - let sep: number; - while ((sep = buffer.indexOf("\n\n")) !== -1) { - const rawFrame = buffer.slice(0, sep); - buffer = replaceRetained(buffer, buffer.slice(sep + 2), "live_transient"); - for (const line of rawFrame.split("\n")) { - const rawData = sseFieldValue(line, "data"); - if (rawData === null) continue; - const data = rawData.trim(); - if (!data || data === "[DONE]") continue; - let parsed: unknown; - try { parsed = JSON.parse(data); } catch { continue; } - if (!isRec(parsed)) continue; - if (isRec(parsed.error)) { - const message = typeof parsed.error.message === "string" - ? parsed.error.message - : "upstream request failed"; - const type = typeof parsed.error.type === "string" ? parsed.error.type : "server_error"; - const code = typeof parsed.error.code === "string" ? parsed.error.code : null; - const status = code === "translation_buffer_limit" - ? 502 - : code === CYBER_POLICY_ERROR_CODE || isCyberPolicyMessage(message) - ? 400 - : streamErrorStatus(message); - const streamError = new ChatCompletionsStreamError(message, { - status, - type: code === "translation_buffer_limit" ? "upstream_error" : type, - code, - }); - throw streamError; - } - if (parsed.usage) usage = parsed.usage; - if (typeof parsed.service_tier === "string") serviceTier = parsed.service_tier; - const choices = Array.isArray(parsed.choices) ? parsed.choices : []; - const choice = isRec(choices[0]) ? choices[0] : null; - if (!choice) continue; - if (typeof choice.finish_reason === "string") finishReason = choice.finish_reason; - const delta = isRec(choice.delta) ? choice.delta : null; - if (!delta) continue; - if (typeof delta.content === "string") content = replaceRetained(content, content + delta.content, "retained_collectors"); - if (delta.refusal !== undefined && delta.refusal !== null) { - if (typeof delta.refusal !== "string") throw refusalTranslationError(); - refusal = replaceRetained(refusal ?? "", (refusal ?? "") + delta.refusal, "retained_collectors"); - } - if (typeof delta.reasoning_content === "string") reasoning = replaceRetained(reasoning, reasoning + delta.reasoning_content, "retained_collectors"); - if (Array.isArray(delta.tool_calls)) { - for (const tc of delta.tool_calls) { - if (!isRec(tc)) continue; - const index = typeof tc.index === "number" ? tc.index : 0; - let current = toolCalls.get(index); - if (!current) { - current = { id: "", name: "", arguments: "", argumentBytes: 0 }; - toolCalls.set(index, current); - translatorBudget.openCall(callScope(index)); - } - if (typeof tc.id === "string") current.id = tc.id; - const fn = isRec(tc.function) ? tc.function : {}; - // Done-frame final arguments are authoritative last-write-wins snapshots. - if (typeof fn.name === "string" && fn.name.length > 0) current.name = fn.name; - if (typeof fn.arguments === "string") { - const replace = fn.arguments.startsWith("{") || fn.arguments.startsWith("[") || current.arguments.length === 0; - const nextBytes = replace - ? Buffer.byteLength(fn.arguments) - : appendedUtf8Bytes(current.arguments, current.argumentBytes, fn.arguments); - const reservation = translatorBudget.reserveTransient(nextBytes, { kind: "tool_args", callId: callScope(index) }); - try { - current.arguments = replace ? fn.arguments : current.arguments + fn.arguments; - reservation.commitRetained(); - translatorBudget.releaseRetained(current.argumentBytes, { kind: "tool_args", callId: callScope(index) }); - current.argumentBytes = nextBytes; - } catch (error) { - reservation.release(); - throw error; - } + buffer.append(decoder.decode(value, { stream: true })); + for (let frame = buffer.next(); frame; frame = buffer.next()) { + const data = sseDataPayload(frame.block)?.trim(); + if (!data || data === "[DONE]") continue; + let parsed: unknown; + try { parsed = JSON.parse(data); } catch { continue; } + if (!isRec(parsed)) continue; + if (isRec(parsed.error)) { + const message = typeof parsed.error.message === "string" + ? parsed.error.message + : "upstream request failed"; + const type = typeof parsed.error.type === "string" ? parsed.error.type : "server_error"; + const code = typeof parsed.error.code === "string" ? parsed.error.code : null; + const status = code === "translation_buffer_limit" + ? 502 + : code === CYBER_POLICY_ERROR_CODE || isCyberPolicyMessage(message) + ? 400 + : streamErrorStatus(message); + const streamError = new ChatCompletionsStreamError(message, { + status, + type: code === "translation_buffer_limit" ? "upstream_error" : type, + code, + }); + throw streamError; + } + if (parsed.usage) usage = parsed.usage; + if (typeof parsed.service_tier === "string") serviceTier = parsed.service_tier; + const choices = Array.isArray(parsed.choices) ? parsed.choices : []; + const choice = isRec(choices[0]) ? choices[0] : null; + if (!choice) continue; + if (typeof choice.finish_reason === "string") finishReason = choice.finish_reason; + const delta = isRec(choice.delta) ? choice.delta : null; + if (!delta) continue; + if (typeof delta.content === "string") content = appendRetained("content", content, delta.content); + if (delta.refusal !== undefined && delta.refusal !== null) { + if (typeof delta.refusal !== "string") throw refusalTranslationError(); + refusal = appendRetained("refusal", refusal ?? "", delta.refusal); + } + if (typeof delta.reasoning_content === "string") reasoning = appendRetained("reasoning", reasoning, delta.reasoning_content); + if (Array.isArray(delta.tool_calls)) { + for (const tc of delta.tool_calls) { + if (!isRec(tc)) continue; + const index = typeof tc.index === "number" ? tc.index : 0; + let current = toolCalls.get(index); + if (!current) { + current = { id: "", name: "", arguments: "", argumentBytes: 0 }; + toolCalls.set(index, current); + translatorBudget.openCall(callScope(index)); + } + if (typeof tc.id === "string") current.id = tc.id; + const fn = isRec(tc.function) ? tc.function : {}; + // Done-frame final arguments are authoritative last-write-wins snapshots. + if (typeof fn.name === "string" && fn.name.length > 0) current.name = fn.name; + if (typeof fn.arguments === "string") { + const replace = fn.arguments.startsWith("{") || fn.arguments.startsWith("[") || current.arguments.length === 0; + const nextBytes = replace + ? Buffer.byteLength(fn.arguments) + : appendedUtf8Bytes(current.arguments, current.argumentBytes, fn.arguments); + const reservation = translatorBudget.reserveTransient(nextBytes, { kind: "tool_args", callId: callScope(index) }); + try { + current.arguments = replace ? fn.arguments : current.arguments + fn.arguments; + reservation.commitRetained(); + translatorBudget.releaseRetained(current.argumentBytes, { kind: "tool_args", callId: callScope(index) }); + current.argumentBytes = nextBytes; + } catch (error) { + reservation.release(); + throw error; } } } } } + buffer.compact(); } } catch (error) { - // Processing may fail between reads; cancel while we still own the lock so the - // upstream translator releases its maps and stops any pending provider read. + // Cancel while we still own the reader so a failed collection releases its upstream. try { await reader.cancel(error); } catch { /* preserve the original failure */ } - translatorBudget.releaseRetained(Buffer.byteLength(refusal ?? ""), { kind: "retained_collectors" }); + releaseCollectors(); // Never leak an open call scope on the error path; the turn budget's // dispose is a backstop, not the owner of this transfer. for (const index of toolCalls.keys()) translatorBudget.closeCall(callScope(index)); @@ -1052,8 +1047,12 @@ export async function collectChatCompletion( code: error.code, }); } - throw error; + if (isChatCompletionsStreamError(error)) throw error; + throw new ChatCompletionsStreamError(error instanceof Error ? error.message : String(error)); } finally { + // Preserve the previous EOF contract: only delimiter-terminated events are collected. + // A partial final frame is discarded, with its retained input ownership released. + buffer.clear(); reader.releaseLock(); } @@ -1085,6 +1084,7 @@ export async function collectChatCompletion( return copy; }); } catch (error) { + releaseCollectors(); for (const copyBytes of chargedCopies) { translatorBudget.releaseRetained(copyBytes, { kind: "retained_collectors" }); } diff --git a/src/server/chat-native-sse.ts b/src/server/chat-native-sse.ts index eb598b7898..e764589f4f 100644 --- a/src/server/chat-native-sse.ts +++ b/src/server/chat-native-sse.ts @@ -7,7 +7,8 @@ import { type TranslatorBudget, } from "../lib/translator-budget"; import type { OcxUsage } from "../types"; -import { nextSseBlock, replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite"; +import { resolveStallTimeoutSec } from "../stall-timeout"; +import { createSseBlockBuffer, replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite"; type Rec = Record; @@ -127,6 +128,7 @@ interface NativeChatSseOptions { requestedModel: string; translatorBudget: TranslatorBudget; signal: AbortSignal; + stallTimeoutSec?: number; onFirstOutput?: () => void; onUsage: (usage: OcxUsage) => void; onTerminal?: (status: number, message?: string) => void; @@ -141,8 +143,11 @@ export function nativeChatSse( const decoder = new TextDecoder(); const encoder = new TextEncoder(); const scope = { kind: "live_transient" as const }; - let buffer = ""; - let bufferBytes = 0; + const buffer = createSseBlockBuffer(options.translatorBudget, nextBytes => { + if (nextBytes > TRANSLATOR_MAX_SSE_EVENT_BYTES) { + throw new Error("upstream SSE event exceeded the safe limit", { cause: { code: "translation_buffer_limit" } }); + } + }); let queuedBytes = 0; let sawFinish = false; let sawDone = false; @@ -150,50 +155,31 @@ export function nativeChatSse( let settled = false; let cancelled = false; let cancelledBySignal = false; + const stallMs = resolveStallTimeoutSec(options.stallTimeoutSec) * 1_000; + // Count only active upstream consumption; downstream backpressure must not + // consume the provider's inactivity allowance. Comments and empty deltas do. + let remainingStallMs = stallMs; + let pullDeadline = 0; + const stalled = new Error("upstream stream stalled without meaningful progress"); const releaseQueued = () => { if (queuedBytes === 0) return; options.translatorBudget.releaseRetained(queuedBytes, scope); queuedBytes = 0; }; - const replaceBuffer = (next: string) => { - const nextBytes = encoder.encode(next).byteLength; - const reservation = options.translatorBudget.reserveTransient(nextBytes, scope); - reservation.commitRetained(); - options.translatorBudget.releaseRetained(bufferBytes, scope); - buffer = next; - bufferBytes = nextBytes; - }; - const appendBuffer = (fragment: string) => { - if (!fragment) return; - const fragmentBytes = encoder.encode(fragment).byteLength; - const nextBytes = bufferBytes + fragmentBytes; - if (nextBytes > TRANSLATOR_MAX_SSE_EVENT_BYTES) { - throw new Error("upstream SSE event exceeded the safe limit", { cause: { code: "translation_buffer_limit" } }); - } - const reservation = options.translatorBudget.reserveTransient(nextBytes, scope); + const enqueue = (controller: ReadableStreamDefaultController, text: string) => { + const byteLength = Buffer.byteLength(text); + const reservation = options.translatorBudget.reserveTransient(byteLength, scope); try { - buffer += fragment; + controller.enqueue(encoder.encode(text)); reservation.commitRetained(); - options.translatorBudget.releaseRetained(bufferBytes, scope); - bufferBytes = nextBytes; + queuedBytes += byteLength; } catch (error) { reservation.release(); throw error; } }; - const enqueue = (controller: ReadableStreamDefaultController, text: string) => { - const bytes = encoder.encode(text); - const reservation = options.translatorBudget.reserveTransient(bytes.byteLength, scope); - controller.enqueue(bytes); - reservation.commitRetained(); - queuedBytes += bytes.byteLength; - }; - const releaseBuffer = () => { - options.translatorBudget.releaseRetained(bufferBytes, scope); - buffer = ""; - bufferBytes = 0; - }; + const releaseBuffer = () => buffer.clear(); const settle = (status: number, message?: string) => { if (settled) return; settled = true; @@ -225,13 +211,20 @@ export function nativeChatSse( ): void => { const payload = sseDataPayload(block); if (payload === null) { + if (performance.now() >= pullDeadline) throw stalled; enqueue(controller, block + delimiter); return; } const trimmed = payload.trim(); + if (trimmed === "") { + if (performance.now() >= pullDeadline) throw stalled; + enqueue(controller, block + delimiter); + return; + } if (trimmed === "[DONE]") { sawDone = true; enqueue(controller, replaceSseDataPayload(block, "[DONE]") + delimiter); + releaseBuffer(); settle(200); try { void reader.cancel().catch(() => {}); } catch { /* already closed */ } controller.close(); @@ -260,6 +253,7 @@ export function nativeChatSse( } const safe = chatCompletionsErrorBody(status, classified.message, classified.type, classified.code); enqueue(controller, replaceSseDataPayload(block, JSON.stringify(safe)) + delimiter); + releaseBuffer(); settle(isCyberPolicyCode(classified.code) ? 400 : status, classified.message); try { void reader.cancel(new Error(classified.message)).catch(() => {}); } catch { /* already closed */ } controller.close(); @@ -268,16 +262,28 @@ export function nativeChatSse( const usage = usageFromChat(parsed.usage); if (usage) options.onUsage(usage); const choices = Array.isArray(parsed.choices) ? parsed.choices : []; - if (choices.some(choice => isRec(choice) && typeof choice.finish_reason === "string" && choice.finish_reason.length > 0)) { - sawFinish = true; - } - if (!firstOutput && choices.some(choice => { + const finished = choices.some(choice => isRec(choice) && typeof choice.finish_reason === "string" && choice.finish_reason.length > 0); + if (finished) sawFinish = true; + const hasProgress = choices.some(choice => { if (!isRec(choice) || !isRec(choice.delta)) return false; const delta = choice.delta; return (typeof delta.content === "string" && delta.content.length > 0) || (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) - || (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0); - })) { + || (typeof delta.reasoning === "string" && delta.reasoning.length > 0) + || (Array.isArray(delta.reasoning_details) && delta.reasoning_details.some(detail => + isRec(detail) && typeof detail.text === "string" && detail.text.length > 0)) + || (typeof delta.refusal === "string" && delta.refusal.length > 0) + || (Array.isArray(delta.tool_calls) && delta.tool_calls.some(tool => { + if (!isRec(tool)) return false; + const fn = isRec(tool.function) ? tool.function : {}; + return (typeof tool.id === "string" && tool.id.length > 0) + || (typeof fn.name === "string" && fn.name.length > 0) + || (typeof fn.arguments === "string" && fn.arguments.length > 0); + })); + }); + if (hasProgress || finished) pullDeadline = performance.now() + stallMs; + else if (performance.now() >= pullDeadline) throw stalled; + if (!firstOutput && hasProgress) { firstOutput = true; options.onFirstOutput?.(); } @@ -286,6 +292,7 @@ export function nativeChatSse( function onAbort() { cancelledBySignal = true; + releaseBuffer(); if (!settled) { settled = true; options.onCancel?.(); @@ -298,15 +305,39 @@ export function nativeChatSse( return new ReadableStream({ async pull(controller) { releaseQueued(); + pullDeadline = performance.now() + remainingStallMs; try { for (;;) { - const next = nextSseBlock(buffer); + if (cancelledBySignal) { + releaseBuffer(); + controller.close(); + return; + } + const next = buffer.next(); if (next) { - replaceBuffer(next.rest); + buffer.compact(); processBlock(controller, next.block, next.delimiter); return; } - const { done, value } = await reader.read(); + if (performance.now() >= pullDeadline) throw stalled; + let timeout: ReturnType | undefined; + let read: Awaited>; + try { + read = await Promise.race([ + reader.read(), + new Promise((_, reject) => { + const check = () => { + const remaining = pullDeadline - performance.now(); + if (remaining <= 0) reject(stalled); + else timeout = setTimeout(check, Math.min(remaining, 2_147_483_647)); + }; + check(); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + const { done, value } = read; if (cancelled) return; if (cancelledBySignal) { releaseBuffer(); @@ -314,11 +345,11 @@ export function nativeChatSse( return; } if (!done) { - appendBuffer(decoder.decode(value, { stream: true })); + buffer.append(decoder.decode(value, { stream: true })); continue; } - appendBuffer(decoder.decode()); - if (buffer.trim().length > 0) { + buffer.append(decoder.decode()); + if (buffer.tail().trim().length > 0) { fail(controller, "upstream SSE ended with an unterminated event", "upstream_sse_unterminated"); return; } @@ -333,13 +364,20 @@ export function nativeChatSse( return; } } catch (error) { + if (cancelled || cancelledBySignal) { + releaseBuffer(); + try { controller.close(); } catch { /* already closed */ } + return; + } const overflow = isTranslatorBudgetExceededError(error) || (error instanceof Error && (error.cause as { code?: unknown } | undefined)?.code === "translation_buffer_limit"); fail( controller, overflow ? "upstream SSE event exceeded the safe limit" : error instanceof Error ? error.message : String(error), - overflow ? "translation_buffer_limit" : "upstream_sse_error", + overflow ? "translation_buffer_limit" : error === stalled ? "upstream_stall_timeout" : "upstream_sse_error", ); + } finally { + remainingStallMs = Math.max(0, pullDeadline - performance.now()); } }, cancel(reason) { diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 9fb24e143a..8cc633665c 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -262,7 +262,7 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio retainedRequestBytes = 0; }; const retainRequest = (request: AdapterRequest) => { - const bytes = new TextEncoder().encode(request.body).byteLength; + const bytes = Buffer.byteLength(request.body); translatorBudget.chargeRetained(bytes, { kind: "request_copies" }); retainedRequestBytes = bytes; }; @@ -501,24 +501,29 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; if (contentType.includes("text/event-stream") && response.body) { if (requestedStream) transferTurnToStream(); + let terminalStatus: number | undefined; const stream = nativeChatSse(response.body, { requestedModel, translatorBudget, signal: upstream.signal, + stallTimeoutSec: config.stallTimeoutSec, onFirstOutput: logIds ? () => recordFirstOutput(logCtx, logIds.start) : undefined, onUsage: usage => { logCtx.usage = usage; attempt.usage = usage; }, + onTerminal: (status: number, message?: string) => { + terminalStatus = status; + if (!requestedStream) return; + try { + cleanupAbort(); + finishLog(status, message, "terminal"); + if (status >= 400) upstream.abort(); + } finally { + releaseStreamTurn(); + } + }, ...(requestedStream ? { - onTerminal: (status: number, message?: string) => { - try { - cleanupAbort(); - finishLog(status, message, "terminal"); - } finally { - releaseStreamTurn(); - } - }, onCancel: () => { try { cleanupAbort(); @@ -543,11 +548,20 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio try { const completion = await collectChatCompletion(stream, requestedModel, translatorBudget); cleanupAbort(); + // A cancelled native relay closes its downstream body. EOF alone must not + // promote the buffered prefix to a successful Chat completion. A terminal + // already accepted by the relay retains precedence over a later abort. + if (req.signal.aborted && terminalStatus === undefined) { + return fail(499, "Client cancelled request", "client_cancelled"); + } finishLog(200); return Response.json(completion); } catch (error) { cleanupAbort(); upstream.abort(); + if (req.signal.aborted && terminalStatus === undefined) { + return fail(499, "Client cancelled request", "client_cancelled"); + } if (isChatCompletionsStreamError(error)) { return fail(error.status, error.message, error.type, error.code); } diff --git a/tests/responses/chat-completions-endpoint.test.ts b/tests/responses/chat-completions-endpoint.test.ts index b622027a97..f39857ca0a 100644 --- a/tests/responses/chat-completions-endpoint.test.ts +++ b/tests/responses/chat-completions-endpoint.test.ts @@ -1,5 +1,4 @@ -import { waitForNativeMainStartupGate } from "../../src/codex/native-profile-startup"; -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -1758,6 +1757,186 @@ test("chat-native request abort releases its active-turn lease and logs 499", as } }); +test("chat-native cancelled non-streaming SSE returns 499 instead of partial success", async () => { + const { handleChatCompletions } = await import("../../src/server/chat-completions"); + const clientAbort = new AbortController(); + let readStarted!: () => void; + const started = new Promise(resolve => { readStarted = resolve; }); + globalThis.fetch = (async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"partial"}}]}\n\n')); + readStarted(); + }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const result = handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", signal: clientAbort.signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), + }), mockConfig("https://provider.example/v1"), {} as Parameters[2]); + await started; + await Bun.sleep(10); + clientAbort.abort("client done"); + const response = await result; + expect(response.status).toBe(499); + expect(await response.json()).toMatchObject({ error: { type: "client_cancelled" } }); +}); + +test("chat-native SSE enforces configured stall timeout despite non-progress frames", async () => { + const { handleChatCompletions } = await import("../../src/server/chat-completions"); + const { clearRequestLogsForTests, getRequestLogEntries } = await import("../../src/server/request-log"); + clearRequestLogsForTests(); + const clientAbort = new AbortController(); + let timer: ReturnType | undefined; + let cancels = 0; + const before = getActiveTurnCount(); + const releaseMisses = activeRegistryMetrics().activeTurns.releaseMisses; + const lease = tryAdmitTurn(); + if (!lease) throw new Error("failed to admit native Chat stall test turn"); + globalThis.fetch = (async () => new Response(new ReadableStream({ + start(controller) { + const wire = new TextEncoder().encode(': heartbeat\n\ndata\n\ndata:\n\ndata: {"choices":[{"delta":{"role":"assistant","content":"","tool_calls":[{"index":0}]}}],"usage":{"prompt_tokens":1,"completion_tokens":0}}\n\n'); + controller.enqueue(wire); + timer = setInterval(() => controller.enqueue(wire), 100); + }, + cancel() { cancels += 1; clearInterval(timer); }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + try { + const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", signal: clientAbort.signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", stream: true, messages: [{ role: "user", content: "hi" }] }), + }), { ...mockConfig("https://provider.example/v1"), stallTimeoutSec: 1 }, {} as Parameters[2], + { requestId: "chat-stall-clock", start: Date.now(), turnAdmissionLease: lease }); + const outcome = await Promise.race([ + response.text(), Bun.sleep(1_600).then(() => "STILL_PENDING"), + ]); + expect(outcome).toContain('"code":"upstream_stall_timeout"'); + expect(outcome).not.toContain("[DONE]"); + clientAbort.abort("late cancellation"); + expect(cancels).toBe(1); + expect(getActiveTurnCount()).toBe(before); + expect(activeRegistryMetrics().activeTurns.releaseMisses).toBe(releaseMisses); + expect(getRequestLogEntries().filter(entry => entry.requestId === "chat-stall-clock")).toHaveLength(1); + expect(getRequestLogEntries().at(-1)?.status).toBe(502); + } finally { clientAbort.abort(); clearInterval(timer); lease.release(); } +}); + +test("chat-native meaningful reasoning keeps a long stream alive and pauses the stall clock under backpressure", async () => { + const { nativeChatSse } = await import("../../src/server/chat-native-sse"); + const budget = createTestTranslatorBudget(); + const abort = new AbortController(); + const encoder = new TextEncoder(); + let source!: ReadableStreamDefaultController; + const body = new ReadableStream({ start(controller) { source = controller; } }); + const stream = nativeChatSse(body, { + requestedModel: "mock/test-model", translatorBudget: budget, signal: abort.signal, + stallTimeoutSec: 1, onUsage() {}, + }); + const reasoning = 'data: {"choices":[{"delta":{"reasoning_content":"thinking"}}]}\n\n'; + source.enqueue(encoder.encode(reasoning)); + // One queued output applies backpressure. That interval is not upstream silence. + await Bun.sleep(1_100); + const output = new Response(stream).text(); + try { + for (let index = 0; index < 3; index++) { + await Bun.sleep(450); + const delta = index === 0 ? { reasoning: "thinking" } + : index === 1 ? { reasoning_details: [{ text: "thinking" }] } + : { reasoning_content: "thinking" }; + source.enqueue(encoder.encode(`data: ${JSON.stringify({ choices: [{ delta }] })}\n\n`)); + } + source.enqueue(encoder.encode('data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n')); + const text = await output; + expect(text.match(/thinking/g)).toHaveLength(4); + expect(text).toContain("[DONE]"); + expect(text).not.toContain("upstream_stall_timeout"); + } finally { abort.abort(); budget.dispose(); } +}); + +test("chat-native valid terminal retains precedence over a late non-streaming abort", async () => { + const { handleChatCompletions } = await import("../../src/server/chat-completions"); + const clientAbort = new AbortController(); + globalThis.fetch = (async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"complete"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n')); + }, + cancel() { clientAbort.abort("late cancellation after terminal"); }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", signal: clientAbort.signal, headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), + }), mockConfig("https://provider.example/v1"), {} as Parameters[2]); + expect(clientAbort.signal.aborted).toBe(true); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ choices: [{ message: { content: "complete" } }] }); +}); + +test("chat-native non-streaming SSE reports a typed stall failure instead of partial success", async () => { + const { handleChatCompletions } = await import("../../src/server/chat-completions"); + let cancels = 0; + globalThis.fetch = (async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"partial"}}]}\n\n')); + }, + cancel() { cancels += 1; }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), + }), { ...mockConfig("https://provider.example/v1"), stallTimeoutSec: 1 }, {} as Parameters[2]); + expect(response.status).toBe(502); + expect(await response.json()).toMatchObject({ error: { type: "upstream_error", code: "upstream_stall_timeout" } }); + expect(cancels).toBe(1); +}); + +test("chat-native SSE only encodes outbound frames and releases buffered tail on cancellation", async () => { + const { nativeChatSse } = await import("../../src/server/chat-native-sse"); + const frames = Array.from({ length: 64 }, () => 'data: {"choices":[{"delta":{"content":"中文😀"}}]}\n\n'); + const wire = new TextEncoder().encode(frames.join("")); + const budget = createTestTranslatorBudget(); + const abort = new AbortController(); + const body = new ReadableStream({ start(controller) { controller.enqueue(wire); } }); + const encode = spyOn(TextEncoder.prototype, "encode"); + try { + const stream = nativeChatSse(body, { + requestedModel: "mock/test-model", translatorBudget: budget, signal: abort.signal, onUsage() {}, + }); + const reader = stream.getReader(); + for (let index = 0; index < 8; index++) expect((await reader.read()).done).toBe(false); + await reader.cancel("done inspecting"); + // No append/suffix-size encoding: every actual encoding is a normalized output frame. + expect(encode.mock.calls.length).toBeGreaterThanOrEqual(8); + for (const [text] of encode.mock.calls) expect(text).toContain('"object":"chat.completion.chunk"'); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { encode.mockRestore(); abort.abort(); budget.dispose(); } +}); + +test("chat-native non-streaming SSE collects CRLF multiline and split UTF-8 frames", async () => { + const { handleChatCompletions } = await import("../../src/server/chat-completions"); + const frames = [ + 'data\r\n\r\ndata:\r\n\r\n', + 'data: {"choices":\r\ndata\r\ndata: [{"delta":{"content":"中文😀"}}]}\r\n\r\n', + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":7,"completion_tokens":3}}\r\n\r\n', + 'data: [DONE]\r\n\r\n', + ]; + const wire = new TextEncoder().encode(frames.join("")); + globalThis.fetch = (async () => new Response(new ReadableStream({ + start(controller) { + for (let offset = 0; offset < wire.byteLength; offset += 7) controller.enqueue(wire.slice(offset, offset + 7)); + controller.close(); + }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), + }), mockConfig("https://provider.example/v1"), {} as Parameters[2]); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + choices: [{ message: { content: "中文😀" }, finish_reason: "stop" }], + usage: { prompt_tokens: 7, completion_tokens: 3 }, + }); +}); + test("chat-native direct streaming without an admission lease does not record a release miss", async () => { const { handleChatCompletions } = await import("../../src/server/chat-completions"); const releaseMisses = activeRegistryMetrics().activeTurns.releaseMisses; @@ -2336,6 +2515,55 @@ test("collectChatCompletion releases every call scope after the final owner is c ); }); +test("collectChatCompletion accounts split surrogate content incrementally and releases it on failure", async () => { + const { collectChatCompletion } = await import("../../src/chat/outbound"); + for (const fail of [false, true]) { + const budget = createTestTranslatorBudget(); + const frames = [ + { choices: [{ delta: { content: "\ud83d", reasoning_content: "\ud83d", refusal: "\ud83d" } }] }, + { choices: [{ delta: { content: "\ude00", reasoning_content: "\ude00", refusal: "\ude00" } }] }, + ...(fail ? [{ error: { message: "upstream request failed", type: "upstream_error", code: "upstream_failure" } }] + : [{ choices: [{ delta: {}, finish_reason: "stop" }] }]), + ]; + const stream = new ReadableStream({ + start(controller) { + for (const frame of frames) controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(frame)}\n\n`)); + controller.close(); + }, + }); + try { + if (fail) { + await expect(collectChatCompletion(stream, "mock/test-model", budget)).rejects.toMatchObject({ code: "upstream_failure" }); + expect(budget.snapshot().currentBytes).toBe(0); + } else { + expect(await collectChatCompletion(stream, "mock/test-model", budget)).toMatchObject({ + choices: [{ message: { content: "😀", reasoning_content: "😀", refusal: "😀" } }], + }); + expect(budget.snapshot().currentBytes).toBe(12); + } + } finally { budget.dispose(); } + } +}); + +test("collectChatCompletion preserves admission for a 20 MiB frame and discards an incomplete EOF frame", async () => { + const { collectChatCompletion } = await import("../../src/chat/outbound"); + const text = "x".repeat(20 * 1024 * 1024); + const completeFrame = `data: ${JSON.stringify({ choices: [{ delta: { content: text }, finish_reason: "stop" }] })}\n\n`; + const incompleteFrame = 'data: {"choices":[{"delta":{"content":"discard me"}}]}'; + const budget = createTestTranslatorBudget(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(completeFrame)); + controller.enqueue(new TextEncoder().encode(incompleteFrame)); + controller.close(); + }, + }); + try { + expect(await collectChatCompletion(stream, "mock/test-model", budget)).toMatchObject({ choices: [{ message: { content: text } }] }); + expect(budget.snapshot().currentBytes).toBe(Buffer.byteLength(text)); + } finally { budget.dispose(); } +}); + test("collectChatCompletion final-copy overflow cleans up scopes and charges", async () => { const module = await import("../../src/chat/outbound"); // Args (100 bytes) fit; args + serialized copy exceed the turn cap, so the From 99337bd9858e79d08f4e2d5bf85aed1ff5bec4f6 Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 00:16:41 -0700 Subject: [PATCH 09/13] docs: describe stream accounting and native Chat lifecycle (cherry picked from commit d7f27e3ed77e19a5e0c290eb8919738db1cc1ce1) --- .../docs/fr/reference/configuration/server.md | 6 +++++- .../docs/ja/reference/configuration/server.md | 6 +++++- .../docs/ko/reference/configuration/server.md | 6 +++++- .../docs/reference/configuration/server.md | 6 +++++- .../docs/ru/reference/configuration/server.md | 6 +++++- .../docs/tr/reference/configuration/server.md | 6 +++++- .../zh-cn/reference/configuration/server.md | 6 +++++- .../zh-tw/reference/configuration/server.md | 6 +++++- structure/adapters/registry.md | 2 +- structure/catalog.md | 2 +- structure/clients/claude-desktop.md | 2 +- structure/clients/integrations.md | 2 ++ structure/data-planes/images.md | 2 +- structure/data-planes/inbound-compat.md | 15 +++++++++++++- structure/gui-and-management-api.md | 2 +- structure/ops/docs-and-release.md | 2 ++ structure/ops/service-and-sidecars.md | 2 +- structure/overview.md | 2 ++ structure/providers/chat-compat.md | 2 ++ structure/providers/cursor.md | 2 ++ structure/providers/xai-grok.md | 2 +- structure/runtime.md | 2 +- structure/subagents.md | 2 +- structure/transports/inventory.md | 2 +- structure/transports/responses.md | 20 +++++++++++++++++++ structure/transports/streaming-health.md | 6 +++++- 26 files changed, 99 insertions(+), 20 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/configuration/server.md b/docs-site/src/content/docs/fr/reference/configuration/server.md index d957d9b50f..cde1b2c5f6 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/server.md +++ b/docs-site/src/content/docs/fr/reference/configuration/server.md @@ -14,7 +14,7 @@ exécute des fonctionnalités d'assistance autour des demandes du fournisseur. | `hostname?` | `string` | `"127.0.0.1"` | Adresse de liaison. Les liaisons hors bouclage nécessitent `OPENCODEX_API_AUTH_TOKEN`. | | `proxy?` | `string` | — | URL du proxy HTTP(S) sortant ou `${ENV_VAR}`. Appliquée à `HTTP_PROXY` / `HTTPS_PROXY` uniquement lorsque ces variables ne sont pas définies ; le bouclage reste dans `NO_PROXY`. | | `emptyCompletionRetry?` | `boolean` | `false` | Active une nouvelle tentative Responses identique lorsqu’une réponse ne contient ni texte ni appel d’outil. Cette tentative peut être facturée. `OCX_EMPTY_COMPLETION_RETRY=0` la désactive sans modifier la configuration ; les combinaisons et les tours de compactage routés restent exclus. | -| `stallTimeoutSec?` | `number` | `300` | Nombre de secondes sans données en amont avant `response.incomplete`. Minimum : 1. | +| `stallTimeoutSec?` | `number` | `300` | Secondes sans progression utile en amont, pour Responses et le Chat natif. Minimum : 1. | | `connectTimeoutMs?` | `number` | `200000` | Délai maximal par tentative pour DNS/TCP/TLS et les en-têtes finaux ; il prend fin avant la génération du corps. | | `shutdownTimeoutMs?` | `number` | `5000` | Délai de vidange gracieux avant l’annulation des tours actifs. | | `websockets?` | `boolean` | `false` | Annonce et autorise la route WebSocket Responses destinée aux clients. La valeur false maintient les clients sur HTTP/SSE ; elle ne désactive pas une optimisation WebSocket canonique admissible vers ChatGPT en amont. | @@ -34,6 +34,10 @@ Si une ancienne version de développement a modifié les métadonnées de l'hist `ocx recover-history --legacy-openai --yes` pour forcer la récupération du fournisseur natif. La commande réétiquette chaque ligne `opencodex` contenant un message utilisateur, y compris l'historique légitime d'un fournisseur dédié ; consultez l'avertissement sur la portée complète dans la référence du cycle de vie avant de l'exécuter. +### Délais et fin de réponse du Chat natif + +Le Chat natif utilise aussi `stallTimeoutSec` pendant l’attente de la sortie amont. Le texte non vide, le raisonnement, le refus, les mises à jour d’outils et les événements de fin renouvellent ce délai ; les commentaires de maintien de connexion, le rôle seul et les statistiques seules ne le renouvellent pas. L’attente d’un client lent suspend le décompte. Un blocage produit `upstream_stall_timeout` : un événement d’erreur en streaming, ou HTTP 502 sans streaming. Une annulation avant le résultat terminal renvoie une erreur d’annulation plutôt qu’une réponse partielle réussie. Le Chat sans streaming accepte les délimiteurs SSE LF et CRLF et les champs data multilignes. + ## Accès à distance La liaison par défaut à `127.0.0.1` est limitée au bouclage. Une adresse hors bouclage telle que `0.0.0.0` diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index e86387f80e..9ce4c6d878 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -14,7 +14,7 @@ description: リスナー、リモート アクセス、アドミッション | `proxy?` | `string` | — |送信 HTTP(S) プロキシ URL または `${ENV_VAR}`。これらの変数が設定されていない場合にのみ、`HTTP_PROXY` / `HTTPS_PROXY` に適用されます。ループバックは `NO_PROXY` に残ります。 | | `emptyCompletionRetry?` | `boolean` | `false` | テキストもツール呼び出しもない Responses ターンを、ターミナルイベント前にストリームが終了した場合も含め、同一リクエストで 1 回再試行するよう明示的に有効化します。再試行は課金対象になる場合があります。`OCX_EMPTY_COMPLETION_RETRY=0` で設定を変更せず無効化できます。combo と routed-compaction turn は対象外です。 | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Codex Responses パススルーから Codex の safety-buffering ヒントを除去します。対象は `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` 応答ヘッダー、`safety_buffering` 型の `response.metadata` SSE イベント、およびその他の SSE イベントにある `safety_buffering` フィールドです。Codex TUI はこれらを、既定の操作でセッションをより弱いモデルに切り替える「より高速なモデルで再試行」プロンプトとして表示します。その他の `x-codex-*` ヘッダーと SSE イベントの内容は、そのフィールドの除去を除いて変更せずに転送されます。既定ではオフです。 | -| `stallTimeoutSec?` | `number` | `300` | `response.incomplete` より前にアップストリーム データがない秒数。最小 1。 +| `stallTimeoutSec?` | `number` | `300` | Responses とネイティブ Chat の有効な上流進捗がない秒数。最小 1 秒。 | | `connectTimeoutMs?` | `number` | `200000` |試行ごとの DNS/TCP/TLS/最終ヘッダーの期限。本体が生成される前に終了します。 | | `shutdownTimeoutMs?` | `number` | `5000` |アクティブなターンが中止される前の正常な排出期限。 | | `websockets?` | `boolean` | `false` | クライアント向け Responses WebSocket パスを広告して許可します。false の場合クライアントは HTTP/SSE を使いますが、対象となる canonical ChatGPT upstream WS 最適化は無効にしません。 | @@ -33,6 +33,10 @@ description: リスナー、リモート アクセス、アドミッション バックアップ サポートが存在する前に古い開発ビルドで再開履歴メタデータが変更された場合は、`ocx recover-history --legacy-openai --yes` を実行してネイティブ プロバイダーの回復を強制します。 このコマンドは、正当な専用プロバイダー履歴を含む、ユーザーメッセージを持つすべての `opencodex` 行を再ラベル付けします。実行前にライフサイクル リファレンスの全範囲に関する警告を確認してください。 +### ネイティブ Chat のタイムアウトと完了 + +ネイティブ Chat も上流出力の待機に `stallTimeoutSec` を使用します。空でないテキスト、推論、拒否内容、ツール更新、完了イベントは待機時間を更新しますが、キープアライブのコメント、ロールのみのイベント、使用量のみのイベントは更新しません。低速クライアントの読み取り待ちは計時を停止します。タイムアウト時は `upstream_stall_timeout` が返り、ストリーミングではエラーイベント、非ストリーミングでは HTTP 502 になります。終端結果より前のキャンセルは成功した部分回答ではなくキャンセルエラーになります。非ストリーミング Chat は LF、CRLF、複数行 data の SSE に対応します。 + ## リモートアクセス デフォルトの `127.0.0.1` バインドはループバックのみです。 `0.0.0.0` などの非ループバック アドレスには、`/api/*` とデータ プレーンの両方でトークン認証が必要です。開始する前にトークンをエクスポートします。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index 5f60fc7c6c..5727ba13d4 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -14,7 +14,7 @@ description: 리스너, 원격 접근, admission 키, 타임아웃, 저장소, | `proxy?` | `string` | — | 송신용 HTTP(S) 프록시 URL 또는 `${ENV_VAR}`입니다. 해당 변수가 비어 있을 때만 `HTTP_PROXY` / `HTTPS_PROXY`에 적용되며, 루프백은 `NO_PROXY`에 그대로 남습니다. | | `emptyCompletionRetry?` | `boolean` | `false` | 텍스트나 도구 호출이 없는 Responses 턴을, 터미널 이벤트 전에 스트림이 종료된 경우를 포함해 동일한 요청으로 한 번 재시도하도록 선택합니다. 재시도에는 비용이 발생할 수 있습니다. `OCX_EMPTY_COMPLETION_RETRY=0`은 설정을 바꾸지 않고 비활성화하며, combo 및 routed-compaction turn은 제외됩니다. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Canonical Codex Responses 응답의 선택적 safety-buffering 헤더 두 개와 SSE 힌트를 제거합니다. 공급자의 안전 정책이나 거절 응답은 바뀌지 않습니다. Native WS 메타데이터와 compact는 제외됩니다. | -| `stallTimeoutSec?` | `number` | `300` | 업스트림 데이터가 없을 때 `response.incomplete`가 되기까지의 초 수입니다. 최소 1입니다. | +| `stallTimeoutSec?` | `number` | `300` | Responses 및 네이티브 Chat에서 유효한 업스트림 진행이 없는 시간(초). 최소 1초. | | `connectTimeoutMs?` | `number` | `200000` | 시도별 DNS/TCP/TLS/최종 헤더 기한입니다. 본문 생성 전에 끝납니다. | | `shutdownTimeoutMs?` | `number` | `5000` | 진행 중인 turn을 중단하기 전에 허용하는 정상 종료 드레인 기한입니다. | | `websockets?` | `boolean` | `false` | 클라이언트용 Responses WebSocket 경로를 광고하고 허용합니다. `false`이면 클라이언트는 HTTP/SSE를 사용하며, 적격 canonical ChatGPT 업스트림 WS 최적화는 비활성화하지 않습니다. | @@ -33,6 +33,10 @@ description: 리스너, 원격 접근, admission 키, 타임아웃, 저장소, 오래된 개발 빌드가 백업 지원이 생기기 전에 resume-history 메타데이터를 바꿨다면, native-provider 복구를 강제로 수행하려면 `ocx recover-history --legacy-openai --yes`를 실행합니다. 이 명령은 정상적인 dedicated-provider history를 포함해 사용자 메시지가 있는 모든 `opencodex` row를 재태깅합니다. 실행하기 전에 lifecycle reference의 전체 범위 경고를 확인하세요. +### 네이티브 Chat 시간 초과와 완료 + +네이티브 Chat도 업스트림 출력을 기다릴 때 `stallTimeoutSec`를 사용합니다. 비어 있지 않은 텍스트, 추론, 거부 내용, 도구 업데이트 및 완료 이벤트는 대기 시간을 갱신하지만 연결 유지 주석, 역할만 있는 이벤트, 사용량만 있는 이벤트는 갱신하지 않습니다. 느린 클라이언트의 읽기를 기다리는 동안에는 시간이 차감되지 않습니다. 시간 초과 시 `upstream_stall_timeout`이 발생하며 스트리밍 요청은 오류 이벤트를, 비스트리밍 요청은 HTTP 502를 받습니다. 종료 결과 전에 취소하면 부분 답변을 성공으로 반환하지 않고 취소 오류를 반환합니다. 비스트리밍 Chat은 LF, CRLF 및 여러 줄 data SSE 형식을 지원합니다. + ## Remote access 기본 `127.0.0.1` 바인드는 루프백 전용입니다. `0.0.0.0`이나 tailnet IP처럼 루프백이 아닌 주소는 `/api/*`와 데이터 플레인 모두에서 토큰 인증이 필요합니다. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 3781d5846e..3fbf619df4 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -16,7 +16,7 @@ runs helper features around provider requests. | `noProxy?` | `string \| string[]` | — | Hosts that bypass `proxy`, merged with inherited `NO_PROXY` and loopback entries. A string may use comma-separated `NO_PROXY` syntax or `${ENV_VAR}`. | | `emptyCompletionRetry?` | `boolean` | `false` | Opt in to one identical Responses retry when a turn has no text or tool call, including a stream that ends before a terminal event. The retry may be billable. `OCX_EMPTY_COMPLETION_RETRY=0` disables it without changing config; combo and routed-compaction turns remain excluded. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Remove optional client-facing hints from canonical Codex Responses passthrough: the two `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` response headers, `response.metadata` events whose metadata type is `safety_buffering`, and top-level `safety_buffering` fields. Other headers, response data, policy refusals and failures are preserved. This does not disable provider safety enforcement or upstream buffering. Native `codex.response.metadata.headers` WebSocket metadata and `/responses/compact` are outside this filter. | -| `stallTimeoutSec?` | `number` | `300` | Seconds without upstream data before `response.incomplete`. Minimum 1. | +| `stallTimeoutSec?` | `number` | `300` | Seconds without meaningful upstream progress (Responses and native Chat). Minimum 1. | | `oauthOpenBrowser?` | `boolean` | `true` | Whether a login may open a browser on the machine running the proxy. Absent and `true` both open, so an existing install is unchanged; only an explicit `false` declines. Decline when you need the authorization link in a different browser profile, or when the dashboard is not on the proxy's machine — the login still starts and the URL is still returned and displayed. `POST /api/oauth/login` and `POST /api/codex-auth/login` accept a per-request `openBrowser` boolean that overrides this, and the dashboard exposes the same choice beside the login button. Device-code flows never open a browser either way. | | `connectTimeoutMs?` | `number` | `200000` | Per-attempt DNS/TCP/TLS/final-header deadline; it ends before body generation. | | `shutdownTimeoutMs?` | `number` | `5000` | Graceful drain deadline before active turns are aborted. | @@ -66,6 +66,10 @@ If an older development build changed resume-history metadata before backup supp It force-relabels every user-message `opencodex` row, including legitimate dedicated-provider history; review the full-scope warning in the lifecycle reference before running it. +### Native Chat timeouts and completion + +Native Chat also uses `stallTimeoutSec` while waiting for upstream output. Nonempty text, reasoning, refusal, tool updates, and finish frames renew the allowance; keepalive comments, role-only frames, and usage alone do not. Waiting for a slow client to read pauses the allowance. A stall produces `upstream_stall_timeout`: an error frame for streaming clients, or HTTP 502 for non-streaming clients. Cancellation before a terminal result returns a cancellation error instead of a successful partial answer. Buffered Chat results accept both LF and CRLF SSE framing, including multiline data. + ## Codex quota network diagnostics The main Codex account row may include `quotaRefresh` when a quota fetch was diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 000a194b5c..b2907d4d84 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -15,7 +15,7 @@ description: Listener, удалённый доступ, admission key, тайм | `proxy?` | `string` | — | URL исходящего HTTP(S)-прокси или `${ENV_VAR}`. Применяется к `HTTP_PROXY` / `HTTPS_PROXY` только когда эти переменные не заданы; loopback всегда остаётся в `NO_PROXY`. | | `emptyCompletionRetry?` | `boolean` | `false` | Явно включает один идентичный повтор Responses, если в turn нет ни текста, ни tool call, включая случай, когда stream завершается до terminal event. Повтор может тарифицироваться. `OCX_EMPTY_COMPLETION_RETRY=0` отключает его без изменения config; combo и routed-compaction turn исключены. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Удаляет подсказки Codex safety-buffering из passthrough-ответов Codex Responses: заголовки `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model`, SSE-события `response.metadata` типа `safety_buffering` и поле `safety_buffering` в других SSE-событиях. Codex TUI отображает их как предложение повторить запрос с более быстрой моделью, действие по умолчанию в котором переключает сессию на более слабую модель. Остальные заголовки `x-codex-*` и содержимое других SSE-событий передаются без изменений, кроме удаления этого поля. По умолчанию выключено. | -| `stallTimeoutSec?` | `number` | `300` | Секунды без upstream-данных до `response.incomplete`. Минимум 1. | +| `stallTimeoutSec?` | `number` | `300` | Секунды без полезного прогресса upstream для Responses и нативного Chat. Минимум 1. | | `connectTimeoutMs?` | `number` | `200000` | Дедлайн одной попытки DNS/TCP/TLS/final-header; он завершается до генерации тела ответа. | | `shutdownTimeoutMs?` | `number` | `5000` | Дедлайн graceful-drain до принудительного прерывания активных turn'ов. | | `websockets?` | `boolean` | `false` | Объявляет и разрешает клиентский WebSocket-путь Responses. При false клиенты используют HTTP/SSE; это не отключает подходящую upstream WS-оптимизацию canonical ChatGPT. | @@ -36,6 +36,10 @@ backup'а, выполните `ocx recover-history --legacy-openai --yes`, чт native-provider history. Команда переименовывает все строки `opencodex` с пользовательским сообщением, включая корректную историю выделенного провайдера; перед запуском прочитайте предупреждение о полном охвате в справочнике lifecycle. +### Тайм-ауты и завершение нативного Chat + +Нативный Chat также использует `stallTimeoutSec` при ожидании вывода upstream. Непустой текст, рассуждения, отказ, обновления инструментов и события завершения обновляют время ожидания; комментарии keepalive, только роль и только статистика использования его не обновляют. Ожидание чтения медленным клиентом приостанавливает отсчёт. При зависании возникает `upstream_stall_timeout`: событие ошибки для потокового клиента или HTTP 502 без потоковой передачи. Отмена до конечного результата возвращает ошибку отмены вместо успешного частичного ответа. Непотоковый Chat поддерживает SSE с LF, CRLF и многострочными полями data. + ## Удалённый доступ По умолчанию bind `127.0.0.1` доступен только на loopback. Не-loopback-адрес, например diff --git a/docs-site/src/content/docs/tr/reference/configuration/server.md b/docs-site/src/content/docs/tr/reference/configuration/server.md index 439cd70606..880af54edb 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/server.md +++ b/docs-site/src/content/docs/tr/reference/configuration/server.md @@ -15,7 +15,7 @@ yardımcı özellikleri nasıl çalıştıracağını kontrol eder. | `hostname?` | `string` | `"127.0.0.1"` | Bağlama adresi. Geri döngü olmayan bağlamalar `OPENCODEX_API_AUTH_TOKEN` gerektirir. | | `proxy?` | `string` | — | Giden HTTP(S) proxy URL'si veya `${ENV_VAR}`. Yalnızca bu değişkenler ayarlanmadığında `HTTP_PROXY` / `HTTPS_PROXY`'ye uygulanır; geri döngü `NO_PROXY` içinde kalır. | | `emptyCompletionRetry?` | `boolean` | `false` | Metin veya araç çağrısı içermeyen bir Responses tamamlamasını aynı istekle bir kez yeniden denemeyi açıkça etkinleştirir. Yeniden deneme ücretlendirilebilir. `OCX_EMPTY_COMPLETION_RETRY=0`, yapılandırmayı değiştirmeden devre dışı bırakır; combo ve routed-compaction turları hariçtir. | -| `stallTimeoutSec?` | `number` | `300` | `response.incomplete` öncesinde yukarı akış verisi olmadan geçen saniye. Minimum 1. | +| `stallTimeoutSec?` | `number` | `300` | Responses ve yerel Chat için anlamlı üst sunucu ilerlemesi olmadan geçen saniye. En az 1. | | `connectTimeoutMs?` | `number` | `200000` | Deneme başına DNS/TCP/TLS/nihai başlık son tarihi; gövde üretiminden önce biter. | | `shutdownTimeoutMs?` | `number` | `5000` | Aktif turlar iptal edilmeden önce zarif boşaltma süresi sınırı. | | `websockets?` | `boolean` | `false` | Responses WebSocket yolu için `supports_websockets` bildirin. False, HTTP/SSE'yi tutar. | @@ -36,6 +36,10 @@ geçmişi meta verilerini değiştirdiyse yerel sağlayıcı kurtarmasını zorl `ocx recover-history --legacy-openai --yes` çalıştırın. Komut, geçerli dedicated-provider geçmişi de dahil olmak üzere kullanıcı iletisi bulunan tüm `opencodex` satırlarını yeniden etiketler; çalıştırmadan önce lifecycle başvurusundaki tam kapsam uyarısını okuyun. +### Yerel Chat zaman aşımı ve tamamlanma + +Yerel Chat de üst sunucu çıktısını beklerken `stallTimeoutSec` kullanır. Boş olmayan metin, akıl yürütme, ret içeriği, araç güncellemeleri ve bitiş olayları süreyi yeniler; bağlantıyı canlı tutan yorumlar, yalnızca rol ve yalnızca kullanım bilgileri yenilemez. Yavaş istemcinin okumasını beklemek süreyi duraklatır. Zaman aşımı `upstream_stall_timeout` üretir: akış istemcileri hata olayı, akışsız istemciler HTTP 502 alır. Sonuç tamamlanmadan iptal edilen istek, başarılı bir kısmi yanıt yerine iptal hatası döndürür. Akışsız Chat, LF ve CRLF ayraçlarını ve çok satırlı data alanlarını destekler. + ## Uzaktan erişim Varsayılan `127.0.0.1` bağlaması yalnızca geri döngüdür. `0.0.0.0` gibi geri diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 8e8050de64..13513481f0 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -15,7 +15,7 @@ description: 监听、远程访问、准入密钥、超时、存储、侧车、 | `proxy?` | `string` | — | 出站 HTTP(S) 代理 URL,或 `${ENV_VAR}`。仅当 `HTTP_PROXY` / `HTTPS_PROXY` 未设置时才会应用;回环地址始终保留在 `NO_PROXY` 中。 | | `emptyCompletionRetry?` | `boolean` | `false` | 显式启用:当 Responses turn 既无文本也无工具调用时,使用相同请求重试一次,包括流在终止事件之前结束的情况。重试可能产生费用。`OCX_EMPTY_COMPLETION_RETRY=0` 可在不修改配置的情况下禁用;combo 与 routed-compaction turn 不参与。 | | `dropCodexSafetyBuffering?` | `boolean` | `false` | 从 Codex Responses 透传响应中移除 Codex safety-buffering 提示:`x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` 响应头、类型为 `safety_buffering` 的 `response.metadata` SSE 事件,以及其他 SSE 事件中的 `safety_buffering` 字段。Codex TUI 会将这些提示显示为“使用更快模型重试”的提示框,其默认操作会把会话切换到较弱的模型。其他 `x-codex-*` 响应头和其他所有 SSE 事件内容均保持不变,但会移除该字段。默认关闭。 | -| `stallTimeoutSec?` | `number` | `300` | 在上游没有数据之前可等待的秒数,超过后返回 `response.incomplete`。最小值为 1。 | +| `stallTimeoutSec?` | `number` | `300` | 上游无有效进展的秒数,适用于 Responses 和原生 Chat;最小 1 秒。 | | `connectTimeoutMs?` | `number` | `200000` | 每次尝试的 DNS/TCP/TLS/最终响应头截止时间;它在正文生成之前结束。 | | `shutdownTimeoutMs?` | `number` | `5000` | 优雅停机截止时间,超过后会中止仍在进行中的请求。 | | `websockets?` | `boolean` | `false` | 声明并允许面向客户端的 Responses WebSocket 路径。设为 false 时客户端使用 HTTP/SSE;它不会禁用符合条件的 canonical ChatGPT 上游 WS 优化。 | @@ -35,6 +35,10 @@ description: 监听、远程访问、准入密钥、超时、存储、侧车、 `ocx recover-history --legacy-openai --yes` 强制使用原生提供方恢复。 此命令会重标所有包含用户消息的 `opencodex` 行,其中包括正常的专用提供方历史记录;执行前请查看生命周期参考中的完整范围警告。 +### 原生 Chat 的超时与完成状态 + +原生 Chat 等待上游输出时也使用 `stallTimeoutSec`。非空文本、推理、拒绝内容、工具更新和完成事件会重置等待额度;保活注释、仅角色事件和单独的用量信息不会。等待慢客户端读取期间暂停计时。超时产生 `upstream_stall_timeout`:流式请求收到错误事件,非流式请求返回 HTTP 502。在终态结果到达前取消请求会返回取消错误,而不会把部分答案当作成功。非流式 Chat 支持 LF、CRLF 及多行 data 的 SSE 格式。 + ## 远程访问 默认的 `127.0.0.1` 绑定仅限回环地址。像 `0.0.0.0` 这样的非回环地址需要 diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md index 402015f060..87307ffe0b 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md @@ -13,7 +13,7 @@ description: 監聽器、遠端存取、許可金鑰、逾時、儲存、sidecar | `hostname?` | `string` | `"127.0.0.1"` | 綁定位址。非回送綁定需要 `OPENCODEX_API_AUTH_TOKEN`。 | | `proxy?` | `string` | — | 對外 HTTP(S) 代理 URL 或 `${ENV_VAR}`。僅在那些變數未設定時套用至 `HTTP_PROXY` / `HTTPS_PROXY`;回送保留在 `NO_PROXY` 中。 | | `emptyCompletionRetry?` | `boolean` | `false` | 明確啟用:當 Responses 完成時沒有文字或工具呼叫,以相同請求重試一次。重試可能產生費用。`OCX_EMPTY_COMPLETION_RETRY=0` 可在不變更設定的情況下停用;combo 與 routed-compaction turn 不適用。 | -| `stallTimeoutSec?` | `number` | `300` | 在 `response.incomplete` 前無上游資料的秒數。最小 1。 | +| `stallTimeoutSec?` | `number` | `300` | 上游無有效進展的秒數,適用於 Responses 與原生 Chat;最小 1 秒。 | | `connectTimeoutMs?` | `number` | `200000` | 每次嘗試的 DNS/TCP/TLS/final-header 截止時間;它在 body 生成前結束。 | | `shutdownTimeoutMs?` | `number` | `5000` | 在中止活躍回合前的優雅排空截止時間。 | | `websockets?` | `boolean` | `false` | 廣告並允許面向 client 的 Responses WebSocket 路徑。False 時 client 使用 HTTP/SSE;不會停用符合條件的 canonical ChatGPT upstream WS 最佳化。 | @@ -32,6 +32,10 @@ description: 監聽器、遠端存取、許可金鑰、逾時、儲存、sidecar 若較舊的開發組建在備份支援存在前變更了 resume-history 中繼資料,請執行 `ocx recover-history --legacy-openai --yes` 以強制原生供應商復原。 此命令會重新標記所有含有使用者訊息的 `opencodex` row,其中也包含正常的專用 provider 歷史;執行前請查看 lifecycle reference 中的完整範圍警告。 +### 原生 Chat 的逾時與完成狀態 + +原生 Chat 等待上游輸出時也使用 `stallTimeoutSec`。非空文字、推理、拒絕內容、工具更新及完成事件會重設等待額度;保活註解、僅角色事件及單獨的用量資訊不會。等待慢速用戶端讀取時暫停計時。逾時產生 `upstream_stall_timeout`:串流請求收到錯誤事件,非串流請求回傳 HTTP 502。終態結果到達前取消請求會回傳取消錯誤,不會將部分答案當成成功。非串流 Chat 支援 LF、CRLF 與多行 data 的 SSE 格式。 + ## 遠端存取 預設的 `127.0.0.1` 綁定僅限回送。如 `0.0.0.0` 的非回送位址需要在 `/api/*` 與 data plane 上都進行 token 認證。在啟動前匯出 token: diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 56a9666ef1..ca43aae480 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## Decision diff --git a/structure/catalog.md b/structure/catalog.md index fc8adf41ba..e02fb32379 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. ## Shared catalog diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index 31edca54d5..85bc0b1bfc 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Codex-native model discovery follows the [shared retirement policy](../catalog.md#shared-catalog). That projection does not migrate existing user-selected Desktop configuration or usage history. -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## Connected Claude Desktop profiles diff --git a/structure/clients/integrations.md b/structure/clients/integrations.md index 1c47462fab..fc797e95a5 100644 --- a/structure/clients/integrations.md +++ b/structure/clients/integrations.md @@ -1,5 +1,7 @@ # Client Integrations +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. + The client-integration subsystem writes one generated OpenCodex provider contribution into a third-party client's existing config without taking ownership of the rest of that file. Its core promise is reversibility: apply snapshots first, writes atomically, records exactly what it owns, diff --git a/structure/data-planes/images.md b/structure/data-planes/images.md index 314fafbceb..f50f92f122 100644 --- a/structure/data-planes/images.md +++ b/structure/data-planes/images.md @@ -7,7 +7,7 @@ Hosted Responses image-tool eligibility uses the shared compatibility policy wit Codex Spark exception; standalone Images retain the separate relay contract below. See [Responses transport](../transports/responses.md#responses-httpsse). -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## Standalone Images diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index 1aef6459ef..fce7ace064 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -47,7 +47,7 @@ Translated Claude timeline reminders use the Chat adapter's on its exact supported route. This is separate from trailing-notice stabilization and from native Chat message passthrough. -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## Chat Completions inbound native path @@ -99,6 +99,19 @@ request-signal cancellation contracts as routed Responses transport. Because call before it binds the adapter; the pick remains inert unless a strategy is configured and the committed key is cooling. See [`responses.md`](../transports/responses.md). +### Native Chat completion lifecycle + +`src/server/chat-native-sse.ts` applies the resolved `stallTimeoutSec` while waiting for upstream +progress. Nonempty text, reasoning, refusal, tool identity/arguments, and finish frames renew the +allowance; comments, role-only frames, empty deltas, and usage alone do not. Downstream backpressure +pauses this wait budget. A stall emits a Chat error with `upstream_stall_timeout` and logs 502; +the non-streaming endpoint returns HTTP 502 rather than a successful partial result. + +`src/chat/outbound.ts` collects LF/CRLF, multiline data, and split UTF-8 through the shared SSE +block buffer and tracks appended output bytes incrementally. A caller cancellation before a native +terminal returns 499 / `client_cancelled`; an already accepted terminal keeps its result. Reader, +timer, turn, and translator ownership are released through the existing lifecycle. + ## Chat conversation identity forwarding `src/server/chat-completions.ts` preserves caller `prompt_cache_key` on the Chat-to-Responses diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 9c8578c2f2..7f64b72b96 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. ## Dashboard serving diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 0aa5928944..5b2e5dc41c 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -3,6 +3,8 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. + ## Public docs The public documentation site lives in `docs-site/` and is built with Astro + Starlight. English is diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index f8ea6c7c0a..907c1154c0 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Service startup and restore use the [catalog retirement policy](../catalog.md#shared-catalog); retirement does not itself change service registration or user-selected model configuration. -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## Background service command selection diff --git a/structure/overview.md b/structure/overview.md index bde954b4b1..a96da8cbb9 100644 --- a/structure/overview.md +++ b/structure/overview.md @@ -3,6 +3,8 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. +Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. + ## Product boundary opencodex is a local proxy for Codex. It does not patch Codex binaries. It changes local Codex diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index cf830a8f1d..0856cba714 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -20,6 +20,8 @@ folding. This is independent of the Claude trailing-notice stabilization option and does not guarantee upstream cache hits. Regression coverage is in `tests/adapters/openai/openai-chat-system-order.test.ts`. +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. + ## Reasoning and tool-result compatibility Kiro groups only consecutive original-message tool results whose raw call ID exactly matches diff --git a/structure/providers/cursor.md b/structure/providers/cursor.md index ab030d8ae7..4bff45c0f6 100644 --- a/structure/providers/cursor.md +++ b/structure/providers/cursor.md @@ -10,6 +10,8 @@ namespace handling retain their provider contract; the bounded native scope live Cursor's direct adapter does not enter the OpenAI Chat serializer's [OpenCode Go instruction ordering](chat-compat.md#opencode-go-chronological-instructions). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. + ## Cursor Native Exec Cursor's experimental live transport can receive server-driven local read/write/delete/ls/grep, diff --git a/structure/providers/xai-grok.md b/structure/providers/xai-grok.md index e01d62a1bd..eb7bb47ae5 100644 --- a/structure/providers/xai-grok.md +++ b/structure/providers/xai-grok.md @@ -7,7 +7,7 @@ Codex-native retirement is scoped to OpenAI catalog/quota evidence. Shared Respo retains xAI provider behavior; see [the catalog boundary](../catalog.md#shared-catalog). -For shared JSON request-body parsing, see [request-copy accounting](../transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. ## xAI Grok hardening (official Grok Build contract parity) diff --git a/structure/runtime.md b/structure/runtime.md index 33238c301f..20478660ef 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -7,7 +7,7 @@ Chat request serialization owns the destination-scoped [OpenCode Go instruction ordering](providers/chat-compat.md#opencode-go-chronological-instructions); it requires no runtime lifecycle change or new configuration option. -For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. ## Entrypoints diff --git a/structure/subagents.md b/structure/subagents.md index 0270523ef6..93b3e65d33 100644 --- a/structure/subagents.md +++ b/structure/subagents.md @@ -25,7 +25,7 @@ Codex treats qualified names literally and defaults absent namespaces to functio declarations inherit their restored namespace container; the compiler never invents an empty encryption marker when the upstream omitted it or returned a nonempty marker. -For shared JSON request-body parsing, see [request-copy accounting](transports/responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. ## Multi-agent surface mode (3-state) diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index 9f13441275..3715536695 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior The Chat adapter's [OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions) changes translated message placement only; endpoint selection and transport stay with their existing owners. -For shared JSON request-body parsing, see [request-copy accounting](responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](responses.md#request-copy-accounting) and [stream-buffer accounting](responses.md#stream-buffer-accounting) contracts. ## Transport inventory diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 1a5b1e40d1..6863138649 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -25,6 +25,26 @@ hard byte cap. Admission limits, parsing, compression, and error envelopes are u `tests/usage/request-decompress.test.ts` covers exact accounting across codecs and Unicode/numeric normalization, UTF-8 counting without encoded copies, and release after malformed or optional empty input. +### Stream-buffer accounting + +`src/server/sse-payload-rewrite.ts` shares an incremental block buffer with native Chat. It scans +only new input, counts consumed blocks rather than remaining suffixes, and preserves LF/CRLF, +partial-event, injection/drop, and EOF behavior. Output admission precedes its single UTF-8 encoding; +failed enqueue and cancellation release the reservation without re-entering a disposed rewrite. +Old/new buffer overlap remains charged against the same translator cap. + +`src/adapters/openai-responses.ts` counts new compaction fragments, including surrogate pairs formed +across deltas, while retaining snapshot/done/delta precedence and existing terminal ownership. +Serialized request and buffered-response observations use byte counts without measurement arrays. +The same rule applies to Anthropic, Google, and Chat response accounting; serialization itself is +preserved where the existing metric is the serialized JSON size. + +`src/lib/translator-budget.ts` admits an event batch atomically from per-event serialized byte sizes +plus exact separators, without joining a second full JSON array. `src/lib/admission.ts` counts and +truncates diagnostic text at UTF-8 code-point boundaries without allocating arrays per character; +byte sizing retains TextEncoder's coercion behavior for legacy non-string runtime callers. +These optimizations do not add request queues, retry policies, or RSS-based admission gates. + ### Credential-bearing HTTP redirects Credential/body-bearing HTTP sends use `redirect: "manual"` at the final executor boundary, diff --git a/structure/transports/streaming-health.md b/structure/transports/streaming-health.md index 1d9741111c..cac95231a5 100644 --- a/structure/transports/streaming-health.md +++ b/structure/transports/streaming-health.md @@ -7,10 +7,14 @@ Codex WebSocket quota-family normalization remains generic; retired-model eviden by the [OpenAI quota owner](../providers/openai-tiers.md#public-provider-contract), not by removing support for non-default WebSocket quota families. -For shared JSON request-body parsing, see [request-copy accounting](responses.md#request-copy-accounting). +Shared parsing and streaming follow the [request-copy](responses.md#request-copy-accounting) and [stream-buffer accounting](responses.md#stream-buffer-accounting) contracts. ## Heartbeat and stall deadline +Native Chat uses the same resolved `stallTimeoutSec` with a pending-upstream-read allowance that +pauses under downstream backpressure. Its Chat error and cancellation contract is documented in +[native Chat completion lifecycle](../data-planes/inbound-compat.md#native-chat-completion-lifecycle). + The HTTP/SSE bridge emits an SSE comment-line keep-alive (`: opencodex heartbeat`) during upstream silence to re-arm Codex's idle timer (Codex's default `stream_idle_timeout` is 300 s and ANY SSE bytes re-arm it). A comment line is discarded by every eventsource parser without producing an event, From c093f9c2379d857c1d5156a7442d2af44214e60f Mon Sep 17 00:00:00 2001 From: olddonkey Date: Sat, 12 Sep 2026 09:28:57 -0700 Subject: [PATCH 10/13] test(chat): pin caller abort precedence over a pending native stall A caller abort that lands while the native Chat stall clock is waiting on upstream silence must be the only reported outcome. The stream closes as a cancellation, onCancel fires once, upstream is cancelled once, and no upstream_stall_timeout terminal surfaces even after the deadline it was racing has elapsed. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 176cbbd2ec1f492c74174eecc54435af2d7cb8db) --- .../chat-completions-endpoint.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/responses/chat-completions-endpoint.test.ts b/tests/responses/chat-completions-endpoint.test.ts index f39857ca0a..482575bb98 100644 --- a/tests/responses/chat-completions-endpoint.test.ts +++ b/tests/responses/chat-completions-endpoint.test.ts @@ -1853,6 +1853,44 @@ test("chat-native meaningful reasoning keeps a long stream alive and pauses the } finally { abort.abort(); budget.dispose(); } }); +test("chat-native caller abort while the stall clock is pending wins over the later deadline", async () => { + const { nativeChatSse } = await import("../../src/server/chat-native-sse"); + const budget = createTestTranslatorBudget(); + const abort = new AbortController(); + const terminals: Array<[number, string | undefined]> = []; + let cancels = 0; + let upstreamCancels = 0; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"partial"}}]}\n\n')); + }, + cancel() { upstreamCancels += 1; }, + }); + const stream = nativeChatSse(body, { + requestedModel: "mock/test-model", translatorBudget: budget, signal: abort.signal, + stallTimeoutSec: 1, onUsage() {}, + onTerminal(status, message) { terminals.push([status, message]); }, + onCancel() { cancels += 1; }, + }); + const reader = stream.getReader(); + try { + const first = await reader.read(); + expect(new TextDecoder().decode(first.value)).toContain("partial"); + // Upstream now stays silent, so the next pull waits on the one-second stall clock. + // A caller abort during that wait must be the only reported outcome, even once the + // deadline it was racing has elapsed. + const pending = reader.read(); + await Bun.sleep(300); + abort.abort("client gone"); + const next = await pending; + expect(next.done).toBe(true); + await Bun.sleep(1_000); + expect(cancels).toBe(1); + expect(upstreamCancels).toBe(1); + expect(terminals).toEqual([]); + } finally { abort.abort(); reader.releaseLock(); budget.dispose(); } +}); + test("chat-native valid terminal retains precedence over a late non-streaming abort", async () => { const { handleChatCompletions } = await import("../../src/server/chat-completions"); const clientAbort = new AbortController(); From 7e2343326a79efab6390af69b34b03c6e0c2d33e Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 13 Sep 2026 16:04:06 +0900 Subject: [PATCH 11/13] docs(structure): split byte accounting out of the Responses transport doc [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry of #4389 by olddonkey onto current dev. structure/transports/responses.md is exactly at its 600-line budget on dev, so the two new owning sections this change needs could not be added there at all: bun run structure:check failed at 630 lines. structure/AGENTS.md says an over-budget doc is split along a topic boundary with its own manifest entry, and that a grace.oversizeDocs entry is only for a split already planned — so this takes the split rather than parking a promise nobody would keep. Request-copy and stream-buffer accounting are one topic and now live in structure/transports/byte-accounting.md with its own manifest entry. responses.md is byte-identical to dev again. Both documents sit in structure/transports/, so the seventeen cross-references this change adds keep their relative prefix and only change file name and are otherwise untouched. structure/gui-and-management-api.md was 602 lines for the same reason. Its cross-reference is dropped instead: the dashboard and management API own neither the request-decompression nor the SSE-rewrite path, which makes it the least load-bearing of the seventeen. The other sixteen are unchanged. Co-authored-by: Olddonkey <22208754+olddonkey@users.noreply.github.com> --- structure/INDEX.md | 7 +++-- structure/adapters/registry.md | 2 +- structure/catalog.md | 2 +- structure/clients/claude-desktop.md | 2 +- structure/clients/integrations.md | 2 +- structure/data-planes/images.md | 2 +- structure/data-planes/inbound-compat.md | 2 +- structure/gui-and-management-api.md | 2 -- structure/manifest.json | 11 ++++++++ structure/ops/docs-and-release.md | 2 +- structure/ops/service-and-sidecars.md | 2 +- structure/overview.md | 2 +- structure/providers/chat-compat.md | 2 +- structure/providers/cursor.md | 2 +- structure/providers/xai-grok.md | 2 +- structure/runtime.md | 2 +- structure/subagents.md | 2 +- structure/transports/byte-accounting.md | 35 ++++++++++++++++++++++++ structure/transports/inventory.md | 2 +- structure/transports/responses.md | 30 -------------------- structure/transports/streaming-health.md | 2 +- 21 files changed, 66 insertions(+), 51 deletions(-) create mode 100644 structure/transports/byte-accounting.md diff --git a/structure/INDEX.md b/structure/INDEX.md index a25322cc1e..43ccdc6cc0 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -36,6 +36,7 @@ The wire surfaces a client actually talks to. | Doc | Scope | | --- | --- | +| [`transports/byte-accounting.md`](transports/byte-accounting.md) | Request-copy and stream-buffer byte accounting shared by parsing, SSE rewriting, the adapters, and the translator budget. | | [`transports/responses.md`](transports/responses.md) | The Responses HTTP/SSE data plane, combo failover, and streaming commit boundaries. | | [`transports/streaming-health.md`](transports/streaming-health.md) | Heartbeat and stall deadlines, plus the opt-in WebSocket transport. | | [`transports/inventory.md`](transports/inventory.md) | The per-provider transport table and diagnostic outbound safety. | @@ -93,7 +94,7 @@ for it; see [`AGENTS.md`](AGENTS.md). | `docs-site/` | [`ops/docs-and-release.md`](ops/docs-and-release.md) | | `gui/` | [`overview.md`](overview.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`design-methodology.md`](design-methodology.md) | | `scripts/` | [`overview.md`](overview.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | -| `src/adapters/` | [`runtime.md`](runtime.md)
[`transports/responses.md`](transports/responses.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers/cursor.md`](providers/cursor.md)
[`providers/chat-compat.md`](providers/chat-compat.md)
[`adapters/registry.md`](adapters/registry.md) | +| `src/adapters/` | [`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses.md`](transports/responses.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers/cursor.md`](providers/cursor.md)
[`providers/chat-compat.md`](providers/chat-compat.md)
[`adapters/registry.md`](adapters/registry.md) | | `src/chat/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md) | | `src/claude/` | [`runtime.md`](runtime.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md) | | `src/cli.ts` | [`runtime.md`](runtime.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | @@ -112,7 +113,7 @@ for it; see [`AGENTS.md`](AGENTS.md). | `src/index.ts` | [`runtime.md`](runtime.md) | | `src/integrations/` | [`clients/integrations.md`](clients/integrations.md) | | `src/lab/` | [`runtime.md`](runtime.md)
[`adapters/compatibility-lab.md`](adapters/compatibility-lab.md) | -| `src/lib/` | [`overview.md`](overview.md)
[`runtime.md`](runtime.md)
[`transports/responses.md`](transports/responses.md)
[`transports/inventory.md`](transports/inventory.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`clients/integrations.md`](clients/integrations.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | +| `src/lib/` | [`overview.md`](overview.md)
[`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses.md`](transports/responses.md)
[`transports/inventory.md`](transports/inventory.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`clients/integrations.md`](clients/integrations.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/oauth/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | | `src/providers/` | [`runtime.md`](runtime.md)
[`subagents.md`](subagents.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | | `src/reasoning-effort.ts` | [`runtime.md`](runtime.md) | @@ -121,7 +122,7 @@ for it; see [`AGENTS.md`](AGENTS.md). | `src/responses/` | [`runtime.md`](runtime.md)
[`transports/responses.md`](transports/responses.md)
[`providers/kiro.md`](providers/kiro.md)
[`providers/xai-grok.md`](providers/xai-grok.md)
[`providers/chat-compat.md`](providers/chat-compat.md) | | `src/router.ts` | [`runtime.md`](runtime.md) | | `src/routing/` | [`catalog.md`](catalog.md) | -| `src/server/` | [`runtime.md`](runtime.md)
[`catalog.md`](catalog.md)
[`subagents.md`](subagents.md)
[`transports/responses.md`](transports/responses.md)
[`transports/streaming-health.md`](transports/streaming-health.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/images.md`](data-planes/images.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers/xai-grok.md`](providers/xai-grok.md)
[`adapters/registry.md`](adapters/registry.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md)
[`ops/service-and-sidecars.md`](ops/service-and-sidecars.md) | +| `src/server/` | [`runtime.md`](runtime.md)
[`catalog.md`](catalog.md)
[`subagents.md`](subagents.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses.md`](transports/responses.md)
[`transports/streaming-health.md`](transports/streaming-health.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/images.md`](data-planes/images.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers/xai-grok.md`](providers/xai-grok.md)
[`adapters/registry.md`](adapters/registry.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md)
[`ops/service-and-sidecars.md`](ops/service-and-sidecars.md) | | `src/service.ts` | [`runtime.md`](runtime.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/stall-timeout.ts` | [`runtime.md`](runtime.md) | | `src/storage/` | [`runtime.md`](runtime.md) | diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index ca43aae480..45988981e4 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Decision diff --git a/structure/catalog.md b/structure/catalog.md index e02fb32379..153b52f111 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Shared catalog diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index 85bc0b1bfc..6b2d1c022e 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Codex-native model discovery follows the [shared retirement policy](../catalog.md#shared-catalog). That projection does not migrate existing user-selected Desktop configuration or usage history. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Connected Claude Desktop profiles diff --git a/structure/clients/integrations.md b/structure/clients/integrations.md index fc797e95a5..eb4ace5975 100644 --- a/structure/clients/integrations.md +++ b/structure/clients/integrations.md @@ -1,6 +1,6 @@ # Client Integrations -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. The client-integration subsystem writes one generated OpenCodex provider contribution into a third-party client's existing config without taking ownership of the rest of that file. Its core diff --git a/structure/data-planes/images.md b/structure/data-planes/images.md index f50f92f122..fdbdee6b32 100644 --- a/structure/data-planes/images.md +++ b/structure/data-planes/images.md @@ -7,7 +7,7 @@ Hosted Responses image-tool eligibility uses the shared compatibility policy wit Codex Spark exception; standalone Images retain the separate relay contract below. See [Responses transport](../transports/responses.md#responses-httpsse). -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Standalone Images diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index fce7ace064..8b27dd5688 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -47,7 +47,7 @@ Translated Claude timeline reminders use the Chat adapter's on its exact supported route. This is separate from trailing-notice stabilization and from native Chat message passthrough. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Chat Completions inbound native path diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 7f64b72b96..bf7e70cd8b 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -3,8 +3,6 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. - ## Dashboard serving The bundled React dashboard is built into `gui/dist` and served by the same Bun proxy. `ocx gui` diff --git a/structure/manifest.json b/structure/manifest.json index 95c2063180..4532d0bd43 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -142,6 +142,17 @@ "src/server/" ] }, + { + "path": "transports/byte-accounting.md", + "tier": 3, + "title": "Byte Accounting", + "scope": "Request-copy and stream-buffer byte accounting shared by parsing, SSE rewriting, the adapters, and the translator budget.", + "documents": [ + "src/adapters/", + "src/lib/", + "src/server/" + ] + }, { "path": "transports/responses.md", "tier": 3, diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 5b2e5dc41c..682e611d1d 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Public docs diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index 907c1154c0..8f414780c8 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior Service startup and restore use the [catalog retirement policy](../catalog.md#shared-catalog); retirement does not itself change service registration or user-selected model configuration. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Background service command selection diff --git a/structure/overview.md b/structure/overview.md index a96da8cbb9..0151115adc 100644 --- a/structure/overview.md +++ b/structure/overview.md @@ -3,7 +3,7 @@ The configuration-only [plaintext V2 contract](subagents.md#plaintext-v2-agent-messages) is scoped to canonical ChatGPT Responses forwarding; other source-area behavior described here is unchanged. -Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Product boundary diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index 0856cba714..2362e46bcd 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -20,7 +20,7 @@ folding. This is independent of the Claude trailing-notice stabilization option and does not guarantee upstream cache hits. Regression coverage is in `tests/adapters/openai/openai-chat-system-order.test.ts`. -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Reasoning and tool-result compatibility diff --git a/structure/providers/cursor.md b/structure/providers/cursor.md index 4bff45c0f6..55f017685b 100644 --- a/structure/providers/cursor.md +++ b/structure/providers/cursor.md @@ -10,7 +10,7 @@ namespace handling retain their provider contract; the bounded native scope live Cursor's direct adapter does not enter the OpenAI Chat serializer's [OpenCode Go instruction ordering](chat-compat.md#opencode-go-chronological-instructions). -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Cursor Native Exec diff --git a/structure/providers/xai-grok.md b/structure/providers/xai-grok.md index eb7bb47ae5..7b8a78dce9 100644 --- a/structure/providers/xai-grok.md +++ b/structure/providers/xai-grok.md @@ -7,7 +7,7 @@ Codex-native retirement is scoped to OpenAI catalog/quota evidence. Shared Respo retains xAI provider behavior; see [the catalog boundary](../catalog.md#shared-catalog). -Shared parsing and streaming follow the [request-copy](../transports/responses.md#request-copy-accounting) and [stream-buffer accounting](../transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](../transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](../transports/byte-accounting.md#stream-buffer-accounting) contracts. ## xAI Grok hardening (official Grok Build contract parity) diff --git a/structure/runtime.md b/structure/runtime.md index 20478660ef..cac23f6854 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -7,7 +7,7 @@ Chat request serialization owns the destination-scoped [OpenCode Go instruction ordering](providers/chat-compat.md#opencode-go-chronological-instructions); it requires no runtime lifecycle change or new configuration option. -Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Entrypoints diff --git a/structure/subagents.md b/structure/subagents.md index 93b3e65d33..98c827ffa7 100644 --- a/structure/subagents.md +++ b/structure/subagents.md @@ -25,7 +25,7 @@ Codex treats qualified names literally and defaults absent namespaces to functio declarations inherit their restored namespace container; the compiler never invents an empty encryption marker when the upstream omitted it or returned a nonempty marker. -Shared parsing and streaming follow the [request-copy](transports/responses.md#request-copy-accounting) and [stream-buffer accounting](transports/responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](transports/byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](transports/byte-accounting.md#stream-buffer-accounting) contracts. ## Multi-agent surface mode (3-state) diff --git a/structure/transports/byte-accounting.md b/structure/transports/byte-accounting.md new file mode 100644 index 0000000000..830d1af7a9 --- /dev/null +++ b/structure/transports/byte-accounting.md @@ -0,0 +1,35 @@ +# Byte Accounting + +How opencodex measures request and stream bytes without allocating copies solely to count +them. These contracts are shared by request parsing, SSE rewriting, the provider adapters and +the translator budget, which is why so many documents link here rather than restating them. + +## Request-copy accounting + +`src/server/request-decompress.ts` observes the UTF-8 sizes of decoded text and reserialized JSON +without allocating encoded byte arrays solely to count them. Parsed-body accounting still uses +`JSON.stringify(parsed)`: numeric normalization can make it larger than the input text. These +observations retain the existing ownership and release lifecycle and do not consume the translator's +hard byte cap. Admission limits, parsing, compression, and error envelopes are unchanged. +`tests/usage/request-decompress.test.ts` covers exact accounting across codecs and Unicode/numeric +normalization, UTF-8 counting without encoded copies, and release after malformed or optional empty input. + +## Stream-buffer accounting + +`src/server/sse-payload-rewrite.ts` shares an incremental block buffer with native Chat. It scans +only new input, counts consumed blocks rather than remaining suffixes, and preserves LF/CRLF, +partial-event, injection/drop, and EOF behavior. Output admission precedes its single UTF-8 encoding; +failed enqueue and cancellation release the reservation without re-entering a disposed rewrite. +Old/new buffer overlap remains charged against the same translator cap. + +`src/adapters/openai-responses.ts` counts new compaction fragments, including surrogate pairs formed +across deltas, while retaining snapshot/done/delta precedence and existing terminal ownership. +Serialized request and buffered-response observations use byte counts without measurement arrays. +The same rule applies to Anthropic, Google, and Chat response accounting; serialization itself is +preserved where the existing metric is the serialized JSON size. + +`src/lib/translator-budget.ts` admits an event batch atomically from per-event serialized byte sizes +plus exact separators, without joining a second full JSON array. `src/lib/admission.ts` counts and +truncates diagnostic text at UTF-8 code-point boundaries without allocating arrays per character; +byte sizing retains TextEncoder's coercion behavior for legacy non-string runtime callers. +These optimizations do not add request queues, retry policies, or RSS-based admission gates. diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index 3715536695..1bb6687cf9 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -6,7 +6,7 @@ is scoped to canonical ChatGPT Responses forwarding; other source-area behavior The Chat adapter's [OpenCode Go instruction ordering](../providers/chat-compat.md#opencode-go-chronological-instructions) changes translated message placement only; endpoint selection and transport stay with their existing owners. -Shared parsing and streaming follow the [request-copy](responses.md#request-copy-accounting) and [stream-buffer accounting](responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](byte-accounting.md#stream-buffer-accounting) contracts. ## Transport inventory diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 6863138649..96ef06d3c0 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -15,36 +15,6 @@ Retired Codex Spark has no model-specific tool or Responses Lite override; gener namespace scrubbing remain shared compatibility behavior. Codex quota/reset evidence follows the [shared/Reserve policy](../providers/openai-tiers.md#public-provider-contract), including suppression of retired model-derived evidence before shared recovery. -### Request-copy accounting - -`src/server/request-decompress.ts` observes the UTF-8 sizes of decoded text and reserialized JSON -without allocating encoded byte arrays solely to count them. Parsed-body accounting still uses -`JSON.stringify(parsed)`: numeric normalization can make it larger than the input text. These -observations retain the existing ownership and release lifecycle and do not consume the translator's -hard byte cap. Admission limits, parsing, compression, and error envelopes are unchanged. -`tests/usage/request-decompress.test.ts` covers exact accounting across codecs and Unicode/numeric -normalization, UTF-8 counting without encoded copies, and release after malformed or optional empty input. - -### Stream-buffer accounting - -`src/server/sse-payload-rewrite.ts` shares an incremental block buffer with native Chat. It scans -only new input, counts consumed blocks rather than remaining suffixes, and preserves LF/CRLF, -partial-event, injection/drop, and EOF behavior. Output admission precedes its single UTF-8 encoding; -failed enqueue and cancellation release the reservation without re-entering a disposed rewrite. -Old/new buffer overlap remains charged against the same translator cap. - -`src/adapters/openai-responses.ts` counts new compaction fragments, including surrogate pairs formed -across deltas, while retaining snapshot/done/delta precedence and existing terminal ownership. -Serialized request and buffered-response observations use byte counts without measurement arrays. -The same rule applies to Anthropic, Google, and Chat response accounting; serialization itself is -preserved where the existing metric is the serialized JSON size. - -`src/lib/translator-budget.ts` admits an event batch atomically from per-event serialized byte sizes -plus exact separators, without joining a second full JSON array. `src/lib/admission.ts` counts and -truncates diagnostic text at UTF-8 code-point boundaries without allocating arrays per character; -byte sizing retains TextEncoder's coercion behavior for legacy non-string runtime callers. -These optimizations do not add request queues, retry policies, or RSS-based admission gates. - ### Credential-bearing HTTP redirects Credential/body-bearing HTTP sends use `redirect: "manual"` at the final executor boundary, diff --git a/structure/transports/streaming-health.md b/structure/transports/streaming-health.md index cac95231a5..e5d4682071 100644 --- a/structure/transports/streaming-health.md +++ b/structure/transports/streaming-health.md @@ -7,7 +7,7 @@ Codex WebSocket quota-family normalization remains generic; retired-model eviden by the [OpenAI quota owner](../providers/openai-tiers.md#public-provider-contract), not by removing support for non-default WebSocket quota families. -Shared parsing and streaming follow the [request-copy](responses.md#request-copy-accounting) and [stream-buffer accounting](responses.md#stream-buffer-accounting) contracts. +Shared parsing and streaming follow the [request-copy](byte-accounting.md#request-copy-accounting) and [stream-buffer accounting](byte-accounting.md#stream-buffer-accounting) contracts. ## Heartbeat and stall deadline From ae48e558544b6699d6ea29f9c3759e18f1548471 Mon Sep 17 00:00:00 2001 From: jeongjin Date: Sun, 13 Sep 2026 13:34:51 +0900 Subject: [PATCH 12/13] fix(devin): restore namespaced tool identities Map each unique Cognition bare tool name back to the request-declared Codex identity before bridge validation. Refuse ambiguous local-name collisions instead of selecting by declaration order. (cherry picked from commit edc6db3ffa6b0857e38589e44a02ebc9fda82dfe) --- src/adapters/devin.ts | 68 +++++++++++++++++++- structure/adapters/registry.md | 5 +- tests/providers/devin-adapter.test.ts | 93 ++++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 3 deletions(-) diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index a662ac0cc6..039339bd29 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -7,6 +7,7 @@ * streams CloudChatEvent into AdapterEvent. */ import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool, OcxToolCall, OcxToolResultMessage, OcxUsage } from "../types"; +import { namespacedToolName } from "../types"; import type { IncomingMeta, ProviderAdapter } from "./base"; import { streamChatEvents, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct"; import { getCachedCatalog } from "./devin/cloud-direct/catalog"; @@ -326,6 +327,65 @@ export function mapOcxToolsToDevin(tools: OcxTool[] | undefined): ToolDef[] | un })); } +/** + * Devin's request mapper advertises the local tool name, so a namespaced Codex tool such as + * `mcp__cua_repl__js` is sent upstream as `js`. Restore a returned bare name to its canonical request + * identity only when exactly one advertised tool owns it. A null owner is an ambiguous catalog and + * must fail before dispatch; an absent owner remains unchanged for the shared undeclared-tool guard + * to reject. + */ +function buildDevinReturnedToolNameMap( + tools: OcxTool[] | undefined, +): ReadonlyMap { + const names = new Map(); + for (const tool of tools ?? []) { + const canonical = namespacedToolName(tool.namespace, tool.name); + if (!names.has(tool.name)) { + names.set(tool.name, canonical); + } else if (names.get(tool.name) !== canonical) { + names.set(tool.name, null); + } + } + return names; +} + +function restoreDevinReturnedToolName( + name: string, + names: ReadonlyMap, +): string | null { + return names.has(name) ? names.get(name)! : name; +} + +type DevinMappedToolCallStart = + | Extract + | Extract; + +function mapDevinToolCallStart( + id: string, + name: string, + names: ReadonlyMap, +): DevinMappedToolCallStart { + const restoredName = restoreDevinReturnedToolName(name, names); + if (restoredName === null) { + return { + type: "error", + message: "Devin emitted a bare client tool name that maps to multiple request-declared tools.", + status: 502, + retryable: false, + }; + } + return { type: "tool_call_start", id, name: restoredName }; +} + +/** Test seam for the request-scoped tool-call event mapping used by runTurn. */ +export function mapDevinToolCallStartForTests( + id: string, + name: string, + tools: OcxTool[] | undefined, +): DevinMappedToolCallStart { + return mapDevinToolCallStart(id, name, buildDevinReturnedToolNameMap(tools)); +} + export function createDevinAdapter( provider: OcxProviderConfig, context: { providerId?: string } = {}, @@ -387,6 +447,7 @@ export function createDevinAdapter( // every RPC to the US server it is not provisioned on. const host = resolveDevinApiServer(provider.baseUrl, credentialProviderId); const modelUid = await resolveWireModelUid(rawModelId, apiKey, host, parsed.options.reasoning); + const returnedToolNames = buildDevinReturnedToolNameMap(parsed.context.tools); let openToolId: string | undefined; let usage: OcxUsage | undefined; let stopReason: string | undefined; @@ -440,8 +501,13 @@ export function createDevinAdapter( } if (event.kind === "tool_call_start") { closeOpenTool(); + const mapped = mapDevinToolCallStart(event.id, event.name, returnedToolNames); + if (mapped.type === "error") { + emit({ ...mapped, ...(usage ? { usage } : {}) }); + return; + } openToolId = event.id; - emit({ type: "tool_call_start", id: event.id, name: event.name }); + emit(mapped); continue; } if (event.kind === "tool_call_args") { diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 45988981e4..77d302581a 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -31,7 +31,10 @@ Some adapters share another adapter's routed-tool semantics while retaining inde `devin-cli` imports that token and the two rows differ only in where the credential came from. `AdapterFactoryContext.providerId` is what keeps them apart: the Cognition tenant is recorded on the credential, not in the registry, so the adapter has to know which row it is serving before it - can resolve a host. + can resolve a host. The adapter advertises bare local tool names to Cognition, so `runTurn` also + owns a request-scoped return map from each unique bare name to the canonical Codex namespace + identity. Unknown names remain subject to the shared undeclared-tool guard; duplicate bare names + fail before dispatch rather than selecting a request tool by declaration order. There is no second Devin transport. An Agent Client Protocol adapter that spawned a local `devin acp` child once existed under the `devin-cli` adapter id and was removed: the CLI's diff --git a/tests/providers/devin-adapter.test.ts b/tests/providers/devin-adapter.test.ts index 262da900d9..2bffa97730 100644 --- a/tests/providers/devin-adapter.test.ts +++ b/tests/providers/devin-adapter.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { createDevinAdapter, mapOcxMessagesToDevin, mapOcxToolsToDevin, resolveWireModelUidForTests } from "../../src/adapters/devin"; +import { createDevinAdapter, mapDevinToolCallStartForTests, mapOcxMessagesToDevin, mapOcxToolsToDevin, resolveWireModelUidForTests } from "../../src/adapters/devin"; import { sanitizeToolDescriptionForCognitionForTests } from "../../src/adapters/devin/cloud-direct/chat"; import { DEVIN_MODEL_CONTEXT_WINDOWS, DEVIN_STATIC_MODELS, collapseDevinModelUid } from "../../src/adapters/devin/live-models"; import { parseCatalogBuffer } from "../../src/adapters/devin/cloud-direct/catalog"; @@ -77,6 +77,97 @@ describe("devin adapter", () => { expect(system).not.toContain("codex_app__list_threads"); }); + test("maps Cognition tool_call_start names to unique canonical request identities", () => { + const tools = [ + { namespace: "mcp__cua_repl", name: "js", description: "control UI", parameters: { type: "object" } }, + { name: "exec", description: "run code", parameters: { type: "object" } }, + ]; + + expect(mapDevinToolCallStartForTests("call_js", "js", tools)).toEqual({ + type: "tool_call_start", + id: "call_js", + name: "mcp__cua_repl__js", + }); + expect(mapDevinToolCallStartForTests("call_canonical", "mcp__cua_repl__js", tools)).toEqual({ + type: "tool_call_start", + id: "call_canonical", + name: "mcp__cua_repl__js", + }); + expect(mapDevinToolCallStartForTests("call_exec", "exec", tools)).toEqual({ + type: "tool_call_start", + id: "call_exec", + name: "exec", + }); + expect(mapDevinToolCallStartForTests("call_unknown", "undeclared", tools)).toEqual({ + type: "tool_call_start", + id: "call_unknown", + name: "undeclared", + }); + }); + + test("maps ambiguous Cognition tool_call_start names to a non-retryable error", () => { + const namespaceCollision = [ + { namespace: "mcp__first", name: "js", description: "first", parameters: { type: "object" } }, + { namespace: "mcp__second", name: "js", description: "second", parameters: { type: "object" } }, + ]; + const bareCollision = [ + { name: "js", description: "bare", parameters: { type: "object" } }, + { namespace: "mcp__cua_repl", name: "js", description: "namespaced", parameters: { type: "object" } }, + ]; + const duplicateIdentity = [ + { namespace: "mcp__cua_repl", name: "js", description: "first copy", parameters: { type: "object" } }, + { namespace: "mcp__cua_repl", name: "js", description: "second copy", parameters: { type: "object" } }, + ]; + + const expected = { + type: "error", + message: "Devin emitted a bare client tool name that maps to multiple request-declared tools.", + status: 502, + retryable: false, + }; + expect(mapDevinToolCallStartForTests("call_1", "js", namespaceCollision)).toEqual(expected); + expect(mapDevinToolCallStartForTests("call_1", "js", [...namespaceCollision].reverse())).toEqual(expected); + expect(mapDevinToolCallStartForTests("call_1", "js", bareCollision)).toEqual(expected); + expect(mapDevinToolCallStartForTests("call_1", "js", duplicateIdentity)).toEqual({ + type: "tool_call_start", + id: "call_1", + name: "mcp__cua_repl__js", + }); + }); + + test("replays a restored namespaced call under the same bare name Cognition was offered", () => { + const parsed: OcxParsedRequest = { + modelId: "swe-2", + stream: true, + context: { + messages: [{ + role: "assistant", + content: [{ + type: "toolCall", + id: "js_0", + namespace: "mcp__cua_repl", + name: "js", + arguments: { code: "1+1" }, + }], + timestamp: 1, + }], + tools: [{ + namespace: "mcp__cua_repl", + name: "js", + description: "control UI", + parameters: { type: "object" }, + }], + }, + options: {}, + }; + + expect(mapOcxMessagesToDevin(parsed).find(item => item.role === "assistant")?.tool_calls).toEqual([{ + id: "js_0", + name: "js", + arguments: JSON.stringify({ code: "1+1" }), + }]); + }); + test("a request with no tools keeps the system prompt exactly as it was", () => { const parsed: OcxParsedRequest = { modelId: "swe-1-7", From 76c720c0521c8128b23722e10ba00182fab240bf Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 13 Sep 2026 16:10:51 +0900 Subject: [PATCH 13/13] fix(devin): fail closed on a canonical/local tool-name collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry of #4457 by jeongjin0, with the unresolved CodeRabbit finding on src/adapters/devin.ts folded in. The return map tracked only advertised local names, but the adapter accepts canonical names on return too, which the existing catalog test pins. With { namespace: "a", name: "x" } and { namespace: "b", name: "a__x" }, a returned a__x is both the first tool's canonical identity and the second tool's advertised name. The map resolved it to b__a__x, so src/bridge.ts dispatched the call through the second tool's identity — a tool the caller may not have named. Register canonical identities as aliases of themselves and mark a conflicting alias ambiguous, so that name now fails before dispatch like any other ambiguous bare name. The unambiguous local name and the unrelated canonical name still resolve, and every existing case is unchanged: a single namespaced tool, a duplicate identical declaration, a bare/namespaced collision, and an undeclared name left for the shared guard. Co-authored-by: Jeongjin Shin <80797980+jeongjin0@users.noreply.github.com> --- src/adapters/devin.ts | 21 ++++++++++++---- structure/adapters/registry.md | 4 +++ tests/providers/devin-adapter.test.ts | 35 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index 039339bd29..45e24cc043 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -333,18 +333,29 @@ export function mapOcxToolsToDevin(tools: OcxTool[] | undefined): ToolDef[] | un * identity only when exactly one advertised tool owns it. A null owner is an ambiguous catalog and * must fail before dispatch; an absent owner remains unchanged for the shared undeclared-tool guard * to reject. + * + * Canonical names are registered as aliases of themselves because the adapter accepts them on return + * too. Tracking only local names let one tool's canonical identity collide with another tool's local + * name and resolve to the wrong owner: with `{ namespace: "a", name: "x" }` and + * `{ namespace: "b", name: "a__x" }`, a returned `a__x` is both the first tool's canonical identity + * and the second tool's advertised name, and it used to map to `b__a__x` — so the bridge dispatched + * the call to the wrong client tool. That case is genuinely ambiguous and now fails closed. */ function buildDevinReturnedToolNameMap( tools: OcxTool[] | undefined, ): ReadonlyMap { const names = new Map(); + const addOwner = (alias: string, canonical: string) => { + if (!names.has(alias)) { + names.set(alias, canonical); + } else if (names.get(alias) !== canonical) { + names.set(alias, null); + } + }; for (const tool of tools ?? []) { const canonical = namespacedToolName(tool.namespace, tool.name); - if (!names.has(tool.name)) { - names.set(tool.name, canonical); - } else if (names.get(tool.name) !== canonical) { - names.set(tool.name, null); - } + addOwner(tool.name, canonical); + addOwner(canonical, canonical); } return names; } diff --git a/structure/adapters/registry.md b/structure/adapters/registry.md index 77d302581a..84ea470a53 100644 --- a/structure/adapters/registry.md +++ b/structure/adapters/registry.md @@ -35,6 +35,10 @@ Some adapters share another adapter's routed-tool semantics while retaining inde owns a request-scoped return map from each unique bare name to the canonical Codex namespace identity. Unknown names remain subject to the shared undeclared-tool guard; duplicate bare names fail before dispatch rather than selecting a request tool by declaration order. + Canonical identities are registered in that map as well, because the adapter accepts them on + return. One tool's canonical identity can be another tool's advertised local name, and resolving + that name to either owner would dispatch the call to a tool the caller may not have named, so it + is treated as ambiguous and fails before dispatch too. There is no second Devin transport. An Agent Client Protocol adapter that spawned a local `devin acp` child once existed under the `devin-cli` adapter id and was removed: the CLI's diff --git a/tests/providers/devin-adapter.test.ts b/tests/providers/devin-adapter.test.ts index 2bffa97730..e8152feb03 100644 --- a/tests/providers/devin-adapter.test.ts +++ b/tests/providers/devin-adapter.test.ts @@ -135,6 +135,41 @@ describe("devin adapter", () => { }); }); + test("fails closed when one tool's canonical identity is another tool's advertised name", () => { + // `a__x` is the first tool's canonical identity and also the second tool's advertised local + // name, whose own canonical identity is `b__a__x`. Both readings are legitimate, so resolving + // to either owner would dispatch the call to a tool the caller may not have named. Before the + // map tracked canonical aliases, a returned `a__x` silently became `b__a__x`. + const aliasCollision = [ + { namespace: "a", name: "x", description: "namespaced", parameters: { type: "object" } }, + { namespace: "b", name: "a__x", description: "lookalike local name", parameters: { type: "object" } }, + ]; + + expect(mapDevinToolCallStartForTests("call_1", "a__x", aliasCollision)).toEqual({ + type: "error", + message: "Devin emitted a bare client tool name that maps to multiple request-declared tools.", + status: 502, + retryable: false, + }); + expect(mapDevinToolCallStartForTests("call_1", "a__x", [...aliasCollision].reverse())).toEqual({ + type: "error", + message: "Devin emitted a bare client tool name that maps to multiple request-declared tools.", + status: 502, + retryable: false, + }); + // The unambiguous local name still resolves, and an unrelated canonical name is untouched. + expect(mapDevinToolCallStartForTests("call_2", "x", aliasCollision)).toEqual({ + type: "tool_call_start", + id: "call_2", + name: "a__x", + }); + expect(mapDevinToolCallStartForTests("call_3", "b__a__x", aliasCollision)).toEqual({ + type: "tool_call_start", + id: "call_3", + name: "b__a__x", + }); + }); + test("replays a restored namespaced call under the same bare name Cognition was offered", () => { const parsed: OcxParsedRequest = { modelId: "swe-2",