From 9bed07855ca9c5e3839661217ed48ba71113f680 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 17:50:23 +0900 Subject: [PATCH 01/34] feat(spend): require one writer lease per state directory for the spend journal The ledger documented a guarantee covering one live proxy process, while the CLI deliberately supports starting a sibling on a different port under the same state directory. Nothing serialized the two. That is not a theoretical overlap: every Responses-serving process is a potential journal writer, including one with no configured ceiling, because a ceiling decides whether to refuse rather than whether to write. Two processes could mint different salts, interleave appends and compaction, and a second process's construction replay declares the first one's open reservations lost while it is still settling them. Startup now takes a state-directory writer lease before any listener can serve, so the exported startServer path is covered rather than only the CLI preflight, and the ledger asserts that lease before replay, append or compaction. Siblings remain supported with independent state directories; a second writer on the same one is refused with a clear message. Ownership is an OS-held SQLite write transaction for the process lifetime, which is the pattern this repository already uses for its other cross-process locks. There is deliberately no stale-owner reclamation: PID reuse, container PID namespaces, restarts and power loss all defeat a PID or timestamp check, and a TTL can evict a live process that was merely paused. Busy means a live owner; any other failure to establish the lock is ambiguous and fails closed. Diagnostics are scalar only on the authenticated health route - ownership, initialized, configured, degraded and bounded counters - and reading them constructs nothing. /healthz is unchanged. --- .../docs/fr/reference/cli/lifecycle.md | 2 +- .../docs/ja/reference/cli/lifecycle.md | 2 +- .../docs/ko/reference/cli/lifecycle.md | 6 +- .../content/docs/reference/cli/lifecycle.md | 10 +- .../docs/reference/configuration/server.md | 2 +- .../docs/ru/reference/cli/lifecycle.md | 6 +- .../docs/tr/reference/cli/lifecycle.md | 8 +- .../docs/zh-cn/reference/cli/lifecycle.md | 2 +- .../docs/zh-tw/reference/cli/lifecycle.md | 2 +- scripts/test-layout/layout.json | 2 + src/cli/dispatch.ts | 6 +- src/cli/index.ts | 9 +- src/lib/spend-ledger-owner.ts | 218 ++++++++++++++++++ src/lib/spend-reservation-ledger.ts | 38 ++- src/server/index.ts | 29 +-- src/server/index/spend-ledger-lifecycle.ts | 44 ++++ src/server/management/system-routes.ts | 2 + structure/config.md | 2 +- structure/gui-and-management-api.md | 2 +- structure/runtime.md | 9 + structure/transports/responses.md | 14 ++ tests/fixtures/test-layout-expected.json | 2 + tests/helpers/spend-ledger-owner-child.ts | 45 ++++ tests/lib/spend-ceiling-enforcement.test.ts | 5 + tests/lib/spend-ledger-owner.test.ts | 210 +++++++++++++++++ .../server/spend-ledger-owner-startup.test.ts | 94 ++++++++ 26 files changed, 730 insertions(+), 41 deletions(-) create mode 100644 src/lib/spend-ledger-owner.ts create mode 100644 src/server/index/spend-ledger-lifecycle.ts create mode 100644 tests/helpers/spend-ledger-owner-child.ts create mode 100644 tests/lib/spend-ledger-owner.test.ts create mode 100644 tests/server/spend-ledger-owner-startup.test.ts diff --git a/docs-site/src/content/docs/fr/reference/cli/lifecycle.md b/docs-site/src/content/docs/fr/reference/cli/lifecycle.md index 148febbf44..c134193f42 100644 --- a/docs-site/src/content/docs/fr/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/fr/reference/cli/lifecycle.md @@ -15,7 +15,7 @@ Assistant de configuration interactif (`setup` est un alias de `init`). Il deman ### `ocx start [--port ] [--socks5 [host:port] | --socks5-off]` -Démarre le serveur proxy, de préférence sur le port `10100`. La commande écrit l’état du PID et du port d’exécution, et refuse de démarrer une deuxième instance active. Lorsque le port préféré est occupé, `start` interroge le processus qui l’occupe puis s’arrête dans tous les cas : elle refuse de démarrer si un processus opencodex y répond et signale sinon que le processus est inconnu. Elle ne déplace jamais l’écouteur vers un autre port d’elle-même, car cela laisserait le premier proxy en cours d’exécution et redirigerait Codex vers le second. Indiquez un autre port avec `--port`, ou définissez `port: 0` dans la configuration pour demander au système d’exploitation d’en attribuer un. Au démarrage, elle synchronise dans le catalogue Codex les modèles de chaque fournisseur. À l’arrêt, elle rétablit le fonctionnement natif de Codex, sauf si le proxy a été lancé comme service géré (`OCX_SERVICE=1`). +Démarre le serveur proxy, de préférence sur le port `10100`. La commande écrit l’état du PID et du port d’exécution, et refuse de démarrer une deuxième instance active. Lorsque le port préféré est occupé, `start` interroge le processus qui l’occupe puis s’arrête dans tous les cas : elle refuse de démarrer si un processus opencodex y répond et signale sinon que le processus est inconnu. Elle ne déplace jamais l’écouteur vers un autre port d’elle-même, car cela laisserait le premier proxy en cours d’exécution et redirigerait Codex vers le second. Un autre `--port` explicite est également refusé avec le même `OPENCODEX_HOME`, car les modes d’observation et de plafond écrivent tous deux dans le même journal de dépenses. Utilisez un `OPENCODEX_HOME` distinct pour une instance sœur indépendante ; `port: 0` ne sépare que l’attribution du port, pas l’état. Au démarrage, elle synchronise dans le catalogue Codex les modèles de chaque fournisseur. À l’arrêt, elle rétablit le fonctionnement natif de Codex, sauf si le proxy a été lancé comme service géré (`OCX_SERVICE=1`). `--socks5` (par défaut `127.0.0.1:10808`) enregistre l’URL SOCKS5 dans `config.proxy` et achemine les requêtes HTTP(S) sortantes dans un véritable tunnel SOCKS5. `--socks5-off` supprime uniquement diff --git a/docs-site/src/content/docs/ja/reference/cli/lifecycle.md b/docs-site/src/content/docs/ja/reference/cli/lifecycle.md index 6939c537d0..bf37bc911b 100644 --- a/docs-site/src/content/docs/ja/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ja/reference/cli/lifecycle.md @@ -15,7 +15,7 @@ description: セットアップ、開始、停止、サービス、診断、同 ### `ocx start [--port ] [--socks5 [host:port] | --socks5-off]` -プロキシ サーバー (優先ポート `10100`) を起動します。PID/ランタイムポートの状態を書き込み、2 番目のライブインスタンスの起動を拒否します。優先ポートが使用中の場合、`start` はそのポートを使用しているプロセスを確認して、どちらの場合も停止します。opencodex が応答していれば起動を拒否し、それ以外は使用しているプロセスを特定できないと報告します。最初のプロキシを実行したまま Codex を 2 番目のプロキシへ向けることになるため、自動でリスナーを別のポートへ移すことはありません。別のポートは `--port` で指定するか、設定で `port: 0` を指定して OS に割り当てを依頼してください。開始時に、各プロバイダーのモデルを Codex のカタログに同期します。マネージド サービス (`OCX_SERVICE=1`) として起動されていない限り、シャットダウン時にネイティブ Codex が復元されます。 +プロキシ サーバー (優先ポート `10100`) を起動します。PID/ランタイムポートの状態を書き込み、2 番目のライブインスタンスの起動を拒否します。優先ポートが使用中の場合、`start` はそのポートを使用しているプロセスを確認して、どちらの場合も停止します。opencodex が応答していれば起動を拒否し、それ以外は使用しているプロセスを特定できないと報告します。最初のプロキシを実行したまま Codex を 2 番目のプロキシへ向けることになるため、自動でリスナーを別のポートへ移すことはありません。同じ `OPENCODEX_HOME` では別の `--port` を明示しても拒否されます。監視のみの構成も上限を適用する構成も同じ支出ジャーナルへ書き込むためです。独立した sibling には別の `OPENCODEX_HOME` を使用してください。`port: 0` はポートだけを OS に割り当てさせ、状態を分離しません。開始時に、各プロバイダーのモデルを Codex のカタログに同期します。マネージド サービス (`OCX_SERVICE=1`) として起動されていない限り、シャットダウン時にネイティブ Codex が復元されます。 `--socks5`（デフォルト `127.0.0.1:10808`）は SOCKS5 URL を `config.proxy` に保存し、送信 HTTP(S) リクエストを実際の SOCKS5 トンネル経由で送信します。`--socks5-off` は保存された SOCKS5 プロキシだけを削除し、HTTP プロキシは削除しません。値は設定に保存されるため、`ocx update` 後も保持されます。URL にユーザー名とパスワードを含めることはできますが、起動ログでは非表示になります。 diff --git a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md index 4db8c09b69..e4f2913c6f 100644 --- a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md @@ -22,8 +22,10 @@ Codex 자동 시작 shim도 설치합니다. 인스턴스는 시작하지 않습니다. 권장 포트가 이미 사용 중이면 `start`가 점유자를 확인한 뒤 어느 경우든 실행을 멈춥니다. opencodex가 응답하면 시작을 거부하고, 그렇지 않으면 점유자를 식별할 수 없다고 알립니다. 첫 번째 프록시를 실행한 채 Codex가 두 번째 프록시를 가리키게 되므로, 리스너를 다른 포트로 -자동 이동하지 않습니다. 다른 포트는 `--port`로 지정하거나, OS에 포트 할당을 요청하려면 구성에서 -`port: 0`을 설정하세요. 시작할 때는 각 공급자의 모델을 Codex 카탈로그로 동기화합니다. 종료할 때는 +자동 이동하지 않습니다. 활성 프록시와 같은 `OPENCODEX_HOME`을 사용하면 다른 `--port`를 명시해도 +시작을 거부합니다. 관찰 전용과 제한 적용 모드 모두 같은 지출 저널에 기록하기 때문입니다. 독립된 형제 +인스턴스에는 별도의 `OPENCODEX_HOME`을 사용하세요. `port: 0`은 포트만 OS에 맡기며 상태를 분리하지 +않습니다. 시작할 때는 각 공급자의 모델을 Codex 카탈로그로 동기화합니다. 종료할 때는 기본 Codex를 복원합니다. 단, 관리형 서비스로 실행한 경우(`OCX_SERVICE=1`)는 예외입니다. `--socks5`(기본값 `127.0.0.1:10808`)는 SOCKS5 URL을 `config.proxy`에 저장하고 실제 SOCKS5 diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index 54c42a9d68..e8380c73ca 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -22,10 +22,12 @@ Start the proxy server (preferred port `10100`). It writes PID/runtime-port stat start a second live instance. When the preferred port is occupied, `start` asks the holder who it is and stops either way: it refuses outright when an opencodex answers there, and reports an unidentified holder otherwise. It never moves the listener to another port on its own, because that -would leave the first proxy running and re-point Codex at the second. Name a different port with -`--port`, or set `port: 0` in the config to ask the OS for one. On start it syncs each provider's -models into Codex's catalog. On shutdown it restores native Codex — unless it was launched as a -managed service (`OCX_SERVICE=1`). +would leave the first proxy running and re-point Codex at the second. An explicit different +`--port` is still refused when the live proxy shares this `OPENCODEX_HOME`, because observe-only +and enforced spend accounting both write the same journal. Use a separate `OPENCODEX_HOME` for an +independent sibling; `port: 0` only asks the OS for that instance's port and does not separate its +state. On start it syncs each provider's models into Codex's catalog. On shutdown it restores +native Codex — unless it was launched as a managed service (`OCX_SERVICE=1`). `--socks5` (default `127.0.0.1:10808`) saves `config.proxy` as a SOCKS5 URL and routes outbound HTTP(S) through a real SOCKS5 tunnel. `--socks5-off` clears only that saved SOCKS5 proxy; it diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index e243f5edd9..8a596e2a96 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -27,7 +27,7 @@ runs helper features around provider requests. | `apiKeys?` | `OcxApiKey[]` | `[]` | Generated `ocx_…` data-plane admission credentials on non-loopback binds. They do not authorize management APIs; management access uses the separate credential documented in the [management reference](/reference/management-api/). Dashboard-managed. | | `storageCleanupPolicy?` | `StorageCleanupPolicy` | disabled | Opt-in archived-session cleanup policy. Never enabled implicitly. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Cap in MiB for evictable app-owned logs, caches, blobs, and continuation payloads. Range 64–4096; not an RSS cap. | -| `spend?` | `{ root?: { maxTokens?: number }; identity?: { maxTokens?: number }; pool?: { maxTokens?: number }; retentionDays?: number }` | unset | Durable token ceilings, off unless you write one. Each scope bounds settled spend plus in-flight reservations plus unresolved spend: `root` is one task including its whole fan-out, `identity` is one account across every task it serves, and `pool` is one provider pool. They intersect, so a request is admitted only when all three have room — which is what holds a ceiling against a client that mints a new task id per request. A reservation is the request's whole input plus its enforceable output ceiling, counted as if every cached prefix misses. Spend survives a restart, so it does not roll forward the way the send-count window does; raising or removing the value is what grants more. `maxTokens` must be a positive integer (0 would refuse everything), `retentionDays` is 1–365 and defaults to 7, and an unknown key in this section is rejected rather than ignored. A refusal is a local HTTP 429 carrying `x-opencodex-local-refusal: workflow_spend_exhausted`, and its message names the scope and the ceiling; no provider is contacted. | +| `spend?` | `{ root?: { maxTokens?: number }; identity?: { maxTokens?: number }; pool?: { maxTokens?: number }; retentionDays?: number }` | unset | Durable token ceilings, off unless you write one. Each scope bounds settled spend plus in-flight reservations plus unresolved spend: `root` is one task including its whole fan-out, `identity` is one account across every task it serves, and `pool` is one provider pool. They intersect, so a request is admitted only when all three have room — which is what holds a ceiling against a client that mints a new task id per request. A reservation is the request's whole input plus its enforceable output ceiling, counted as if every cached prefix misses. Observe-only mode still journals, so every server owns the state directory's single-writer lease; an explicit sibling must use a separate `OPENCODEX_HOME`. Spend survives an ordinary process restart when its writes reached the filesystem, but the journal does not promise survival across host power loss because each append is not fsynced. Raising or removing the value is what grants more. `maxTokens` must be a positive integer (0 would refuse everything), `retentionDays` is 1–365 and defaults to 7, and an unknown key in this section is rejected rather than ignored. A refusal is a local HTTP 429 carrying `x-opencodex-local-refusal: workflow_spend_exhausted`, and its message names the scope and the ceiling; no provider is contacted. | | `codexAutoStart?` | `boolean` | `true` | Let the Codex shim run `ocx ensure` before launching Codex. False makes ensure a no-op. | | `codexShimAutoRestore?` | `boolean` | `true` | Restore an installed shim after a completed external Codex update replaces it. Environment opt-out: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | | `codexDesktopAuthless?` | `boolean` | `false` | Opt-in authless Codex Desktop routing on a loopback bind: inject the dedicated `opencodex` provider with `requires_openai_auth = false` so Desktop opens without a ChatGPT login. Ignored on non-loopback binds. `ocx system settings --desktop-authless on`. See [Codex integration](/guides/codex-integration/#authless-codex-desktop-opt-in). | diff --git a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md index 5bdde33ae2..f338d75388 100644 --- a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md @@ -25,8 +25,10 @@ PID/runtime-port, а попытка поднять второй живой эк занят, `start` проверяет, кто его удерживает, и в любом случае останавливается: он отказывается от запуска, если там отвечает opencodex, а иначе сообщает о неопознанном владельце порта. Прокси никогда сам не переносит listener на другой порт, поскольку тогда первый прокси продолжил бы -работать, а Codex был бы перенаправлен на второй. Укажите другой порт через `--port` или задайте -`port: 0` в конфигурации, чтобы порт назначила ОС. На старте прокси синхронизирует модели каждого +работать, а Codex был бы перенаправлен на второй. Явно указанный другой `--port` также отклоняется +при общем `OPENCODEX_HOME`: режим наблюдения и режим с лимитами оба пишут в один журнал расходов. +Для независимого соседнего экземпляра используйте отдельный `OPENCODEX_HOME`; `port: 0` поручает +ОС выбрать только порт и не разделяет состояние. На старте прокси синхронизирует модели каждого провайдера в каталог Codex. При shutdown он восстанавливает native Codex — если только прокси не был запущен как managed service (`OCX_SERVICE=1`). diff --git a/docs-site/src/content/docs/tr/reference/cli/lifecycle.md b/docs-site/src/content/docs/tr/reference/cli/lifecycle.md index f2b72af51c..ffa5b19df5 100644 --- a/docs-site/src/content/docs/tr/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/tr/reference/cli/lifecycle.md @@ -26,9 +26,11 @@ durumunu yazar ve ikinci bir canlı örneği başlatmayı reddeder. Tercih edile doluysa `start`, portu tutan süreci sorgular ve her iki durumda da durur: orada bir opencodex yanıt veriyorsa başlatmayı reddeder, aksi halde portu tutan sürecin tanımlanamadığını bildirir. İlk proxy'yi çalışır durumda bırakıp Codex'i ikinciye -yönlendireceği için dinleyiciyi kendiliğinden başka bir porta taşımaz. `--port` ile -farklı bir port belirtin veya işletim sisteminden bir port istemek için yapılandırmada -`port: 0` ayarlayın. Başlangıçta her sağlayıcının modellerini Codex'in kataloğuna +yönlendireceği için dinleyiciyi kendiliğinden başka bir porta taşımaz. Aynı +`OPENCODEX_HOME` kullanılırken farklı bir `--port` açıkça verilse de başlangıç reddedilir; +yalnızca gözlem ve sınır uygulama kiplerinin ikisi de aynı harcama günlüğüne yazar. Bağımsız +bir kardeş örnek için ayrı bir `OPENCODEX_HOME` kullanın. `port: 0` yalnızca port seçimini +işletim sistemine bırakır, durumu ayırmaz. Başlangıçta her sağlayıcının modellerini Codex'in kataloğuna senkronize eder. Kapatıldığında — yönetilen bir servis olarak başlatılmadığı sürece (`OCX_SERVICE=1`) — yerel Codex'i geri yükler. diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md index f482b1701e..30687e2b81 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md @@ -15,7 +15,7 @@ description: 安装、启动、停止、服务、诊断、同步和更新命令 ### `ocx start [--port ] [--socks5 [host:port] | --socks5-off]` -启动代理服务器（首选端口 `10100`）。它会写入 PID/运行时端口状态，并拒绝启动第二个存活实例。当首选端口已被占用时，`start` 会探测占用者，并且无论结果如何都会停止：如果那里响应的是 opencodex，它会直接拒绝启动；否则会报告无法识别的占用者。它绝不会自行把监听地址移到其他端口，因为那会让第一个代理继续运行，并将 Codex 重新指向第二个代理。使用 `--port` 指定其他端口，或在配置中设置 `port: 0`，让操作系统分配端口。启动时，它会把每个提供方的模型同步到 Codex 的目录中。关闭时，它会恢复原生 Codex，除非它是作为受管服务启动的（`OCX_SERVICE=1`）。 +启动代理服务器（首选端口 `10100`）。它会写入 PID/运行时端口状态，并拒绝启动第二个存活实例。当首选端口已被占用时，`start` 会探测占用者，并且无论结果如何都会停止：如果那里响应的是 opencodex，它会直接拒绝启动；否则会报告无法识别的占用者。它绝不会自行把监听地址移到其他端口，因为那会让第一个代理继续运行，并将 Codex 重新指向第二个代理。即使显式指定不同的 `--port`，共用同一个 `OPENCODEX_HOME` 时也会拒绝启动，因为仅观察模式和启用上限的模式都会写入同一个支出日志。独立的同级实例必须使用单独的 `OPENCODEX_HOME`；`port: 0` 只让操作系统分配端口，并不会隔离状态。启动时，它会把每个提供方的模型同步到 Codex 的目录中。关闭时，它会恢复原生 Codex，除非它是作为受管服务启动的（`OCX_SERVICE=1`）。 `--socks5`（默认 `127.0.0.1:10808`）会将 SOCKS5 URL 保存到 `config.proxy`，并通过真正的 SOCKS5 隧道转发出站 HTTP(S) 请求。`--socks5-off` 只会清除已保存的 SOCKS5 代理，不会删除 HTTP 代理。该值保存在配置中，因此会在 `ocx update` 后保留。URL 可以包含用户名和密码，但启动日志会将其隐藏。 diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md b/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md index da2a7e0847..d9c104d3c9 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md @@ -15,7 +15,7 @@ description: 安裝、啟動、停止、服務、診斷、同步與更新指令 ### `ocx start [--port ] [--socks5 [host:port] | --socks5-off]` -啟動代理伺服器（偏好連接埠 `10100`）。它寫入 PID/runtime-port 狀態，並拒絕啟動第二個即時實例。偏好連接埠被佔用時，`start` 會探測佔用者，且無論結果如何都會停止：若回應的是 opencodex，它會直接拒絕啟動；否則會回報無法識別的佔用者。它絕不會自行將監聽位置移到其他連接埠，因為這會讓第一個代理繼續執行，並將 Codex 重新指向第二個代理。請用 `--port` 指定其他連接埠，或在設定中設為 `port: 0`，讓作業系統指派連接埠。啟動時它將每個供應商的模型同步到 Codex 目錄。關閉時它還原原生 Codex——除非它是作為受管服務啟動的（`OCX_SERVICE=1`）。 +啟動代理伺服器（偏好連接埠 `10100`）。它寫入 PID/runtime-port 狀態，並拒絕啟動第二個即時實例。偏好連接埠被佔用時，`start` 會探測佔用者，且無論結果如何都會停止：若回應的是 opencodex，它會直接拒絕啟動；否則會回報無法識別的佔用者。它絕不會自行將監聽位置移到其他連接埠，因為這會讓第一個代理繼續執行，並將 Codex 重新指向第二個代理。即使明確指定不同的 `--port`，共用同一個 `OPENCODEX_HOME` 時仍會拒絕啟動，因為僅觀察模式和啟用上限的模式都會寫入同一份支出日誌。獨立的同層實例必須使用不同的 `OPENCODEX_HOME`；`port: 0` 只讓作業系統指派連接埠，不會隔離狀態。啟動時它將每個供應商的模型同步到 Codex 目錄。關閉時它還原原生 Codex——除非它是作為受管服務啟動的（`OCX_SERVICE=1`）。 `--socks5`（預設 `127.0.0.1:10808`）會將 SOCKS5 URL 儲存到 `config.proxy`，並透過真正的 SOCKS5 通道轉送對外 HTTP(S) 請求。`--socks5-off` 只會清除已儲存的 SOCKS5 代理，不會刪除 HTTP 代理。此值儲存在設定中，因此會在 `ocx update` 後保留。URL 可以包含使用者名稱和密碼，但啟動記錄會隱藏它們。 diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 906114492d..45ccb64cc1 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1345,6 +1345,8 @@ "sponsor-presets.test.ts": "providers", "spend-ceiling-enforcement.test.ts": "lib", "spend-ledger-file-journal.test.ts": "lib", + "spend-ledger-owner-startup.test.ts": "server", + "spend-ledger-owner.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sse-client-frame-bounds.test.ts": "responses", "sse-decoder.test.ts": "responses", diff --git a/src/cli/dispatch.ts b/src/cli/dispatch.ts index d152d673f5..029336f66c 100644 --- a/src/cli/dispatch.ts +++ b/src/cli/dispatch.ts @@ -945,9 +945,9 @@ export type StartOwnerDecision = "refuse" | "service-stay-out" | "sibling"; * * The #3106 guard exists so a bare `start` cannot shadow a healthy configured-port * proxy with an ephemeral-port copy. An interactive `--port X` naming a DIFFERENT - * port than the live proxy's is an explicit sibling request, not that shadow — and - * refusing it also broke every spawned-launcher test on a machine running a real - * proxy, because the probe reaches the machine-global port across sandbox homes. + * port than the live proxy's is an explicit sibling request, not that shadow. The + * state-directory spend-ledger lease makes the final same-home refusal; keeping this + * decision allows isolated homes on one machine to remain independent. * The service wrapper always passes the configured port and keeps its exact * stay-out-of-the-way semantics: it never takes the sibling path. */ diff --git a/src/cli/index.ts b/src/cli/index.ts index fb68ae09f2..00d89469fd 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -78,6 +78,7 @@ import { } from "./tray-proxy"; import { requestBoundSystemRestart } from "./system-restart-client"; import { installCrashGuards } from "../lib/crash-guard"; +import { SpendLedgerOwnerError } from "../lib/spend-ledger-owner"; import { redactUrlForLog } from "../lib/redact"; import { dispatchCommand, decideBusyPreferredPort, decideStartWithLiveOwner } from "./dispatch"; import { AuxiliaryListenerBindError, findAvailablePort, isAddrInUse, PortUnavailableError, shouldPersistSelectedPort, waitForPortAvailable } from "../server/ports"; @@ -418,8 +419,8 @@ async function handleStart(options: { block?: boolean } = {}) { // to bake the service (observed: a probe on 10198 left the service pinned there). siblingStart = true; console.warn( - `Proxy already running on port ${owner.live.port}; starting a second instance on requested port ${requestedPort}. ` - + `The new instance takes over this home's pid/runtime records and Codex config while it runs.`, + `Proxy already running on port ${owner.live.port}; requested a second instance on port ${requestedPort}. ` + + `Startup continues only for an independent OPENCODEX_HOME; one state directory has one spend-ledger writer.`, ); } @@ -463,6 +464,10 @@ async function handleStart(options: { block?: boolean } = {}) { scheduleCatalogPrewarm(); break; } catch (err) { + if (err instanceof SpendLedgerOwnerError) { + console.error(`❌ ${err.message}`); + process.exit(1); + } if (err instanceof AuxiliaryListenerBindError || !isAddrInUse(err) || attempt >= 2) throw err; if (requestedPort !== undefined) { console.log(`⚠️ Port ${port} was taken while starting; waiting to retry the same port...`); diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts new file mode 100644 index 0000000000..ccc9509f21 --- /dev/null +++ b/src/lib/spend-ledger-owner.ts @@ -0,0 +1,218 @@ +/** Process-lifetime single-writer ownership for the shared spend journal. */ +import { Database } from "bun:sqlite"; +import { chmodSync, closeSync, lstatSync, mkdirSync, openSync, realpathSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { getConfigDir } from "../config/paths"; +import { recordOwnedConfigPath } from "./config-ownership"; +import { assertNotRealHomeUnderTest } from "./test-home-guard"; +import { hardenSecretDir, hardenSecretPath } from "./windows-secret-acl"; + +export const SPEND_LEDGER_OWNER_FILENAME = "spend-ledger-owner.sqlite"; +const OWNER_SIDECARS = ["-journal", "-wal", "-shm"] as const; + +export type SpendLedgerOwnerErrorCode = + | "SPEND_LEDGER_OWNER_BUSY" + | "SPEND_LEDGER_OWNER_UNAVAILABLE" + | "SPEND_LEDGER_OWNER_HOME_CONFLICT" + | "SPEND_LEDGER_OWNER_NOT_HELD"; + +export class SpendLedgerOwnerError extends Error { + constructor(readonly code: SpendLedgerOwnerErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.name = "SpendLedgerOwnerError"; + } +} + +export interface SpendLedgerOwnerLease { + release(): void; +} + +interface ActiveOwner { + readonly home: string; + readonly database: Database; + references: number; +} + +let activeOwner: ActiveOwner | null = null; +let boundLedgerHome: string | null = null; + +function errorCode(error: unknown): unknown { + return error !== null && typeof error === "object" && "code" in error ? error.code : undefined; +} + +function assertPrivateFile(path: string): void { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger ownership could not be established safely.", + ); + } + if (process.platform !== "win32" && stat.uid !== process.getuid!()) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger ownership could not be established safely.", + ); + } +} + +function prepareOwnerPath(configDir: string): { home: string; path: string } { + const requested = resolve(configDir); + // First statement before mutation: tests must never acquire against the real user home. + assertNotRealHomeUnderTest(requested); + mkdirSync(requested, { recursive: true, mode: 0o700 }); + const home = realpathSync.native(requested); + if (process.platform !== "win32") chmodSync(home, 0o700); + hardenSecretDir(home, { required: true }); + const path = join(home, SPEND_LEDGER_OWNER_FILENAME); + try { closeSync(openSync(path, "wx", 0o600)); } + catch (error) { if (errorCode(error) !== "EEXIST") throw error; } + assertPrivateFile(path); + if (process.platform !== "win32") chmodSync(path, 0o600); + hardenSecretPath(path, { required: true }); + assertPrivateFile(path); + recordOwnedConfigPath(home, path); + for (const suffix of OWNER_SIDECARS) recordOwnedConfigPath(home, `${path}${suffix}`); + return { home: process.platform === "win32" ? home.toLowerCase() : home, path }; +} + +function isBusy(error: unknown): boolean { + const code = errorCode(error); + const message = error instanceof Error ? error.message : ""; + return code === "SQLITE_BUSY" || code === "SQLITE_LOCKED" + || /database (?:is|table is) locked/i.test(message); +} + +function stateDirectoryIdentity(configDir: string): string { + const requested = resolve(configDir); + assertNotRealHomeUnderTest(requested); + let canonical = requested; + try { canonical = realpathSync.native(requested); } catch { /* acquisition owns creation */ } + return process.platform === "win32" ? canonical.toLowerCase() : canonical; +} + +/** + * Hold one SQLite write transaction until the final in-process reference releases it. + * SQLite and the OS release a crashed process; no PID, timestamp, TTL or lock-file unlink + * can evict a live owner or mistake a reused process identity for this lease. + */ +export function acquireSpendLedgerOwner(configDir = getConfigDir()): SpendLedgerOwnerLease { + const requestedHome = stateDirectoryIdentity(configDir); + if (boundLedgerHome !== null && boundLedgerHome !== requestedHome) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_HOME_CONFLICT", + "This process already owns the spend ledger for a different state directory.", + ); + } + if (activeOwner) { + if (activeOwner.home !== requestedHome) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_HOME_CONFLICT", + "This process already owns the spend ledger for a different state directory.", + ); + } + activeOwner.references += 1; + return leaseFor(activeOwner); + } + + let prepared: { home: string; path: string }; + try { + prepared = prepareOwnerPath(configDir); + } catch (cause) { + if (cause instanceof SpendLedgerOwnerError) throw cause; + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger ownership could not be established safely.", + { cause }, + ); + } + + let database: Database | undefined; + try { + database = new Database(prepared.path, { create: true }); + database.exec("PRAGMA locking_mode = NORMAL; PRAGMA busy_timeout = 0; BEGIN IMMEDIATE"); + } catch (cause) { + try { database?.close(); } catch { /* preserve acquisition failure */ } + if (isBusy(cause)) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_BUSY", + "Another OpenCodex process already owns this spend ledger. Use a separate OPENCODEX_HOME for an independent instance.", + ); + } + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger ownership could not be established safely.", + { cause }, + ); + } + + activeOwner = { home: prepared.home, database, references: 1 }; + return leaseFor(activeOwner); +} + +function leaseFor(owner: ActiveOwner): SpendLedgerOwnerLease { + let released = false; + return Object.freeze({ + release(): void { + if (released) return; + released = true; + if (activeOwner !== owner || owner.references < 1) return; + owner.references -= 1; + if (owner.references > 0) return; + activeOwner = null; + let failure: unknown; + try { owner.database.exec("ROLLBACK"); } catch (error) { failure = error; } + try { owner.database.close(); } catch (error) { failure ??= error; } + if (failure !== undefined) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger ownership could not be released cleanly.", + { cause: failure }, + ); + } + }, + }); +} + +/** The singleton journal may be touched only while its matching state directory is owned. */ +export function assertSpendLedgerOwnerHeld(configDir = getConfigDir()): void { + let home: string; + try { + const canonical = realpathSync.native(resolve(configDir)); + home = process.platform === "win32" ? canonical.toLowerCase() : canonical; + } catch (cause) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + { cause }, + ); + } + if (!activeOwner || activeOwner.home !== home || activeOwner.references < 1) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + ); + } +} + +/** Permanently bind the live process-wide singleton to its first constructed home. */ +export function bindSpendLedgerOwnerHome(configDir = getConfigDir()): void { + assertSpendLedgerOwnerHeld(configDir); + const home = stateDirectoryIdentity(configDir); + if (boundLedgerHome !== null && boundLedgerHome !== home) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_HOME_CONFLICT", + "This process already owns the spend ledger for a different state directory.", + ); + } + boundLedgerHome = home; +} + +export function spendLedgerOwnerSnapshot(): { readonly ownership: "held" | "unheld" } { + return { ownership: activeOwner ? "held" : "unheld" }; +} + +/** Test seam paired with discarding the process-wide ledger singleton. */ +export function resetSpendLedgerOwnerBindingForTest(): void { + boundLedgerHome = null; +} diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 5c10efca77..b5f3f79fa4 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -24,11 +24,10 @@ * released. Minting a new root id mints no new budget because the identity and pool scopes * still hold the spend. * - * SUPPORTED TOPOLOGY: this guarantees a single proxy process against its own journal. The - * file is append-friendly, but nothing here serializes two live processes writing it, so a - * second proxy sharing the same OPENCODEX_HOME is explicitly outside the guarantee -- that - * needs a shared store with cross-process atomicity and is declared out of scope rather - * than implied. + * SUPPORTED TOPOLOGY: one live writer owns one OPENCODEX_HOME journal. Every server and + * direct shared-ledger caller must hold the state-directory SQLite lease before replay, + * append or compaction. Independent homes remain independent; multi-host shared storage + * still needs a distributed transaction boundary and is outside this local lease. * * Five properties this file owes its callers. Each one was absent in the first draft, and a * budget that can be bypassed is worse than no budget because it looks like protection: @@ -66,6 +65,7 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; +import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, resetSpendLedgerOwnerBindingForTest, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; /** @@ -1036,6 +1036,8 @@ export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined): SpendR * configures no ceiling must not open a journal merely because the server started. */ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void { + assertSpendLedgerOwnerHeld(); + if (sharedLedger) bindSpendLedgerOwnerHome(); sharedPolicy = policy; sharedLedger?.reconfigure(policy); } @@ -1046,6 +1048,8 @@ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void * disk. */ export function sharedSpendLedger(): SpendReservationLedger { + assertSpendLedgerOwnerHeld(); + bindSpendLedgerOwnerHome(); if (!sharedLedger) { const home = getConfigDir(); sharedLedger = createSpendReservationLedger({ @@ -1057,8 +1061,32 @@ export function sharedSpendLedger(): SpendReservationLedger { return sharedLedger; } +const MAX_DIAGNOSTIC_ERROR_COUNT = 1_000_000; + +/** Scalar-only and side-effect-free: reading diagnostics never constructs or replays. */ +export function spendLedgerDiagnosticsSnapshot(): { + readonly ownership: "held" | "unheld"; + readonly initialized: boolean; + readonly configured: boolean; + readonly degraded: boolean; + readonly persistFailures: number; + readonly corruptRecords: number; +} { + const ledger = sharedLedger; + const bounded = (value: number): number => Math.min(MAX_DIAGNOSTIC_ERROR_COUNT, Math.max(0, value)); + return { + ...spendLedgerOwnerSnapshot(), + initialized: ledger !== undefined, + configured: spendCeilingsConfigured(sharedPolicy), + degraded: ledger?.degraded ?? false, + persistFailures: bounded(ledger?.persistFailures ?? 0), + corruptRecords: bounded(ledger?.corruptRecords ?? 0), + }; +} + /** Test seam. Production never discards the ledger: that would reset a spent budget. */ export function resetSharedSpendLedgerForTest(): void { sharedLedger = undefined; sharedPolicy = DEFAULT_SPEND_RESERVATION_POLICY; + resetSpendLedgerOwnerBindingForTest(); } diff --git a/src/server/index.ts b/src/server/index.ts index 47323e0f68..af20051d9e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -116,7 +116,6 @@ import { type RequestLogEntry, } from "./request-log"; import { sessionLaneIdFromRequest } from "./request-log-conversation"; -import { configureSharedSpendLedger, spendPolicyFromConfig } from "../lib/spend-reservation-ledger"; import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse, type WorkflowRefusalLog } from "./workflow-refusal"; export { addFinalRequestLog, @@ -205,8 +204,15 @@ import { import { detectInstall } from "../update/index"; import { createServeOptions, type ServerIngress } from "./index/serve-options"; import { inspectStartupOwnership, setStartupCacheInvalidationWrite, warnAgentTaskRecoveryStartup, warnPlaintextV2AgentMessagesStartup, type StartServerDeps } from "./index/startup-warnings"; +import { acquireSpendLedgerServerLifecycle, type SpendLedgerServerLifecycle } from "./index/spend-ledger-lifecycle"; export function startServer(port?: number, deps: StartServerDeps = {}): Server { + const spendLedgerLifecycle = acquireSpendLedgerServerLifecycle(getConfigDir()); + try { return startServerWithSpendLedgerOwner(port, deps, spendLedgerLifecycle); } + catch (error) { spendLedgerLifecycle.releaseAfterFailedStart(); throw error; } +} + +function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartServerDeps, spendLedgerLifecycle: SpendLedgerServerLifecycle): Server { const localAttestationSecret = deps.localAttestationSecret ?? createLocalAttestationSecret(); // Captured before loadConfig() starts the optional ACL flight so stop() drains the same dir // even if OPENCODEX_HOME changes underneath a long-lived process. @@ -299,12 +305,8 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server({ ...serveOptions, port: listenPort, hostname: bindHost }); + server = spendLedgerLifecycle.track(Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost })); // Both binds are one startup transaction (#1102). If the loopback bind fails after the // public one succeeded, leaving the public listener up would strand it: the CLI's port @@ -711,11 +713,11 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server({ + loopbackServer = spendLedgerLifecycle.track(Bun.serve({ ...serveOptions, port: loopbackListenerPort, hostname: "127.0.0.1", - }); + })); } catch (error) { try { // startServer is synchronous, so this rollback cannot await. Bun begins closing the @@ -731,11 +733,11 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server({ + managementIngressServer = spendLedgerLifecycle.track(Bun.serve({ ...serveOptions, port: managementIngressPort, hostname: "127.0.0.1", - }); + })); } catch (error) { // Preserve the management bind failure while synchronously initiating rollback of every // listener already opened in this startup transaction. startServer must not become async. @@ -792,7 +794,8 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server }>(server: T): T; + release(): void; + releaseAfterFailedStart(): void; +} + +/** Acquire before config loading so every later startup failure has one rollback owner. */ +export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedgerServerLifecycle { + const owner: SpendLedgerOwnerLease = acquireSpendLedgerOwner(configDir); + const failedStartStops: Array<() => void> = []; + let released = false; + const release = (): void => { + if (released) return; + released = true; + owner.release(); + }; + return { + configure(spend): void { + configureSharedSpendLedger(spendPolicyFromConfig(spend)); + }, + track }>(server: T): T { + // Capture the raw stop before startServer replaces the public method with full teardown. + const stop = server.stop.bind(server); + failedStartStops.push(() => { try { void stop(true); } catch { /* preserve startup failure */ } }); + return server; + }, + release, + releaseAfterFailedStart(): void { + for (const stop of failedStartStops.reverse()) stop(); + release(); + }, + }; +} diff --git a/src/server/management/system-routes.ts b/src/server/management/system-routes.ts index 3819a8e52d..04d9f56621 100644 --- a/src/server/management/system-routes.ts +++ b/src/server/management/system-routes.ts @@ -39,6 +39,7 @@ import { } from "../../lib/codex-restart-contract"; import { jsonResponse } from "../auth-cors"; import { getInspectionCounters } from "../relay"; +import { spendLedgerDiagnosticsSnapshot } from "../../lib/spend-reservation-ledger"; import type { performCodexRestart, readCodexAppServerState, @@ -60,6 +61,7 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise { const lines: string[] = []; @@ -80,17 +81,21 @@ const watched = (inner: SpendReservationLedger, asked: string[]): SpendReservati let home = ""; let previousHome: string | undefined; +let owner: SpendLedgerOwnerLease | null = null; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-spend-ceiling-")); process.env.OPENCODEX_HOME = home; + owner = acquireSpendLedgerOwner(); resetSharedSpendLedgerForTest(); resetWorkflowBudgetsForTest(); }); afterEach(() => { resetSharedSpendLedgerForTest(); + owner?.release(); + owner = null; if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; removeTreeWithRetry(home); diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts new file mode 100644 index 0000000000..e2e542f372 --- /dev/null +++ b/tests/lib/spend-ledger-owner.test.ts @@ -0,0 +1,210 @@ +/** Cross-process ownership for the process-wide spend journal (#5123). */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + acquireSpendLedgerOwner, + SPEND_LEDGER_OWNER_FILENAME, + SpendLedgerOwnerError, + spendLedgerOwnerSnapshot, + type SpendLedgerOwnerLease, +} from "../../src/lib/spend-ledger-owner"; +import { + SPEND_LEDGER_JOURNAL_FILENAME, + SPEND_LEDGER_SALT_FILENAME, + resetSharedSpendLedgerForTest, + sharedSpendLedger, + spendLedgerDiagnosticsSnapshot, +} from "../../src/lib/spend-reservation-ledger"; +import { helperPath } from "../helpers/repo-root"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS } from "../helpers/test-budget"; + +const childPath = helperPath("spend-ledger-owner-child.ts"); +let root = ""; +let home = ""; +let previousHome: string | undefined; +const children = new Set>(); + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "ocx-spend-owner-")); + home = join(root, "state-a"); + previousHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + resetSharedSpendLedgerForTest(); +}); + +afterEach(async () => { + for (const child of children) { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await child.exited; + } + children.clear(); + resetSharedSpendLedgerForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(root); +}); + +function spawnHolder(targetHome: string, mode: "observe" | "enforced", suffix: string) { + const holdMarker = join(root, `held-${suffix}`); + const releaseMarker = join(root, `release-${suffix}`); + const child = Bun.spawn([process.execPath, childPath], { + env: { + ...process.env, + OPENCODEX_HOME: targetHome, + OCX_SPEND_OWNER_CHILD: JSON.stringify({ holdMarker, releaseMarker, mode }), + }, + stdout: "pipe", + stderr: "pipe", + }); + children.add(child); + return { child, holdMarker, releaseMarker }; +} + +async function waitForMarker(path: string, child: ReturnType): Promise { + const deadline = Date.now() + INTERNAL_DEADLINE_MS; + while (Date.now() < deadline) { + if (existsSync(path)) return; + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`owner child exited before holding: ${await new Response(child.stderr).text()}`); + } + await Bun.sleep(10); + } + throw new Error("timed out waiting for spend-ledger owner child"); +} + +async function childResult(child: ReturnType) { + const [stdout] = await Promise.all([new Response(child.stdout).text(), child.exited]); + children.delete(child); + return JSON.parse(stdout.trim().split("\n").filter(Boolean).at(-1) ?? "{}") as { + status: string; + code?: string; + message?: string; + }; +} + +function busyError(): SpendLedgerOwnerError { + try { acquireSpendLedgerOwner(); } + catch (error) { + if (error instanceof SpendLedgerOwnerError) return error; + throw error; + } + throw new Error("expected spend-ledger ownership refusal"); +} + +describe("real process ownership", () => { + for (const [holderMode, contenderMode] of [["observe", "enforced"], ["enforced", "observe"]] as const) { + test(`${holderMode} and ${contenderMode} configurations contend identically`, async () => { + const holder = spawnHolder(home, holderMode, `${holderMode}-${contenderMode}`); + await waitForMarker(holder.holdMarker, holder.child); + const refusal = busyError(); + expect(refusal.code).toBe("SPEND_LEDGER_OWNER_BUSY"); + writeFileSync(holder.releaseMarker, "release"); + expect((await childResult(holder.child)).status).toBe("acquired"); + }, SPAWN_BUDGET_MS); + } + + test("independent state directories are independent", async () => { + const holder = spawnHolder(home, "observe", "independent"); + await waitForMarker(holder.holdMarker, holder.child); + const other = acquireSpendLedgerOwner(join(root, "state-b")); + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + other.release(); + writeFileSync(holder.releaseMarker, "release"); + await childResult(holder.child); + }, SPAWN_BUDGET_MS); + + test("graceful release lets the next process acquire", async () => { + const holder = spawnHolder(home, "observe", "graceful"); + await waitForMarker(holder.holdMarker, holder.child); + writeFileSync(holder.releaseMarker, "release"); + await childResult(holder.child); + const next = acquireSpendLedgerOwner(); + next.release(); + }, SPAWN_BUDGET_MS); + + test("an abruptly killed owner is reacquirable without replacing the lock file", async () => { + const holder = spawnHolder(home, "observe", "killed"); + await waitForMarker(holder.holdMarker, holder.child); + const lockPath = join(home, SPEND_LEDGER_OWNER_FILENAME); + const journalPath = join(home, SPEND_LEDGER_JOURNAL_FILENAME); + const saltPath = join(home, SPEND_LEDGER_SALT_FILENAME); + const before = [lockPath, journalPath, saltPath].map(path => ({ path, stat: statSync(path) })); + holder.child.kill("SIGKILL"); + await holder.child.exited; + children.delete(holder.child); + const next = acquireSpendLedgerOwner(); + for (const entry of before) { + const after = statSync(entry.path); + expect(after.size).toBe(entry.stat.size); + if (process.platform !== "win32") expect(after.ino).toBe(entry.stat.ino); + } + next.release(); + }, SPAWN_BUDGET_MS); +}); + +describe("in-process references and privacy", () => { + let leases: SpendLedgerOwnerLease[] = []; + afterEach(() => { + for (const lease of leases.splice(0).reverse()) lease.release(); + }); + + test("two leases share ownership and one release does not free it", async () => { + const first = acquireSpendLedgerOwner(); + const second = acquireSpendLedgerOwner(); + leases.push(first, second); + first.release(); + const holder = spawnHolder(home, "observe", "references"); + expect((await childResult(holder.child)).code).toBe("SPEND_LEDGER_OWNER_BUSY"); + second.release(); + expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); + }, SPAWN_BUDGET_MS); + + test("one process refuses a second different state directory", () => { + leases.push(acquireSpendLedgerOwner()); + let failure: unknown; + try { acquireSpendLedgerOwner(join(root, "state-b")); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_HOME_CONFLICT"); + }); + + test("a constructed singleton keeps its home after the final lease releases", () => { + const first = acquireSpendLedgerOwner(); + leases.push(first); + sharedSpendLedger(); + first.release(); + process.env.OPENCODEX_HOME = join(root, "state-b"); + let failure: unknown; + try { acquireSpendLedgerOwner(); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_HOME_CONFLICT"); + }); + + test("busy refusal contains no private identity or filesystem data", async () => { + const holder = spawnHolder(home, "observe", "privacy"); + await waitForMarker(holder.holdMarker, holder.child); + const refusal = busyError(); + const text = `${refusal.name} ${refusal.code} ${refusal.message}`.toLowerCase(); + for (const secret of [home.toLowerCase(), String(process.pid), "spend-ledger.jsonl", "child-root", "account", "scope", "request"]) { + expect(text).not.toContain(secret); + } + writeFileSync(holder.releaseMarker, "release"); + await childResult(holder.child); + }, SPAWN_BUDGET_MS); + + test("diagnostics do not construct or create ledger files", () => { + leases.push(acquireSpendLedgerOwner()); + expect(spendLedgerDiagnosticsSnapshot()).toEqual({ + ownership: "held", + initialized: false, + configured: false, + degraded: false, + persistFailures: 0, + corruptRecords: 0, + }); + expect(existsSync(join(home, SPEND_LEDGER_JOURNAL_FILENAME))).toBe(false); + expect(existsSync(join(home, SPEND_LEDGER_SALT_FILENAME))).toBe(false); + }); +}); diff --git a/tests/server/spend-ledger-owner-startup.test.ts b/tests/server/spend-ledger-owner-startup.test.ts new file mode 100644 index 0000000000..d93baf5c52 --- /dev/null +++ b/tests/server/spend-ledger-owner-startup.test.ts @@ -0,0 +1,94 @@ +/** startServer owns and releases the shared spend journal lease (#5123). */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../../src/config"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import { flushNativeMainStartupReleases } from "../../src/codex/native-profile-startup"; +import { + SPEND_LEDGER_JOURNAL_FILENAME, + SPEND_LEDGER_SALT_FILENAME, + resetSharedSpendLedgerForTest, +} from "../../src/lib/spend-reservation-ledger"; +import { acquireSpendLedgerOwner, spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; +import { startServer } from "../../src/server"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +let home = ""; +let previousHome: string | undefined; +let codexHome: IsolatedCodexHome | null = null; + +function config(): OcxConfig { + return { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "kimi", + providers: { kimi: { adapter: "openai-chat", baseUrl: "https://kimi.test/v1", models: ["k3"] } }, + }; +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-spend-owner-startup-")); + previousHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + codexHome = installIsolatedCodexHome("ocx-spend-owner-codex-"); + resetSharedSpendLedgerForTest(); + saveConfig(config()); +}); + +afterEach(async () => { + await flushNativeMainStartupReleases(); + await flushConfigDirHardeningForTests(); + resetSharedSpendLedgerForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome?.restore(); + codexHome = null; + removeTreeWithRetry(home); +}); + +test("startServer acquires before serving and final stop releases", async () => { + const server = startServer(0); + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + try { + const adminToken = readFileSync(join(home, "admin-api-token"), "utf8").trim(); + const liveness = await fetch(new URL("/healthz", server.url)); + expect(await liveness.json()).not.toHaveProperty("spendLedger"); + const health = await fetch(new URL("/api/system/health", server.url), { + headers: { "x-opencodex-api-key": adminToken }, + }); + expect(await health.json()).toMatchObject({ + spendLedger: { + ownership: "held", + initialized: false, + configured: false, + degraded: false, + persistFailures: 0, + corruptRecords: 0, + }, + }); + expect(existsSync(join(home, SPEND_LEDGER_JOURNAL_FILENAME))).toBe(false); + expect(existsSync(join(home, SPEND_LEDGER_SALT_FILENAME))).toBe(false); + } finally { + await server.stop(true); + } + expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); +}); + +test("a partial start that bound public before an auxiliary failure releases ownership", () => { + const blocker = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("blocked") }); + try { + const candidate = config(); + candidate.unauthenticatedLoopbackListener = { enabled: true, port: blocker.port }; + saveConfig(candidate); + expect(() => startServer(0)).toThrow(); + expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); + const next = acquireSpendLedgerOwner(); + next.release(); + } finally { + blocker.stop(true); + } +}); From 192d0c5223d7322a6202b4059f46d3a54d980f55 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 18:24:35 +0900 Subject: [PATCH 02/34] fix(spend): bind journal mutations to a live lease, not to the handle Pre-publication review found three gaps in the writer lease. The guard sat where the shared ledger was handed out, so a caller holding an existing ledger or journal facade could keep mutating after the final release, once another process legitimately owned the home. Ownership is now asserted at the mutations themselves: every reservation state change, every file-backed append and compaction rewrite, and salt creation. A retained handle refuses to write, which the new negative case pins with a second owner in place. Two differently configured homes could alias one journal or salt through a link while holding separate owner databases, so both could write the same file. The backing files must now be regular, single-link, self-owned files; links, owner files and unusable owner databases are refused. This is the cooperative configuration contract, not a claim of isolation against a hostile same-UID process. The unsupervised restart starts its replacement before the parent exits. With a zero busy timeout and every owner error terminal, the child would exit busy while the parent still owned the home, and the parent would then exit leaving nothing serving. The deferred restart child now carries a one-use parent marker, accepted only when it matches its actual parent, and waits a bounded five seconds for ownership. An ordinary sibling still fails closed immediately. --- src/lib/spend-ledger-owner.ts | 22 +++- src/lib/spend-reservation-ledger.ts | 78 +++++++++++-- src/server/management/system-restart.ts | 9 +- tests/lib/spend-ledger-owner.test.ts | 148 +++++++++++++++++++++++- 4 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index ccc9509f21..ad956a5c28 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -8,6 +8,8 @@ import { assertNotRealHomeUnderTest } from "./test-home-guard"; import { hardenSecretDir, hardenSecretPath } from "./windows-secret-acl"; export const SPEND_LEDGER_OWNER_FILENAME = "spend-ledger-owner.sqlite"; +export const SPEND_LEDGER_RESTART_PARENT_ENV = "OCX_SPEND_LEDGER_RESTART_PARENT_PID"; +export const SPEND_LEDGER_RESTART_WAIT_MS = 5_000; const OWNER_SIDECARS = ["-journal", "-wal", "-shm"] as const; export type SpendLedgerOwnerErrorCode = @@ -91,6 +93,23 @@ function stateDirectoryIdentity(configDir: string): string { return process.platform === "win32" ? canonical.toLowerCase() : canonical; } +function restartHandoffWaitMs(): number { + const markedParent = process.env[SPEND_LEDGER_RESTART_PARENT_ENV]; + delete process.env[SPEND_LEDGER_RESTART_PARENT_ENV]; + return markedParent === String(process.ppid) ? SPEND_LEDGER_RESTART_WAIT_MS : 0; +} + +/** Mark only a parent-exit restart child for bounded lease acquisition. */ +export function spendLedgerRestartEnvironment( + source: NodeJS.ProcessEnv, + parentPid?: number, +): NodeJS.ProcessEnv { + const env = { ...source }; + if (parentPid === undefined) delete env[SPEND_LEDGER_RESTART_PARENT_ENV]; + else env[SPEND_LEDGER_RESTART_PARENT_ENV] = String(parentPid); + return env; +} + /** * Hold one SQLite write transaction until the final in-process reference releases it. * SQLite and the OS release a crashed process; no PID, timestamp, TTL or lock-file unlink @@ -98,6 +117,7 @@ function stateDirectoryIdentity(configDir: string): string { */ export function acquireSpendLedgerOwner(configDir = getConfigDir()): SpendLedgerOwnerLease { const requestedHome = stateDirectoryIdentity(configDir); + const busyTimeout = restartHandoffWaitMs(); if (boundLedgerHome !== null && boundLedgerHome !== requestedHome) { throw new SpendLedgerOwnerError( "SPEND_LEDGER_OWNER_HOME_CONFLICT", @@ -130,7 +150,7 @@ export function acquireSpendLedgerOwner(configDir = getConfigDir()): SpendLedger let database: Database | undefined; try { database = new Database(prepared.path, { create: true }); - database.exec("PRAGMA locking_mode = NORMAL; PRAGMA busy_timeout = 0; BEGIN IMMEDIATE"); + database.exec(`PRAGMA locking_mode = NORMAL; PRAGMA busy_timeout = ${busyTimeout}; BEGIN IMMEDIATE`); } catch (cause) { try { database?.close(); } catch { /* preserve acquisition failure */ } if (isBusy(cause)) { diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index b5f3f79fa4..e122166680 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -53,7 +53,7 @@ * that are re-applied to an EXISTING file rather than trusted from its creation. */ -import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; +import { appendFileSync, chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; // Definition-site import, not the ../config barrel -- same reasoning as @@ -65,7 +65,7 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, resetSpendLedgerOwnerBindingForTest, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; +import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; /** @@ -391,7 +391,21 @@ function hardenLedgerFile(path: string, options: { readonly force?: boolean } = } catch { /* best-effort: a non-owner cannot chmod */ } } -export function createFileSpendJournal(path: string): SpendJournal { +function assertSafeLedgerFile(path: string): void { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 + || (process.platform !== "win32" && stat.uid !== process.getuid!())) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger storage could not be opened safely.", + ); + } +} + +export function createFileSpendJournal( + path: string, + options: { readonly assertMutation?: () => void } = {}, +): SpendJournal { const ensureDir = (): string => { const dir = dirname(path); // The guard runs before any mutation so a rejected write leaves nothing behind. @@ -402,25 +416,37 @@ export function createFileSpendJournal(path: string): SpendJournal { return { read(): string[] { if (!existsSync(path)) return []; + assertSafeLedgerFile(path); // Replay is once per process and is the moment a journal inherited from an older build // or a restored backup first passes through here. hardenLedgerFile(path, { force: true }); return readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0); }, append(line: string): void { + options.assertMutation?.(); ensureDir(); const created = !existsSync(path); + if (!created) assertSafeLedgerFile(path); appendFileSync(path, line + "\n", { encoding: "utf8", mode: 0o600 }); + assertSafeLedgerFile(path); hardenLedgerFile(path, { force: created }); }, rewrite(lines: string[]): void { + options.assertMutation?.(); ensureDir(); // Same directory, so the rename is atomic on the same filesystem: a crash mid-compaction // leaves either the old journal or the new one, never a half-written ledger. - const temp = `${path}.compact-${process.pid}`; - writeFileSync(temp, lines.map((line) => line + "\n").join(""), { encoding: "utf8", mode: 0o600 }); + if (existsSync(path)) assertSafeLedgerFile(path); + const temp = `${path}.compact-${process.pid}-${randomBytes(6).toString("hex")}`; + writeFileSync(temp, lines.map((line) => line + "\n").join(""), { + encoding: "utf8", + mode: 0o600, + flag: "wx", + }); + assertSafeLedgerFile(temp); hardenLedgerFile(temp, { force: true }); renameSync(temp, path); + assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); }, }; @@ -433,17 +459,27 @@ export function createFileSpendJournal(path: string): SpendJournal { * recorded spend, which would hand every scope a fresh allowance -- so it is a file, not a * per-process value. */ -export function loadOrCreateSpendLedgerSalt(path: string): string { +export function loadOrCreateSpendLedgerSalt( + path: string, + options: { readonly assertMutation?: () => void } = {}, +): string { if (existsSync(path)) { + assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); const existing = readFileSync(path, "utf8").trim(); if (/^[0-9a-f]{32,}$/.test(existing)) return existing; + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger storage could not be opened safely.", + ); } const dir = dirname(path); + options.assertMutation?.(); assertNotRealHomeUnderTest(dir); mkdirSync(dir, { recursive: true, mode: 0o700 }); const salt = randomBytes(32).toString("hex"); - writeFileSync(path, salt + "\n", { encoding: "utf8", mode: 0o600 }); + writeFileSync(path, salt + "\n", { encoding: "utf8", mode: 0o600, flag: "wx" }); + assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); return salt; } @@ -529,6 +565,8 @@ export function createSpendReservationLedger(options: { * no file anyone could correlate. */ readonly salt?: string; + /** Shared production ledgers assert their live state-directory lease before mutation. */ + readonly assertMutationOwner?: () => void; } = {}): SpendReservationLedger { // Mutable because the ceilings are operator configuration, and configuration is reloadable. // The three bounds below are read through functions for the same reason: a value captured @@ -538,6 +576,7 @@ export function createSpendReservationLedger(options: { const journal = options.journal; const now = options.now ?? (() => Date.now()); const salt = options.salt ?? ""; + const assertMutationOwner = options.assertMutationOwner; const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; @@ -592,7 +631,8 @@ export function createSpendReservationLedger(options: { journal.append(JSON.stringify(record)); recordsOnDisk += 1; return true; - } catch { + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; // In-memory state still bounds this process; the counter is how a caller learns the // restart guarantee degraded instead of discovering it after the fact. persistFailures += 1; @@ -819,7 +859,8 @@ export function createSpendReservationLedger(options: { try { rewrite.call(journal, [JSON.stringify(checkpoint)]); recordsOnDisk = 1; - } catch { + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; // Compaction is maintenance, not accounting: a failed rewrite leaves the previous // journal intact and every figure in it still replayable. persistFailures += 1; @@ -850,6 +891,7 @@ export function createSpendReservationLedger(options: { get policy() { return policy; }, reserve(request: SpendReservationRequest): SpendReservationDecision { + assertMutationOwner?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); const at = request.at ?? now(); const send = aliasFor("send", request.sendId); @@ -904,6 +946,7 @@ export function createSpendReservationLedger(options: { }, markDispatched(sendId: string): boolean { + assertMutationOwner?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || reservation.status !== "open") return false; @@ -914,6 +957,7 @@ export function createSpendReservationLedger(options: { }, abandon(sendId: string): boolean { + assertMutationOwner?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for @@ -926,6 +970,7 @@ export function createSpendReservationLedger(options: { }, settle(sendId: string, usage: SpendUsage): boolean { + assertMutationOwner?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; @@ -937,6 +982,7 @@ export function createSpendReservationLedger(options: { }, markLost(sendId: string): boolean { + assertMutationOwner?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; @@ -967,6 +1013,7 @@ export function createSpendReservationLedger(options: { }, prune(at: number = now()): void { + assertMutationOwner?.(); // Removal requires BOTH inactive and not exhausted inside the window. An // exhausted-but-idle scope that was dropped would be recreated fresh under the // same id -- the exact laundering the ceiling exists to stop. @@ -975,6 +1022,7 @@ export function createSpendReservationLedger(options: { }, reconfigure(next: SpendReservationPolicy): void { + assertMutationOwner?.(); policy = next; }, }; @@ -1052,10 +1100,18 @@ export function sharedSpendLedger(): SpendReservationLedger { bindSpendLedgerOwnerHome(); if (!sharedLedger) { const home = getConfigDir(); + const journalPath = join(home, SPEND_LEDGER_JOURNAL_FILENAME); + const saltPath = join(home, SPEND_LEDGER_SALT_FILENAME); + const assertMutationOwner = (): void => { + assertSpendLedgerOwnerHeld(home); + if (existsSync(journalPath)) assertSafeLedgerFile(journalPath); + if (existsSync(saltPath)) assertSafeLedgerFile(saltPath); + }; sharedLedger = createSpendReservationLedger({ - journal: createFileSpendJournal(join(home, SPEND_LEDGER_JOURNAL_FILENAME)), - salt: loadOrCreateSpendLedgerSalt(join(home, SPEND_LEDGER_SALT_FILENAME)), + journal: createFileSpendJournal(journalPath, { assertMutation: assertMutationOwner }), + salt: loadOrCreateSpendLedgerSalt(saltPath, { assertMutation: assertMutationOwner }), policy: sharedPolicy, + assertMutationOwner, }); } return sharedLedger; diff --git a/src/server/management/system-restart.ts b/src/server/management/system-restart.ts index cfcc9ca416..f88c4209b2 100644 --- a/src/server/management/system-restart.ts +++ b/src/server/management/system-restart.ts @@ -36,6 +36,7 @@ import { isServiceViable } from "../../service"; import { readRuntimePort } from "../../config/process-state"; import { withProcessRuntimeProvenance } from "../../lib/bun-runtime"; import { selfLaunchArgv } from "../../lib/self-launch-argv"; +import { spendLedgerRestartEnvironment } from "../../lib/spend-ledger-owner"; import { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS, @@ -225,8 +226,12 @@ function spawnDetachedStart( return new Promise((resolve, reject) => { let child: ReturnType; try { - const env: NodeJS.ProcessEnv = { ...process.env }; - delete env.OCX_SERVICE; + const sourceEnv: NodeJS.ProcessEnv = { ...process.env }; + delete sourceEnv.OCX_SERVICE; + const env = spendLedgerRestartEnvironment( + sourceEnv, + waitForHealthBeforeParentExit ? undefined : process.pid, + ); child = spawn(process.execPath, launchArgs, { detached: true, stdio: "ignore", diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index e2e542f372..1e46347a0b 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -1,12 +1,14 @@ /** Cross-process ownership for the process-wide spend journal (#5123). */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, statSync, writeFileSync } from "node:fs"; +import { existsSync, linkSync, mkdirSync, mkdtempSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireSpendLedgerOwner, SPEND_LEDGER_OWNER_FILENAME, + SPEND_LEDGER_RESTART_PARENT_ENV, SpendLedgerOwnerError, + spendLedgerRestartEnvironment, spendLedgerOwnerSnapshot, type SpendLedgerOwnerLease, } from "../../src/lib/spend-ledger-owner"; @@ -47,12 +49,18 @@ afterEach(async () => { removeTreeWithRetry(root); }); -function spawnHolder(targetHome: string, mode: "observe" | "enforced", suffix: string) { +function spawnHolder( + targetHome: string, + mode: "observe" | "enforced", + suffix: string, + extraEnv: Record = {}, +) { const holdMarker = join(root, `held-${suffix}`); const releaseMarker = join(root, `release-${suffix}`); const child = Bun.spawn([process.execPath, childPath], { env: { ...process.env, + ...extraEnv, OPENCODEX_HOME: targetHome, OCX_SPEND_OWNER_CHILD: JSON.stringify({ holdMarker, releaseMarker, mode }), }, @@ -95,6 +103,15 @@ function busyError(): SpendLedgerOwnerError { } describe("real process ownership", () => { + test("only a parent-exit restart environment carries the handoff marker", () => { + const source = { OCX_SPEND_LEDGER_RESTART_PARENT_PID: "stale", KEEP_ME: "yes" }; + expect(spendLedgerRestartEnvironment(source)).toEqual({ KEEP_ME: "yes" }); + expect(spendLedgerRestartEnvironment(source, 4242)).toEqual({ + KEEP_ME: "yes", + OCX_SPEND_LEDGER_RESTART_PARENT_PID: "4242", + }); + }); + for (const [holderMode, contenderMode] of [["observe", "enforced"], ["enforced", "observe"]] as const) { test(`${holderMode} and ${contenderMode} configurations contend identically`, async () => { const holder = spawnHolder(home, holderMode, `${holderMode}-${contenderMode}`); @@ -125,6 +142,22 @@ describe("real process ownership", () => { next.release(); }, SPAWN_BUDGET_MS); + test("a marked restart child waits for the parent lease while an ordinary sibling fails immediately", async () => { + const parent = acquireSpendLedgerOwner(); + const ordinary = spawnHolder(home, "observe", "ordinary-sibling"); + expect((await childResult(ordinary.child)).code).toBe("SPEND_LEDGER_OWNER_BUSY"); + + const restart = spawnHolder(home, "observe", "restart-child", { + [SPEND_LEDGER_RESTART_PARENT_ENV]: String(process.pid), + }); + await Bun.sleep(25); + expect(restart.child.exitCode).toBeNull(); + parent.release(); + await waitForMarker(restart.holdMarker, restart.child); + writeFileSync(restart.releaseMarker, "release"); + expect((await childResult(restart.child)).status).toBe("acquired"); + }, SPAWN_BUDGET_MS); + test("an abruptly killed owner is reacquirable without replacing the lock file", async () => { const holder = spawnHolder(home, "observe", "killed"); await waitForMarker(holder.holdMarker, holder.child); @@ -182,6 +215,29 @@ describe("in-process references and privacy", () => { expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_HOME_CONFLICT"); }); + test("a retained shared ledger refuses mutation after its final lease releases", async () => { + const lease = acquireSpendLedgerOwner(); + const retained = sharedSpendLedger(); + lease.release(); + const holder = spawnHolder(home, "observe", "retained-handle"); + await waitForMarker(holder.holdMarker, holder.child); + let failure: unknown; + try { + retained.reserve({ + sendId: "retained", + scopes: { rootId: "retained-root" }, + inputTokens: 1, + outputCeilingTokens: 1, + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_NOT_HELD"); + writeFileSync(holder.releaseMarker, "release"); + await childResult(holder.child); + }, SPAWN_BUDGET_MS); + test("busy refusal contains no private identity or filesystem data", async () => { const holder = spawnHolder(home, "observe", "privacy"); await waitForMarker(holder.holdMarker, holder.child); @@ -208,3 +264,91 @@ describe("in-process references and privacy", () => { expect(existsSync(join(home, SPEND_LEDGER_SALT_FILENAME))).toBe(false); }); }); + +describe("backing file identity", () => { + const expectBackingAliasesRefused = (kind: "hardlink" | "symlink"): void => { + const first = acquireSpendLedgerOwner(home); + sharedSpendLedger().reserve({ + sendId: "first", + scopes: { rootId: "first-root" }, + inputTokens: 1, + outputCeilingTokens: 1, + }); + first.release(); + resetSharedSpendLedgerForTest(); + + for (const filename of [SPEND_LEDGER_JOURNAL_FILENAME, SPEND_LEDGER_SALT_FILENAME]) { + const otherHome = join(root, `${kind}-${filename}`); + const prepared = acquireSpendLedgerOwner(otherHome); + prepared.release(); + const source = join(home, filename); + const destination = join(otherHome, filename); + if (kind === "hardlink") linkSync(source, destination); + else symlinkSync(source, destination); + const owner = acquireSpendLedgerOwner(otherHome); + process.env.OPENCODEX_HOME = otherHome; + let failure: unknown; + try { sharedSpendLedger(); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_UNAVAILABLE"); + owner.release(); + resetSharedSpendLedgerForTest(); + } + }; + + test("hard-linked owner files fail closed", () => { + const linkedHome = join(root, "owner-hardlink"); + mkdirSync(linkedHome, { recursive: true }); + const target = join(root, "owner-hardlink-target"); + writeFileSync(target, "owner"); + linkSync(target, join(linkedHome, SPEND_LEDGER_OWNER_FILENAME)); + let failure: unknown; + try { acquireSpendLedgerOwner(linkedHome); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_UNAVAILABLE"); + }); + + test.skipIf(process.platform === "win32")("symbolically linked owner files fail closed", () => { + const linkedHome = join(root, "owner-symlink"); + mkdirSync(linkedHome, { recursive: true }); + const target = join(root, "owner-symlink-target"); + writeFileSync(target, "owner"); + symlinkSync(target, join(linkedHome, SPEND_LEDGER_OWNER_FILENAME)); + let failure: unknown; + try { acquireSpendLedgerOwner(linkedHome); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_UNAVAILABLE"); + }); + + test("invalid or unusable owner databases fail closed", () => { + const invalidHome = join(root, "owner-invalid"); + mkdirSync(invalidHome, { recursive: true }); + writeFileSync(join(invalidHome, SPEND_LEDGER_OWNER_FILENAME), "not sqlite"); + let invalid: unknown; + try { acquireSpendLedgerOwner(invalidHome); } catch (error) { invalid = error; } + expect(invalid).toBeInstanceOf(SpendLedgerOwnerError); + expect((invalid as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_UNAVAILABLE"); + + const unusableHome = join(root, "owner-unusable"); + mkdirSync(join(unusableHome, SPEND_LEDGER_OWNER_FILENAME), { recursive: true }); + let unusable: unknown; + try { acquireSpendLedgerOwner(unusableHome); } catch (error) { unusable = error; } + expect(unusable).toBeInstanceOf(SpendLedgerOwnerError); + expect((unusable as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_UNAVAILABLE"); + }); + + test("linked journals and salts cannot cross independent homes", () => { + expectBackingAliasesRefused("hardlink"); + }); + + test.skipIf(process.platform === "win32")("symbolically linked journals and salts fail closed", () => { + expectBackingAliasesRefused("symlink"); + }); + + test("the shared ledger requires a live lease", () => { + let failure: unknown; + try { sharedSpendLedger(); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_NOT_HELD"); + }); +}); From 06fda1c3150f4381a666cf151cc2a948206574c9 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 18:35:25 +0900 Subject: [PATCH 03/34] fix(spend): release the ledger singleton with the lease that owned it The binding outlived ownership, so a process that served one state directory and then legitimately served another was refused for a conflict it no longer had. Every test process that starts servers against per-test homes is that shape, and so is a restart handoff inside one process. Releasing the last lease now discards the in-memory ledger along with the binding. Nothing is lost: the journal on disk is the durable record and the next construction replays it, which is what a restart already does. Two homes owned at the same time are still refused, which is the invariant that matters. --- src/lib/spend-ledger-owner.ts | 17 +++++++++++++++++ src/lib/spend-reservation-ledger.ts | 7 ++++++- tests/lib/spend-ledger-owner.test.ts | 18 ++++++++++++------ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index ad956a5c28..20ae4c47bd 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -37,6 +37,19 @@ interface ActiveOwner { let activeOwner: ActiveOwner | null = null; let boundLedgerHome: string | null = null; +const releaseHooks: Array<() => void> = []; + +/** + * Run when the last lease on a state directory goes away. + * + * The ledger singleton is bound to the home it was built for, so it has to go when ownership + * does; otherwise a process that starts a second server against a different state directory + * either writes the old home's journal or is refused for a conflict it no longer has. The + * journal on disk is the durable record and construction replays it, so nothing is lost. + */ +export function onSpendLedgerOwnerReleased(hook: () => void): void { + releaseHooks.push(hook); +} function errorCode(error: unknown): unknown { return error !== null && typeof error === "object" && "code" in error ? error.code : undefined; @@ -180,6 +193,10 @@ function leaseFor(owner: ActiveOwner): SpendLedgerOwnerLease { owner.references -= 1; if (owner.references > 0) return; activeOwner = null; + boundLedgerHome = null; + for (const hook of releaseHooks) { + try { hook(); } catch { /* a discard hook must not mask a release failure */ } + } let failure: unknown; try { owner.database.exec("ROLLBACK"); } catch (error) { failure = error; } try { owner.database.close(); } catch (error) { failure ??= error; } diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index e122166680..8860f8b215 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -65,7 +65,12 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; +import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; + +// The singleton belongs to the state directory it was built for. Releasing ownership hands that +// directory to whoever comes next, so the in-memory copy goes with it and the next construction +// replays the journal. +onSpendLedgerOwnerReleased(() => { sharedLedger = undefined; }); export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; /** diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index 1e46347a0b..1b74d2bc45 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -203,16 +203,22 @@ describe("in-process references and privacy", () => { expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_HOME_CONFLICT"); }); - test("a constructed singleton keeps its home after the final lease releases", () => { + test("releasing the final lease frees the process to own a different state directory", () => { + // The refusal above is about two homes owned at once. Once ownership is gone the singleton + // has no directory to belong to, so it is discarded with the lease: the next home builds its + // own ledger by replaying its own journal. Keeping the old binding instead would strand a + // process that legitimately serves one home and then another. const first = acquireSpendLedgerOwner(); - leases.push(first); sharedSpendLedger(); first.release(); + expect(spendLedgerDiagnosticsSnapshot()).toMatchObject({ ownership: "unheld", initialized: false }); + process.env.OPENCODEX_HOME = join(root, "state-b"); - let failure: unknown; - try { acquireSpendLedgerOwner(); } catch (error) { failure = error; } - expect(failure).toBeInstanceOf(SpendLedgerOwnerError); - expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_HOME_CONFLICT"); + const second = acquireSpendLedgerOwner(); + leases.push(second); + sharedSpendLedger(); + expect(spendLedgerDiagnosticsSnapshot()).toMatchObject({ ownership: "held", initialized: true }); + expect(existsSync(join(root, "state-b", SPEND_LEDGER_OWNER_FILENAME))).toBe(true); }); test("a retained shared ledger refuses mutation after its final lease releases", async () => { From 4d6d8966491429e97649b353753ddca5ad4a1d1d Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 18:57:20 +0900 Subject: [PATCH 04/34] fix(spend): prove exact ownership on every accounting read and change Review found a retained ledger coming back to life. The guard compared the state directory, which cannot tell "still the ownership I was built under" from "the same directory, owned again since" - so a handle kept across a release and a reacquire resumed with totals from before the gap, over a journal another writer may have appended to. Ownership now has an identity: each acquisition that takes the lock mints one, the ledger captures it at construction, and every accounting read and change proves that exact value. A handle from the previous ownership is refused; a fresh handle replays what happened in between. Reads are covered as well as writes, because reporting figures from a journal this process no longer owns is the same error with a quieter symptom. The file-backed journal and salt writers no longer accept an absent owner check. The parameter is required, so a caller that owns its own temporary file writes that decision down instead of inheriting it by omission. A link whose target does not exist read as "no file" through existsSync, so the safety check was skipped and the append created that target elsewhere. Entry presence is now decided with lstat, which sees the link itself. --- src/lib/spend-ledger-owner.ts | 31 ++++++++ src/lib/spend-reservation-ledger.ts | 88 ++++++++++++++------- structure/runtime.md | 5 +- structure/transports/responses.md | 11 ++- tests/lib/spend-ledger-file-journal.test.ts | 14 +++- tests/lib/spend-ledger-owner.test.ts | 40 ++++++++++ 6 files changed, 153 insertions(+), 36 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index 20ae4c47bd..3e63d5800c 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -37,6 +37,15 @@ interface ActiveOwner { let activeOwner: ActiveOwner | null = null; let boundLedgerHome: string | null = null; +/** + * Identity of the current ownership, not just of the home it covers. + * + * A home name alone cannot tell "still the lease I was built under" from "the same directory, + * owned again since". A ledger that only checked the home therefore came back to life across a + * release and a reacquire, writing totals it had accumulated before another writer owned the + * journal. Every acquisition that actually takes the lock mints a new value. + */ +let ownerGeneration = 0; const releaseHooks: Array<() => void> = []; /** @@ -180,6 +189,7 @@ export function acquireSpendLedgerOwner(configDir = getConfigDir()): SpendLedger } activeOwner = { home: prepared.home, database, references: 1 }; + ownerGeneration += 1; return leaseFor(activeOwner); } @@ -211,6 +221,27 @@ function leaseFor(owner: ActiveOwner): SpendLedgerOwnerLease { }); } +/** The ownership a ledger was built under, to be handed back on every later use. */ +export function currentSpendLedgerOwnerGeneration(): number { + return ownerGeneration; +} + +/** + * Refuse a handle built under an ownership that has since ended. + * + * The state directory may be the same one; what matters is that the lock was dropped and taken + * again in between, because another process could have written the journal while it was free. + */ +export function assertSpendLedgerOwnerGeneration(generation: number, configDir = getConfigDir()): void { + assertSpendLedgerOwnerHeld(configDir); + if (generation !== ownerGeneration) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + ); + } +} + /** The singleton journal may be touched only while its matching state directory is owned. */ export function assertSpendLedgerOwnerHeld(configDir = getConfigDir()): void { let home: string; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 8860f8b215..46b0c49b9c 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -53,7 +53,7 @@ * that are re-applied to an EXISTING file rather than trusted from its creation. */ -import { appendFileSync, chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; +import { appendFileSync, chmodSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; // Definition-site import, not the ../config barrel -- same reasoning as @@ -65,7 +65,7 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; +import { assertSpendLedgerOwnerGeneration, assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, currentSpendLedgerOwnerGeneration, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; // The singleton belongs to the state directory it was built for. Releasing ownership hands that // directory to whoever comes next, so the in-memory copy goes with it and the next construction @@ -396,6 +396,22 @@ function hardenLedgerFile(path: string, options: { readonly force?: boolean } = } catch { /* best-effort: a non-owner cannot chmod */ } } +/** + * Does a directory entry exist here, whatever it points at? + * + * `existsSync` follows the link, so a symlink whose target is absent reads as "no file" and an + * append then creates that target somewhere else entirely. The entry itself is what decides + * whether the safety check runs. + */ +function ledgerEntryExists(path: string): boolean { + try { + lstatSync(path); + return true; + } catch { + return false; + } +} + function assertSafeLedgerFile(path: string): void { const stat = lstatSync(path); if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 @@ -409,7 +425,7 @@ function assertSafeLedgerFile(path: string): void { export function createFileSpendJournal( path: string, - options: { readonly assertMutation?: () => void } = {}, + options: { readonly assertMutation: () => void }, ): SpendJournal { const ensureDir = (): string => { const dir = dirname(path); @@ -420,7 +436,7 @@ export function createFileSpendJournal( }; return { read(): string[] { - if (!existsSync(path)) return []; + if (!ledgerEntryExists(path)) return []; assertSafeLedgerFile(path); // Replay is once per process and is the moment a journal inherited from an older build // or a restored backup first passes through here. @@ -428,20 +444,20 @@ export function createFileSpendJournal( return readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0); }, append(line: string): void { - options.assertMutation?.(); + options.assertMutation(); ensureDir(); - const created = !existsSync(path); + const created = !ledgerEntryExists(path); if (!created) assertSafeLedgerFile(path); appendFileSync(path, line + "\n", { encoding: "utf8", mode: 0o600 }); assertSafeLedgerFile(path); hardenLedgerFile(path, { force: created }); }, rewrite(lines: string[]): void { - options.assertMutation?.(); + options.assertMutation(); ensureDir(); // Same directory, so the rename is atomic on the same filesystem: a crash mid-compaction // leaves either the old journal or the new one, never a half-written ledger. - if (existsSync(path)) assertSafeLedgerFile(path); + if (ledgerEntryExists(path)) assertSafeLedgerFile(path); const temp = `${path}.compact-${process.pid}-${randomBytes(6).toString("hex")}`; writeFileSync(temp, lines.map((line) => line + "\n").join(""), { encoding: "utf8", @@ -466,9 +482,9 @@ export function createFileSpendJournal( */ export function loadOrCreateSpendLedgerSalt( path: string, - options: { readonly assertMutation?: () => void } = {}, + options: { readonly assertMutation: () => void }, ): string { - if (existsSync(path)) { + if (ledgerEntryExists(path)) { assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); const existing = readFileSync(path, "utf8").trim(); @@ -479,7 +495,7 @@ export function loadOrCreateSpendLedgerSalt( ); } const dir = dirname(path); - options.assertMutation?.(); + options.assertMutation(); assertNotRealHomeUnderTest(dir); mkdirSync(dir, { recursive: true, mode: 0o700 }); const salt = randomBytes(32).toString("hex"); @@ -570,8 +586,12 @@ export function createSpendReservationLedger(options: { * no file anyone could correlate. */ readonly salt?: string; - /** Shared production ledgers assert their live state-directory lease before mutation. */ - readonly assertMutationOwner?: () => void; + /** + * Shared production ledgers prove their exact ownership before reading or changing + * accounting. Identity, not just the directory name: a handle kept across a release and a + * reacquire describes a journal another writer may have changed in between. + */ + readonly assertOwnedAccounting?: () => void; } = {}): SpendReservationLedger { // Mutable because the ceilings are operator configuration, and configuration is reloadable. // The three bounds below are read through functions for the same reason: a value captured @@ -581,7 +601,7 @@ export function createSpendReservationLedger(options: { const journal = options.journal; const now = options.now ?? (() => Date.now()); const salt = options.salt ?? ""; - const assertMutationOwner = options.assertMutationOwner; + const assertOwnedAccounting = options.assertOwnedAccounting; const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; @@ -896,7 +916,7 @@ export function createSpendReservationLedger(options: { get policy() { return policy; }, reserve(request: SpendReservationRequest): SpendReservationDecision { - assertMutationOwner?.(); + assertOwnedAccounting?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); const at = request.at ?? now(); const send = aliasFor("send", request.sendId); @@ -951,7 +971,7 @@ export function createSpendReservationLedger(options: { }, markDispatched(sendId: string): boolean { - assertMutationOwner?.(); + assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || reservation.status !== "open") return false; @@ -962,7 +982,7 @@ export function createSpendReservationLedger(options: { }, abandon(sendId: string): boolean { - assertMutationOwner?.(); + assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for @@ -975,7 +995,7 @@ export function createSpendReservationLedger(options: { }, settle(sendId: string, usage: SpendUsage): boolean { - assertMutationOwner?.(); + assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; @@ -987,7 +1007,7 @@ export function createSpendReservationLedger(options: { }, markLost(sendId: string): boolean { - assertMutationOwner?.(); + assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; @@ -1002,6 +1022,9 @@ export function createSpendReservationLedger(options: { }, snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { + // Reading accounting from a handle whose ownership has ended is as wrong as writing it: + // the figures describe a journal this process no longer owns. + assertOwnedAccounting?.(); const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); if (!state) return undefined; return { @@ -1013,12 +1036,13 @@ export function createSpendReservationLedger(options: { }, exhausted(scope: SpendScope, scopeId: string): boolean { + assertOwnedAccounting?.(); const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); return state !== undefined && isExhausted(scope, state); }, prune(at: number = now()): void { - assertMutationOwner?.(); + assertOwnedAccounting?.(); // Removal requires BOTH inactive and not exhausted inside the window. An // exhausted-but-idle scope that was dropped would be recreated fresh under the // same id -- the exact laundering the ceiling exists to stop. @@ -1027,7 +1051,7 @@ export function createSpendReservationLedger(options: { }, reconfigure(next: SpendReservationPolicy): void { - assertMutationOwner?.(); + assertOwnedAccounting?.(); policy = next; }, }; @@ -1098,7 +1122,9 @@ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void /** * Process-wide ledger backed by the journal under OPENCODEX_HOME. Created lazily so * importing the module -- or running a request path that never reserves -- touches no - * disk. + * disk. One directory at a time, not one directory for the life of the process: the singleton + * is discarded when its ownership ends, so a later directory replays its own journal rather + * than inheriting figures from the previous one. */ export function sharedSpendLedger(): SpendReservationLedger { assertSpendLedgerOwnerHeld(); @@ -1107,16 +1133,20 @@ export function sharedSpendLedger(): SpendReservationLedger { const home = getConfigDir(); const journalPath = join(home, SPEND_LEDGER_JOURNAL_FILENAME); const saltPath = join(home, SPEND_LEDGER_SALT_FILENAME); - const assertMutationOwner = (): void => { - assertSpendLedgerOwnerHeld(home); - if (existsSync(journalPath)) assertSafeLedgerFile(journalPath); - if (existsSync(saltPath)) assertSafeLedgerFile(saltPath); + // Captured once, checked on every later use. The home says which directory; this says + // which ownership of it, so a handle kept across a release and a reacquire is refused + // rather than resuming with totals from before another writer held the journal. + const generation = currentSpendLedgerOwnerGeneration(); + const assertOwnedAccounting = (): void => { + assertSpendLedgerOwnerGeneration(generation, home); + if (ledgerEntryExists(journalPath)) assertSafeLedgerFile(journalPath); + if (ledgerEntryExists(saltPath)) assertSafeLedgerFile(saltPath); }; sharedLedger = createSpendReservationLedger({ - journal: createFileSpendJournal(journalPath, { assertMutation: assertMutationOwner }), - salt: loadOrCreateSpendLedgerSalt(saltPath, { assertMutation: assertMutationOwner }), + journal: createFileSpendJournal(journalPath, { assertMutation: assertOwnedAccounting }), + salt: loadOrCreateSpendLedgerSalt(saltPath, { assertMutation: assertOwnedAccounting }), policy: sharedPolicy, - assertMutationOwner, + assertOwnedAccounting, }); } return sharedLedger; diff --git a/structure/runtime.md b/structure/runtime.md index 8c6131a42b..dbcb0d0e21 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -165,7 +165,10 @@ pin through `src/server/port-reclaim.ts` — and a configured `port: 0` still me Every `startServer` invocation acquires the `src/lib/spend-ledger-owner.ts` SQLite writer lease for its resolved OpenCodex state directory before loading configuration or binding a listener. References share one lease only inside one process and one directory; a different directory in -that process is refused because the shared ledger is process-wide. A second process on the same +that process is refused while the lease is held, because the shared ledger is process-wide. The +refusal is about two directories owned at once, not forever: releasing the final reference +discards the singleton with its binding, so the same process may then own a different directory +and build a ledger by replaying that directory's own journal. A second process on the same directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop releases the final reference after listener teardown, and every thrown startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, diff --git a/structure/transports/responses.md b/structure/transports/responses.md index e831c04704..a07e8c6840 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -1063,8 +1063,15 @@ The shared journal has one live writer per state directory. `startServer` acquir records, and `configureSharedSpendLedger` asserts it before changing a live singleton. This applies identically with and without configured ceilings: observe-only still appends, settles and compacts. Two servers in one process and one directory share a reference-counted lease; that process cannot -switch the process-wide singleton to another directory. A separate process may use a separate -directory. SQLite crash release permits the next owner without stale-PID or TTL reclamation. +hold two directories at once. Sequential ownership is allowed and concurrent ownership is not: +releasing the final reference discards the singleton, so a later directory replays its own +journal instead of inheriting figures. A ledger records the ownership it was built under and +proves that exact identity on every accounting read and change, so a handle kept across a release +and a reacquire of the same directory is refused rather than resuming over writes another owner +may have made. File-backed journal and salt writers are owner-bound at construction, and a +directory entry that is a link -- including one whose target does not exist -- is refused instead +of followed. A separate process may use a separate directory. SQLite crash release permits the +next owner without stale-PID or TTL reclamation. The journal survives an ordinary process restart once its writes reached the filesystem. It does not claim host power-loss durability: the append path does not fsync each record, so power loss can diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index f2c10952fd..1a270de0f6 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -9,6 +9,12 @@ import { /** POSIX mode bits do not describe a Windows ACL, where hardenSecretPath does the work. */ const posixModes = process.platform !== "win32"; +/** + * These fixtures own their own temporary journal, so there is no shared state directory to + * prove ownership of. The parameter is required precisely so opting out is written down here + * rather than defaulted into a production caller by omission. + */ +const unowned = { assertMutation: (): void => { /* fixture-owned journal */ } }; const modeOf = (path: string): number => statSync(path).mode & 0o777; const line = (send: string): string => JSON.stringify({ v: 1, kind: "lost", send, at: 1 }); @@ -16,7 +22,7 @@ describe("spend ledger file journal", () => { test.skipIf(!posixModes)("a journal that already exists is re-hardened, not trusted", () => { const dir = mkdtempSync(join(tmpdir(), "ocx-spend-journal-")); const path = join(dir, "spend-ledger.jsonl"); - const journal = createFileSpendJournal(path); + const journal = createFileSpendJournal(path, unowned); journal.append(line("alias-one")); expect(modeOf(path)).toBe(0o600); @@ -36,7 +42,7 @@ describe("spend ledger file journal", () => { test("compaction replaces the journal atomically and leaves no temp behind", () => { const dir = mkdtempSync(join(tmpdir(), "ocx-spend-compact-")); const path = join(dir, "spend-ledger.jsonl"); - const journal = createFileSpendJournal(path); + const journal = createFileSpendJournal(path, unowned); journal.append(line("alias-one")); journal.append(line("alias-two")); @@ -55,11 +61,11 @@ describe("spend ledger file journal", () => { const dir = mkdtempSync(join(tmpdir(), "ocx-spend-salt-")); const path = join(dir, "spend-ledger.salt"); - const minted = loadOrCreateSpendLedgerSalt(path); + const minted = loadOrCreateSpendLedgerSalt(path, unowned); expect(minted).toMatch(/^[0-9a-f]{64}$/); // Stability is the whole contract: a salt that changed per process would alias the same // root id differently after a restart and hand every scope a fresh allowance. - expect(loadOrCreateSpendLedgerSalt(path)).toBe(minted); + expect(loadOrCreateSpendLedgerSalt(path, unowned)).toBe(minted); if (posixModes) expect(modeOf(path)).toBe(0o600); }); }); diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index 1b74d2bc45..3c3e240d9f 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -221,6 +221,46 @@ describe("in-process references and privacy", () => { expect(existsSync(join(root, "state-b", SPEND_LEDGER_OWNER_FILENAME))).toBe(true); }); + test("a retained ledger stays refused after the same home is owned again", () => { + // The dangerous case is not a different directory, it is the same one owned again. The + // retained handle carries totals from before the gap, and another writer may have appended + // to the journal while nobody held the lock. + const first = acquireSpendLedgerOwner(); + const retained = sharedSpendLedger(); + first.release(); + + const second = acquireSpendLedgerOwner(); + leases.push(second); + let failure: unknown; + try { retained.snapshot("root", "r1"); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_NOT_HELD"); + + let mutation: unknown; + try { + retained.reserve({ sendId: "s1", targets: [{ scope: "root", scopeId: "r1" }], tokens: 1 }); + } catch (error) { mutation = error; } + expect(mutation).toBeInstanceOf(SpendLedgerOwnerError); + + // A handle taken under the new ownership works and sees the journal as it is now. + expect(() => sharedSpendLedger().snapshot("root", "r1")).not.toThrow(); + }); + + test("a dangling journal symlink is refused rather than followed", () => { + const lease = acquireSpendLedgerOwner(); + leases.push(lease); + const journal = join(home, SPEND_LEDGER_JOURNAL_FILENAME); + const target = join(root, "elsewhere.jsonl"); + symlinkSync(target, journal); + + let failure: unknown; + try { sharedSpendLedger().reserve({ sendId: "s1", targets: [{ scope: "root", scopeId: "r1" }], tokens: 1 }); } + catch (error) { failure = error; } + expect(failure).toBeInstanceOf(SpendLedgerOwnerError); + // The point of the case: the link's target must not have been created by following it. + expect(existsSync(target)).toBe(false); + }); + test("a retained shared ledger refuses mutation after its final lease releases", async () => { const lease = acquireSpendLedgerOwner(); const retained = sharedSpendLedger(); From 20a959a3c0d82f26e2eda619807b4983889b89ad Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 19:23:54 +0900 Subject: [PATCH 05/34] fix(spend): make ownership proof something only the owner module can mint The previous round kept adding checks at each caller, and the review was right that this converged on one boundary rather than a set of gaps. A required guard supplied by the caller proves nothing, because the caller can supply one that does nothing - which is exactly what the fixtures here did. Production journal and salt storage is now minted by the owner module from the directory it actually owns. The mint takes a file name rather than a path, so nothing outside chooses the destination; the returned value carries the exact ownership it was minted under and re-proves it on every read, append, rewrite, salt read and salt create; and a brand means a look-alike object is refused on identity rather than on shape. There is no longer an exported entrypoint that writes a caller-chosen path. Every ledger member now proves ownership too, including knows and the policy, degraded, persistFailures and corruptRecords getters. Reporting a figure from a journal this process no longer owns is the same error as writing one. The generic in-memory and injected factory stays usable without any of this and does not pretend to enforce process ownership. The file-journal cases that cover real persistence, hardening and compaction now take a real lease over a throwaway state directory and go through the production entrypoints, rather than being replaced by a stand-in. Also corrects the reserve request shape in the reacquire regression to scopes with input and output-ceiling tokens, and the stale comment claiming production never discards the singleton. --- src/lib/spend-ledger-owner.ts | 61 ++++++++++++++++ src/lib/spend-reservation-ledger.ts | 67 ++++++++++-------- tests/lib/spend-ledger-file-journal.test.ts | 78 +++++++++++++++------ tests/lib/spend-ledger-owner.test.ts | 53 ++++++++++++-- 4 files changed, 205 insertions(+), 54 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index 3e63d5800c..1a2beceeb4 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -226,6 +226,67 @@ export function currentSpendLedgerOwnerGeneration(): number { return ownerGeneration; } +const STORAGE_BRAND: unique symbol = Symbol("spend-ledger-storage"); + +/** + * Permission to touch one file under the owned state directory, for as long as this exact + * ownership lasts. + * + * A callback supplied by the caller cannot be this proof: the caller can pass one that does + * nothing, which is how a "required guard" still allowed an unowned write. Only this module + * mints one, it derives the path from the directory actually owned rather than accepting a + * path to trust, and the brand means a structurally similar object is not accepted in its place. + */ +export interface SpendLedgerStorage { + readonly [STORAGE_BRAND]: true; + readonly path: string; + /** Throws unless the ownership this was minted under is still the current one. */ + assert(): void; +} + +/** + * Mint storage permission for one file name directly under the owned state directory. + * + * The name is a file name, never a path: joining a caller-supplied path would let the caller + * choose the destination, which is the thing ownership is supposed to decide. + */ +export function mintSpendLedgerStorage(fileName: string): SpendLedgerStorage { + if (fileName.includes("/") || fileName.includes("\\") || fileName === "." || fileName === "..") { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger storage could not be established safely.", + ); + } + assertSpendLedgerOwnerHeld(); + const owner = activeOwner; + if (!owner) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + ); + } + const generation = ownerGeneration; + const home = owner.home; + return Object.freeze({ + [STORAGE_BRAND]: true as const, + path: join(home, fileName), + assert(): void { + assertSpendLedgerOwnerGeneration(generation, home); + }, + }); +} + +/** Reject anything that did not come from {@link mintSpendLedgerStorage}. */ +export function assertMintedStorage(storage: SpendLedgerStorage): void { + if (storage?.[STORAGE_BRAND] !== true) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + ); + } + storage.assert(); +} + /** * Refuse a handle built under an ownership that has since ended. * diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 46b0c49b9c..6e178ccff5 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -65,7 +65,7 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertSpendLedgerOwnerGeneration, assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, currentSpendLedgerOwnerGeneration, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "./spend-ledger-owner"; +import { assertMintedStorage, assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, type SpendLedgerStorage } from "./spend-ledger-owner"; // The singleton belongs to the state directory it was built for. Releasing ownership hands that // directory to whoever comes next, so the in-memory copy goes with it and the next construction @@ -423,10 +423,13 @@ function assertSafeLedgerFile(path: string): void { } } -export function createFileSpendJournal( - path: string, - options: { readonly assertMutation: () => void }, -): SpendJournal { +/** + * The production journal. Its location comes from the owned state directory and every touch + * proves that ownership, so there is no entrypoint here that writes a caller-chosen path. + */ +export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJournal { + assertMintedStorage(storage); + const path = storage.path; const ensureDir = (): string => { const dir = dirname(path); // The guard runs before any mutation so a rejected write leaves nothing behind. @@ -436,6 +439,7 @@ export function createFileSpendJournal( }; return { read(): string[] { + storage.assert(); if (!ledgerEntryExists(path)) return []; assertSafeLedgerFile(path); // Replay is once per process and is the moment a journal inherited from an older build @@ -444,7 +448,7 @@ export function createFileSpendJournal( return readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0); }, append(line: string): void { - options.assertMutation(); + storage.assert(); ensureDir(); const created = !ledgerEntryExists(path); if (!created) assertSafeLedgerFile(path); @@ -453,7 +457,7 @@ export function createFileSpendJournal( hardenLedgerFile(path, { force: created }); }, rewrite(lines: string[]): void { - options.assertMutation(); + storage.assert(); ensureDir(); // Same directory, so the rename is atomic on the same filesystem: a crash mid-compaction // leaves either the old journal or the new one, never a half-written ledger. @@ -480,10 +484,9 @@ export function createFileSpendJournal( * recorded spend, which would hand every scope a fresh allowance -- so it is a file, not a * per-process value. */ -export function loadOrCreateSpendLedgerSalt( - path: string, - options: { readonly assertMutation: () => void }, -): string { +export function loadOrCreateSpendLedgerSalt(storage: SpendLedgerStorage): string { + assertMintedStorage(storage); + const path = storage.path; if (ledgerEntryExists(path)) { assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); @@ -495,7 +498,7 @@ export function loadOrCreateSpendLedgerSalt( ); } const dir = dirname(path); - options.assertMutation(); + storage.assert(); assertNotRealHomeUnderTest(dir); mkdirSync(dir, { recursive: true, mode: 0o700 }); const salt = randomBytes(32).toString("hex"); @@ -910,10 +913,12 @@ export function createSpendReservationLedger(options: { }; return { - get persistFailures() { return persistFailures; }, - get corruptRecords() { return corruptRecords; }, - get degraded() { return persistFailures > 0 || corruptRecords > 0; }, - get policy() { return policy; }, + // Every figure this ledger reports describes a journal it must still own. Reporting one + // after ownership ended is the same error as writing then, with a quieter symptom. + get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, + get corruptRecords() { assertOwnedAccounting?.(); return corruptRecords; }, + get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, + get policy() { assertOwnedAccounting?.(); return policy; }, reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); @@ -1018,6 +1023,7 @@ export function createSpendReservationLedger(options: { }, knows(sendId: string): boolean { + assertOwnedAccounting?.(); return reservations.has(aliasFor("send", sendId)); }, @@ -1130,21 +1136,18 @@ export function sharedSpendLedger(): SpendReservationLedger { assertSpendLedgerOwnerHeld(); bindSpendLedgerOwnerHome(); if (!sharedLedger) { - const home = getConfigDir(); - const journalPath = join(home, SPEND_LEDGER_JOURNAL_FILENAME); - const saltPath = join(home, SPEND_LEDGER_SALT_FILENAME); - // Captured once, checked on every later use. The home says which directory; this says - // which ownership of it, so a handle kept across a release and a reacquire is refused - // rather than resuming with totals from before another writer held the journal. - const generation = currentSpendLedgerOwnerGeneration(); + // Minted by the owner module from the directory it actually owns, and carrying the exact + // ownership they were minted under. Nothing here chooses a path or supplies its own guard. + const journalStorage = mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME); + const saltStorage = mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME); const assertOwnedAccounting = (): void => { - assertSpendLedgerOwnerGeneration(generation, home); - if (ledgerEntryExists(journalPath)) assertSafeLedgerFile(journalPath); - if (ledgerEntryExists(saltPath)) assertSafeLedgerFile(saltPath); + journalStorage.assert(); + if (ledgerEntryExists(journalStorage.path)) assertSafeLedgerFile(journalStorage.path); + if (ledgerEntryExists(saltStorage.path)) assertSafeLedgerFile(saltStorage.path); }; sharedLedger = createSpendReservationLedger({ - journal: createFileSpendJournal(journalPath, { assertMutation: assertOwnedAccounting }), - salt: loadOrCreateSpendLedgerSalt(saltPath, { assertMutation: assertOwnedAccounting }), + journal: createOwnedFileSpendJournal(journalStorage), + salt: loadOrCreateSpendLedgerSalt(saltStorage), policy: sharedPolicy, assertOwnedAccounting, }); @@ -1175,7 +1178,13 @@ export function spendLedgerDiagnosticsSnapshot(): { }; } -/** Test seam. Production never discards the ledger: that would reset a spent budget. */ +/** + * Test seam for discarding the singleton outright. + * + * Production discards it too, but only with the ownership it belongs to, which is what + * `onSpendLedgerOwnerReleased` does. Neither path resets a spent budget: the journal is the + * durable record and the next construction replays it. + */ export function resetSharedSpendLedgerForTest(): void { sharedLedger = undefined; sharedPolicy = DEFAULT_SPEND_RESERVATION_POLICY; diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index 1a270de0f6..f3e358e6dc 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -1,28 +1,63 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { chmodSync, mkdtempSync, readFileSync, readdirSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { - createFileSpendJournal, + createOwnedFileSpendJournal, loadOrCreateSpendLedgerSalt, + SPEND_LEDGER_JOURNAL_FILENAME, + SPEND_LEDGER_SALT_FILENAME, + resetSharedSpendLedgerForTest, } from "../../src/lib/spend-reservation-ledger"; +import { + acquireSpendLedgerOwner, + mintSpendLedgerStorage, + type SpendLedgerOwnerLease, +} from "../../src/lib/spend-ledger-owner"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; /** POSIX mode bits do not describe a Windows ACL, where hardenSecretPath does the work. */ const posixModes = process.platform !== "win32"; -/** - * These fixtures own their own temporary journal, so there is no shared state directory to - * prove ownership of. The parameter is required precisely so opting out is written down here - * rather than defaulted into a production caller by omission. - */ -const unowned = { assertMutation: (): void => { /* fixture-owned journal */ } }; const modeOf = (path: string): number => statSync(path).mode & 0o777; const line = (send: string): string => JSON.stringify({ v: 1, kind: "lost", send, at: 1 }); +const homes: string[] = []; +const leases: SpendLedgerOwnerLease[] = []; +let previousHome: string | undefined; + +/** + * A real lease over a throwaway state directory. + * + * These cases cover the production persistence, hardening and compaction paths, so they use the + * production entrypoints rather than a stand-in: storage is minted by the owner module from the + * directory it owns, which is the only way to obtain it. + */ +function ownedHome(prefix: string): string { + const home = mkdtempSync(join(tmpdir(), prefix)); + homes.push(home); + previousHome ??= process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + resetSharedSpendLedgerForTest(); + leases.push(acquireSpendLedgerOwner()); + return home; +} + +afterEach(() => { + for (const lease of leases.splice(0)) { + try { lease.release(); } catch { /* a failed release must not mask the case's result */ } + } + resetSharedSpendLedgerForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + previousHome = undefined; + for (const home of homes.splice(0)) removeTreeWithRetry(home); +}); + describe("spend ledger file journal", () => { test.skipIf(!posixModes)("a journal that already exists is re-hardened, not trusted", () => { - const dir = mkdtempSync(join(tmpdir(), "ocx-spend-journal-")); - const path = join(dir, "spend-ledger.jsonl"); - const journal = createFileSpendJournal(path, unowned); + const dir = ownedHome("ocx-spend-journal-"); + const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); journal.append(line("alias-one")); expect(modeOf(path)).toBe(0o600); @@ -40,9 +75,9 @@ describe("spend ledger file journal", () => { }); test("compaction replaces the journal atomically and leaves no temp behind", () => { - const dir = mkdtempSync(join(tmpdir(), "ocx-spend-compact-")); - const path = join(dir, "spend-ledger.jsonl"); - const journal = createFileSpendJournal(path, unowned); + const dir = ownedHome("ocx-spend-compact-"); + const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); journal.append(line("alias-one")); journal.append(line("alias-two")); @@ -52,20 +87,23 @@ describe("spend ledger file journal", () => { expect(readFileSync(path, "utf8")).toBe(line("checkpoint-stand-in") + "\n"); expect(journal.read()).toHaveLength(1); - // The temp file is renamed over the journal, never left in the home directory. - expect(readdirSync(dir)).toEqual(["spend-ledger.jsonl"]); + // The temp file is renamed over the journal, never left in the home directory. Asserted as + // the absence of a compaction temp rather than an exact listing, because the owned state + // directory also holds the lease database this case had to acquire to write at all. + expect(readdirSync(dir).filter(name => name.includes(".compact-"))).toEqual([]); + expect(readdirSync(dir)).toContain(SPEND_LEDGER_JOURNAL_FILENAME); if (posixModes) expect(modeOf(path)).toBe(0o600); }); test("the alias salt is minted once and reused, so replay still matches live requests", () => { - const dir = mkdtempSync(join(tmpdir(), "ocx-spend-salt-")); - const path = join(dir, "spend-ledger.salt"); + const dir = ownedHome("ocx-spend-salt-"); + const path = join(dir, SPEND_LEDGER_SALT_FILENAME); - const minted = loadOrCreateSpendLedgerSalt(path, unowned); + const minted = loadOrCreateSpendLedgerSalt(mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME)); expect(minted).toMatch(/^[0-9a-f]{64}$/); // Stability is the whole contract: a salt that changed per process would alias the same // root id differently after a restart and hand every scope a fresh allowance. - expect(loadOrCreateSpendLedgerSalt(path, unowned)).toBe(minted); + expect(loadOrCreateSpendLedgerSalt(mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME))).toBe(minted); if (posixModes) expect(modeOf(path)).toBe(0o600); }); }); diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index 3c3e240d9f..c5eaa599ca 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { acquireSpendLedgerOwner, SPEND_LEDGER_OWNER_FILENAME, + mintSpendLedgerStorage, SPEND_LEDGER_RESTART_PARENT_ENV, SpendLedgerOwnerError, spendLedgerRestartEnvironment, @@ -15,6 +16,8 @@ import { import { SPEND_LEDGER_JOURNAL_FILENAME, SPEND_LEDGER_SALT_FILENAME, + createOwnedFileSpendJournal, + loadOrCreateSpendLedgerSalt, resetSharedSpendLedgerForTest, sharedSpendLedger, spendLedgerDiagnosticsSnapshot, @@ -24,6 +27,13 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS } from "../helpers/test-budget"; const childPath = helperPath("spend-ledger-owner-child.ts"); +/** The shape the ledger actually takes: scopes plus the two token figures it books against. */ +const reserveRequest = (sendId: string) => ({ + sendId, + scopes: { rootId: "r1" }, + inputTokens: 1, + outputCeilingTokens: 1, +}); let root = ""; let home = ""; let previousHome: string | undefined; @@ -237,15 +247,49 @@ describe("in-process references and privacy", () => { expect((failure as SpendLedgerOwnerError).code).toBe("SPEND_LEDGER_OWNER_NOT_HELD"); let mutation: unknown; - try { - retained.reserve({ sendId: "s1", targets: [{ scope: "root", scopeId: "r1" }], tokens: 1 }); - } catch (error) { mutation = error; } + try { retained.reserve(reserveRequest("s1")); } catch (error) { mutation = error; } expect(mutation).toBeInstanceOf(SpendLedgerOwnerError); + // Reads are refused for the same reason writes are: the figures describe a journal this + // handle no longer owns. + for (const read of [ + () => retained.knows("s1"), + () => retained.exhausted("root", "r1"), + () => retained.policy, + () => retained.degraded, + () => retained.persistFailures, + () => retained.corruptRecords, + ]) { + expect(read).toThrow(SpendLedgerOwnerError); + } + // A handle taken under the new ownership works and sees the journal as it is now. expect(() => sharedSpendLedger().snapshot("root", "r1")).not.toThrow(); }); + test("storage this module did not mint is not accepted as proof", () => { + // The defect this closes: a required guard the caller supplies can be a guard that does + // nothing, so a look-alike object must be refused on identity rather than on shape. + const lease = acquireSpendLedgerOwner(); + leases.push(lease); + const forged = { + path: join(home, SPEND_LEDGER_JOURNAL_FILENAME), + assert(): void { /* a caller-supplied guard proves nothing */ }, + } as unknown as Parameters[0]; + + expect(() => createOwnedFileSpendJournal(forged)).toThrow(SpendLedgerOwnerError); + expect(() => loadOrCreateSpendLedgerSalt(forged)).toThrow(SpendLedgerOwnerError); + expect(existsSync(join(home, SPEND_LEDGER_JOURNAL_FILENAME))).toBe(false); + }); + + test("minting refuses a name that is not a plain file in the owned directory", () => { + const lease = acquireSpendLedgerOwner(); + leases.push(lease); + for (const name of ["../escape.jsonl", "nested/child.jsonl", ".."]) { + expect(() => mintSpendLedgerStorage(name)).toThrow(SpendLedgerOwnerError); + } + }); + test("a dangling journal symlink is refused rather than followed", () => { const lease = acquireSpendLedgerOwner(); leases.push(lease); @@ -254,8 +298,7 @@ describe("in-process references and privacy", () => { symlinkSync(target, journal); let failure: unknown; - try { sharedSpendLedger().reserve({ sendId: "s1", targets: [{ scope: "root", scopeId: "r1" }], tokens: 1 }); } - catch (error) { failure = error; } + try { sharedSpendLedger().reserve(reserveRequest("s1")); } catch (error) { failure = error; } expect(failure).toBeInstanceOf(SpendLedgerOwnerError); // The point of the case: the link's target must not have been created by following it. expect(existsSync(target)).toBe(false); From 6da37f4de999f8f883cdd910f61f17d9d83b42e4 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 19:35:12 +0900 Subject: [PATCH 06/34] fix(spend): key ownership proof on identity, not on a marker the token carries A marker on the object was still forgeable. Spreading a real token copies its own symbol, so { ...minted, path: elsewhere, assert() {} } passed the brand check and then ran the caller's replacement guard against the caller's path - the same no-op guard the required-callback version allowed, reached another way. The token now carries nothing at all. Its path and the ownership it was minted under live in a module-private table keyed by the token's identity, which a copy cannot reproduce because a copy is a different object. Callers ask this module for the path and for the check rather than reading either off the value they were handed. The negative case covers all three shapes: a bare look-alike, a hand-built object with the right fields, and a spread of a genuine token with the path redirected and the guard replaced. It also asserts the redirected file was never created, and that a genuine token still works so the refusal is about identity rather than refusing everything. A separate case pins that a real token stops working once its ownership ends. --- src/lib/spend-ledger-owner.ts | 63 +++++++++++++++++----------- src/lib/spend-reservation-ledger.ts | 26 ++++++------ tests/lib/spend-ledger-owner.test.ts | 40 ++++++++++++++---- 3 files changed, 83 insertions(+), 46 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index 1a2beceeb4..21654d38cc 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -226,22 +226,32 @@ export function currentSpendLedgerOwnerGeneration(): number { return ownerGeneration; } -const STORAGE_BRAND: unique symbol = Symbol("spend-ledger-storage"); - /** * Permission to touch one file under the owned state directory, for as long as this exact * ownership lasts. * * A callback supplied by the caller cannot be this proof: the caller can pass one that does - * nothing, which is how a "required guard" still allowed an unowned write. Only this module - * mints one, it derives the path from the directory actually owned rather than accepting a - * path to trust, and the brand means a structurally similar object is not accepted in its place. + * nothing, which is how a "required guard" still allowed an unowned write. A marker carried ON + * the object is not proof either, because an object spread copies it: `{ ...minted, path: + * elsewhere, assert() {} }` would look minted while pointing somewhere else. So the token + * carries nothing. The path and the ownership it was minted under live in a table only this + * module can read, keyed by the token's identity, and a copy is simply not that key. */ export interface SpendLedgerStorage { - readonly [STORAGE_BRAND]: true; - readonly path: string; - /** Throws unless the ownership this was minted under is still the current one. */ - assert(): void; + readonly __spendLedgerStorage: unique symbol; +} + +const mintedStorage = new WeakMap(); + +function trustedStorage(storage: SpendLedgerStorage): { readonly path: string; readonly generation: number; readonly home: string } { + const entry = typeof storage === "object" && storage !== null ? mintedStorage.get(storage) : undefined; + if (!entry) { + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_NOT_HELD", + "Spend-ledger ownership is required before the shared ledger can be used.", + ); + } + return entry; } /** @@ -267,24 +277,27 @@ export function mintSpendLedgerStorage(fileName: string): SpendLedgerStorage { } const generation = ownerGeneration; const home = owner.home; - return Object.freeze({ - [STORAGE_BRAND]: true as const, - path: join(home, fileName), - assert(): void { - assertSpendLedgerOwnerGeneration(generation, home); - }, - }); + // The token is deliberately empty: everything trustworthy about it is held here, under its + // identity, where a spread cannot reach. + const token = Object.freeze({}) as unknown as SpendLedgerStorage; + mintedStorage.set(token, { path: join(home, fileName), generation, home }); + return token; } -/** Reject anything that did not come from {@link mintSpendLedgerStorage}. */ -export function assertMintedStorage(storage: SpendLedgerStorage): void { - if (storage?.[STORAGE_BRAND] !== true) { - throw new SpendLedgerOwnerError( - "SPEND_LEDGER_OWNER_NOT_HELD", - "Spend-ledger ownership is required before the shared ledger can be used.", - ); - } - storage.assert(); +/** The file this token stands for, as this module recorded it at mint time. */ +export function spendLedgerStoragePath(storage: SpendLedgerStorage): string { + return trustedStorage(storage).path; +} + +/** + * Refuse unless this exact token was minted here and its ownership is still current. + * + * Both halves matter: identity rules out a copy or a look-alike, and the generation rules out a + * token minted before the lock was dropped and taken again. + */ +export function assertStorageOwned(storage: SpendLedgerStorage): void { + const entry = trustedStorage(storage); + assertSpendLedgerOwnerGeneration(entry.generation, entry.home); } /** diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 6e178ccff5..96bcc92030 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -65,7 +65,7 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertMintedStorage, assertSpendLedgerOwnerHeld, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, type SpendLedgerStorage } from "./spend-ledger-owner"; +import { assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerStorage } from "./spend-ledger-owner"; // The singleton belongs to the state directory it was built for. Releasing ownership hands that // directory to whoever comes next, so the in-memory copy goes with it and the next construction @@ -428,8 +428,8 @@ function assertSafeLedgerFile(path: string): void { * proves that ownership, so there is no entrypoint here that writes a caller-chosen path. */ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJournal { - assertMintedStorage(storage); - const path = storage.path; + assertStorageOwned(storage); + const path = spendLedgerStoragePath(storage); const ensureDir = (): string => { const dir = dirname(path); // The guard runs before any mutation so a rejected write leaves nothing behind. @@ -439,7 +439,7 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ }; return { read(): string[] { - storage.assert(); + assertStorageOwned(storage); if (!ledgerEntryExists(path)) return []; assertSafeLedgerFile(path); // Replay is once per process and is the moment a journal inherited from an older build @@ -448,7 +448,7 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ return readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0); }, append(line: string): void { - storage.assert(); + assertStorageOwned(storage); ensureDir(); const created = !ledgerEntryExists(path); if (!created) assertSafeLedgerFile(path); @@ -457,7 +457,7 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ hardenLedgerFile(path, { force: created }); }, rewrite(lines: string[]): void { - storage.assert(); + assertStorageOwned(storage); ensureDir(); // Same directory, so the rename is atomic on the same filesystem: a crash mid-compaction // leaves either the old journal or the new one, never a half-written ledger. @@ -485,8 +485,8 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ * per-process value. */ export function loadOrCreateSpendLedgerSalt(storage: SpendLedgerStorage): string { - assertMintedStorage(storage); - const path = storage.path; + assertStorageOwned(storage); + const path = spendLedgerStoragePath(storage); if (ledgerEntryExists(path)) { assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); @@ -498,7 +498,7 @@ export function loadOrCreateSpendLedgerSalt(storage: SpendLedgerStorage): string ); } const dir = dirname(path); - storage.assert(); + assertStorageOwned(storage); assertNotRealHomeUnderTest(dir); mkdirSync(dir, { recursive: true, mode: 0o700 }); const salt = randomBytes(32).toString("hex"); @@ -1140,10 +1140,12 @@ export function sharedSpendLedger(): SpendReservationLedger { // ownership they were minted under. Nothing here chooses a path or supplies its own guard. const journalStorage = mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME); const saltStorage = mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME); + const journalPath = spendLedgerStoragePath(journalStorage); + const saltPath = spendLedgerStoragePath(saltStorage); const assertOwnedAccounting = (): void => { - journalStorage.assert(); - if (ledgerEntryExists(journalStorage.path)) assertSafeLedgerFile(journalStorage.path); - if (ledgerEntryExists(saltStorage.path)) assertSafeLedgerFile(saltStorage.path); + assertStorageOwned(journalStorage); + if (ledgerEntryExists(journalPath)) assertSafeLedgerFile(journalPath); + if (ledgerEntryExists(saltPath)) assertSafeLedgerFile(saltPath); }; sharedLedger = createSpendReservationLedger({ journal: createOwnedFileSpendJournal(journalStorage), diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index c5eaa599ca..de7d64190c 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -268,18 +268,40 @@ describe("in-process references and privacy", () => { }); test("storage this module did not mint is not accepted as proof", () => { - // The defect this closes: a required guard the caller supplies can be a guard that does - // nothing, so a look-alike object must be refused on identity rather than on shape. + // Three ways to try: a bare look-alike, and - the one a marker on the object cannot stop - + // a spread of a real token with the path redirected and the guard replaced. Identity is the + // only thing a copy cannot reproduce. const lease = acquireSpendLedgerOwner(); leases.push(lease); - const forged = { - path: join(home, SPEND_LEDGER_JOURNAL_FILENAME), - assert(): void { /* a caller-supplied guard proves nothing */ }, - } as unknown as Parameters[0]; + const elsewhere = join(root, "elsewhere.jsonl"); + const minted = mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME); + const forgeries = [ + {} as unknown as typeof minted, + { path: elsewhere, assert(): void { /* proves nothing */ } } as unknown as typeof minted, + { ...minted, path: elsewhere, assert(): void { /* proves nothing */ } } as unknown as typeof minted, + ]; + + for (const forged of forgeries) { + expect(() => createOwnedFileSpendJournal(forged)).toThrow(SpendLedgerOwnerError); + expect(() => loadOrCreateSpendLedgerSalt(forged)).toThrow(SpendLedgerOwnerError); + } + expect(existsSync(elsewhere)).toBe(false); - expect(() => createOwnedFileSpendJournal(forged)).toThrow(SpendLedgerOwnerError); - expect(() => loadOrCreateSpendLedgerSalt(forged)).toThrow(SpendLedgerOwnerError); - expect(existsSync(join(home, SPEND_LEDGER_JOURNAL_FILENAME))).toBe(false); + // A token this module did mint keeps working, so the refusal is about identity and not + // about refusing everything. + expect(() => createOwnedFileSpendJournal(minted)).not.toThrow(); + }); + + test("a minted token stops working once its ownership ends", () => { + const first = acquireSpendLedgerOwner(); + const minted = mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(minted); + first.release(); + + const second = acquireSpendLedgerOwner(); + leases.push(second); + expect(() => journal.append("{}")).toThrow(SpendLedgerOwnerError); + expect(() => journal.read()).toThrow(SpendLedgerOwnerError); }); test("minting refuses a name that is not a plain file in the owned directory", () => { From 7c47530d0b8c59952e2dc81858eaded94f2438c0 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 19:42:20 +0900 Subject: [PATCH 07/34] fix(spend): take ownership on demand instead of refusing to serve Hosted CI showed the guard was asserting the wrong rule. Requiring ownership to touch the journal is right; requiring it to have been taken by startServer is not, and handleResponses is an equally supported entry point that never calls startServer - 72 test files exercise exactly that shape, and an embedder can too. The result was a 502 wherever a turn reached its first physical send without a server having started. The shared ledger now takes the lease itself when nobody holds one. The guarantee becomes "no unowned writer" rather than "no writer outside one entry point", and it is not weakened: a directory another process owns is refused exactly as before, and the call fails closed with it. The lease lives as long as a server's would and returns through the same release path. Applying a policy no longer demands ownership either. Recording a value touches no journal; only reconfiguring a ledger that already exists does, because that ledger is a live view of an owned directory. --- src/lib/spend-reservation-ledger.ts | 37 +++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 96bcc92030..f596ab5168 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -65,12 +65,25 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerStorage } from "./spend-ledger-owner"; +import { acquireSpendLedgerOwner, assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerOwnerLease, type SpendLedgerStorage } from "./spend-ledger-owner"; + +/** + * A lease this module took because the journal was needed and nobody held one. + * + * Kept for the life of the process, like the lease a server takes: the release path is the + * same one, and SQLite hands the directory back if the process dies. + */ +let onDemandLease: SpendLedgerOwnerLease | undefined; // The singleton belongs to the state directory it was built for. Releasing ownership hands that // directory to whoever comes next, so the in-memory copy goes with it and the next construction // replays the journal. -onSpendLedgerOwnerReleased(() => { sharedLedger = undefined; }); +onSpendLedgerOwnerReleased(() => { + sharedLedger = undefined; + // The release that triggered this hook is the one that ended that lease; dropping the + // reference here must not try to release it a second time. + onDemandLease = undefined; +}); export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; /** @@ -1119,8 +1132,12 @@ export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined): SpendR * configures no ceiling must not open a journal merely because the server started. */ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void { - assertSpendLedgerOwnerHeld(); - if (sharedLedger) bindSpendLedgerOwnerHome(); + // Recording a policy value touches no journal, so it needs no ownership. Changing a ledger + // that already exists does, because that ledger is a live view of an owned directory. + if (sharedLedger) { + assertSpendLedgerOwnerHeld(); + bindSpendLedgerOwnerHome(); + } sharedPolicy = policy; sharedLedger?.reconfigure(policy); } @@ -1133,6 +1150,13 @@ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void * than inheriting figures from the previous one. */ export function sharedSpendLedger(): SpendReservationLedger { + // Ownership is required to touch the journal, but requiring it to have been taken ELSEWHERE + // is a different rule, and a wrong one: `startServer` takes the lease up front while + // `handleResponses` is an equally supported entry point that does not. Take it here when no + // one has, so the guarantee is "no unowned writer" rather than "no writer outside one + // entrypoint". On demand does not mean optional: a directory another process owns is refused + // exactly as before, and this call fails closed with it. + if (spendLedgerOwnerSnapshot().ownership !== "held") onDemandLease ??= acquireSpendLedgerOwner(); assertSpendLedgerOwnerHeld(); bindSpendLedgerOwnerHome(); if (!sharedLedger) { @@ -1188,6 +1212,11 @@ export function spendLedgerDiagnosticsSnapshot(): { * durable record and the next construction replays it. */ export function resetSharedSpendLedgerForTest(): void { + const lease = onDemandLease; + onDemandLease = undefined; + if (lease) { + try { lease.release(); } catch { /* a failed release must not mask the reset */ } + } sharedLedger = undefined; sharedPolicy = DEFAULT_SPEND_RESERVATION_POLICY; resetSpendLedgerOwnerBindingForTest(); From 64dae869b21a5ea0a67679f9cf9860c57cd38170 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 19:50:07 +0900 Subject: [PATCH 08/34] fix(spend): drop the implicit lease and own the directory in the fixtures instead The on-demand lease was wrong twice over. Its premise - that handleResponses is an equally supported entry point - does not hold: the package exports only the root, src/index.ts exports startServer and not this handler, and every production caller is server-internal beneath that lease. A direct import of an internal module in a test is not a public contract. It also leaked: the lease took its own reference, so server.stop left one behind, blocking final cleanup and the sequential directory switch the design allows. Ownership is required again, with no exception carved for callers that skipped startServer. The cases that dispatch without a server now take the real lease through a shared helper and release it after each case, so the production rule is exercised rather than relaxed for tests. The exact-token, startup and restart boundaries are unchanged. --- src/lib/spend-reservation-ledger.ts | 29 ++----------------- .../anthropic-quota-dispatch.test.ts | 5 ++++ ...anthropic-sidecar-account-failover.test.ts | 4 +++ tests/helpers/owned-spend-home.ts | 29 +++++++++++++++++++ 4 files changed, 40 insertions(+), 27 deletions(-) create mode 100644 tests/helpers/owned-spend-home.ts diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index f596ab5168..3c71d4dd6b 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -65,25 +65,12 @@ import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; -import { acquireSpendLedgerOwner, assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerOwnerLease, type SpendLedgerStorage } from "./spend-ledger-owner"; - -/** - * A lease this module took because the journal was needed and nobody held one. - * - * Kept for the life of the process, like the lease a server takes: the release path is the - * same one, and SQLite hands the directory back if the process dies. - */ -let onDemandLease: SpendLedgerOwnerLease | undefined; +import { assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerStorage } from "./spend-ledger-owner"; // The singleton belongs to the state directory it was built for. Releasing ownership hands that // directory to whoever comes next, so the in-memory copy goes with it and the next construction // replays the journal. -onSpendLedgerOwnerReleased(() => { - sharedLedger = undefined; - // The release that triggered this hook is the one that ended that lease; dropping the - // reference here must not try to release it a second time. - onDemandLease = undefined; -}); +onSpendLedgerOwnerReleased(() => { sharedLedger = undefined; }); export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; /** @@ -1150,13 +1137,6 @@ export function configureSharedSpendLedger(policy: SpendReservationPolicy): void * than inheriting figures from the previous one. */ export function sharedSpendLedger(): SpendReservationLedger { - // Ownership is required to touch the journal, but requiring it to have been taken ELSEWHERE - // is a different rule, and a wrong one: `startServer` takes the lease up front while - // `handleResponses` is an equally supported entry point that does not. Take it here when no - // one has, so the guarantee is "no unowned writer" rather than "no writer outside one - // entrypoint". On demand does not mean optional: a directory another process owns is refused - // exactly as before, and this call fails closed with it. - if (spendLedgerOwnerSnapshot().ownership !== "held") onDemandLease ??= acquireSpendLedgerOwner(); assertSpendLedgerOwnerHeld(); bindSpendLedgerOwnerHome(); if (!sharedLedger) { @@ -1212,11 +1192,6 @@ export function spendLedgerDiagnosticsSnapshot(): { * durable record and the next construction replays it. */ export function resetSharedSpendLedgerForTest(): void { - const lease = onDemandLease; - onDemandLease = undefined; - if (lease) { - try { lease.release(); } catch { /* a failed release must not mask the reset */ } - } sharedLedger = undefined; sharedPolicy = DEFAULT_SPEND_RESERVATION_POLICY; resetSpendLedgerOwnerBindingForTest(); diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 19eeed80a1..6026f90011 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -1,5 +1,6 @@ /** Physical response attribution through the real adapter and response/search loops. */ import { afterEach, beforeEach, expect, mock, test } from "bun:test"; +import { useOwnedSpendHome } from "../../helpers/owned-spend-home"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -59,6 +60,10 @@ beforeEach(() => { clearResponseStateForTests(); }); +// This case dispatches without starting a server, so it takes the spend-journal lease itself. +// Registered after the hook above so the isolated home is already in place. +useOwnedSpendHome(); + afterEach(() => { adapterRequestsFollow = false; try { diff --git a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts index 8d094db631..4fce8689c7 100644 --- a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +++ b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts @@ -15,6 +15,7 @@ import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oa import { clearAccountQuotaCache, getCachedProviderAccountQuota, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { useOwnedSpendHome } from "../../helpers/owned-spend-home"; const previousHome = process.env.OPENCODEX_HOME; let testHome = ""; @@ -110,6 +111,9 @@ afterEach(() => { removeTreeWithRetry(testHome); }); +// Dispatches without starting a server, so it takes the spend-journal lease itself. +useOwnedSpendHome(); + afterAll(() => { if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; diff --git a/tests/helpers/owned-spend-home.ts b/tests/helpers/owned-spend-home.ts new file mode 100644 index 0000000000..e13b3c7753 --- /dev/null +++ b/tests/helpers/owned-spend-home.ts @@ -0,0 +1,29 @@ +import { afterEach, beforeEach } from "bun:test"; +import { acquireSpendLedgerOwner, type SpendLedgerOwnerLease } from "../../src/lib/spend-ledger-owner"; +import { resetSharedSpendLedgerForTest } from "../../src/lib/spend-reservation-ledger"; + +/** + * Hold the spend-journal writer lease for a case that dispatches without starting a server. + * + * `startServer` takes this lease before anything can serve, so production traffic always + * reaches the ledger owning its state directory. A case that calls an internal handler directly + * skips that step, and the ledger refuses to write for a process that owns nothing. Taking the + * real lease here keeps the production rule intact instead of teaching the ledger to make an + * exception for tests. + * + * The lease is released after every case, so a later case under a different state directory + * finds the directory free. + */ +export function useOwnedSpendHome(): void { + let lease: SpendLedgerOwnerLease | undefined; + beforeEach(() => { + resetSharedSpendLedgerForTest(); + lease = acquireSpendLedgerOwner(); + }); + afterEach(() => { + const held = lease; + lease = undefined; + try { held?.release(); } catch { /* a failed release must not mask the case's result */ } + resetSharedSpendLedgerForTest(); + }); +} From 2c763017578fac1b0d54a9f2d1d95f87520e5e0b Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 19:54:10 +0900 Subject: [PATCH 09/34] test(spend): order the lease release ahead of each fixture's cleanup Hook registration order differed between the two fixtures, so neither FIFO nor LIFO execution could guarantee the lease closed before the directory holding it was removed - a failed removal on Windows, an unlinked live database on POSIX. The helper now returns an idempotent release and each fixture calls it first in its own teardown, so the ordering is stated where it matters instead of inferred from where a hook happened to be registered. Adds the same treatment to the Claude native-affinity fixture, which CI showed reaching the ledger through combo dispatch. --- .../anthropic-quota-dispatch.test.ts | 12 ++++--- ...anthropic-sidecar-account-failover.test.ts | 11 +++--- .../claude-native-affinity.test.ts | 7 ++++ tests/helpers/owned-spend-home.ts | 35 +++++++++++-------- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 6026f90011..3e641d9371 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -1,6 +1,6 @@ /** Physical response attribution through the real adapter and response/search loops. */ import { afterEach, beforeEach, expect, mock, test } from "bun:test"; -import { useOwnedSpendHome } from "../../helpers/owned-spend-home"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -38,6 +38,7 @@ const originalHome = process.env.OPENCODEX_HOME; let originalFetch: typeof globalThis.fetch; let unexpectedGlobalFetches = 0; let home: string; +let releaseSpendHome: (() => void) | undefined; let sent: { authorization: string | null; apiKey: string | null; body: Record }[]; beforeEach(() => { @@ -58,13 +59,14 @@ beforeEach(() => { clearAccountQuotaCache(); resetProviderQuotaReconcileStateForTests(); clearResponseStateForTests(); + // Dispatching without starting a server means taking the spend-journal lease here, and + // releasing it before this case's directory is removed. + releaseSpendHome = acquireOwnedSpendHome(); }); -// This case dispatches without starting a server, so it takes the spend-journal lease itself. -// Registered after the hook above so the isolated home is already in place. -useOwnedSpendHome(); - afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; adapterRequestsFollow = false; try { // Provider code may catch the guard's rejection; the attempted network call still fails the test. diff --git a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts index 4fce8689c7..7328d081f4 100644 --- a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +++ b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts @@ -15,10 +15,11 @@ import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oa import { clearAccountQuotaCache, getCachedProviderAccountQuota, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; -import { useOwnedSpendHome } from "../../helpers/owned-spend-home"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; const previousHome = process.env.OPENCODEX_HOME; let testHome = ""; +let releaseSpendHome: (() => void) | undefined; let handleResponses: typeof import("../../../src/server/responses")["handleResponses"]; let observedKeys: string[] = []; let sidecarMode = false; @@ -101,9 +102,14 @@ beforeEach(() => { clearGenericFailoverHealth(); clearAccountQuotaCache(); resetProviderQuotaReconcileStateForTests(); + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before the directory below is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; clearAnthropicAccountPoolState(); clearGenericFailoverHealth(); clearAccountQuotaCache(); @@ -111,9 +117,6 @@ afterEach(() => { removeTreeWithRetry(testHome); }); -// Dispatches without starting a server, so it takes the spend-journal lease itself. -useOwnedSpendHome(); - afterAll(() => { if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; diff --git a/tests/claude-integration/claude-native-affinity.test.ts b/tests/claude-integration/claude-native-affinity.test.ts index d3cb071e5b..b646e6bef7 100644 --- a/tests/claude-integration/claude-native-affinity.test.ts +++ b/tests/claude-integration/claude-native-affinity.test.ts @@ -13,6 +13,7 @@ import type { OcxConfig } from "../../src/types"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; const metadata = "user_test_account__session_conversation-native"; @@ -23,6 +24,7 @@ let isolated: IsolatedCodexHome; let home: string; let previousHome: string | undefined; let token: string; +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; @@ -33,8 +35,13 @@ beforeEach(() => { writeFileSync(join(isolated.path, "auth.json"), JSON.stringify({ tokens: { access_token: token, account_id: "fixture-native-main" } })); clearComboSelectionState(); clearComboTargetCooldowns(); + // Dispatches without starting a server, so the spend-journal lease is taken here. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before the directory is removed, so no live database sits inside it. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearComboSelectionState(); clearComboTargetCooldowns(); diff --git a/tests/helpers/owned-spend-home.ts b/tests/helpers/owned-spend-home.ts index e13b3c7753..2a70746c98 100644 --- a/tests/helpers/owned-spend-home.ts +++ b/tests/helpers/owned-spend-home.ts @@ -1,4 +1,3 @@ -import { afterEach, beforeEach } from "bun:test"; import { acquireSpendLedgerOwner, type SpendLedgerOwnerLease } from "../../src/lib/spend-ledger-owner"; import { resetSharedSpendLedgerForTest } from "../../src/lib/spend-reservation-ledger"; @@ -11,19 +10,25 @@ import { resetSharedSpendLedgerForTest } from "../../src/lib/spend-reservation-l * real lease here keeps the production rule intact instead of teaching the ledger to make an * exception for tests. * - * The lease is released after every case, so a later case under a different state directory - * finds the directory free. + * Returns an idempotent release. Call it FIRST in the case's own teardown, before the state + * directory is removed and before the home variable is restored: an open SQLite lease inside a + * directory being deleted fails the removal on Windows and leaves an unlinked live database on + * POSIX. Ordering is stated by the caller rather than inferred from hook registration order, + * which differs between these fixtures and is not a contract either way. */ -export function useOwnedSpendHome(): void { - let lease: SpendLedgerOwnerLease | undefined; - beforeEach(() => { - resetSharedSpendLedgerForTest(); - lease = acquireSpendLedgerOwner(); - }); - afterEach(() => { - const held = lease; - lease = undefined; - try { held?.release(); } catch { /* a failed release must not mask the case's result */ } - resetSharedSpendLedgerForTest(); - }); +export function acquireOwnedSpendHome(): () => void { + resetSharedSpendLedgerForTest(); + const lease: SpendLedgerOwnerLease = acquireSpendLedgerOwner(); + let released = false; + return () => { + if (released) return; + released = true; + try { + lease.release(); + } catch { + /* a failed release must not mask the case's own result */ + } finally { + resetSharedSpendLedgerForTest(); + } + }; } From 4bb4d80b0d9292f6c58e19cc6b6c47922204c81a Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:02:27 +0900 Subject: [PATCH 10/34] test(spend): let a failed lease release fail the case The release swallowed whatever lease.release() threw. A rollback or close that fails is a defect in the thing under test, and hiding it leaves a green run over a lease that never let go - the exact state the single-writer rule exists to prevent. The reset stays in finally so the next case still starts from a discarded singleton. --- tests/helpers/owned-spend-home.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/owned-spend-home.ts b/tests/helpers/owned-spend-home.ts index 2a70746c98..f954d1d0b2 100644 --- a/tests/helpers/owned-spend-home.ts +++ b/tests/helpers/owned-spend-home.ts @@ -23,10 +23,10 @@ export function acquireOwnedSpendHome(): () => void { return () => { if (released) return; released = true; + // A rollback or close failure is a real defect in the thing under test, so it is allowed to + // fail the case. Swallowing it would leave a green run over a lease that never let go. try { lease.release(); - } catch { - /* a failed release must not mask the case's own result */ } finally { resetSharedSpendLedgerForTest(); } From 567b6737a089821442b804b4ff5980495d3e5a1a Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:02:36 +0900 Subject: [PATCH 11/34] test(spend): own the state directory in the six fixtures CI named Every dispatch charges the ledger: reserveDispatch consults the spend observer before it books, so any case that reaches prepareAdapterExchange touches the shared journal. These six call the internal handler directly and never take the lease startServer takes, so the ledger refused and the cases saw 502/529 instead of their own contract. Each one takes the real lease at the end of its own beforeEach, after its home is in place, and releases it at the top of its own afterEach, before that home is removed. Ordering is stated at the call site because hook registration order differs between these fixtures and is a contract in neither direction. Diagnosed from the hosted shard logs at 654bcee, not guessed: claude-inbound- cache-stabilize (shard 1), main-account-hard-lock-auth (shard 2), reserve-auth- context and reserve-dispatch (shard 3), github-copilot-account-origin and kiro-auth-context-continuation (shard 4). Shards stop at the first failed batch, so this is a partial view by construction and the remaining fixtures are being inventoried rather than discovered one run at a time. --- .../claude-inbound-cache-stabilize.test.ts | 9 +++++++++ .../main-account-hard-lock-auth.test.ts | 10 ++++++++++ tests/codex-integration/reserve-auth-context.test.ts | 10 ++++++++++ tests/codex-integration/reserve-dispatch.test.ts | 10 ++++++++++ .../github-copilot-account-origin.test.ts | 10 ++++++++++ .../kiro/kiro-auth-context-continuation.test.ts | 10 ++++++++++ 6 files changed, 59 insertions(+) diff --git a/tests/claude-integration/claude-inbound-cache-stabilize.test.ts b/tests/claude-integration/claude-inbound-cache-stabilize.test.ts index b11cbe3aaa..09b55cc329 100644 --- a/tests/claude-integration/claude-inbound-cache-stabilize.test.ts +++ b/tests/claude-integration/claude-inbound-cache-stabilize.test.ts @@ -7,6 +7,7 @@ import { handleClaudeMessages } from "../../src/server/claude-messages"; import type { OcxConfig, OcxClaudeCodeConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { beforeEach, afterEach, describe, expect, test } from "bun:test"; import { stabilizeClaudeInstructionsForPromptCache } from "../../src/claude/inbound-cache-stabilize"; import { anthropicToResponsesTranslation } from "../../src/claude/inbound"; @@ -404,14 +405,22 @@ describe("Messages operator opt-in at the outbound boundary", () => { let isolatedHome: IsolatedCodexHome | undefined; let previousHome: string | undefined; let configHome: string | undefined; + let releaseSpendHome: (() => void) | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; isolatedHome = installIsolatedCodexHome("ocx-prefix-contract-"); configHome = mkdtempSync(join(tmpdir(), "ocx-prefix-config-")); process.env.OPENCODEX_HOME = configHome; + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; isolatedHome?.restore(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/codex-integration/main-account-hard-lock-auth.test.ts b/tests/codex-integration/main-account-hard-lock-auth.test.ts index 6b25b58abb..3645380c93 100644 --- a/tests/codex-integration/main-account-hard-lock-auth.test.ts +++ b/tests/codex-integration/main-account-hard-lock-auth.test.ts @@ -35,6 +35,7 @@ import { setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; import type { OcxConfig } from "../../src/types"; import { COLD_SPAWN_WARMUP_HOOK_BUDGET_MS, warmModuleGraph } from "../helpers/cold-spawn-warmup"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { helperPath, repoRoot } from "../helpers/repo-root"; import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS } from "../helpers/test-budget"; @@ -106,6 +107,8 @@ function addAlternative(cfg: OcxConfig): void { }); } +let releaseSpendHome: (() => void) | undefined; + beforeEach(() => { tokenExpiry = Math.floor(Date.now() / 1000) + 86_400; previousHome = process.env.OPENCODEX_HOME; @@ -122,9 +125,16 @@ beforeEach(() => { clearAccountNeedsReauth("hard-lock-pool"); mainAccount.setMainAccountPlan(null); writeMain(); + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; mock.restore(); clearAccountQuota(); clearThreadAccountMap(); diff --git a/tests/codex-integration/reserve-auth-context.test.ts b/tests/codex-integration/reserve-auth-context.test.ts index b66c2377bf..c5ab020d0d 100644 --- a/tests/codex-integration/reserve-auth-context.test.ts +++ b/tests/codex-integration/reserve-auth-context.test.ts @@ -25,6 +25,7 @@ import type { DataPlaneAdmission } from "../../src/server/auth-cors"; import type { WhamUsageResponse } from "../../src/codex/quota-types"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const MAIN = mainAccount.MAIN_CODEX_ACCOUNT_ID; const accountId = "reserve-workspace-fixture"; @@ -88,6 +89,8 @@ function prohibitPhysicalReads(): void { spyOn(mainAccount, "getValidMainAccountToken").mockImplementation(fail); } +let releaseSpendHome: (() => void) | undefined; + beforeEach(() => { oldHome = process.env.OPENCODEX_HOME; oldCodexHome = process.env.CODEX_HOME; @@ -133,9 +136,16 @@ beforeEach(() => { } throw new Error("unexpected outbound fixture destination"); }, { preconnect() {} })); + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(async () => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; mock.restore(); clearAccountQuota(); // Cancels this fixture's pending persistence timer before deleting its home. clearMainAccountInfoCache(); diff --git a/tests/codex-integration/reserve-dispatch.test.ts b/tests/codex-integration/reserve-dispatch.test.ts index f8da3b4af0..bd75b0af64 100644 --- a/tests/codex-integration/reserve-dispatch.test.ts +++ b/tests/codex-integration/reserve-dispatch.test.ts @@ -27,6 +27,7 @@ import { flushConfigDirHardeningForTests } from "../../src/config/paths"; import type { DataPlaneAdmission } from "../../src/server/auth-cors"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const accountId = "reserve-dispatch-workspace"; const accessToken = "reserve-dispatch-owned-fixture"; @@ -68,6 +69,8 @@ async function authorize() { return { ctx, cfg, guard }; } +let releaseSpendHome: (() => void) | undefined; + beforeEach(() => { oldHome = process.env.OPENCODEX_HOME; oldCodexHome = process.env.CODEX_HOME; @@ -113,9 +116,16 @@ beforeEach(() => { } throw new Error("unexpected dispatch fixture destination"); }, { preconnect() {} })); + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(async () => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; mock.restore(); clearAccountQuota(); clearMainAccountInfoCache(); diff --git a/tests/providers/github-copilot/github-copilot-account-origin.test.ts b/tests/providers/github-copilot/github-copilot-account-origin.test.ts index 4da89c816e..845d3e29cb 100644 --- a/tests/providers/github-copilot/github-copilot-account-origin.test.ts +++ b/tests/providers/github-copilot/github-copilot-account-origin.test.ts @@ -9,6 +9,7 @@ import { saveConfig } from "../../../src/config"; import { setActiveProviderApiKey } from "../../../src/providers/api-keys"; import type { OcxConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; const ACCOUNT_A_ORIGIN = "https://a.githubcopilot.com"; const ACCOUNT_B_ORIGIN = "https://b.githubcopilot.com"; @@ -187,15 +188,24 @@ function installFetch(options: { return { dispatches }; } +let releaseSpendHome: (() => void) | undefined; + beforeEach(() => { beforeBuildReturns = undefined; beforePacingReturns = undefined; home = mkdtempSync(join(tmpdir(), "ocx-copilot-origin-")); process.env.OPENCODEX_HOME = home; clearGenericFailoverHealth(); + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearGenericFailoverHealth(); if (originalHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/providers/kiro/kiro-auth-context-continuation.test.ts b/tests/providers/kiro/kiro-auth-context-continuation.test.ts index 83e22d0b48..567ce21ffd 100644 --- a/tests/providers/kiro/kiro-auth-context-continuation.test.ts +++ b/tests/providers/kiro/kiro-auth-context-continuation.test.ts @@ -15,6 +15,7 @@ import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-f import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; import type { AdapterEvent, OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; const previousHome = process.env.OPENCODEX_HOME; let testHome = ""; @@ -84,15 +85,24 @@ beforeAll(async () => { ({ handleResponses } = await import("../../../src/server/responses")); }); +let releaseSpendHome: (() => void) | undefined; + beforeEach(() => { testHome = mkdtempSync(join(tmpdir(), "ocx-kiro-continuation-auth-")); process.env.OPENCODEX_HOME = testHome; kiroBuilds = []; clearAnthropicAccountPoolState(); clearGenericFailoverHealth(); + // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken + // last because the lease binds the home in effect at the moment it is taken. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before this case's home is removed: an open lease inside a directory being + // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. + releaseSpendHome?.(); + releaseSpendHome = undefined; clearAnthropicAccountPoolState(); clearGenericFailoverHealth(); removeTreeWithRetry(testHome); From d3cca4e3438081c76ce32f45e7bf78b4a063443e Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:12:42 +0900 Subject: [PATCH 12/34] test(spend): own the state directory in the merged send-budget count rows These rows arrived with #5152 after this branch was cut, so no CI run has ever seen them against the ownership rule. Every one of them dispatches through the handler directly and therefore charges the shared ledger. The lease is taken per row, not per file. Two rows install their own OPENCODEX_HOME and the rest inherit the preload sandbox, and a lease is bound to the directory in effect when it was taken, so a block-level hook would either bind the wrong directory or conflict with the rows that swap one in. The two own-home rows drop it inside their existing finally, ahead of the environment restore and the directory removal. The file's afterEach drops it as a backstop. Without that, a row that throws mid-assertion leaves the lease behind and the next row's different home reports an ownership conflict instead of the failure that actually happened. Local checks: NOT RUN. --- .../responses-send-budget-counts.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/responses/responses-send-budget-counts.test.ts b/tests/responses/responses-send-budget-counts.test.ts index d6119caa0f..2c9888fb76 100644 --- a/tests/responses/responses-send-budget-counts.test.ts +++ b/tests/responses/responses-send-budget-counts.test.ts @@ -14,6 +14,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { saveCredential } from "../../src/oauth/store"; import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; @@ -31,6 +32,16 @@ import { createRequestExecutionBudget } from "../../src/lib/request-execution-bu */ const originalFetch = globalThis.fetch; +// Every dispatching row below calls the handler directly, so it takes the spend-journal writer +// lease that startServer would have taken for it. Per row rather than per file: two rows install +// their own OPENCODEX_HOME, and a lease is bound to the directory in effect when it was taken. +// The teardown drop is a backstop for a row that throws mid-assertion, because a lease left +// behind makes the NEXT row's different home read as an ownership conflict rather than as this +// row's failure. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome = acquireOwnedSpendHome(); }; +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; + beforeEach(() => { clearComboSelectionState(); clearComboTargetCooldowns(); @@ -38,6 +49,7 @@ beforeEach(() => { }); afterEach(() => { + dropSpendHome(); globalThis.fetch = originalFetch; setCachedCatalogForTests(null); clearComboSelectionState(); @@ -105,6 +117,9 @@ describe("upstream sends per logical request", () => { const previousJwtFlag = process.env.OPENCODEX_DEVIN_SEND_USER_JWT; const home = mkdtempSync(join(tmpdir(), "devin-send-count-")); process.env.OPENCODEX_HOME = home; + // Taken on the home this row just installed, and dropped in its finally before that home + // is removed: an open lease inside a directory being deleted fails the removal on Windows. + takeSpendHome(); delete process.env.OPENCODEX_DEVIN_SEND_USER_JWT; const apiKey = "devin-count-test"; // Devin is an OAuth-kind provider: the key the adapter ends up using is injected onto the @@ -149,6 +164,7 @@ describe("upstream sends per logical request", () => { totalSends: totalSends(logCtx), }, `status ${response.status}: ${body.slice(0, 200)}`).toEqual({ chatCalls: 1, totalSends: 1 }); } finally { + dropSpendHome(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (previousJwtFlag === undefined) delete process.env.OPENCODEX_DEVIN_SEND_USER_JWT; @@ -168,6 +184,7 @@ describe("upstream sends per logical request", () => { const previousJwtFlag = process.env.OPENCODEX_DEVIN_SEND_USER_JWT; const home = mkdtempSync(join(tmpdir(), "devin-send-denied-")); process.env.OPENCODEX_HOME = home; + takeSpendHome(); delete process.env.OPENCODEX_DEVIN_SEND_USER_JWT; const apiKey = "devin-denied-test"; await saveCredential("devin", { @@ -230,6 +247,7 @@ describe("upstream sends per logical request", () => { refused: true, }); } finally { + dropSpendHome(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; if (previousJwtFlag === undefined) delete process.env.OPENCODEX_DEVIN_SEND_USER_JWT; @@ -243,6 +261,7 @@ describe("upstream sends per logical request", () => { const upstream = alwaysFailing(502, "upstream busy"); const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses( responsesRequest("t0/model-t0"), { defaultProvider: "t0", providers: { t0: transientChatProvider("t0") } } as unknown as OcxConfig, @@ -262,6 +281,7 @@ describe("upstream sends per logical request", () => { const upstream = alwaysFailing(502, "upstream busy"); const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), comboOverTargets(1), logCtx); expect(response.status).toBe(502); @@ -276,6 +296,7 @@ describe("upstream sends per logical request", () => { const upstream = alwaysFailing(502, "upstream busy"); const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), comboOverTargets(3), logCtx); expect(response.status).toBe(502); @@ -306,6 +327,7 @@ describe("upstream sends per logical request", () => { const upstream = alwaysFailing(502, "upstream busy"); const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), comboOverTargets(13), logCtx); expect(response.status).toBe(502); @@ -343,6 +365,7 @@ describe("ambiguous reset safety across Responses recovery", () => { throw Object.assign(new Error("The socket connection was closed unexpectedly."), { code: "ECONNRESET" }); }) as typeof fetch; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses( responsesRequest(combo ? "combo/fan" : "t0/model-t0"), config, logCtx, ); @@ -367,6 +390,7 @@ describe("ambiguous reset safety across Responses recovery", () => { throw Object.assign(new Error("connection reset by peer"), { code: "ECONNRESET" }); }) as typeof fetch; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), comboOverTargets(2), logCtx); expect(response.status).toBe(429); expect((await response.json()).error.code).toBe("upstream_reset_replay_refused"); @@ -382,6 +406,7 @@ describe("ambiguous reset safety across Responses recovery", () => { sends += 1; throw Object.assign(new Error("reset"), { code: "ECONNRESET" }); }) as typeof fetch; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), config, { model: "", provider: "" }); expect(response.status).toBe(429); expect((await response.json()).error.code).toBe("upstream_reset_replay_refused"); @@ -402,6 +427,7 @@ describe("ambiguous reset safety after outer recovery", () => { throw Object.assign(new Error("connection reset by peer"), { code: "ECONNRESET" }); }) as typeof fetch; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("combo/fan"), config, logCtx); expect(response.status).toBe(429); expect((await response.json()).error.code).toBe("upstream_reset_replay_refused"); @@ -430,6 +456,7 @@ describe("ambiguous reset safety after outer recovery", () => { throw Object.assign(new Error("connection reset by peer"), { code: "ECONNRESET" }); }) as typeof fetch; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(responsesRequest("t0/model-t0"), config, logCtx); expect(response.status).toBe(429); From dba082c75d1df88de6f811042bcef120250f82d8 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:26:15 +0900 Subject: [PATCH 13/34] test(spend): take the writer lease in every fixture that dispatches directly Thirty-one fixtures call a response handler without going through startServer, so none of them held the spend-journal writer lease and every dispatch came back as "Spend-ledger ownership is required before the shared ledger can be used". CI could only ever show a handful of these at a time, because a shard stops at its first failed batch, so the set was derived by tracing the dispatching call blocks rather than by re-running until the next one appeared. Placement is stated at each call site, never inferred from hook order: - A fixture that installs its own OPENCODEX_HOME takes the lease after that assignment and drops it before the environment is restored and the directory removed. An open lease inside a directory being deleted fails the removal on Windows and leaves an unlinked live database on POSIX. - A fixture that inherits the preload sandbox home takes the lease at the dispatch and drops it in the file's afterEach, so a row that throws mid-assertion cannot leave a lease behind and make the next row's different home read as an ownership conflict. - Files that mix both shapes get neither a file-level nor a block-level lease. Each dispatching case owns its own. Cases that never reach a physical dispatch are deliberately untouched: admission refusals, management-API rows, in-memory ledger rows and direct persistence rows. No assertion, expected value, mock, fixture or timeout is changed anywhere in this commit, and no file-size cap moves. Five fixtures sit at exactly their ratchet cap and are NOT in this batch: openai-responses-passthrough, responses-compaction-routing, usage/request-log, responses-custom-tool-repair and responses-undeclared-tool-guard. A cap only moves down, so they cannot take an additive lease and need an extraction first. That is its own change rather than blank-line churn smuggled into this one. Local checks: NOT RUN. --- ...rminal-continuation-owner-rotation.test.ts | 7 +++++ .../model-pinned-effort.test.ts | 7 +++++ tests/images/z-handler-activation.test.ts | 7 +++++ .../adapter-event-oauth-failover.test.ts | 7 +++++ tests/oauth/oauth-account-attribution.test.ts | 5 ++++ .../github-copilot-stream-contract.test.ts | 8 ++++++ .../github-copilot-wire-defaults.test.ts | 12 ++++++++- tests/providers/opencode-go-luna-wire.test.ts | 19 +++++++++++-- tests/providers/rate-limit-retry.test.ts | 10 +++++++ tests/responses/empty-completion-core.test.ts | 20 ++++++++++++++ .../responses/fresh-connection-optout.test.ts | 5 ++++ .../responses/responses-account-label.test.ts | 15 +++++++++++ .../responses-console-go-upload-retry.test.ts | 7 +++++ ...responses-forward-incomplete-quota.test.ts | 5 ++++ .../responses-native-main-refresh.test.ts | 7 +++++ .../responses-opaque-blob-recovery.test.ts | 7 +++++ .../responses-pool-401-refresh.test.ts | 7 +++++ .../responses-preview-main-read-fence.test.ts | 26 +++++++++++------- ...sponses-reasoning-effort-downgrade.test.ts | 7 +++++ .../responses-shadow-intercept.test.ts | 15 +++++++++++ .../responses-show-thinking-summary.test.ts | 19 ++++++++++++- .../probe-lease-dispatch-wiring.test.ts | 5 ++++ .../routing-policy-surface-parity.test.ts | 10 +++++++ ...subagent-fallback-handle-responses.test.ts | 7 +++++ .../server/context-history-ownership.test.ts | 7 +++++ ...plaintext-v2-agent-messages-server.test.ts | 27 +++++++++++++++++++ ...combo-reasoning-replay-eligibility.test.ts | 11 ++++++++ .../server-combo-zero-output-failover.test.ts | 7 +++++ tests/server/terminal-guard-server.test.ts | 25 +++++++++++++++++ .../web-search-passthrough-bridge.test.ts | 5 ++++ tests/web-search/web-search.test.ts | 11 +++++++- 31 files changed, 323 insertions(+), 14 deletions(-) diff --git a/tests/adapters/terminal-continuation-owner-rotation.test.ts b/tests/adapters/terminal-continuation-owner-rotation.test.ts index 458f8237b6..69ce097aca 100644 --- a/tests/adapters/terminal-continuation-owner-rotation.test.ts +++ b/tests/adapters/terminal-continuation-owner-rotation.test.ts @@ -16,6 +16,7 @@ import type { OcxParsedRequest, OcxProviderConfig, } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; interface BuildObservation { @@ -110,18 +111,24 @@ describe("terminal continuation provider-owner rotation", () => { let originalFetch: typeof fetch; let previousHome: string | undefined; let testHome = ""; + let releaseSpendHome: (() => void) | undefined; beforeEach(() => { originalFetch = globalThis.fetch; previousHome = process.env.OPENCODEX_HOME; testHome = mkdtempSync(join(tmpdir(), "ocx-terminal-owner-")); process.env.OPENCODEX_HOME = testHome; + // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); builds = []; clearKeyCooldowns(); clearResponseStateForTests(); }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; diff --git a/tests/codex-integration/model-pinned-effort.test.ts b/tests/codex-integration/model-pinned-effort.test.ts index a808b86767..9aa8063f57 100644 --- a/tests/codex-integration/model-pinned-effort.test.ts +++ b/tests/codex-integration/model-pinned-effort.test.ts @@ -12,6 +12,7 @@ import { parseRequest } from "../../src/responses/parser"; import { routeModel } from "../../src/router"; import { createTestTranslatorBudget, withTestTranslatorBudget } from "../helpers/translator-budget"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; @@ -265,11 +266,14 @@ describe("operator pins on the actual request wire", () => { let failFirst: boolean; let failureStatus: number; let onFirstSend: (() => void) | undefined; + let releaseSpendHome: (() => void) | undefined; beforeEach(() => { savedHome = process.env.OPENCODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-pin-wire-")); process.env.OPENCODEX_HOME = home; + // Take the writer lease after this block installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); codexHome = installIsolatedCodexHome("ocx-pin-wire-codex-"); captured = []; failFirst = false; @@ -309,6 +313,9 @@ describe("operator pins on the actual request wire", () => { }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; if (savedHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = savedHome; diff --git a/tests/images/z-handler-activation.test.ts b/tests/images/z-handler-activation.test.ts index 5b6f578a79..b7e836802d 100644 --- a/tests/images/z-handler-activation.test.ts +++ b/tests/images/z-handler-activation.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { randomUUID } from "node:crypto"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import type { ProviderAdapter } from "../../src/adapters/base"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** * Dispatch-priority regression test for the image bridge (PR #424). @@ -39,9 +40,12 @@ let runTurnCalled = false; let mockWsPlan: unknown = undefined; let handleResponses: typeof import("../../src/server/responses")["handleResponses"]; +let releaseSpendHome: (() => void) | undefined; beforeAll(async () => { process.env.OPENCODEX_HOME = join(tmpdir(), "ocx-test-" + randomUUID()); + // Take the writer lease after this suite installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); const actualResolver = await import("../../src/server/adapter-resolve"); mock.module("../../src/server/adapter-resolve", () => ({ @@ -112,6 +116,9 @@ beforeAll(async () => { }); afterAll(() => { + // Release before restoring the home to prevent the old directory from retaining a live ledger lease. + releaseSpendHome?.(); + releaseSpendHome = undefined; if (PREV_HOME === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = PREV_HOME; mock.restore(); diff --git a/tests/oauth/adapter-event-oauth-failover.test.ts b/tests/oauth/adapter-event-oauth-failover.test.ts index 35fe353d45..2cd12caf12 100644 --- a/tests/oauth/adapter-event-oauth-failover.test.ts +++ b/tests/oauth/adapter-event-oauth-failover.test.ts @@ -6,6 +6,7 @@ import type { ProviderAdapter } from "../../src/adapters/base"; import { clearGenericFailoverHealth } from "../../src/oauth/generic-account-failover"; import { getAccountSet, getCredential, saveCredential, setActiveAccount } from "../../src/oauth/store"; import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const actualResolver = await import("../../src/server/adapter-resolve"); @@ -57,6 +58,7 @@ mock.module("../../src/server/adapter-resolve", () => ({ const { handleResponses } = await import("../../src/server/responses"); const originalHome = process.env.OPENCODEX_HOME; let home = ""; +let releaseSpendHome: (() => void) | undefined; /** * `enabled: undefined` is the case that matters after #2568d — the key absent entirely, which is @@ -100,6 +102,8 @@ async function seedAccounts(count: number): Promise { beforeEach(() => { home = mkdtempSync(join(tmpdir(), "ocx-adapter-event-failover-")); process.env.OPENCODEX_HOME = home; + // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); clearGenericFailoverHealth(); attempts = []; attemptKeys = []; @@ -111,6 +115,9 @@ beforeEach(() => { }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearGenericFailoverHealth(); if (originalHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/oauth/oauth-account-attribution.test.ts b/tests/oauth/oauth-account-attribution.test.ts index ff6a838462..68b444a983 100644 --- a/tests/oauth/oauth-account-attribution.test.ts +++ b/tests/oauth/oauth-account-attribution.test.ts @@ -14,6 +14,7 @@ import { summarizeUsage } from "../../src/usage/summary"; import type { RequestLogContext } from "../../src/server/request-log"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; /** @@ -66,9 +67,13 @@ async function withHome(run: (home: string) => Promise): Promise { const prevCodex = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + // Take the writer lease after this helper installs its home so direct handler dispatch can open the spend journal. + const releaseSpendHome = acquireOwnedSpendHome(); try { return await run(home); } finally { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome(); globalThis.fetch = originalFetch; removeTreeWithRetry(home); if (prevOpencodex === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/providers/github-copilot/github-copilot-stream-contract.test.ts b/tests/providers/github-copilot/github-copilot-stream-contract.test.ts index 21ba36336e..29340342b2 100644 --- a/tests/providers/github-copilot/github-copilot-stream-contract.test.ts +++ b/tests/providers/github-copilot/github-copilot-stream-contract.test.ts @@ -11,6 +11,9 @@ import { providerConfigSeed } from "../../../src/providers/derive"; import { getProviderRegistryEntry } from "../../../src/providers/registry"; import { handleResponses } from "../../../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; interface SseEvent { event?: string; @@ -164,6 +167,9 @@ describe("GitHub Copilot Responses client stream contract", () => { const originalFetch = globalThis.fetch; afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -177,6 +183,8 @@ describe("GitHub Copilot Responses client stream contract", () => { providers: { "github-copilot": copilotProvider() }, } as unknown as OcxConfig; + // Direct dispatch needs the writer lease that startServer normally owns for this home. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/providers/github-copilot/github-copilot-wire-defaults.test.ts b/tests/providers/github-copilot/github-copilot-wire-defaults.test.ts index 50019d02cf..bdf8a6e2db 100644 --- a/tests/providers/github-copilot/github-copilot-wire-defaults.test.ts +++ b/tests/providers/github-copilot/github-copilot-wire-defaults.test.ts @@ -17,6 +17,9 @@ import { getProviderRegistryEntry } from "../../../src/providers/registry"; import { resolveWireProtocolOverride } from "../../../src/server/adapter-resolve"; import { handleResponses } from "../../../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; const RESPONSES_ONLY = [ "gpt-5.3-codex", @@ -137,7 +140,12 @@ describe("the registry default is isolated to the copilot provider", () => { describe("the wire default survives the handleResponses replay", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); function captureUpstreamUrl(): string[] { const urls: string[] = []; @@ -154,6 +162,8 @@ describe("the wire default survives the handleResponses replay", () => { async function drive(model: string, inboundWire?: "responses" | "chat" | "anthropic"): Promise { const urls = captureUpstreamUrl(); const config = { providers: { "github-copilot": copilotProvider() } } as unknown as OcxConfig; + // Direct dispatch needs the writer lease that startServer normally owns for this home. + releaseSpendHome = acquireOwnedSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/providers/opencode-go-luna-wire.test.ts b/tests/providers/opencode-go-luna-wire.test.ts index 7866ec9415..a07c66ff2d 100644 --- a/tests/providers/opencode-go-luna-wire.test.ts +++ b/tests/providers/opencode-go-luna-wire.test.ts @@ -13,9 +13,15 @@ import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { createResponsesPassthroughAdapter } from "../../src/adapters/openai-responses"; import { parseRequest } from "../../src/responses/parser"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const MODEL = "gpt-5.6-luna"; const GO_RESPONSES_MODELS = [MODEL, "grok-4.6", "muse-spark-1.3-contributor"]; +let releaseSpendHome: (() => void) | undefined; + +// Direct dispatch needs the writer lease that startServer normally owns for this home. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; function opencodeGo(overrides: Partial = {}): OcxProviderConfig { const entry = getProviderRegistryEntry("opencode-go"); @@ -102,7 +108,11 @@ describe("OpenCode Go stateless Responses", () => { describe("OpenCode Go stateless reasoning and continuation routes", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + dropSpendHome(); + globalThis.fetch = originalFetch; + }); const continuations = [ { id: "full", name: "full history", fullHistory: true, summary: "auto" }, @@ -147,6 +157,7 @@ describe("OpenCode Go stateless reasoning and continuation routes", () => { }) as typeof fetch; const config = { providers: { "opencode-go": opencodeGo() } } as unknown as OcxConfig; const drive = async (body: Record) => { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: `opencode-go/${model}`, stream: streaming, reasoning: { summary: continuation.summary }, @@ -214,7 +225,10 @@ describe("OpenCode Go stateless reasoning and continuation routes", () => { describe("OpenCode Go Luna Responses route (#1482)", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + dropSpendHome(); + globalThis.fetch = originalFetch; + }); test("handleResponses sends Luna to the documented /responses endpoint", async () => { const requests: Array<{ url: string; body: Record }> = []; @@ -234,6 +248,7 @@ describe("OpenCode Go Luna Responses route (#1482)", () => { const config = { providers: { "opencode-go": opencodeGo() }, } as unknown as OcxConfig; + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/providers/rate-limit-retry.test.ts b/tests/providers/rate-limit-retry.test.ts index dab8ec84b9..861929f72e 100644 --- a/tests/providers/rate-limit-retry.test.ts +++ b/tests/providers/rate-limit-retry.test.ts @@ -5,6 +5,9 @@ import { } from "../../src/providers/key-failover"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; describe("rateLimitRetryPolicyFor", () => { test("null when absent or explicitly disabled", () => { @@ -140,6 +143,9 @@ describe("retry loop client-abort handling", () => { const originalFetch = globalThis.fetch; afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -170,6 +176,8 @@ describe("retry loop client-abort handling", () => { }, } as OcxConfig; + // Direct dispatch needs the writer lease that startServer normally owns for this home. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -217,6 +225,7 @@ describe("retry loop client-abort handling", () => { } as OcxConfig; const abort = new AbortController(); + releaseSpendHome = acquireOwnedSpendHome(); const pending = handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -273,6 +282,7 @@ describe("retry loop client-abort handling", () => { } as OcxConfig; const abort = new AbortController(); + releaseSpendHome = acquireOwnedSpendHome(); const pending = handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/responses/empty-completion-core.test.ts b/tests/responses/empty-completion-core.test.ts index 6ec5541f12..6a645c8da5 100644 --- a/tests/responses/empty-completion-core.test.ts +++ b/tests/responses/empty-completion-core.test.ts @@ -8,6 +8,7 @@ import { } from "../../src/providers/request-pacing"; import type { RequestLogContext } from "../../src/server/request-log"; import type { AdapterEvent, OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const actualResolver = await import("../../src/server/adapter-resolve"); const actualResolveAdapter = actualResolver.resolveAdapter; @@ -20,6 +21,10 @@ let builtBodies: string[] = []; let customRunTurn: ProviderAdapter["runTurn"] | undefined; let passthroughFetchCalls = 0; let bodyObservationReleaseCalls = 0; +let releaseSpendHome: (() => void) | undefined; + +// Direct dispatch needs the writer lease that startServer normally owns for this home. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; function attemptAt(index: number): AdapterEvent[] { return attemptEvents[index] ?? [{ type: "error", message: `missing fixture attempt ${index}` }]; @@ -165,6 +170,9 @@ beforeEach(() => { }); afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + releaseSpendHome?.(); + releaseSpendHome = undefined; resetProviderRequestPacingForTest(); delete process.env.OCX_EMPTY_COMPLETION_RETRY; }); @@ -176,6 +184,7 @@ describe("empty-completion core integration", () => { // and that transport already falls back to HTTP SSE for exactly these bodies (#2473), with // an 18.2 MB HTTP 200 observed in #2426. Unset must mean "send it", not "guess a ceiling". const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses( request(false, "x".repeat(20 * 1024 * 1024)), config("test-passthrough"), @@ -220,6 +229,7 @@ describe("empty-completion core integration", () => { }); test("a normal-sized passthrough body still reaches upstream", async () => { + takeSpendHome(); const response = await handleResponses( request(false), config("test-passthrough", { maxUpstreamBodyBytes: 4_096 }), @@ -232,6 +242,7 @@ describe("empty-completion core integration", () => { }); test("an explicit zero limit lets an oversized turn reach upstream", async () => { + takeSpendHome(); const response = await handleResponses( request(false, "x".repeat(512)), config("test-passthrough", { maxUpstreamBodyBytes: 0 }), @@ -266,6 +277,7 @@ describe("empty-completion core integration", () => { emit({ type: "done" }); }; + takeSpendHome(); const response = await handleResponses(request(true), paced, { model: "", provider: "" }); expect(response.status).toBe(200); const reader = response.body!.getReader(); @@ -297,6 +309,7 @@ describe("empty-completion core integration", () => { ]; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(request(stream), config("test-run-turn"), logCtx); const body = await response.text(); @@ -319,6 +332,7 @@ describe("empty-completion core integration", () => { ]; const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(request(stream), config("test-http"), logCtx); const body = await response.text(); @@ -352,6 +366,7 @@ describe("empty-completion core integration", () => { const paced = config("test-http"); paced.providers.fixture!.requestPacing = { enabled: true, minIntervalMs: 100 }; + takeSpendHome(); const response = await handleResponses(request(false), paced, { model: "", provider: "" }); expect(response.status).toBe(200); expect(httpCalls).toBe(2); @@ -369,6 +384,7 @@ describe("empty-completion core integration", () => { const tierGated = config("test-http"); tierGated.providers.fixture!.supportsServiceTier = false; + takeSpendHome(); const response = await handleResponses( request(false, "please answer", { service_tier: "priority" }), tierGated, @@ -388,6 +404,7 @@ describe("empty-completion core integration", () => { const disabled = config("test-run-turn"); delete disabled.emptyCompletionRetry; + takeSpendHome(); const response = await handleResponses(request(false), disabled, { model: "", provider: "" }); await response.text(); @@ -410,6 +427,7 @@ describe("empty-completion core integration", () => { emit({ type: "done" }); }; + takeSpendHome(); const response = await handleResponses( request(true), config("test-run-turn", { stallTimeoutSec: 0.05 }), @@ -424,6 +442,7 @@ describe("empty-completion core integration", () => { test("combo and routed-compaction turns are excluded from the retry", async () => { attemptEvents = [[{ type: "done" }], [{ type: "text_delta", text: "must not run" }, { type: "done" }]]; + takeSpendHome(); const combo = await handleResponses( request(false), config("test-run-turn"), @@ -472,6 +491,7 @@ describe("empty-completion core integration", () => { }), }); + takeSpendHome(); const response = await handleResponses(req, guarded, { model: "", provider: "" }); const body = await response.text(); diff --git a/tests/responses/fresh-connection-optout.test.ts b/tests/responses/fresh-connection-optout.test.ts index 944afbd222..0cde659563 100644 --- a/tests/responses/fresh-connection-optout.test.ts +++ b/tests/responses/fresh-connection-optout.test.ts @@ -9,6 +9,7 @@ import { handleResponses } from "../../src/server/responses"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; describe("wantsFreshConnection", () => { test("returns false when env is unset or empty", () => { @@ -277,6 +278,8 @@ describe("the OAuth dispatch boundary", () => { process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; process.env.OCX_FRESH_CONNECTION_HOSTS = freshHost; + // Taken after this case installs its home so the direct dispatch owns that journal. + const releaseSpendHome = acquireOwnedSpendHome(); const sends: Array<{ url: string; init?: RequestInit }> = []; try { @@ -330,6 +333,8 @@ describe("the OAuth dispatch boundary", () => { expect(headers.get("x-grok-req-id")).toBeTruthy(); } } finally { + // Released before restoring or removing the home so Windows can delete its lease files. + releaseSpendHome(); globalThis.fetch = nativeFetch; if (previousHosts === undefined) delete process.env.OCX_FRESH_CONNECTION_HOSTS; else process.env.OCX_FRESH_CONNECTION_HOSTS = previousHosts; diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index 08dacdc59f..bfccb8afb1 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -15,10 +15,15 @@ import type { RequestLogContext } from "../../src/server/request-log"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { CodexWsMetadata } from "../../src/server/responses/codex-ws-metadata"; import { applyAccountQuotaFromUpstreamHeaders, getAccountQuotaHistory } from "../../src/codex/quota"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; + +// Taken only by callbacks that physically dispatch, after withPoolHome installs their home. +const takeSpendHome = (): void => { releaseSpendHome = acquireOwnedSpendHome(); }; function poolConfig(accountIds: string[]): OcxConfig { return { @@ -80,6 +85,9 @@ async function withPoolHome(run: (home: string) => Promise): Promise { try { return await run(home); } finally { + // Released before the helper removes or restores the home so its lease cannot outlive it. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); clearThreadAccountMap(); @@ -133,6 +141,7 @@ describe("Responses account usage attribution", () => { const originalWebSocket = globalThis.WebSocket; try { await withPoolHome(async home => { + takeSpendHome(); writeFileSync(join(home, "auth.json"), JSON.stringify({ tokens: { access_token: "main-access-token", account_id: "main-account" }, })); @@ -198,6 +207,7 @@ describe("Responses account usage attribution", () => { const finalQuotaAllowed = new Promise(resolve => { releaseFinalQuota = resolve; }); try { await withPoolHome(async () => { + takeSpendHome(); savePoolCredential("pool-ws-replaced"); class MetadataSocket { listeners = new Map void>>(); @@ -255,6 +265,7 @@ describe("Responses account usage attribution", () => { test("main-pool and legacy added accounts carry their effective labels", async () => { await withPoolHome(async home => { + takeSpendHome(); writeFileSync(join(home, "auth.json"), JSON.stringify({ tokens: { access_token: "main-access-token", account_id: "main-account" }, })); @@ -280,6 +291,7 @@ describe("Responses account usage attribution", () => { test("a pre-stream quota retry updates attribution to the serving alternate account", async () => { await withPoolHome(async () => { + takeSpendHome(); const config = poolConfig(["pool-a", "pool-b"]); for (const id of ["pool-a", "pool-b"]) { savePoolCredential(id); @@ -310,6 +322,7 @@ describe("Responses account usage attribution", () => { test("a quota message wrapped in HTTP 502 cools the account and retries an alternate", async () => { await withPoolHome(async () => { + takeSpendHome(); const config = poolConfig(["pool-a", "pool-b"]); for (const id of ["pool-a", "pool-b"]) { savePoolCredential(id); @@ -349,6 +362,7 @@ describe("Responses account usage attribution", () => { // refused nothing -- and the cooldown outlives the request that invented it. test("a refused reset replay is not quota evidence and invites no client retry", async () => { await withPoolHome(async () => { + takeSpendHome(); const config = poolConfig(["pool-a"]); savePoolCredential("pool-a"); updateAccountQuota("pool-a", 10); @@ -372,6 +386,7 @@ describe("Responses account usage attribution", () => { test("a wrapped quota failure cools a sole account when no alternate exists", async () => { await withPoolHome(async () => { + takeSpendHome(); const config = poolConfig(["pool-a"]); savePoolCredential("pool-a"); updateAccountQuota("pool-a", 10); diff --git a/tests/responses/responses-console-go-upload-retry.test.ts b/tests/responses/responses-console-go-upload-retry.test.ts index 68fc1972eb..75805f3c64 100644 --- a/tests/responses/responses-console-go-upload-retry.test.ts +++ b/tests/responses/responses-console-go-upload-retry.test.ts @@ -6,6 +6,7 @@ import { markResponseNonReplayable } from "../../src/lib/upstream-retry"; import { handleResponses } from "../../src/server/responses/core"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const originalFetch = globalThis.fetch; @@ -32,13 +33,19 @@ const EFFORT_REFUSAL = JSON.stringify({ }); let testDir = ""; +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-console-go-upload-retry-")); process.env.OPENCODEX_HOME = testDir; + // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = originalOpenCodexHome; diff --git a/tests/responses/responses-forward-incomplete-quota.test.ts b/tests/responses/responses-forward-incomplete-quota.test.ts index 0d93e3d5e4..ee79fd9c18 100644 --- a/tests/responses/responses-forward-incomplete-quota.test.ts +++ b/tests/responses/responses-forward-incomplete-quota.test.ts @@ -26,6 +26,7 @@ import { isEagerRelaySseResponse } from "../../src/server/relay"; import { sendResponseToWebSocket, type WsData } from "../../src/server/ws-bridge"; import { installIsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { INTERNAL_DEADLINE_MS, SERVER_BUDGET_MS } from "../helpers/test-budget"; const provider: OcxProviderConfig = { @@ -116,6 +117,8 @@ async function exerciseSpawnReporter(path: ReporterPath): Promise { const home = mkdtempSync(join(tmpdir(), "ocx-incomplete-quota-")); const codexHome = installIsolatedCodexHome("ocx-incomplete-quota-codex-"); process.env.OPENCODEX_HOME = home; + // Taken after this reporter installs its home so every direct dispatch owns its journal. + const releaseSpendHome = acquireOwnedSpendHome(); const accountId = "incomplete-quota-endpoint"; const model = "gpt-test"; const config: OcxConfig = { @@ -303,6 +306,8 @@ async function exerciseSpawnReporter(path: ReporterPath): Promise { expect(wsDispatches).toBe(path === "guarded-ws" ? 2 : 0); expect(httpDispatches).toBe(path === "guarded-ws" ? 0 : 2); } finally { + // Released before restoring or removing the home so the lease files are not left open. + releaseSpendHome(); client?.close(); await endpoint.stop(true); await upstream.stop(true); diff --git a/tests/responses/responses-native-main-refresh.test.ts b/tests/responses/responses-native-main-refresh.test.ts index f70bc07f1e..3a318cc083 100644 --- a/tests/responses/responses-native-main-refresh.test.ts +++ b/tests/responses/responses-native-main-refresh.test.ts @@ -14,12 +14,14 @@ import { tryAdmitTurn } from "../../src/server/lifecycle"; import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const originalFetch = globalThis.fetch; let home = ""; let previousOcxHome: string | undefined; let previousCodexHome: string | undefined; +let releaseSpendHome: (() => void) | undefined; const OTHER_ACCOUNT_ID = "other"; function config(options: { secondAccount?: boolean } = {}): OcxConfig { @@ -57,6 +59,8 @@ beforeEach(() => { previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); @@ -71,6 +75,9 @@ beforeEach(() => { }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearAccountNeedsReauth(OTHER_ACCOUNT_ID); diff --git a/tests/responses/responses-opaque-blob-recovery.test.ts b/tests/responses/responses-opaque-blob-recovery.test.ts index a47a155f38..48290dad1b 100644 --- a/tests/responses/responses-opaque-blob-recovery.test.ts +++ b/tests/responses/responses-opaque-blob-recovery.test.ts @@ -14,6 +14,7 @@ import { } from "../../src/server/responses/core"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { markBodyNonPersistable, rememberResponseState, previousResponseProviderState } from "../../src/responses/state"; @@ -66,15 +67,21 @@ const CALLER_MISMATCH_BLOB_ERROR = JSON.stringify({ }); let testDir = ""; +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-opaque-blob-recovery-")); process.env.OPENCODEX_HOME = testDir; + // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. + releaseSpendHome = acquireOwnedSpendHome(); clearReasoningReplayCacheForTests(); resetThoughtSignatureReplayForTests(); }); afterEach(() => { + // Release before restoring or removing the home to prevent Windows removal failures and POSIX unlinked databases. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearReasoningReplayCacheForTests(); resetThoughtSignatureReplayForTests(); diff --git a/tests/responses/responses-pool-401-refresh.test.ts b/tests/responses/responses-pool-401-refresh.test.ts index b1043cc2dd..2fc4409c13 100644 --- a/tests/responses/responses-pool-401-refresh.test.ts +++ b/tests/responses/responses-pool-401-refresh.test.ts @@ -33,6 +33,7 @@ import { encryptedInput, recoverySse, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; /** @@ -51,6 +52,7 @@ const originalFetch = globalThis.fetch; let home = ""; let previousOcxHome: string | undefined; let previousCodexHome: string | undefined; +let releaseSpendHome: (() => void) | undefined; function config(options: { secondAccount?: boolean } = {}): OcxConfig { return { @@ -306,6 +308,8 @@ beforeEach(() => { previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + // Direct handler dispatches need the writer lease that startServer normally holds. + releaseSpendHome = acquireOwnedSpendHome(); clearAccountNeedsReauth(ACCOUNT_ID); clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); @@ -316,6 +320,9 @@ beforeEach(() => { }); afterEach(() => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearCompactHandoffRoutesForTests(); clearAccountNeedsReauth(ACCOUNT_ID); diff --git a/tests/responses/responses-preview-main-read-fence.test.ts b/tests/responses/responses-preview-main-read-fence.test.ts index b015629345..d7380c778d 100644 --- a/tests/responses/responses-preview-main-read-fence.test.ts +++ b/tests/responses/responses-preview-main-read-fence.test.ts @@ -40,6 +40,7 @@ import { encryptedInput, recoverySse, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; /** @@ -184,15 +185,22 @@ async function postSpawn( const requestHeaders = new Headers(headers); requestHeaders.set("content-type", "application/json"); requestHeaders.set("x-openai-subagent", "collab_spawn"); - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: requestHeaders, - body: JSON.stringify({ model, input, stream: false }), - }), config, logCtx, options); - // handleResponses owns its translator budget through the returned body lifecycle. Draining the - // body also lets completed Responses schedule their state write before afterEach cancels it. - await response.arrayBuffer(); - return response; + // Acquire on this case's installed home so direct dispatch can open the shared spend journal. + const releaseSpendHome = acquireOwnedSpendHome(); + try { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: requestHeaders, + body: JSON.stringify({ model, input, stream: false }), + }), config, logCtx, options); + // handleResponses owns its translator budget through the returned body lifecycle. Draining the + // body also lets completed Responses schedule their state write before afterEach cancels it. + await response.arrayBuffer(); + return response; + } finally { + // Release before afterEach removes this home to prevent Windows removal failures. + releaseSpendHome(); + } } beforeEach(() => { diff --git a/tests/responses/responses-reasoning-effort-downgrade.test.ts b/tests/responses/responses-reasoning-effort-downgrade.test.ts index ee18288fce..e609d2f6e8 100644 --- a/tests/responses/responses-reasoning-effort-downgrade.test.ts +++ b/tests/responses/responses-reasoning-effort-downgrade.test.ts @@ -6,6 +6,7 @@ import { handleResponses } from "../../src/server/responses/core"; import { resetReasoningMetadataCachesForTests } from "../../src/providers/reasoning-metadata"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** * Rejected-rung learning on the request path: a rung the catalog advertises can still be refused @@ -27,6 +28,7 @@ const REFUSAL = JSON.stringify({ const UNRELATED = JSON.stringify({ error: { type: "invalid_request_error", message: "Invalid upload request." } }); let testDir = ""; +let releaseSpendHome: (() => void) | undefined; function writeSnapshot(values: string[]): void { writeFileSync(join(testDir, "reasoning-metadata-cache.json"), JSON.stringify({ @@ -101,10 +103,15 @@ function success(): Response { beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-reasoning-downgrade-")); process.env.OPENCODEX_HOME = testDir; + // Direct handler dispatches need the writer lease that startServer normally holds. + releaseSpendHome = acquireOwnedSpendHome(); resetReasoningMetadataCachesForTests(); }); afterEach(() => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; resetReasoningMetadataCachesForTests(); if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/responses/responses-shadow-intercept.test.ts b/tests/responses/responses-shadow-intercept.test.ts index feafd404df..d5cbdd680c 100644 --- a/tests/responses/responses-shadow-intercept.test.ts +++ b/tests/responses/responses-shadow-intercept.test.ts @@ -14,10 +14,17 @@ import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; import { catalogConvergenceFactory } from "../helpers/catalog-convergence"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; +// Taken only by rows that reach upstream through the direct handler helper. +const takeSpendHome = (): void => { releaseSpendHome = acquireOwnedSpendHome(); }; afterEach(() => { + // Released first so a failed dispatch cannot leak the writer lease into the next row. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -137,6 +144,7 @@ async function post( describe("shadow call intercept request path (issue #311)", () => { test("rewrites a gpt-5.6-luna helper call without overriding configured effort (#2706)", async () => { + takeSpendHome(); const bodies: Array> = []; globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { bodies.push(JSON.parse(String(init?.body ?? "{}")) as Record); @@ -157,6 +165,7 @@ describe("shadow call intercept request path (issue #311)", () => { }); test("a self-target is a no-op instead of an intercept loop (#2706)", async () => { + takeSpendHome(); const bodies: Array> = []; const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { @@ -183,6 +192,7 @@ describe("shadow call intercept request path (issue #311)", () => { }); test("rewrites a gpt-5.6-luna turn request too (#1684)", async () => { + takeSpendHome(); const bodies: Array> = []; const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { @@ -206,6 +216,7 @@ describe("shadow call intercept request path (issue #311)", () => { // Recording the operator-configured prefix instead of the caller's raw string removes the // class, rather than adding one more pattern to a deny-list. test("the recorded marker is the configured prefix, never the caller's raw model string", async () => { + takeSpendHome(); const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async () => new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], @@ -222,6 +233,7 @@ describe("shadow call intercept request path (issue #311)", () => { }); test("a configured non-default prefix is recorded as itself", async () => { + takeSpendHome(); const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async () => new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], @@ -296,6 +308,7 @@ function chatOk(text: string): Response { describe("a combo shadow-call target enters the failover loop (#4129)", () => { test("a helper call rewritten to a combo hops past a 429 to the second target", async () => { + takeSpendHome(); const urls: string[] = []; const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async (url: unknown) => { @@ -327,6 +340,7 @@ describe("a combo shadow-call target enters the failover loop (#4129)", () => { }); test("a combo whose first target intersects the source still routes as a combo", async () => { + takeSpendHome(); const urls: string[] = []; const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async (url: unknown) => { @@ -360,6 +374,7 @@ describe("a combo shadow-call target enters the failover loop (#4129)", () => { }); test("a non-combo replacement still takes the ordinary late intercept", async () => { + takeSpendHome(); const urls: string[] = []; const logCtx: RequestLogContext = { model: "", provider: "" }; globalThis.fetch = (async (url: unknown) => { diff --git a/tests/responses/responses-show-thinking-summary.test.ts b/tests/responses/responses-show-thinking-summary.test.ts index a749518b63..2fe5e8d5f0 100644 --- a/tests/responses/responses-show-thinking-summary.test.ts +++ b/tests/responses/responses-show-thinking-summary.test.ts @@ -6,6 +6,11 @@ import { providerConfigSeed } from "../../src/providers/derive"; import { getProviderRegistryEntry } from "../../src/providers/registry"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseInheritedSpendHome: (() => void) | undefined; +// Taken per inherited-home dispatch so mixed custom-home rows bind to their own directory. +const takeInheritedSpendHome = (): void => { releaseInheritedSpendHome = acquireOwnedSpendHome(); }; // Provider-opted visible thinking (showThinkingSummary): a provider that serves // genuine user-facing reasoning surfaces it on the summary channel even when the @@ -57,9 +62,15 @@ async function runHandleResponses(body: Record, seed: OcxProvid describe("showThinkingSummary provider option", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Released first so a failed row cannot leak ownership into the next sandbox case. + releaseInheritedSpendHome?.(); + releaseInheritedSpendHome = undefined; + globalThis.fetch = originalFetch; + }); test("provider opt-in never relabels raw content as a summary", async () => { + takeInheritedSpendHome(); const response = await runHandleResponses( { model: "deepseek-v4-flash", input: "ping", stream: true }, shownSeed(), @@ -70,6 +81,7 @@ describe("showThinkingSummary provider option", () => { }); test("explicit client summary none keeps thinking hidden", async () => { + takeInheritedSpendHome(); const response = await runHandleResponses( { model: "deepseek-v4-flash", input: "ping", stream: true, reasoning: { summary: "none" } }, shownSeed(), @@ -80,6 +92,7 @@ describe("showThinkingSummary provider option", () => { }); test("without the provider option, omitted summary stays hidden", async () => { + takeInheritedSpendHome(); const seed = { ...providerConfigSeed(getProviderRegistryEntry("deepseek")!), apiKey: "sk-test" } as OcxProviderConfig; const response = await runHandleResponses( { model: "deepseek-v4-flash", input: "ping", stream: true }, @@ -101,6 +114,8 @@ describe("showThinkingSummary provider option", () => { const home = mkdtempSync(join(tmpdir(), "ocx-show-thinking-")); const prevHome = process.env.OPENCODEX_HOME; process.env.OPENCODEX_HOME = home; + // Taken after this row installs its home so the direct dispatch owns that journal. + const releaseSpendHome = acquireOwnedSpendHome(); writeFileSync(join(home, "auth.json"), JSON.stringify({ "google-antigravity": { activeAccountId: "active", @@ -165,6 +180,8 @@ describe("showThinkingSummary provider option", () => { if (stream) expect(text).toContain("response.completed"); expect(text).toContain("OK"); } finally { + // Released before restoring or removing the home so its lease files can be deleted. + releaseSpendHome(); if (prevHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = prevHome; rmSync(home, { recursive: true, force: true }); diff --git a/tests/routing/probe-lease-dispatch-wiring.test.ts b/tests/routing/probe-lease-dispatch-wiring.test.ts index 00c4a6b942..e1d372ddb3 100644 --- a/tests/routing/probe-lease-dispatch-wiring.test.ts +++ b/tests/routing/probe-lease-dispatch-wiring.test.ts @@ -30,6 +30,7 @@ import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { clearAccountQuota, updateAccountQuota } from "../../src/codex/auth-api"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; /** @@ -104,6 +105,8 @@ describe("recovery limiter wiring is reachable from production (#4701)", () => { }, } as OcxConfig; + // Acquire on the installed test home so this physical dispatch can open the spend journal. + const releaseSpendHome = acquireOwnedSpendHome(); try { const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", @@ -115,6 +118,8 @@ describe("recovery limiter wiring is reachable from production (#4701)", () => { expect(response.status).toBe(200); expect(sharedPoolBackpressure().state().initialSends).toBe(1); } finally { + // Release before afterEach removes the home to prevent Windows removal failures. + releaseSpendHome(); globalThis.fetch = originalFetch; } }); diff --git a/tests/routing/routing-policy-surface-parity.test.ts b/tests/routing/routing-policy-surface-parity.test.ts index d73c5ff483..5bdd93161a 100644 --- a/tests/routing/routing-policy-surface-parity.test.ts +++ b/tests/routing/routing-policy-surface-parity.test.ts @@ -11,6 +11,7 @@ import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types import { clearRequestLogsForTests, type RequestLogContext } from "../../src/server/request-log"; import { readUsageEntries } from "../../src/usage/log"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const MODEL = "policy/daily"; const EXPECTED_RICH_EVIDENCE = { @@ -125,8 +126,15 @@ mock.module("../../src/server/adapter-resolve", () => ({ const { handleResponses, handleResponsesCompact } = await import("../../src/server/responses"); const { handleChatCompletions } = await import("../../src/server/chat-completions"); const { handleClaudeMessages } = await import("../../src/server/claude-messages"); +let releaseSpendHome: (() => void) | undefined; + +// Taken only by handler rows whose fixture adapter produces a dispatched response. +const takeSpendHome = (): void => { releaseSpendHome = acquireOwnedSpendHome(); }; afterEach(() => { + // Released first so a failed handler row cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; adapterFactory = undefined; }); @@ -224,6 +232,7 @@ describe("routing policy request evidence parity (via dev handlers)", () => { } }); test("rich evidence (tools + image) produces identical route decision across all three surfaces", async () => { + takeSpendHome(); adapterFactory = minimalSuccessAdapter; const config = testConfig(); @@ -345,6 +354,7 @@ describe("routing policy request evidence parity (via dev handlers)", () => { }); test("plain text with no tools produces no hard requirements on every surface", async () => { + takeSpendHome(); adapterFactory = minimalSuccessAdapter; const config = testConfig(); diff --git a/tests/routing/subagent-fallback-handle-responses.test.ts b/tests/routing/subagent-fallback-handle-responses.test.ts index fb236802e1..3b042433c1 100644 --- a/tests/routing/subagent-fallback-handle-responses.test.ts +++ b/tests/routing/subagent-fallback-handle-responses.test.ts @@ -43,6 +43,7 @@ import { encryptedInput as recoverableEncryptedInput, recoverySse, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; setDefaultTimeout(30_000); @@ -52,6 +53,7 @@ const originalNow = Date.now; let testDir: string; let previousOpencodexHome: string | undefined; let previousCodexHome: string | undefined; +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-hr-")); @@ -59,6 +61,8 @@ beforeEach(() => { previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = testDir; process.env.CODEX_HOME = testDir; + // Direct handler dispatches need the writer lease that startServer normally holds. + releaseSpendHome = acquireOwnedSpendHome(); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -71,6 +75,9 @@ beforeEach(() => { }); afterEach(() => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; Date.now = originalNow; clearThreadAccountMap(); diff --git a/tests/server/context-history-ownership.test.ts b/tests/server/context-history-ownership.test.ts index 3a1df4253f..04c1047584 100644 --- a/tests/server/context-history-ownership.test.ts +++ b/tests/server/context-history-ownership.test.ts @@ -16,6 +16,7 @@ import { resetContextRelayActivationForTests } from "../../src/codex/context-com const principal = "principal-a"; const keyAdmission: DataPlaneAdmission = { kind: "configured", keyId: "k1", source: "dedicated", contextPrincipalId: principal }; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const destination = "https://chatgpt.com/backend-api/codex"; @@ -25,6 +26,7 @@ let previousCodexHome: string | undefined; let home = ""; let sent: Array<{ url: string; headers: Headers }> = []; let failFirstAccount: string | undefined; +let releaseSpendHome: (() => void) | undefined; function install(id: string, owner: string, token = `${id}-token`): void { saveCodexAccountCredential(id, { accessToken: token, refreshToken: `${id}-refresh`, @@ -75,6 +77,8 @@ beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; previousCodexHome = process.env.CODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-context-owner-")); process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + // Direct handler dispatches need the writer lease that startServer normally holds. + releaseSpendHome = acquireOwnedSpendHome(); setContextFeature(true); clearContextSessionOwnersForTests(); clearAccountQuota(); clearThreadAccountMap(); clearCodexUpstreamHealth(); for (const id of ["pool-a", "pool-b", "__main__"]) clearAccountNeedsReauth(id); @@ -98,6 +102,9 @@ beforeEach(() => { }); afterEach(() => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; clearContextSessionOwnersForTests(); clearAccountQuota(); clearThreadAccountMap(); clearCodexUpstreamHealth(); removeTreeWithRetry(home); diff --git a/tests/server/plaintext-v2-agent-messages-server.test.ts b/tests/server/plaintext-v2-agent-messages-server.test.ts index 4de9f24bfc..c94cb9404c 100644 --- a/tests/server/plaintext-v2-agent-messages-server.test.ts +++ b/tests/server/plaintext-v2-agent-messages-server.test.ts @@ -15,10 +15,17 @@ import { import { clearResponseStateForTests, expandPreviousResponseInput } from "../../src/responses/state"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +let releaseInheritedSpendHome: (() => void) | undefined; +// Taken per inherited-home dispatch because the pool retry row installs a different home. +const takeInheritedSpendHome = (): void => { releaseInheritedSpendHome = acquireOwnedSpendHome(); }; beforeEach(() => { clearResponseStateForTests(); }); afterEach(() => { + // Released first so a failed row cannot leak its writer lease into the next case. + releaseInheritedSpendHome?.(); + releaseInheritedSpendHome = undefined; globalThis.fetch = originalFetch; clearResponseStateForTests(); }); @@ -89,12 +96,16 @@ async function withPoolHome(run: () => Promise): Promise { const previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + // Taken after the pool case installs its home so its direct dispatch owns that journal. + const releaseSpendHome = acquireOwnedSpendHome(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearAccountQuota(); try { return await run(); } finally { + // Released before removal or env restoration so the lease cannot outlive this home. + releaseSpendHome(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearAccountQuota(); @@ -137,6 +148,7 @@ function overLimitResponsePayload(id = "resp-plaintext-v2-overflow") { describe("plaintext v2 agent messages at the Responses server boundary", () => { test.each(["json", "legacy-tee", "eager-relay"] as const)("null namespace restores before %s delivery and continuation storage", async mode => { + takeInheritedSpendHome(); const id = `resp-null-namespace-${mode}`; const item = { ...completedResponsePayload(id).output[0]!, namespace: null }; const payload = { id, status: "completed", output: [item] }; @@ -156,6 +168,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("rewrites the canonical request and restores every SSE response snapshot", async () => { + takeInheritedSpendHome(); const sentBodies: string[] = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { sentBodies.push(typeof init?.body === "string" ? init.body : ""); @@ -207,6 +220,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("restores the namespace in bounded JSON responses", async () => { + takeInheritedSpendHome(); globalThis.fetch = (async () => new Response(JSON.stringify(completedResponsePayload()), { status: 200, headers: { "content-type": "application/json" }, @@ -225,6 +239,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("rejects an unclassified successful response while restoration is required", async () => { + takeInheritedSpendHome(); globalThis.fetch = (async () => new Response( JSON.stringify(completedResponsePayload()), { status: 200 }, @@ -244,6 +259,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("restores aliases after SSE snapshot repair copies request tools and tool choice", async () => { + takeInheritedSpendHome(); globalThis.fetch = (async () => { const response = completedResponsePayload("resp-snapshot-sse"); return new Response( @@ -279,6 +295,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("restores aliases after bounded JSON snapshot repair", async () => { + takeInheritedSpendHome(); globalThis.fetch = (async () => new Response( JSON.stringify(completedResponsePayload("resp-snapshot-json")), { status: 200, headers: { "content-type": "application/json" } }, @@ -304,6 +321,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("keeps the marker and reserved namespace when the option is disabled", async () => { + takeInheritedSpendHome(); const sentBodies: string[] = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { sentBodies.push(typeof init?.body === "string" ? init.body : ""); @@ -324,6 +342,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("keeps the whole request unchanged when tool-search history conflicts with the alias", async () => { + takeInheritedSpendHome(); const sentBodies: string[] = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { sentBodies.push(typeof init?.body === "string" ? init.body : ""); @@ -425,6 +444,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("fails closed for over-limit streamed responses in both relay modes", async () => { + takeInheritedSpendHome(); for (const streamMode of ["legacy-tee", "eager-relay"] as const) { globalThis.fetch = (async () => new Response( `event: response.completed\ndata: ${JSON.stringify({ @@ -451,6 +471,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("rejects over-limit bounded JSON before HTTP or WebSocket reframing", async () => { + takeInheritedSpendHome(); const fixtureId = "plaintext-v2-bounded-json-fixture"; const fixtureModel = "fixture-model"; const mutableRegistry = PROVIDER_REGISTRY as unknown as Array>; @@ -503,6 +524,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test("rejects an over-limit JSON response and does not retain it for continuation", async () => { + takeInheritedSpendHome(); const sentBodies: string[] = []; let requestIndex = 0; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { @@ -547,6 +569,7 @@ describe("plaintext v2 agent messages at the Responses server boundary", () => { }); test.each([undefined, "websocket"] as const)("stores the client namespace across an option change on %s", async inboundTransport => { + takeInheritedSpendHome(); const sentBodies: string[] = []; let requestIndex = 0; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { @@ -605,6 +628,7 @@ test("plaintext startup warning requires explicit opt-in and names retention", ( for (const streamMode of ["legacy-tee", "eager-relay"] as const) { for (const refusal of ["malformed", "unknown-alias", "conflicting-binding"] as const) { test(`${streamMode} ${refusal} is refused without caching or retry`, async () => { + takeInheritedSpendHome(); let sends = 0; const sent: string[] = []; globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { @@ -635,6 +659,7 @@ for (const streamMode of ["legacy-tee", "eager-relay"] as const) { } test("malformed bounded JSON is a single-attempt 502", async () => { + takeInheritedSpendHome(); let sends = 0; globalThis.fetch = (async () => { sends += 1; return new Response("{malformed", { headers: { "content-type": "application/json" } }); }) as typeof fetch; const response = await handleResponses(collaborationRequest(), config(true), { model: "", provider: "" }); @@ -644,6 +669,7 @@ test("malformed bounded JSON is a single-attempt 502", async () => { }); test("cross-coordinate namespace conflict cannot publish continuation", async () => { + takeInheritedSpendHome(); let sends = 0; globalThis.fetch = (async () => { sends += 1; @@ -663,6 +689,7 @@ test("cross-coordinate namespace conflict cannot publish continuation", async () test("concurrent native requests do not share plaintext alias metadata", async () => { + takeInheritedSpendHome(); const pending: Array<{ enabled: boolean; resolve: (value: Response) => void }> = []; let bothReady!: () => void; const ready = new Promise(resolve => { bothReady = resolve; }); diff --git a/tests/server/server-combo-reasoning-replay-eligibility.test.ts b/tests/server/server-combo-reasoning-replay-eligibility.test.ts index 5f39d0a925..38451a7f95 100644 --- a/tests/server/server-combo-reasoning-replay-eligibility.test.ts +++ b/tests/server/server-combo-reasoning-replay-eligibility.test.ts @@ -15,6 +15,7 @@ import { clearComboRecallForTests } from "../../src/server/responses/combo-sessi import { TARGET_INCOMPATIBLE_MESSAGE } from "../../src/server/responses/core-errors"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; type HandleOptions = NonNullable[3]>; @@ -23,6 +24,10 @@ let testDir = ""; let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; const servers: Array> = []; +let releaseSpendHome: (() => void) | undefined; + +// Acquire only for rows that physically dispatch, after their temporary home is installed. +const takeSpendHome = (): void => { releaseSpendHome = acquireOwnedSpendHome(); }; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; @@ -38,6 +43,9 @@ beforeEach(() => { }); afterEach(async () => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; let responseStatePending = true; try { for (const server of servers.splice(0)) await server.stop(true); @@ -172,6 +180,7 @@ describe("combo mandatory reasoning replay failover", () => { }); test("upstream_server_error failover to strict Responses preserves existing reasoning_text", async () => { + takeSpendHome(); const failed = serve(() => Response.json({ error: { type: "server_error", code: "upstream_server_error", message: "busy" }, }, { status: 500 })); @@ -208,6 +217,7 @@ describe("combo mandatory reasoning replay failover", () => { }); test("a foreign opaque-only replay skips the strict target without forwarding or fabrication", async () => { + takeSpendHome(); let firstTargetFails = false; const first = serve(() => firstTargetFails ? Response.json({ error: { type: "server_error", code: "upstream_server_error", message: "busy" } }, { status: 500 }) @@ -257,6 +267,7 @@ describe("combo mandatory reasoning replay failover", () => { }); test("an exhausted combo reports target_incompatible when mandatory plaintext is unavailable", async () => { + takeSpendHome(); let firstTargetFails = false; const first = serve(() => firstTargetFails ? Response.json({ error: { type: "server_error", code: "upstream_server_error", message: "busy" } }, { status: 500 }) diff --git a/tests/server/server-combo-zero-output-failover.test.ts b/tests/server/server-combo-zero-output-failover.test.ts index 88adaf522d..a5a7a9fa29 100644 --- a/tests/server/server-combo-zero-output-failover.test.ts +++ b/tests/server/server-combo-zero-output-failover.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { ManagementRequest as Request } from "../helpers/management-auth"; import { comboProviderFactory } from "../helpers/combo-provider"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; import { clearComboRecallForTests } from "../../src/server/responses/combo-session-recall"; @@ -43,12 +44,15 @@ let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; const servers: Array> = []; const provider = comboProviderFactory(() => undefined); +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-combo-zero-output-codex-"); testDir = mkdtempSync(join(tmpdir(), "ocx-combo-zero-output-")); process.env.OPENCODEX_HOME = testDir; + // Direct handler dispatches need the writer lease that startServer normally holds. + releaseSpendHome = acquireOwnedSpendHome(); clearComboSelectionState(); clearComboRecallForTests(); clearComboTargetCooldowns(); @@ -59,6 +63,9 @@ beforeEach(() => { }); afterEach(async () => { + // Release before home teardown to prevent Windows removal failures and a live unlinked database. + releaseSpendHome?.(); + releaseSpendHome = undefined; let responseStatePending = true; try { for (const server of servers.splice(0)) await server.stop(true); diff --git a/tests/server/terminal-guard-server.test.ts b/tests/server/terminal-guard-server.test.ts index e95a380ef4..22bac135c2 100644 --- a/tests/server/terminal-guard-server.test.ts +++ b/tests/server/terminal-guard-server.test.ts @@ -7,6 +7,11 @@ import { clearKeyCooldowns } from "../../src/providers/key-failover"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseInheritedSpendHome: (() => void) | undefined; +// Taken per inherited-home dispatch because one row below installs a different home. +const takeInheritedSpendHome = (): void => { releaseInheritedSpendHome = acquireOwnedSpendHome(); }; const config = { port: 0, @@ -98,10 +103,14 @@ describe("server terminal guard integration", () => { }); afterEach(() => { + // Released first so a failed row cannot carry its writer lease into the next case. + releaseInheritedSpendHome?.(); + releaseInheritedSpendHome = undefined; globalThis.fetch = originalFetch; }); test("re-asks Claude once inside the same Responses turn and forwards the tool call", async () => { + takeInheritedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -124,6 +133,7 @@ describe("server terminal guard integration", () => { }); test("terminal-guard continuation 429 replays on the same key before surfacing", async () => { + takeInheritedSpendHome(); const retryConfig = { ...config, providers: { @@ -175,6 +185,8 @@ describe("server terminal guard integration", () => { const previousHome = process.env.OPENCODEX_HOME; const home = mkdtempSync(join(tmpdir(), "ocx-terminal-guard-failover-")); process.env.OPENCODEX_HOME = home; + // Taken after this case installs its home so the direct dispatch owns that journal. + const releaseSpendHome = acquireOwnedSpendHome(); clearKeyCooldowns("claude-se"); const budgetConfig = { ...config, @@ -221,6 +233,8 @@ describe("server terminal guard integration", () => { // A per-iteration budget would replay on the second key too (5+ sends). expect(sends).toBe(4); } finally { + // Released before restoring or removing the home so its lease files can be deleted. + releaseSpendHome(); clearKeyCooldowns("claude-se"); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; @@ -229,6 +243,7 @@ describe("server terminal guard integration", () => { }); test("terminal-guard continuation shares the request-wide 429 budget with the main loop", async () => { + takeInheritedSpendHome(); const budgetConfig = { ...config, providers: { @@ -279,6 +294,7 @@ describe("server terminal guard integration", () => { }); test("terminal-guard continuation preserves structured cyber_policy semantics", async () => { + takeInheritedSpendHome(); const secret = `OpenAI flagged this request for potential high-risk cybersecurity activity. Authorization: ${["Bear", "er"].join("")} continuationsecret123456`; let sends = 0; globalThis.fetch = (async () => { @@ -316,6 +332,7 @@ describe("server terminal guard integration", () => { }); test("terminal-guard continuation abort during the 429 wait yields 499 without replaying", async () => { + takeInheritedSpendHome(); const abortConfig = { ...config, providers: { @@ -364,6 +381,7 @@ describe("server terminal guard integration", () => { }); test("a stalled continuation body reports 504 even when cancelling it aborts the client signal", async () => { + takeInheritedSpendHome(); // Cancelling the stalled source can disconnect the client in the same tick. The // classifier has to read the thrown error first, or this timeout is reported as a // client cancellation and the caller loses the upstream stall signal. @@ -397,6 +415,7 @@ describe("server terminal guard integration", () => { }); test("a stalled initial body fails with a 504 upstream error instead of a proxy error", async () => { + takeInheritedSpendHome(); // The initial stream has no continuation classifier: without one the bridge catch // reports this upstream timeout as a 500 proxy_error. const stallConfig = { ...config, stallTimeoutSec: 1 } as unknown as OcxConfig; @@ -424,6 +443,7 @@ describe("server terminal guard integration", () => { }); test("terminal-guard 429 wait longer than the stall budget still succeeds (heartbeats)", async () => { + takeInheritedSpendHome(); const stallConfig = { ...config, stallTimeoutSec: 1, @@ -471,6 +491,7 @@ describe("server terminal guard integration", () => { }, 5_000); test("openai-chat provider without terminalContinuationGuard does not re-ask", async () => { + takeInheritedSpendHome(); const chatConfig = openAiChatConfig(); let sends = 0; globalThis.fetch = (async () => { @@ -497,6 +518,7 @@ describe("server terminal guard integration", () => { }); test("openai-chat provider with terminalContinuationGuard false does not re-ask", async () => { + takeInheritedSpendHome(); const chatConfig = openAiChatConfig(false); let sends = 0; globalThis.fetch = (async () => { @@ -522,6 +544,7 @@ describe("server terminal guard integration", () => { }); test("openai-chat provider with terminalContinuationGuard re-asks once and forwards the tool call", async () => { + takeInheritedSpendHome(); const chatConfig = openAiChatConfig(true); let sends = 0; const bodies: Record[] = []; @@ -553,6 +576,7 @@ describe("server terminal guard integration", () => { }); test("combo attempts do not run an opted-in openai-chat terminal guard", async () => { + takeInheritedSpendHome(); const comboConfig = { ...openAiChatConfig(true), combos: { @@ -586,6 +610,7 @@ describe("server terminal guard integration", () => { }); test("routed compaction does not run an opted-in openai-chat terminal guard", async () => { + takeInheritedSpendHome(); const chatConfig = openAiChatConfig(true); let sends = 0; globalThis.fetch = (async () => { diff --git a/tests/web-search/web-search-passthrough-bridge.test.ts b/tests/web-search/web-search-passthrough-bridge.test.ts index 92d8a2e9ed..05d06f4332 100644 --- a/tests/web-search/web-search-passthrough-bridge.test.ts +++ b/tests/web-search/web-search-passthrough-bridge.test.ts @@ -32,6 +32,7 @@ import { waitForProviderRequestSlot, } from "../../src/providers/request-pacing"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig, ProviderWebSearchBridgeBackend, ProviderWebSearchBridgeConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** One SSE event block without its blank-line delimiter. */ function frame(type: string, payload: Record): string { @@ -1420,6 +1421,8 @@ describe("the reported turn, end to end through handleResponses", () => { hooks.onProviderResponse?.(leg); return new Response(text, { headers: { "content-type": "text/event-stream" } }); }) as unknown as typeof fetch; + // Direct dispatch needs the writer lease that startServer normally owns for this home. + const releaseSpendHome = acquireOwnedSpendHome(); try { const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", @@ -1428,6 +1431,8 @@ describe("the reported turn, end to end through handleResponses", () => { }), ocxConfig, { model: "", provider: "" }); return { body: await response.text(), outbound, destinations, searches, searchUrls, searchHeaders }; } finally { + // Release before later teardown can replace or remove the preload sandbox home. + releaseSpendHome(); globalThis.fetch = savedFetch; } } diff --git a/tests/web-search/web-search.test.ts b/tests/web-search/web-search.test.ts index 0041ff2646..ebc3ecd405 100644 --- a/tests/web-search/web-search.test.ts +++ b/tests/web-search/web-search.test.ts @@ -15,6 +15,7 @@ import type { OcxMessage, OcxParsedRequest } from "../../src/types"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; import { withUpstreamHttpVersion } from "../../src/lib/upstream-http-version"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** * Wrap a fetch so it applies the provider's HTTP-version pin the way `providerFetch` does in @@ -566,7 +567,13 @@ describe("web-search sidecar planning", () => { }); const originalFetch = globalThis.fetch; -afterEach(() => { globalThis.fetch = originalFetch; }); +let releaseSpendHome: (() => void) | undefined; +afterEach(() => { + // Release the preload-home lease before later teardown can replace or remove that home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; +}); test("issue #2885 — Zhipu-shaped web-search routing preserves the provider HTTP version pin", async () => { let routedProtocol: string | undefined; @@ -606,6 +613,8 @@ test("issue #2885 — Zhipu-shaped web-search routing preserves the provider HTT throw new Error("the routed web-search leg must use the provider fetch"); }) as typeof fetch; + // Direct dispatch needs the writer lease that startServer normally owns for this home. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { From d209eb8e7cbd439afeca9c8ff86a1e21461dac83 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:33:45 +0900 Subject: [PATCH 14/34] test(spend): finish the lease migration and remove the image suite's own home Twenty more fixtures dispatch directly and inherit the preload sandbox home, so each takes the writer lease at its dispatch and drops it in its own afterEach. Three files that had no afterEach get one whose only job is that drop: a case that throws mid-assertion would otherwise leave the lease behind and make the next case's home read as an ownership conflict instead of reporting its own failure. Cases that never reach a physical dispatch stay lease-free, and the boundary is drawn at the seam rather than by file. v2-agent-message-failfast keeps its bare post() for the two rows that assert dispatch never happens and routes the rest through a dispatchPost() that takes the lease; abort-race leaves the build-time abort and buildRequest-throw rows alone; opencode-go-session-header leaves the policy-fallback rows, whose injected runCore answers without an adapter. The image activation suite also now removes the home it owns. Taking the lease creates the state directory, and that suite names a fresh one per run, so before this it left a directory behind on every run. Release, then remove, then restore: the removal has to happen while OPENCODEX_HOME still names it. No assertion, expected value, mock, fixture or timeout changes, and no cap moves. Local checks: NOT RUN. --- tests/adapters/abort-race.test.ts | 11 +++++ .../empty-tool-output-annotation.test.ts | 11 ++++- ...laude-code-thought-signature-scope.test.ts | 9 ++++ tests/images/z-handler-activation.test.ts | 12 ++++- tests/providers/deepseek-inbound-wire.test.ts | 21 +++++++- .../deepseek-responses-item-id-repair.test.ts | 11 ++++- .../opencode-go-session-header.test.ts | 16 +++++- .../responses-function-tool-repair.test.ts | 15 +++++- .../responses-image-gen-repair.test.ts | 17 ++++++- .../responses-inbound-store-default.test.ts | 11 ++++- .../responses-muse-tool-name-alias.test.ts | 18 ++++++- ...nses-reasoning-summary-passthrough.test.ts | 12 ++++- ...sponses-self-named-namespace-scrub.test.ts | 20 +++++++- ...ses-stateless-dangling-call-repair.test.ts | 11 ++++- .../responses-tool-search-repair.test.ts | 15 +++++- .../fastwire-characterization-wire.test.ts | 9 ++++ tests/routing/fastwire-observability.test.ts | 12 ++++- tests/server/cancel-body-on-abort.test.ts | 13 ++++- tests/server/response-model-identity.test.ts | 7 +++ .../server/v2-agent-message-failfast.test.ts | 49 +++++++++++++------ tests/service/service-tier-capability.test.ts | 14 +++++- 21 files changed, 281 insertions(+), 33 deletions(-) diff --git a/tests/adapters/abort-race.test.ts b/tests/adapters/abort-race.test.ts index 33a6da1ab4..7843e1c087 100644 --- a/tests/adapters/abort-race.test.ts +++ b/tests/adapters/abort-race.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, mock, test } from "bun:test"; import type { ProviderAdapter } from "../../src/adapters/base"; import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const actualResolver = await import("../../src/server/adapter-resolve"); let adapterFactory: ((provider: OcxProviderConfig) => ProviderAdapter) | undefined; @@ -13,8 +14,15 @@ mock.module("../../src/server/adapter-resolve", () => ({ })); const { handleResponses } = await import("../../src/server/responses"); +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; afterEach(() => { + // Release first so a failed dispatch cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; adapterFactory = undefined; }); @@ -63,6 +71,7 @@ describe("Responses abort guards", () => { }, }); + takeSpendHome(); const response = await post("test-run-turn", false); const body = await response.text(); @@ -107,6 +116,7 @@ describe("Responses abort guards", () => { }, }); + takeSpendHome(); const response = await post("test-fetch", true, clientAbort.signal); await response.text(); await new Promise(resolve => setImmediate(resolve)); @@ -175,6 +185,7 @@ describe("Responses abort guards", () => { }, }); + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/adapters/empty-tool-output-annotation.test.ts b/tests/adapters/empty-tool-output-annotation.test.ts index d0acc8213b..8538834f7f 100644 --- a/tests/adapters/empty-tool-output-annotation.test.ts +++ b/tests/adapters/empty-tool-output-annotation.test.ts @@ -5,6 +5,7 @@ import { getProviderRegistryEntry } from "../../src/providers/registry"; import { routedProviderConfig } from "../../src/router"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig, OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const ANNOTATION = "[ocx] empty tool output: the tool ran but produced no stdout or return value; do not treat this as success, failure, or user-provided input."; @@ -148,7 +149,13 @@ describe("openai-chat empty tool output annotation", () => { describe("openai-responses empty tool output annotation", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + let releaseSpendHome: (() => void) | undefined; + afterEach(() => { + // Release first so a failed dispatch cannot leak writer ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); async function drive(config: OcxConfig, input: unknown[]): Promise<{ body: Record }> { const requests: Array<{ body: Record }> = []; @@ -156,6 +163,8 @@ describe("openai-responses empty tool output annotation", () => { requests.push({ body: JSON.parse(String(init?.body ?? "{}")) as Record }); return Response.json({ id: "resp_test", object: "response", status: "completed", output: [] }); }) as typeof fetch; + // Direct dispatch needs the writer lease to prevent spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/claude-integration/claude-code-thought-signature-scope.test.ts b/tests/claude-integration/claude-code-thought-signature-scope.test.ts index b3d981c327..814ce7ceb4 100644 --- a/tests/claude-integration/claude-code-thought-signature-scope.test.ts +++ b/tests/claude-integration/claude-code-thought-signature-scope.test.ts @@ -11,6 +11,7 @@ import { afterEach, describe, expect, mock, test } from "bun:test"; import type { ProviderAdapter } from "../../src/adapters/base"; import type { AdapterEvent, OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const actualResolver = await import("../../src/server/adapter-resolve"); @@ -24,8 +25,15 @@ mock.module("../../src/server/adapter-resolve", () => ({ })); const { handleResponses } = await import("../../src/server/responses"); +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; afterEach(() => { + // Release first so a failed dispatch cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; adapterFactory = undefined; }); @@ -72,6 +80,7 @@ async function drive(options: { }; if (options.promptCacheKey !== undefined) body.prompt_cache_key = options.promptCacheKey; + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/images/z-handler-activation.test.ts b/tests/images/z-handler-activation.test.ts index b7e836802d..6c07f2a5d3 100644 --- a/tests/images/z-handler-activation.test.ts +++ b/tests/images/z-handler-activation.test.ts @@ -5,6 +5,7 @@ import { randomUUID } from "node:crypto"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import type { ProviderAdapter } from "../../src/adapters/base"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; /** * Dispatch-priority regression test for the image bridge (PR #424). @@ -41,9 +42,13 @@ let mockWsPlan: unknown = undefined; let handleResponses: typeof import("../../src/server/responses")["handleResponses"]; let releaseSpendHome: (() => void) | undefined; +// Retained so teardown can remove it. Nothing created this directory before the lease did: +// taking ownership mkdirs the state directory, so the suite now owns its removal too. +let ownedHome = ""; beforeAll(async () => { - process.env.OPENCODEX_HOME = join(tmpdir(), "ocx-test-" + randomUUID()); + ownedHome = join(tmpdir(), "ocx-test-" + randomUUID()); + process.env.OPENCODEX_HOME = ownedHome; // Take the writer lease after this suite installs its home so direct handler dispatch can open the spend journal. releaseSpendHome = acquireOwnedSpendHome(); @@ -116,9 +121,12 @@ beforeAll(async () => { }); afterAll(() => { - // Release before restoring the home to prevent the old directory from retaining a live ledger lease. + // Release, then remove, then restore. An open lease inside a directory being deleted fails + // the removal on Windows and leaves an unlinked live database on POSIX, and the removal has + // to happen while OPENCODEX_HOME still names the directory being removed. releaseSpendHome?.(); releaseSpendHome = undefined; + if (ownedHome) removeTreeWithRetry(ownedHome); if (PREV_HOME === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = PREV_HOME; mock.restore(); diff --git a/tests/providers/deepseek-inbound-wire.test.ts b/tests/providers/deepseek-inbound-wire.test.ts index fa2b686a2d..a9bf4730b7 100644 --- a/tests/providers/deepseek-inbound-wire.test.ts +++ b/tests/providers/deepseek-inbound-wire.test.ts @@ -25,6 +25,7 @@ import { MAX_SYNTHESIZED_OUTPUT_ITEMS } from "../../src/server/responses-json-ev import type { ResponsesTerminalRepairScheduler } from "../../src/server/responses-terminal-repair"; import { sendResponseToWebSocket } from "../../src/server/ws-bridge"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; const createResponsesPassthroughAdapter = (...args: Parameters) => @@ -33,6 +34,13 @@ const createResponsesPassthroughAdapter = (...args: Parameters void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +// Release before the next case so a failed dispatch cannot leave an ownership conflict. +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; class ManualTerminalScheduler implements ResponsesTerminalRepairScheduler { private current = 0; @@ -165,7 +173,10 @@ describe("DeepSeek wire selection is scoped to the inbound protocol", () => { describe("the inbound scope survives the handleResponses replay", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + dropSpendHome(); + globalThis.fetch = originalFetch; + }); function captureUpstreamRequests(): Array<{ url: string; body: Record }> { const requests: Array<{ url: string; body: Record }> = []; @@ -190,6 +201,7 @@ describe("the inbound scope survives the handleResponses replay", () => { ): Promise<{ url: string; body: Record }> { const requests = captureUpstreamRequests(); const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; + takeSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -250,6 +262,7 @@ describe("the inbound scope survives the handleResponses replay", () => { abortSignal: testAbort.signal, responsesTerminalRepairScheduler: scheduler, } as Parameters[3]; + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -329,6 +342,7 @@ describe("the inbound scope survives the handleResponses replay", () => { }) as typeof fetch; const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; const abort = new AbortController(); + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -441,6 +455,7 @@ describe("the inbound scope survives the handleResponses replay", () => { const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; const deadline = AbortSignal.timeout(5_000); + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -498,6 +513,7 @@ describe("the inbound scope survives the handleResponses replay", () => { // The plain provider seed carries no explicit repair config; the registry's // { repairInvalidIds: true } policy must reach the live route via backfill. const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -525,6 +541,7 @@ describe("the inbound scope survives the handleResponses replay", () => { })) as typeof fetch; const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; const abort = new AbortController(); + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -610,6 +627,7 @@ describe("the bounded-JSON mechanism stays alive behind a synthetic registry ent }); }); afterEach(() => { + dropSpendHome(); globalThis.fetch = originalFetch; const index = mutableRegistry.findIndex(entry => entry.id === FIXTURE_ID); if (index >= 0) mutableRegistry.splice(index, 1); @@ -655,6 +673,7 @@ describe("the bounded-JSON mechanism stays alive behind a synthetic registry ent options: { stream?: boolean; websocket?: boolean } = {}, ): Promise { const config = { providers: { [FIXTURE_ID]: provider } } as unknown as OcxConfig; + takeSpendHome(); return handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/providers/deepseek-responses-item-id-repair.test.ts b/tests/providers/deepseek-responses-item-id-repair.test.ts index ed95f2394c..fad2542661 100644 --- a/tests/providers/deepseek-responses-item-id-repair.test.ts +++ b/tests/providers/deepseek-responses-item-id-repair.test.ts @@ -8,6 +8,7 @@ import { } from "../../src/server/responses-item-id-repair"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; const UUID_MSG = "9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d"; @@ -166,7 +167,13 @@ describe("registry-derived DeepSeek repair policy (#938)", () => { describe("streamed HTTP path carries canonical ids (#938)", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + let releaseSpendHome: (() => void) | undefined; + afterEach(() => { + // Release first so a failed stream cannot leak writer ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); test("the relayed SSE contains no upstream UUID item ids (un-enriched saved seed)", async () => { // The live path must backfill the registry policy through routedProviderConfig — @@ -199,6 +206,8 @@ describe("streamed HTTP path carries canonical ids (#938)", () => { }) as typeof fetch; const config = { providers: { deepseek: plainSeed } } as unknown as OcxConfig; + // Direct dispatch needs the writer lease to prevent spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/providers/opencode-go-session-header.test.ts b/tests/providers/opencode-go-session-header.test.ts index 4f51b581c0..d972344803 100644 --- a/tests/providers/opencode-go-session-header.test.ts +++ b/tests/providers/opencode-go-session-header.test.ts @@ -9,6 +9,7 @@ import { getOrAllocateRequestSessionLane } from "../../src/server/request-log-co import { handleChatCompletions } from "../../src/server/chat-completions"; import { handleClaudeMessages } from "../../src/server/claude-messages"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const MUSE_MODEL = "muse-spark-1.3-contributor"; const CHAT_MODEL = "glm-5.2"; @@ -28,6 +29,13 @@ const RESPONSES_SESSION_VECTORS = { prefixed: "ocx_c974cef031af8717276b933929f0c073", codex: "ocx_a0cfe09ee92e4bfa2e560579bc46c50e", } as const; +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +// Release before the next case so a failed dispatch cannot leave an ownership conflict. +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; function opencodeGo(overrides: Partial = {}): OcxProviderConfig { const entry = getProviderRegistryEntry("opencode-go"); @@ -96,6 +104,7 @@ async function captureRequest(input: { const config = { providers: { [providerName]: input.provider ?? opencodeGo() }, } as unknown as OcxConfig; + takeSpendHome(); const response = input.claude ? await handleClaudeMessages( new Request("http://localhost/v1/messages", { method: "POST", @@ -136,7 +145,10 @@ async function captureRequest(input: { describe("OpenCode Go session affinity (#3344)", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + dropSpendHome(); + globalThis.fetch = originalFetch; + }); for (const model of [CHAT_MODEL, MUSE_MODEL]) { // The policy target is deliberately renamed, so provider-name wire defaults do not apply; @@ -183,6 +195,7 @@ describe("OpenCode Go session affinity (#3344)", () => { // Preliminary route checks the first target; dispatch independently picks Go. entropy.mockReturnValueOnce(0); try { + takeSpendHome(); const response = await handleClaudeMessages(new Request("http://localhost/v1/messages", { method: "POST", headers: { "content-type": "application/json", ...identity.headers } as Record, body: JSON.stringify({ model: "combo/affinity", max_tokens: 64, stream: false, @@ -243,6 +256,7 @@ describe("OpenCode Go session affinity (#3344)", () => { } as unknown as OcxConfig; const entropy = spyOn(Math, "random").mockReturnValue(0.9).mockReturnValueOnce(0); try { + takeSpendHome(); const response = await handleClaudeMessages(new Request("http://localhost/v1/messages", { method: "POST", headers: { "content-type": "application/json", [SESSION_HEADER]: "client-session-a" }, body: JSON.stringify({ model: "combo/affinity", max_tokens: 64, stream: false, diff --git a/tests/responses/responses-function-tool-repair.test.ts b/tests/responses/responses-function-tool-repair.test.ts index 2f9343cd3d..599e9cd9fa 100644 --- a/tests/responses/responses-function-tool-repair.test.ts +++ b/tests/responses/responses-function-tool-repair.test.ts @@ -1,6 +1,6 @@ import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { collectFunctionCallRepairSchemas, repairFunctionCalls, @@ -10,6 +10,18 @@ import { createResponsesFunctionToolRepairBlockRewrite } from "../../src/server/ import { createTranslatorBudget, TranslatorBudgetExceededError } from "../../src/lib/translator-budget"; import { sseDataPayload } from "../../src/server/sse-payload-rewrite"; import { currentTurnWireToolCatalogBody } from "../../src/server/responses-undeclared-tool-guard"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +afterEach(() => { + // Release first so a failed dispatch cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); const parameters = { type: "object", properties: { cell_id: { type: "string" }, yield_time_ms: { type: "integer" }, @@ -391,6 +403,7 @@ test("native Responses JSON/SSE and replay share the original function schema re method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "fixture/grok-probe", stream, input: [{ role: "user", content: "synthetic" }], tools, ...extra }), }); + takeSpendHome(); const response = await handleResponses(request(), config, { model: "", provider: "" }); expect(response.status).toBe(200); const raw = await response.text(); diff --git a/tests/responses/responses-image-gen-repair.test.ts b/tests/responses/responses-image-gen-repair.test.ts index f5f358081e..dfafde3725 100644 --- a/tests/responses/responses-image-gen-repair.test.ts +++ b/tests/responses/responses-image-gen-repair.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { imageGenToolCallAliases, relaySseWithImageGenCallRestore as relaySseWithImageGenCallRestoreProduction, @@ -8,6 +8,18 @@ import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { finalizeTranslatorBudgetResponse } from "../../src/lib/translator-budget"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +afterEach(() => { + // Release first so a failed dispatch cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); function relaySseWithImageGenCallRestore( body: ReadableStream, @@ -185,6 +197,7 @@ describe("Responses image-gen call restoration", () => { } as OcxConfig; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -245,6 +258,7 @@ describe("Responses image-gen call restoration", () => { } as OcxConfig; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -317,6 +331,7 @@ describe("Responses image-gen call restoration", () => { } as OcxConfig; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/responses/responses-inbound-store-default.test.ts b/tests/responses/responses-inbound-store-default.test.ts index 23bcc6aee1..73e6890b2b 100644 --- a/tests/responses/responses-inbound-store-default.test.ts +++ b/tests/responses/responses-inbound-store-default.test.ts @@ -18,6 +18,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { CODEX_FORWARD_BASE_URL } from "../../src/providers/openai-tiers"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; function providerConfig(overrides: Partial = {}): OcxConfig { return { @@ -36,7 +37,13 @@ function providerConfig(overrides: Partial = {}): OcxConfig { describe("/v1/responses defaults store:false only for the canonical forward Codex backend", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + let releaseSpendHome: (() => void) | undefined; + afterEach(() => { + // Release first so a failed dispatch cannot leak writer ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); function captureUpstream(): { urls: string[]; bodies: string[] } { const urls: string[] = []; @@ -61,6 +68,8 @@ describe("/v1/responses defaults store:false only for the canonical forward Code store: unknown, ): Promise<{ url: string; body: Record | null }> { const { urls, bodies } = captureUpstream(); + // Direct dispatch needs the writer lease to prevent spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/responses/responses-muse-tool-name-alias.test.ts b/tests/responses/responses-muse-tool-name-alias.test.ts index 655165d394..30f76330d4 100644 --- a/tests/responses/responses-muse-tool-name-alias.test.ts +++ b/tests/responses/responses-muse-tool-name-alias.test.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { createResponsesPassthroughAdapter as createResponsesPassthroughAdapterProduction } from "../../src/adapters/openai-responses"; import { buildMuseToolNameAliasPlan, @@ -11,8 +11,20 @@ import { import { expandPreviousResponseInput } from "../../src/responses/state"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; +let releaseSpendHome: (() => void) | undefined; + +// Direct physical dispatch needs the writer lease to prevent spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +afterEach(() => { + // Release first so a failed dispatch cannot leak ownership into the next case. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); + const createResponsesPassthroughAdapter = (...args: Parameters) => withTestTranslatorBudget(createResponsesPassthroughAdapterProduction(...args)); @@ -270,6 +282,7 @@ describe("muse tool-name inbound restore through handleResponses", () => { }), { headers: { "content-type": "application/json" } }); }) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -304,6 +317,7 @@ describe("muse tool-name inbound restore through handleResponses", () => { return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); }) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -338,6 +352,7 @@ describe("muse tool-name inbound restore through handleResponses", () => { return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); }) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -389,6 +404,7 @@ describe("muse tool-name inbound restore through handleResponses", () => { return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); }) as typeof fetch; try { + takeSpendHome(); const turn1 = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/responses/responses-reasoning-summary-passthrough.test.ts b/tests/responses/responses-reasoning-summary-passthrough.test.ts index ab6e099bcc..dfebadfd06 100644 --- a/tests/responses/responses-reasoning-summary-passthrough.test.ts +++ b/tests/responses/responses-reasoning-summary-passthrough.test.ts @@ -3,6 +3,9 @@ import { providerConfigSeed } from "../../src/providers/derive"; import { getProviderRegistryEntry } from "../../src/providers/registry"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; /** * The passthrough relay for DeepSeek's native /responses endpoint emits @@ -59,6 +62,8 @@ async function runHandleResponses(body: Record, upstreamBody: u { status: 200, headers: { "content-type": contentType } }, )) as typeof fetch; const config = { providers: { deepseek: deepseekSeed() } } as unknown as OcxConfig; + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); return handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -73,7 +78,12 @@ async function runHandleResponses(body: Record, upstreamBody: u describe("passthrough reasoning summary rewrite honors hideThinkingSummary", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); test("SSE: hidden thinking stays on the content channel", async () => { // No reasoning.summary in the request -> parseRequest sets hideThinkingSummary. diff --git a/tests/responses/responses-self-named-namespace-scrub.test.ts b/tests/responses/responses-self-named-namespace-scrub.test.ts index df187d48d1..410ed394ba 100644 --- a/tests/responses/responses-self-named-namespace-scrub.test.ts +++ b/tests/responses/responses-self-named-namespace-scrub.test.ts @@ -10,9 +10,18 @@ import { afterEach, expect, test } from "bun:test"; import { handleResponses } from "../../src/server/responses"; import { scrubSelfNamedToolCallNamespace } from "../../src/server/responses-self-named-namespace-scrub"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; -afterEach(() => { globalThis.fetch = originalFetch; }); +let releaseSpendHome: (() => void) | undefined; +// Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; +}); function forwardConfig(): OcxConfig { return { @@ -79,6 +88,7 @@ test("a self-named namespace on a passthrough custom_tool_call is scrubbed befor status: 200, headers: { "content-type": "text/event-stream" }, })) as typeof fetch; + takeSpendHome(); const res = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -116,6 +126,7 @@ test("a self-named namespace declared by the current turn is preserved", async ( ], }; + takeSpendHome(); const res = await handleResponses(request(body), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -143,6 +154,7 @@ test("the reserved functions namespace does not create a same-name collision", a ], }; + takeSpendHome(); const res = await handleResponses(request(body), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -155,6 +167,7 @@ test("a self-named namespace on a passthrough bare function_call is scrubbed", a status: 200, headers: { "content-type": "text/event-stream" }, })) as typeof fetch; + takeSpendHome(); const res = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -178,6 +191,7 @@ test("a Chat-shaped function declaration still authorizes the bare function scru status: 200, headers: { "content-type": "text/event-stream" }, })) as typeof fetch; + takeSpendHome(); const res = await handleResponses(request(chatShaped), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -214,6 +228,7 @@ test("tool_choice for a namespaced custom tool cannot authorize a colliding bare ], }; + takeSpendHome(); const res = await handleResponses(request(body), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -248,6 +263,7 @@ test("mixed custom and function collisions are scoped to the response item type" ], }; + takeSpendHome(); const res = await handleResponses(request(body), forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const text = await res.text(); @@ -271,6 +287,7 @@ test("a genuine MCP namespace on a passthrough call is left alone", async () => status: 200, headers: { "content-type": "text/event-stream" }, })) as typeof fetch; + takeSpendHome(); const res = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); const text = await res.text(); expect(text).toContain('"namespace":"mcp__docs"'); @@ -287,6 +304,7 @@ test("the bounded JSON (stream:false) passthrough path scrubs the same shape (#3 headers: { "content-type": "application/json", authorization: "Bearer test", "chatgpt-account-id": "acct" }, body: JSON.stringify({ ...requestBody, stream: false }), }); + takeSpendHome(); const res = await handleResponses(req, forwardConfig(), { model: "", provider: "" }); expect(res.status).toBe(200); const body = await res.json() as { output: Array> }; diff --git a/tests/responses/responses-stateless-dangling-call-repair.test.ts b/tests/responses/responses-stateless-dangling-call-repair.test.ts index 13b30573d9..8e8c52ba15 100644 --- a/tests/responses/responses-stateless-dangling-call-repair.test.ts +++ b/tests/responses/responses-stateless-dangling-call-repair.test.ts @@ -13,8 +13,10 @@ import { providerConfigSeed } from "../../src/providers/derive"; import { getProviderRegistryEntry } from "../../src/providers/registry"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const MODEL = "deepseek-v4-flash"; +let releaseSpendHome: (() => void) | undefined; function deepseekProvider(): ReturnType & { apiKey: string } { return { ...providerConfigSeed(getProviderRegistryEntry("deepseek")!), apiKey: "sk-test" }; @@ -22,7 +24,12 @@ function deepseekProvider(): ReturnType & { apiKey: s describe("stateless Responses wire repairs orphaned tool calls", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); async function drive(input: unknown[]): Promise<{ url: string; body: Record }> { const requests: Array<{ url: string; body: Record }> = []; @@ -34,6 +41,8 @@ describe("stateless Responses wire repairs orphaned tool calls", () => { return Response.json({ id: "resp_deepseek", object: "response", status: "completed", output: [] }); }) as typeof fetch; const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/responses/responses-tool-search-repair.test.ts b/tests/responses/responses-tool-search-repair.test.ts index d415fff16e..392aa00a5e 100644 --- a/tests/responses/responses-tool-search-repair.test.ts +++ b/tests/responses/responses-tool-search-repair.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { isTranslatorBudgetExceededError } from "../../src/lib/translator-budget"; import { restoreRoutedToolSearchCallsInJson, @@ -7,8 +7,16 @@ import { import { createRoutedToolSearchRestoreBlockRewrite } from "../../src/server/responses-tool-search-repair"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +let releaseSpendHome: (() => void) | undefined; +afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); + function frame(event: string, payload: Record): string { return `event: ${event}\ndata: ${JSON.stringify({ type: event, ...payload })}`; } @@ -349,6 +357,8 @@ describe("routed Responses tool-search compatibility", () => { }) as typeof fetch; try { + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -395,6 +405,7 @@ describe("routed Responses tool-search compatibility", () => { }) as typeof fetch; try { + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -433,6 +444,7 @@ describe("routed Responses tool-search compatibility", () => { }) as typeof fetch; try { + releaseSpendHome = acquireOwnedSpendHome(); await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -495,6 +507,7 @@ describe("routed Responses tool-search compatibility", () => { }), { headers: { "content-type": "application/json" } })) as typeof fetch; try { + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/routing/fastwire-characterization-wire.test.ts b/tests/routing/fastwire-characterization-wire.test.ts index 7d3c01d0dc..1ee18ef05f 100644 --- a/tests/routing/fastwire-characterization-wire.test.ts +++ b/tests/routing/fastwire-characterization-wire.test.ts @@ -6,10 +6,17 @@ import * as adapterResolveModule from "../../src/server/adapter-resolve"; import type { RequestLogContext } from "../../src/server/request-log"; import { handleResponses } from "../../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; +// Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -45,6 +52,7 @@ async function driveResponses(args: { ...(args.callerTier === undefined ? {} : { service_tier: args.callerTier }), }; + takeSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -355,6 +363,7 @@ describe("FastWire characterization: rawBody observation point", () => { }), }); + takeSpendHome(); await handleResponses(request, config, { model: "", provider: "" }, {}); expect(outboundBody?.service_tier).toBe("priority"); expect(adapterRawBody?.service_tier).toBe("flex"); diff --git a/tests/routing/fastwire-observability.test.ts b/tests/routing/fastwire-observability.test.ts index 0ced61dd44..587f98242f 100644 --- a/tests/routing/fastwire-observability.test.ts +++ b/tests/routing/fastwire-observability.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import type { AdapterRequest } from "../../src/adapters/base"; import { createOpenAIChatAdapter } from "../../src/adapters/openai-chat"; import { createResponsesPassthroughAdapter } from "../../src/adapters/openai-responses"; @@ -26,6 +26,14 @@ import { estimateComboCost, serviceTierContextFromOutcome } from "../../src/usag import type { ExpectedPriceOverlay } from "../../src/usage/expected-prices"; import { normalizeUsageEntryForTest } from "../../src/usage/log"; import { createTestTranslatorBudget, withTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; +afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); const SERVICE_WIRE = { kind: "service-tier" as const, @@ -491,6 +499,8 @@ describe("FastWire logging and persistence", () => { }) as typeof fetch; const logCtx: RequestLogContext = { model: "", provider: "" }; try { + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/server/cancel-body-on-abort.test.ts b/tests/server/cancel-body-on-abort.test.ts index 075fb80c08..24f1df143c 100644 --- a/tests/server/cancel-body-on-abort.test.ts +++ b/tests/server/cancel-body-on-abort.test.ts @@ -1,8 +1,16 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { cancelBodyOnAbort } from "../../src/lib/abort"; import { handleLive, readBodyCapped } from "../../src/server/live"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; +afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); function bodyWithCancelSpy(): { body: ReadableStream; cancelled: () => boolean } { let cancelled = false; @@ -197,6 +205,8 @@ describe("readBodyCapped settles the stream when a read throws", () => { } as OcxConfig; try { + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -258,6 +268,7 @@ describe("readBodyCapped settles the stream when a read throws", () => { } as OcxConfig; try { + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/server/response-model-identity.test.ts b/tests/server/response-model-identity.test.ts index 11d021563d..319018c231 100644 --- a/tests/server/response-model-identity.test.ts +++ b/tests/server/response-model-identity.test.ts @@ -2,10 +2,15 @@ import { afterEach, describe, expect, test } from "bun:test"; import { handleResponses } from "../../src/server/responses/core"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -85,6 +90,8 @@ async function post(args: { }) as typeof fetch; const logCtx = { model: "", provider: "" } as RequestLogContext; + // Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. + releaseSpendHome = acquireOwnedSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/server/v2-agent-message-failfast.test.ts b/tests/server/v2-agent-message-failfast.test.ts index 0e0dc66a24..18aba6211b 100644 --- a/tests/server/v2-agent-message-failfast.test.ts +++ b/tests/server/v2-agent-message-failfast.test.ts @@ -7,8 +7,12 @@ import { } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; +// Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; /** * Structurally faithful Fernet fixture: version + timestamp + IV + one AES-CBC @@ -40,6 +44,9 @@ const ROUTING_ENVELOPE = [ const MESSAGE_ROUTING_ENVELOPE = ROUTING_ENVELOPE.replace("NEW_TASK", "MESSAGE"); afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; }); @@ -128,6 +135,16 @@ async function post( }), config, { model: "", provider: "" }); } +async function dispatchPost( + config: OcxConfig, + model: string, + input: unknown[], + headers: HeadersInit = {}, +): Promise { + takeSpendHome(); + return post(config, model, input, headers); +} + describe("V2 routed agent-message ciphertext guard", () => { test("blocks a pure Fernet-only agent task", () => { expect(hasUnreadableEncryptedAgentTask(agentMessage([ @@ -233,7 +250,7 @@ describe("V2 routed agent-message ciphertext guard", () => { }); }) as typeof fetch; - const response = await post( + const response = await dispatchPost( mixedComboConfig(), "combo/mixed", agentMessage([ @@ -313,7 +330,7 @@ describe("V2 routed agent-message ciphertext guard", () => { }); }) as typeof fetch; - const response = await post( + const response = await dispatchPost( config, "combo/mixed", agentMessage([ @@ -500,7 +517,7 @@ describe("V2 routed agent-message ciphertext guard", () => { { type: "input_text", text: ROUTING_ENVELOPE }, { type: "encrypted_content", encrypted_content: FERNET_TASK }, ]); - const response = await post(nativeConfig(), "gpt-5.5", input, { + const response = await dispatchPost(nativeConfig(), "gpt-5.5", input, { authorization: "Bearer caller-codex-token", }); @@ -585,7 +602,7 @@ describe("routed Responses agent-message ciphertext repair", () => { test("repairs a mixed child result replayed behind a later user turn", async () => { const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [mixedChildResult(), userTurn]); + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [mixedChildResult(), userTurn]); expect(response.status).toBe(200); expect(outbound()).toHaveLength(1); @@ -604,7 +621,7 @@ describe("routed Responses agent-message ciphertext repair", () => { expect(hasUnreadableEncryptedAgentTask(input)).toBe(false); const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", input); + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", input); expect(response.status).toBe(200); expect(outbound()[0]).not.toContain(FERNET_TASK); @@ -618,7 +635,7 @@ describe("routed Responses agent-message ciphertext repair", () => { expect(hasUnreadableEncryptedAgentTask(input)).toBe(false); const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", input); + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", input); expect(response.status).toBe(200); expect(outbound()[0]).not.toContain(FERNET_TASK); @@ -638,7 +655,7 @@ describe("routed Responses agent-message ciphertext repair", () => { test("leaves a fully readable child result exactly as the adapter already lowered it", async () => { const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", @@ -666,7 +683,7 @@ describe("routed Responses agent-message ciphertext repair", () => { }); }) as typeof fetch; - const response = await post(routedConfig(), "xai/grok-4.5", [mixedChildResult(), userTurn]); + const response = await dispatchPost(routedConfig(), "xai/grok-4.5", [mixedChildResult(), userTurn]); expect(response.status).toBe(200); expect(forwardedBody).toContain("the child finished the migration"); @@ -688,7 +705,7 @@ describe("routed Responses agent-message ciphertext repair", () => { }); }) as typeof fetch; - const response = await post(nativeConfig(), "gpt-5.5", [mixedChildResult(), userTurn], { + const response = await dispatchPost(nativeConfig(), "gpt-5.5", [mixedChildResult(), userTurn], { authorization: "Bearer caller-codex-token", }); @@ -710,7 +727,7 @@ describe("routed Responses agent-message ciphertext repair", () => { } as OcxConfig; const outbound = captureOutbound("relay-model"); - const response = await post(config, "relayfwd/child-model", [mixedChildResult(), userTurn]); + const response = await dispatchPost(config, "relayfwd/child-model", [mixedChildResult(), userTurn]); expect(response.status).toBe(200); expect(outbound()[0]).not.toContain(FERNET_TASK); @@ -736,7 +753,7 @@ describe("routed Responses agent-message ciphertext repair", () => { } as OcxConfig; const outbound = captureOutbound("relay-model"); - const response = await post(config, "combo/routed", [mixedChildResult(), userTurn]); + const response = await dispatchPost(config, "combo/routed", [mixedChildResult(), userTurn]); expect(response.status).toBe(200); expect(outbound()).toHaveLength(1); @@ -755,7 +772,7 @@ describe("routed Responses agent-message ciphertext repair", () => { expect(structurallyValidFernetTokens(`${first}${second}`)).toEqual([FERNET_TASK]); const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", @@ -776,7 +793,7 @@ describe("routed Responses agent-message ciphertext repair", () => { test("repairs a token embedded inside a text part and keeps the prose around it", async () => { const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", @@ -801,7 +818,7 @@ describe("routed Responses agent-message ciphertext repair", () => { for (const [name, text] of Object.entries(readable)) { const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", @@ -819,7 +836,7 @@ describe("routed Responses agent-message ciphertext repair", () => { // two ordinary encoded fragments do not become a marker merely by being adjacent. const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", @@ -847,7 +864,7 @@ describe("routed Responses agent-message ciphertext repair", () => { expect(structurallyValidFernetTokens(blob)).toEqual([]); const outbound = captureOutbound("relay-model"); - const response = await post(routedResponsesConfig(), "relay/child-model", [{ + const response = await dispatchPost(routedResponsesConfig(), "relay/child-model", [{ type: "agent_message", author: "/root/child", recipient: "/root", diff --git a/tests/service/service-tier-capability.test.ts b/tests/service/service-tier-capability.test.ts index febdfc7a6e..20b3dde0bd 100644 --- a/tests/service/service-tier-capability.test.ts +++ b/tests/service/service-tier-capability.test.ts @@ -26,6 +26,11 @@ import { import { candidateCapabilityEvidence } from "../../src/routing/capability"; import { resolveProductionBehaviorValues } from "../../src/routing/compatibility/behavior"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +let releaseSpendHome: (() => void) | undefined; +// Direct dispatch needs the writer lease that prevents spend-ledger ownership failures. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; describe("registry capability reaches saved configs without overriding them", () => { test("the registry holds the defaults; the seed stays free of them so explicit config stays distinguishable", () => { @@ -389,7 +394,12 @@ describe("routing evidence uses the final model adapter", () => { describe("the gate fires on the live handleResponses path", () => { const originalFetch = globalThis.fetch; - afterEach(() => { globalThis.fetch = originalFetch; }); + afterEach(() => { + // Release the lease before later teardown can replace the preload sandbox home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + }); function captureBody(): { bodies: Record[] } { const bodies: Record[] = []; @@ -409,6 +419,7 @@ describe("the gate fires on the live handleResponses path", () => { ): Promise> { const { bodies } = captureBody(); const config = { providers: { [providerName]: provider }, ...(fastMode === undefined ? {} : { fastMode }) } as unknown as OcxConfig; + takeSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -458,6 +469,7 @@ describe("the gate fires on the live handleResponses path", () => { test("DeepSeek clears a stripped caller tier from request logging", async () => { const { bodies } = captureBody(); const logCtx: RequestLogContext = { model: "", provider: "" }; + takeSpendHome(); await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", From afd95493eae0737d6129eab8cf9bcef24446d55a Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:42:51 +0900 Subject: [PATCH 15/34] test(spend): make room for the lease in the five fixtures at their cap A file-size cap only ever moves down, so these five could not take an additive lease. The repository answer to that is an extraction, not deleted blank lines or two statements on one line, and each one here removes more than the lease costs. Four move a helper to a sibling module, verbatim, so the cases that called it read the same behaviour through a different name: the SSE stream reader and builder out of the undeclared-tool guard (nine files had written their own copy of readAll), the request-log row builder, the key-auth URL builder out of the Responses passthrough, and the config/request/upstream fixtures out of compaction routing. responses-custom-tool-repair is split instead, because its twelve dispatching cases are one contiguous block and no helper in it is worth enough lines. They move to responses-custom-tool-repair-dispatch.test.ts unchanged, registered in both scripts/test-layout/layout.json and tests/fixtures/test-layout-expected.json, with the fixtures both halves need in a shared helper so the two copies cannot drift. The original keeps the cases that drive the compat functions directly and no longer dispatches at all, so it needs no lease. Compaction routing takes the lease per dispatching describe rather than per file: five of its cases install a home inside the case body, and each of those drops the block lease and takes one for its own directory, then drops that before the directory is removed. The one describe that only exercises a pure function takes no lease. One placement is worth naming. In the passthrough file the shared call arrow takes the lease inside its body, not beside it: beside it the acquire would run while the describe was being collected, and the first case's teardown would drop it for every case after. No assertion, expected value, mock, fixture or timeout changes, and no cap moves. Local checks: NOT RUN. --- scripts/test-layout/layout.json | 1 + tests/fixtures/test-layout-expected.json | 1 + tests/helpers/compaction-routing-fixtures.ts | 107 ++ tests/helpers/custom-tool-repair-fixtures.ts | 20 + tests/helpers/passthrough-key-url.ts | 26 + tests/helpers/request-log-entry.ts | 21 + tests/helpers/sse-stream.ts | 35 + .../openai-responses-passthrough.test.ts | 46 +- .../responses-compaction-routing.test.ts | 149 +-- ...ponses-custom-tool-repair-dispatch.test.ts | 979 ++++++++++++++++++ .../responses-custom-tool-repair.test.ts | 962 +---------------- .../responses-undeclared-tool-guard.test.ts | 48 +- tests/usage/request-log.test.ts | 19 +- 13 files changed, 1290 insertions(+), 1124 deletions(-) create mode 100644 tests/helpers/compaction-routing-fixtures.ts create mode 100644 tests/helpers/custom-tool-repair-fixtures.ts create mode 100644 tests/helpers/passthrough-key-url.ts create mode 100644 tests/helpers/request-log-entry.ts create mode 100644 tests/helpers/sse-stream.ts create mode 100644 tests/responses/responses-custom-tool-repair-dispatch.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 45ccb64cc1..7db22d3c25 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1215,6 +1215,7 @@ "responses-console-go-upload-retry.test.ts": "responses", "responses-context-overflow.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses", + "responses-custom-tool-repair-dispatch.test.ts": "responses", "responses-custom-tool-repair.test.ts": "responses", "responses-custom-tool-stream-consistency.test.ts": "responses", "responses-fetch-helpers-boundary.test.ts": "responses", diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 7297245513..b88224e089 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1041,6 +1041,7 @@ "responses-console-go-upload-retry.test.ts": "responses", "responses-context-overflow.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses", + "responses-custom-tool-repair-dispatch.test.ts": "responses", "responses-custom-tool-repair.test.ts": "responses", "responses-custom-tool-stream-consistency.test.ts": "responses", "responses-default-namespace-emit-normalize.test.ts": "responses", diff --git a/tests/helpers/compaction-routing-fixtures.ts b/tests/helpers/compaction-routing-fixtures.ts new file mode 100644 index 0000000000..fb4db7b627 --- /dev/null +++ b/tests/helpers/compaction-routing-fixtures.ts @@ -0,0 +1,107 @@ +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; + +/** + * Config, request and upstream-response fixtures for the compaction-routing suite. + * + * Moved verbatim out of tests/responses/responses-compaction-routing.test.ts: that file sits at + * its file-size cap, and the repository answer to a cap is a sibling helper rather than + * compressed control flow. Nothing here decides anything; every value is the one its callers + * were already building inline. + */ +export function keyProviderConfig(overrides: Partial = {}): OcxConfig { + return { + defaultProvider: "gw", + providers: { + gw: { + adapter: "openai-responses", + baseUrl: "https://gateway.example/v1", + authMode: "key", + apiKey: "test-key", + ...overrides, + }, + }, + } as unknown as OcxConfig; +} + +export function nativePoolConfig(): OcxConfig { + return { + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + codexAccounts: [{ + id: "pool-a", + email: "pool@example.test", + isMain: false, + chatgptAccountId: "pool_acc", + }], + } as OcxConfig; +} + +/** Two-account pool: the alternate-attempt tests need somewhere for the retry to go. */ +export function twoAccountPoolConfig(): OcxConfig { + const config = nativePoolConfig(); + config.codexAccounts = [ + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ] as OcxConfig["codexAccounts"]; + return config; +} + +export function compactionRequest( + body: Record, + signal?: AbortSignal, + extraHeaders: Record = {}, +): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", ...extraHeaders }, + body: JSON.stringify(body), + signal, + }); +} + +export function baseCompactionBody(extra: Record = {}): Record { + return { + model: "gw/some-model", + stream: false, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "earlier turn" }] }, + { type: "compaction_trigger" }, + ], + tools: [{ type: "function", name: "shell" }], + tool_choice: "auto", + parallel_tool_calls: true, + ...extra, + }; +} + +export function jsonResponse(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +export function completedPayload(text: string): Record { + return { + id: "resp_1", + status: "completed", + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + }; +} + +export function sseResponse(events: Array>): Response { + const body = events.map(e => `event: ${String(e.type)}\ndata: ${JSON.stringify(e)}\n\n`).join(""); + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} diff --git a/tests/helpers/custom-tool-repair-fixtures.ts b/tests/helpers/custom-tool-repair-fixtures.ts new file mode 100644 index 0000000000..cedd309490 --- /dev/null +++ b/tests/helpers/custom-tool-repair-fixtures.ts @@ -0,0 +1,20 @@ +/** + * Shared fixtures for the routed custom-tool repair suites. + * + * Both halves of that suite read the same SSE data line, build the same event block and compare + * the same decorated and canonical apply_patch bodies. They live here because the suite was split + * in two and duplicating a fixture is how two copies of it drift apart. + */ +export function dataPayload(block: string): Record { + const line = block.split(/\r?\n/).find(entry => entry.startsWith("data:")); + if (!line) throw new Error("missing SSE data line"); + return JSON.parse(line.slice(5).trim()) as Record; +} + +export function frame(event: string, payload: Record): string { + return `event: ${event}\ndata: ${JSON.stringify({ type: event, ...payload })}`; +} + +export const DECORATED_PATCH = "*** Begin Patch ***\n*** Update File: README.md\n@@\n-old\n+new\n*** End Patch ***"; +export const CANONICAL_PATCH = "*** Begin Patch\n*** Update File: README.md\n@@\n-old\n+new\n*** End Patch"; +export const WRAPPED_DECORATED_PATCH = JSON.stringify({ input: DECORATED_PATCH }); diff --git a/tests/helpers/passthrough-key-url.ts b/tests/helpers/passthrough-key-url.ts new file mode 100644 index 0000000000..15716abdd6 --- /dev/null +++ b/tests/helpers/passthrough-key-url.ts @@ -0,0 +1,26 @@ +import { createResponsesPassthroughAdapter } from "../../src/adapters/openai-responses"; +import { withTestTranslatorBudget } from "./translator-budget"; + +/** + * The upstream URL a key-auth Responses passthrough adapter would actually request. + * + * Moved out of tests/responses/openai-responses-passthrough.test.ts verbatim, including the + * test translator budget its local adapter wrapper applied: that file is at its file-size cap, + * and the repository answer to a cap is a sibling helper rather than compressed control flow. + */ +export function buildKeyAuthUrl(baseUrl: string, responsesPath?: string): string { + const adapter = withTestTranslatorBudget(createResponsesPassthroughAdapter({ + adapter: "openai-responses", + baseUrl, + authMode: "key" as const, + apiKey: "sk-test", + ...(responsesPath === undefined ? {} : { responsesPath }), + })); + return adapter.buildRequest({ + modelId: "test-model", + context: { messages: [] }, + stream: true, + options: {}, + _rawBody: { model: "test-model", input: "ping" }, + }, { headers: new Headers() }).url; +} diff --git a/tests/helpers/request-log-entry.ts b/tests/helpers/request-log-entry.ts new file mode 100644 index 0000000000..4ccafb3ed4 --- /dev/null +++ b/tests/helpers/request-log-entry.ts @@ -0,0 +1,21 @@ +import type { RequestLogEntry } from "../../src/server"; + +/** + * One request-log row with every required field already filled in. + * + * Moved out of tests/usage/request-log.test.ts verbatim: that file sits at its file-size cap, and + * the repository answer to a cap is a sibling helper rather than compressed control flow. The + * defaults are the ones its twenty-seven call sites were already relying on. + */ +export function log(overrides: Partial): RequestLogEntry { + return { + requestId: "ocx-test", + timestamp: 1, + model: "gpt-test", + provider: "openai", + status: 200, + durationMs: 10, + usageStatus: "unreported", + ...overrides, + }; +} diff --git a/tests/helpers/sse-stream.ts b/tests/helpers/sse-stream.ts new file mode 100644 index 0000000000..63e5ccb639 --- /dev/null +++ b/tests/helpers/sse-stream.ts @@ -0,0 +1,35 @@ +/** + * Read one SSE body end to end, and build one from a string. + * + * Both are the same two functions nine test files had written out for themselves, which is how + * they came to live here: a case at its file-size cap needed room for one more line, and the + * repository answer to that is a sibling helper rather than compressed control flow. Moved + * verbatim from tests/responses/responses-undeclared-tool-guard.test.ts, so a case that used the + * local copies is reading exactly the same behaviour through a different name. + */ +export function streamFromText(text: string): ReadableStream { + const chunk = new TextEncoder().encode(text); + let sent = false; + return new ReadableStream({ + pull(controller) { + if (sent) { + controller.close(); + return; + } + sent = true; + controller.enqueue(chunk); + }, + }); +} + +export async function readAll(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text; +} diff --git a/tests/responses/openai-responses-passthrough.test.ts b/tests/responses/openai-responses-passthrough.test.ts index 42ace1b77a..f9c2029f38 100644 --- a/tests/responses/openai-responses-passthrough.test.ts +++ b/tests/responses/openai-responses-passthrough.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, spyOn, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { Buffer } from "node:buffer"; import { createOpenAIChatAdapter } from "../../src/adapters/openai-chat"; import { createResponsesPassthroughAdapter as createResponsesPassthroughAdapterProduction } from "../../src/adapters/openai-responses"; @@ -22,9 +22,17 @@ import { import { createTranslatorBudget } from "../../src/lib/translator-budget"; import type { AdapterEvent, OcxConfig } from "../../src/types"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { buildKeyAuthUrl } from "../helpers/passthrough-key-url"; import { restoreRoutedNamespaceCalls } from "../../src/responses/namespace-tool-compat"; import { restoreRoutedCustomCalls } from "../../src/responses/custom-tool-compat"; +// A case that calls handleResponses directly never takes the writer lease startServer takes, so +// its dispatch is refused. Dropped in teardown so a throwing case cannot leave the lease behind. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +afterEach(() => { releaseSpendHome?.(); releaseSpendHome = undefined; }); + const createResponsesPassthroughAdapter = (...args: Parameters) => withTestTranslatorBudget(createResponsesPassthroughAdapterProduction(...args)); @@ -559,6 +567,7 @@ test("noncanonical Responses preserves provider-owned safety-buffering hints", a dropCodexSafetyBuffering: true, providers: { fixture: providerConfig }, } as OcxConfig; + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -692,23 +701,6 @@ test("passthrough serialized-body observation releases after the request settles budget.dispose(); }); -function buildKeyAuthUrl(baseUrl: string, responsesPath?: string): string { - const adapter = createResponsesPassthroughAdapter({ - adapter: "openai-responses", - baseUrl, - authMode: "key" as const, - apiKey: "sk-test", - ...(responsesPath === undefined ? {} : { responsesPath }), - }); - return adapter.buildRequest({ - modelId: "test-model", - context: { messages: [] }, - stream: true, - options: {}, - _rawBody: { model: "test-model", input: "ping" }, - }, { headers: new Headers() }).url; -} - describe("OpenAI Responses key-auth URL construction", () => { test("BUG-R289 preserves legacy /v1/responses URL when responsesPath is absent", () => { for (const [baseUrl, expectedUrl] of [ @@ -4283,6 +4275,7 @@ describe("routed namespace and custom-tool identity", () => { }); try { + takeSpendHome(); const jsonResponse = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -4303,6 +4296,7 @@ describe("routed namespace and custom-tool identity", () => { arguments: "{}", }); + takeSpendHome(); const sseResponse = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -4367,6 +4361,7 @@ describe("routed namespace and custom-tool identity", () => { }) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -4713,11 +4708,15 @@ describe("raw usage passthrough on the forward path (#41980 parity, #37138 adjac stream, input: [{ role: "user", content: [{ type: "input_text", text: "hi" }] }], }); - const call = (stream: boolean) => handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: requestBody(stream), - }), config, { model: "", provider: "" }); + // Inside the arrow: taken beside it, at collection time, the first teardown drops it for good. + const call = (stream: boolean) => { + takeSpendHome(); + return handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody(stream), + }), config, { model: "", provider: "" }); + }; test("streamed response.completed with usage extras reaches the client byte-identical", async () => { const savedFetch = globalThis.fetch; @@ -4794,6 +4793,7 @@ test("canonical Responses hint suppression is opt-in at the request boundary", a const config = { port: 0, dropCodexSafetyBuffering, providers: { openai: { ...provider, codexAccountMode: "direct", upstreamWebsocket: false, } } } as OcxConfig; + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer fixture-forward-token" }, body: JSON.stringify({ model: "openai/gpt-5.6-sol", input: "ping", stream: true }), diff --git a/tests/responses/responses-compaction-routing.test.ts b/tests/responses/responses-compaction-routing.test.ts index 3250e696c6..45e174abed 100644 --- a/tests/responses/responses-compaction-routing.test.ts +++ b/tests/responses/responses-compaction-routing.test.ts @@ -7,7 +7,7 @@ import { sessionLaneIdFromRequest } from "../../src/server/request-log-conversat * contract; every other gateway has to be driven as a plain summarizer, or Codex * fatals on a compaction turn that came back as an ordinary message. */ -import { afterEach, describe, expect, jest, spyOn, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, jest, spyOn, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -42,111 +42,24 @@ import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { clearComboRecallForTests, recallComboForLane, rememberComboForLane } from "../../src/server/responses/combo-session-recall"; import { captureConfigGeneration } from "../../src/lib/state-store-sweeper"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { baseCompactionBody, compactionRequest, completedPayload, jsonResponse, keyProviderConfig, nativePoolConfig, sseResponse, twoAccountPoolConfig } from "../helpers/compaction-routing-fixtures"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; +// A case that calls a handler directly never runs startServer, so it never takes the +// spend-journal writer lease and its dispatch is refused before it reaches its own contract. +// Taken per dispatching block rather than for the whole file: five cases install a home of their +// own inside the case body, and a lease binds the directory in effect when it was taken. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; + afterEach(() => { + dropSpendHome(); globalThis.fetch = originalFetch; }); -function keyProviderConfig(overrides: Partial = {}): OcxConfig { - return { - defaultProvider: "gw", - providers: { - gw: { - adapter: "openai-responses", - baseUrl: "https://gateway.example/v1", - authMode: "key", - apiKey: "test-key", - ...overrides, - }, - }, - } as unknown as OcxConfig; -} - -function nativePoolConfig(): OcxConfig { - return { - defaultProvider: "openai", - activeCodexAccountId: "pool-a", - providers: { - openai: { - adapter: "openai-responses", - baseUrl: "https://chatgpt.com/backend-api/codex", - authMode: "forward", - codexAccountMode: "pool", - }, - }, - codexAccounts: [{ - id: "pool-a", - email: "pool@example.test", - isMain: false, - chatgptAccountId: "pool_acc", - }], - } as OcxConfig; -} - -/** Two-account pool: the alternate-attempt tests need somewhere for the retry to go. */ -function twoAccountPoolConfig(): OcxConfig { - const config = nativePoolConfig(); - config.codexAccounts = [ - { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, - { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, - ] as OcxConfig["codexAccounts"]; - return config; -} - -function compactionRequest( - body: Record, - signal?: AbortSignal, - extraHeaders: Record = {}, -): Request { - return new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json", ...extraHeaders }, - body: JSON.stringify(body), - signal, - }); -} - -function baseCompactionBody(extra: Record = {}): Record { - return { - model: "gw/some-model", - stream: false, - input: [ - { type: "message", role: "user", content: [{ type: "input_text", text: "earlier turn" }] }, - { type: "compaction_trigger" }, - ], - tools: [{ type: "function", name: "shell" }], - tool_choice: "auto", - parallel_tool_calls: true, - ...extra, - }; -} - -function jsonResponse(payload: unknown): Response { - return new Response(JSON.stringify(payload), { - status: 200, - headers: { "content-type": "application/json" }, - }); -} - -function completedPayload(text: string): Record { - return { - id: "resp_1", - status: "completed", - output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], - usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, - }; -} - -function sseResponse(events: Array>): Response { - const body = events.map(e => `event: ${String(e.type)}\ndata: ${JSON.stringify(e)}\n\n`).join(""); - return new Response(body, { - status: 200, - headers: { "content-type": "text/event-stream" }, - }); -} - describe("supportsNativeResponsesCompactEndpoint (#422)", () => { const canonicalForward = { adapter: "openai-responses", @@ -183,6 +96,7 @@ describe("supportsNativeResponsesCompactEndpoint (#422)", () => { }); describe("Codex auth-context error parity (#2392)", () => { + beforeEach(takeSpendHome); const cases: Array<{ label: string; createError: () => Error; @@ -344,6 +258,7 @@ describe("Codex auth-context error parity (#2392)", () => { }); describe("native compact usage reporting", () => { + beforeEach(takeSpendHome); test("the buffered upstream body fills the request log usage and stays intact for the client", async () => { const config = { defaultProvider: "openai-apikey", @@ -374,6 +289,9 @@ describe("native compact usage reporting", () => { const previousOpencodexHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = testDir; + // This case serves from its own directory, so the block lease cannot cover it. + dropSpendHome(); + takeSpendHome(); process.env.CODEX_HOME = testDir; try { const mainConfig = nativePoolConfig(); @@ -402,6 +320,8 @@ describe("native compact usage reporting", () => { } finally { globalThis.fetch = originalFetch; clearAccountQuota(); + // Released before the directory holding it is removed. + dropSpendHome(); removeTreeWithRetry(testDir); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -412,6 +332,7 @@ describe("native compact usage reporting", () => { }); describe("native Codex pool compaction", () => { + beforeEach(takeSpendHome); test("ignores a retired Spark reset without cooling later compact requests", async () => { const testDir = mkdtempSync(join(tmpdir(), "ocx-compact-scope-")); const previousOpencodexHome = process.env.OPENCODEX_HOME; @@ -421,6 +342,9 @@ describe("native Codex pool compaction", () => { let sparkPhase = true; try { process.env.OPENCODEX_HOME = testDir; + // This case serves from its own directory, so the block lease cannot cover it. + dropSpendHome(); + takeSpendHome(); process.env.CODEX_HOME = testDir; clearCodexUpstreamHealth(); saveCodexAccountCredential("pool-a", { @@ -471,6 +395,8 @@ describe("native Codex pool compaction", () => { } finally { globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + // Released before the directory holding it is removed. + dropSpendHome(); removeTreeWithRetry(testDir); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -494,6 +420,9 @@ describe("native Codex pool compaction", () => { const bodyReleased = new Promise(resolve => { releaseBody = resolve; }); try { process.env.OPENCODEX_HOME = testDir; + // This case serves from its own directory, so the block lease cannot cover it. + dropSpendHome(); + takeSpendHome(); process.env.CODEX_HOME = testDir; Date.now = () => now; clearCodexUpstreamHealth(); @@ -541,6 +470,8 @@ describe("native Codex pool compaction", () => { Date.now = originalNow; globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + // Released before the directory holding it is removed. + dropSpendHome(); removeTreeWithRetry(testDir); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -562,6 +493,9 @@ describe("native Codex pool compaction", () => { const fetchStarted = new Promise(resolve => { markFetchStarted = resolve; }); try { process.env.OPENCODEX_HOME = testDir; + // This case serves from its own directory, so the block lease cannot cover it. + dropSpendHome(); + takeSpendHome(); process.env.CODEX_HOME = testDir; Date.now = () => now; clearCodexUpstreamHealth(); @@ -606,6 +540,8 @@ describe("native Codex pool compaction", () => { Date.now = originalNow; globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + // Released before the directory holding it is removed. + dropSpendHome(); removeTreeWithRetry(testDir); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -616,6 +552,7 @@ describe("native Codex pool compaction", () => { }); describe("routed compaction for key-mode openai-responses (#422)", () => { + beforeEach(takeSpendHome); test("rewrites the wire: no trigger, no tools, summarizer prompt present", async () => { const bodies: Array> = []; globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { @@ -736,6 +673,7 @@ describe("routed compaction for key-mode openai-responses (#422)", () => { }); describe("bare native compaction model without canonical openai (#2901)", () => { + beforeEach(takeSpendHome); /** A GitHub-Copilot-style operator: one third-party provider, no `openai` row at all. */ function copilotOnlyConfig(): OcxConfig { return { @@ -842,6 +780,7 @@ describe("bare native compaction model without canonical openai (#2901)", () => }); describe("compaction terminal handling (#422)", () => { + beforeEach(takeSpendHome); test("an upstream failure does not become an empty compaction", async () => { globalThis.fetch = (async () => jsonResponse({ id: "resp_1", @@ -924,11 +863,15 @@ describe("compaction terminal handling (#422)", () => { * fired, three means it recursed. */ describe("compact alternate-account attempt (#913)", () => { + beforeEach(takeSpendHome); function withPoolEnv(name: string, run: (config: OcxConfig) => Promise): Promise { const testDir = mkdtempSync(join(tmpdir(), name)); const previousOpencodexHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; process.env.OPENCODEX_HOME = testDir; + // This case serves from its own directory, so the block lease cannot cover it. + dropSpendHome(); + takeSpendHome(); process.env.CODEX_HOME = testDir; clearCodexUpstreamHealth(); clearUpstreamHostHealth(); @@ -947,6 +890,8 @@ describe("compact alternate-account attempt (#913)", () => { clearCodexUpstreamHealth(); clearUpstreamHostHealth(); clearAccountQuota(); + // Released before the directory holding it is removed. + dropSpendHome(); removeTreeWithRetry(testDir); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -1821,6 +1766,7 @@ describe("compact alternate-account attempt (#913)", () => { }); describe("compaction combo recall after combo switch (#3891)", () => { + beforeEach(takeSpendHome); afterEach(() => clearComboRecallForTests()); function comboTestConfig(): OcxConfig { @@ -2270,6 +2216,7 @@ test("a no-eligible policy compact request persists the evaluation trace", async * already degrade an unpaired output to "[tool output for unknown call]" on their own. */ describe("computer screenshot output translation boundary", () => { + beforeEach(takeSpendHome); const screenshot = { type: "computer_call_output", call_id: "call_screen", output: { type: "computer_screenshot", image_url: "https://example.com/screen.png" }, @@ -2368,6 +2315,7 @@ describe("computer screenshot output translation boundary", () => { }); describe("external task-input envelopes (#3735)", () => { + beforeEach(takeSpendHome); // Synthetic charset/length fixture: short plaintext in this slot is deliberately // normalized to input_text before parsing, so it cannot exercise opaque rejection. const opaqueOutput = `g${"A".repeat(127)}`; @@ -2479,6 +2427,7 @@ describe("external task-input envelopes (#3735)", () => { }); describe("established-history external task input (#3807)", () => { + beforeEach(takeSpendHome); // Synthetic complete envelope from the #3735 contract; #3807's history rendering // is not a captured outbound request. Keep the real tool pair distinct from delivery. const deliveryText = " Follow up on the earlier tool result.\n"; @@ -2613,6 +2562,7 @@ describe("established-history external task input (#3807)", () => { }); describe("unpaired tool result boundary (#3259)", () => { + beforeEach(takeSpendHome); function unpairedBody(item: Record): Record { return { model: "gw/some-model", @@ -2730,6 +2680,7 @@ describe("unpaired tool result boundary (#3259)", () => { }); describe("unusable-call_id task-input seed (#3807)", () => { + beforeEach(takeSpendHome); const seed = (extra: Record) => ({ type: "function_call_output", id: "fc_seed", name: "create_thread", namespace: "codex", output: "continue", ...extra, diff --git a/tests/responses/responses-custom-tool-repair-dispatch.test.ts b/tests/responses/responses-custom-tool-repair-dispatch.test.ts new file mode 100644 index 0000000000..fd8d34349f --- /dev/null +++ b/tests/responses/responses-custom-tool-repair-dispatch.test.ts @@ -0,0 +1,979 @@ +/** + * The routed custom-tool repair contract as handleResponses actually serves it. + * + * Split out of responses-custom-tool-repair.test.ts, which keeps the cases that exercise the + * compat functions directly. These are the cases that dispatch, and a dispatching case needs the + * spend-journal writer lease that startServer would have taken; the file it came from is at its + * file-size cap and had no room to take it. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { CANONICAL_PATCH, DECORATED_PATCH, dataPayload, frame } from "../helpers/custom-tool-repair-fixtures"; + +// Taken at each dispatch rather than per case, and dropped in teardown so a case that throws +// mid-assertion cannot leave the lease behind for the next one to trip over. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +afterEach(() => { releaseSpendHome?.(); releaseSpendHome = undefined; }); + +describe("routed Responses custom-tool repair through handleResponses", () => { + test("handleResponses sends an upstream-safe exec function and restores client SSE", async () => { + const savedFetch = globalThis.fetch; + let outboundBody: Record | undefined; + const upstreamItem = { + type: "function_call", + id: "fc_exec", + call_id: "call_exec", + name: "exec", + arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", + status: "completed", + }; + const upstream = [ + frame("response.output_item.added", { output_index: 0, item: { ...upstreamItem, arguments: "", status: "in_progress" } }), + frame("response.function_call_arguments.done", { output_index: 0, item_id: "fc_exec", arguments: upstreamItem.arguments }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { response: { id: "resp_1", status: "completed", output: [upstreamItem] } }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + globalThis.fetch = (async (_input, init) => { + outboundBody = JSON.parse(String(init?.body)) as Record; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: true, + input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], + tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], + }), + }), config, { model: "", provider: "" }); + const clientSse = await response.text(); + const outboundTools = outboundBody?.tools as Array> | undefined; + + expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "exec" }); + expect(clientSse).toContain('"type":"custom_tool_call"'); + expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); + expect(clientSse).toContain('"input":"const apps = await sky.list_apps();"'); + expect(clientSse).not.toContain("response.function_call_arguments.done"); + expect(clientSse).not.toContain('"type":"function_call"'); + expect(clientSse).toContain("data: [DONE]"); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses lowers and restores apply_patch when the destination denies custom tools", async () => { + const savedFetch = globalThis.fetch; + let outboundBody: Record | undefined; + const upstreamItem = { + type: "function_call", + id: "fc_patch_next", + call_id: "call_patch_next", + name: "apply_patch", + arguments: JSON.stringify({ input: "*** Begin Patch\n*** End Patch" }), + status: "completed", + }; + const upstream = [ + frame("response.output_item.added", { + output_index: 0, + item: { ...upstreamItem, arguments: "", status: "in_progress" }, + }), + frame("response.function_call_arguments.done", { + output_index: 0, + item_id: upstreamItem.id, + arguments: upstreamItem.arguments, + }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { + response: { id: "resp_patch", status: "completed", output: [upstreamItem] }, + }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + globalThis.fetch = (async (_input, init) => { + outboundBody = JSON.parse(String(init?.body)) as Record; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + supportsResponsesCustomTools: false, + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/grok-4.6", + stream: true, + input: [ + { + type: "custom_tool_call", + id: "ctc_patch_prior", + call_id: "call_patch_prior", + name: "apply_patch", + input: "noop", + }, + { type: "custom_tool_call_output", call_id: "call_patch_prior", output: "done" }, + ], + tools: [{ + type: "custom", + name: "apply_patch", + description: "Apply a patch", + format: { type: "grammar", syntax: "lark" }, + }], + }), + }), config, { model: "", provider: "" }); + const clientSse = await response.text(); + const outboundTools = outboundBody?.tools as Array> | undefined; + const outboundInput = outboundBody?.input as Array> | undefined; + + expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "apply_patch" }); + expect(outboundInput?.[0]).toMatchObject({ + type: "function_call", + call_id: "call_patch_prior", + name: "apply_patch", + arguments: JSON.stringify({ input: "noop" }), + }); + expect(outboundInput?.[1]).toMatchObject({ + type: "function_call_output", + call_id: "call_patch_prior", + output: "done", + }); + expect(clientSse).toContain('"type":"custom_tool_call"'); + expect(clientSse).toContain('"id":"ctc_patch_next"'); + expect(clientSse).toContain('"call_id":"call_patch_next"'); + expect(clientSse).toContain('"name":"apply_patch"'); + expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); + expect(clientSse).toContain("data: [DONE]"); + expect(clientSse).not.toContain('"type":"function_call"'); + expect(clientSse).not.toContain("response.function_call_arguments.done"); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses lowers apply_patch for a noncanonical forward destination that denies custom tools", async () => { + const savedFetch = globalThis.fetch; + let outboundBody: Record | undefined; + let outboundAuthorization: string | null = null; + let outboundUrl = ""; + const upstreamItem = { + type: "function_call", + id: "fc_patch_next", + call_id: "call_patch_next", + name: "apply_patch", + arguments: JSON.stringify({ input: "*** Begin Patch\n*** End Patch" }), + status: "completed", + }; + const upstream = [ + frame("response.output_item.added", { + output_index: 0, + item: { ...upstreamItem, arguments: "", status: "in_progress" }, + }), + frame("response.function_call_arguments.done", { + output_index: 0, + item_id: upstreamItem.id, + arguments: upstreamItem.arguments, + }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { + response: { id: "resp_patch", status: "completed", output: [upstreamItem] }, + }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + globalThis.fetch = (async (input, init) => { + outboundUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + outboundBody = JSON.parse(String(init?.body)) as Record; + outboundAuthorization = new Headers(init?.headers).get("authorization"); + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://provider.example/v1", + authMode: "forward", + headers: { authorization: "Bearer provider-static" }, + supportsResponsesCustomTools: false, + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer caller-secret" }, + body: JSON.stringify({ + model: "fixture/grok-4.6", + stream: true, + input: [ + { + type: "custom_tool_call", + id: "ctc_patch_prior", + call_id: "call_patch_prior", + name: "apply_patch", + input: "noop", + }, + { type: "custom_tool_call_output", call_id: "call_patch_prior", output: "done" }, + ], + tools: [{ + type: "custom", + name: "apply_patch", + description: "Apply a patch", + format: { type: "grammar", syntax: "lark" }, + }], + }), + }), config, { model: "", provider: "" }); + const clientSse = await response.text(); + const outboundTools = outboundBody?.tools as Array> | undefined; + const outboundInput = outboundBody?.input as Array> | undefined; + + expect(outboundUrl).toBe("https://provider.example/v1/responses"); + expect(outboundAuthorization).toBe("Bearer provider-static"); + expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "apply_patch" }); + expect(outboundInput?.[0]).toMatchObject({ + type: "function_call", + call_id: "call_patch_prior", + name: "apply_patch", + arguments: JSON.stringify({ input: "noop" }), + }); + expect(outboundInput?.[1]).toMatchObject({ + type: "function_call_output", + call_id: "call_patch_prior", + output: "done", + }); + expect(clientSse).toContain('"type":"custom_tool_call"'); + expect(clientSse).toContain('"id":"ctc_patch_next"'); + expect(clientSse).toContain('"call_id":"call_patch_next"'); + expect(clientSse).toContain('"name":"apply_patch"'); + expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); + expect(clientSse).toContain("data: [DONE]"); + expect(clientSse).not.toContain('"type":"function_call"'); + expect(clientSse).not.toContain("response.function_call_arguments.done"); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses continuation rewrites custom_tool_call_output and keeps call_id ordered", async () => { + const savedFetch = globalThis.fetch; + const outboundBodies: Array> = []; + const firstUpstreamItem = { + type: "function_call", + id: "fc_exec", + call_id: "call_exec", + name: "exec", + arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", + status: "completed", + }; + const secondUpstreamMessage = { + type: "message", + id: "msg_2", + role: "assistant", + content: [{ type: "output_text", text: "27 apps" }], + status: "completed", + }; + let turn = 0; + globalThis.fetch = (async (_input, init) => { + outboundBodies.push(JSON.parse(String(init?.body)) as Record); + turn += 1; + if (turn === 1) { + const upstream = [ + frame("response.output_item.added", { output_index: 0, item: { ...firstUpstreamItem, arguments: "", status: "in_progress" } }), + frame("response.function_call_arguments.done", { output_index: 0, item_id: "fc_exec", arguments: firstUpstreamItem.arguments }), + frame("response.output_item.done", { output_index: 0, item: firstUpstreamItem }), + frame("response.completed", { response: { id: "resp_1", status: "completed", output: [firstUpstreamItem] } }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + } + const upstream = [ + frame("response.output_item.added", { output_index: 0, item: { ...secondUpstreamMessage, content: [], status: "in_progress" } }), + frame("response.output_item.done", { output_index: 0, item: secondUpstreamMessage }), + frame("response.completed", { response: { id: "resp_2", status: "completed", output: [secondUpstreamMessage] } }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + const tools = [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }]; + + try { + takeSpendHome(); + const first = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: true, + input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], + tools, + }), + }), config, { model: "", provider: "" }); + const firstSse = await first.text(); + expect(firstSse).toContain('"type":"custom_tool_call"'); + expect(firstSse).toContain('"call_id":"call_exec"'); + expect(firstSse).not.toContain('"type":"function_call"'); + + takeSpendHome(); + const second = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: true, + input: [ + { role: "user", content: [{ type: "input_text", text: "list apps" }] }, + { + type: "custom_tool_call", + id: "ctc_exec", + call_id: "call_exec", + name: "exec", + input: "const apps = await sky.list_apps();", + }, + { type: "custom_tool_call_output", call_id: "call_exec", output: "27 apps" }, + { type: "custom_tool_call_output", call_id: "call_other", output: "wrong pairing must stay distinct" }, + ], + tools, + }), + }), config, { model: "", provider: "" }); + const secondSse = await second.text(); + const continuationInput = outboundBodies[1]?.input as Array>; + expect(outboundBodies).toHaveLength(2); + expect(continuationInput).toEqual(expect.arrayContaining([ + expect.objectContaining({ + type: "function_call", + call_id: "call_exec", + name: "exec", + arguments: JSON.stringify({ input: "const apps = await sky.list_apps();" }), + }), + expect.objectContaining({ + type: "function_call_output", + call_id: "call_exec", + output: "27 apps", + }), + ])); + const execOutput = continuationInput.find(item => item.type === "function_call_output" && item.call_id === "call_exec"); + const otherOutput = continuationInput.find(item => item.call_id === "call_other"); + expect(execOutput).toMatchObject({ type: "function_call_output", output: "27 apps" }); + expect(otherOutput).toMatchObject({ type: "custom_tool_call_output", call_id: "call_other" }); + expect(continuationInput.filter(item => item.type === "function_call_output")).toHaveLength(1); + expect(secondSse).toContain('"text":"27 apps"'); + expect(secondSse).toContain('"id":"resp_2"'); + expect(secondSse).not.toContain('"type":"function_call"'); + expect(secondSse.indexOf("resp_2")).toBeLessThan(secondSse.indexOf("data: [DONE]")); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses restores routed custom calls in non-streaming JSON", async () => { + const savedFetch = globalThis.fetch; + const upstreamItem = { + type: "function_call", + id: "fc_exec", + call_id: "call_exec", + name: "exec", + arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", + status: "completed", + }; + globalThis.fetch = (async () => new Response(JSON.stringify({ + id: "resp_json", + status: "completed", + output: [upstreamItem], + }), { headers: { "content-type": "application/json" } })) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: false, + input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], + tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], + }), + }), config, { model: "", provider: "" }); + const body = await response.json() as { output: Array> }; + + expect(body.output[0]).toMatchObject({ + type: "custom_tool_call", + id: "ctc_exec", + name: "exec", + input: "const apps = await sky.list_apps();", + }); + expect(body.output[0]).not.toHaveProperty("arguments"); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses does not restore routed custom calls excluded by request policy", async () => { + const savedFetch = globalThis.fetch; + const upstreamItem = { + type: "function_call", + id: "fc_exec", + call_id: "call_exec", + name: "exec", + arguments: "{\"input\":\"ignored policy\"}", + status: "completed", + }; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + const execTool = { + type: "custom", + name: "exec", + description: "Run JavaScript", + format: { type: "grammar", syntax: "lark" }, + }; + const ordinaryTool = { + type: "function", + name: "ordinary", + description: "Ordinary function", + parameters: { type: "object" }, + }; + const cases: Array<{ + name: string; + stream: boolean; + tools: Array>; + toolChoice?: unknown; + metadata?: unknown; + /** The upstream call names a tool this request never declared at all (#1700). */ + undeclared?: boolean; + }> = [ + { + name: "streaming none", + stream: true, + tools: [execTool], + toolChoice: "none", + }, + { + name: "streaming allowlist", + stream: true, + tools: [execTool, ordinaryTool], + toolChoice: { + type: "allowed_tools", + mode: "required", + tools: [{ type: "function", name: "ordinary" }], + }, + }, + { + name: "named ordinary function", + stream: false, + tools: [execTool, ordinaryTool], + toolChoice: { type: "function", name: "ordinary" }, + }, + { + name: "custom-looking metadata without a declared tool", + stream: false, + tools: [ordinaryTool], + metadata: { nested: { type: "custom", name: "exec" } }, + undeclared: true, + }, + ]; + + globalThis.fetch = (async (_input, init) => { + const outboundBody = JSON.parse(String(init?.body)) as { stream?: boolean }; + if (outboundBody.stream === true) { + const upstream = [ + frame("response.output_item.added", { + output_index: 0, + item: { ...upstreamItem, arguments: "", status: "in_progress" }, + }), + frame("response.function_call_arguments.done", { + output_index: 0, + item_id: "fc_exec", + arguments: upstreamItem.arguments, + }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { + response: { id: "resp_policy", status: "completed", output: [upstreamItem] }, + }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + } + return new Response(JSON.stringify({ + id: "resp_policy", + status: "completed", + output: [upstreamItem], + }), { headers: { "content-type": "application/json" } }); + }) as typeof fetch; + + try { + for (const policyCase of cases) { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: policyCase.stream, + input: [{ role: "user", content: [{ type: "input_text", text: policyCase.name }] }], + tools: policyCase.tools, + ...(policyCase.toolChoice !== undefined ? { tool_choice: policyCase.toolChoice } : {}), + ...(policyCase.metadata !== undefined ? { metadata: policyCase.metadata } : {}), + }), + }), config, { model: "", provider: "" }); + + if (policyCase.stream) { + const clientSse = await response.text(); + expect(clientSse).toContain('"type":"function_call"'); + expect(clientSse).toContain('"id":"fc_exec"'); + expect(clientSse).toContain("response.function_call_arguments.done"); + expect(clientSse).not.toContain("custom_tool_call"); + expect(clientSse).not.toContain("ctc_exec"); + } else if (policyCase.undeclared) { + // #1700: this request's catalog holds only `ordinary` — a metadata blob that merely + // looks like a tool declaration declares nothing — so a call to `exec` is refused + // instead of relayed. The restore contract still holds either way: it never became + // a custom_tool_call. + expect(response.status).toBe(502); + const body = await response.json() as { error: { message: string } }; + expect(body.error.message).toContain('undeclared client tool "exec"'); + } else { + const body = await response.json() as { output: Array> }; + expect(body.output[0]).toEqual(upstreamItem); + } + } + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses preserves native apply_patch calls that were never converted", async () => { + const savedFetch = globalThis.fetch; + const upstreamItem = { + type: "function_call", + id: "fc_patch", + call_id: "call_patch", + name: "apply_patch", + arguments: "{\"patch\":\"*** Begin Patch\"}", + status: "completed", + }; + globalThis.fetch = (async () => new Response(JSON.stringify({ + id: "resp_patch", + status: "completed", + output: [upstreamItem], + }), { headers: { "content-type": "application/json" } })) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: false, + input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], + tools: [{ + type: "custom", + name: "apply_patch", + description: "Apply a patch", + format: { type: "grammar", syntax: "lark" }, + }], + }), + }), config, { model: "", provider: "" }); + const body = await response.json() as { output: Array> }; + + expect(body.output[0]).toEqual(upstreamItem); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses preserves disallowed native apply_patch input in JSON and SSE", async () => { + const savedFetch = globalThis.fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + const upstreamItem = { + type: "custom_tool_call", + id: "ctc_patch", + call_id: "call_patch", + name: "apply_patch", + input: DECORATED_PATCH, + status: "completed", + }; + + globalThis.fetch = (async (_input, init) => { + const outbound = JSON.parse(String(init?.body)) as { stream?: boolean }; + if (outbound.stream === true) { + const upstream = [ + frame("response.output_item.added", { + output_index: 0, + item: { ...upstreamItem, input: "", status: "in_progress" }, + }), + frame("response.custom_tool_call_input.done", { + output_index: 0, + item_id: "ctc_patch", + input: DECORATED_PATCH, + }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { + response: { id: "resp_patch_stream", status: "completed", output: [upstreamItem] }, + }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + } + return Response.json({ id: "resp_patch_json", status: "completed", output: [upstreamItem] }); + }) as typeof fetch; + + try { + for (const stream of [false, true]) { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream, + input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], + tools: [ + { + type: "custom", + name: "apply_patch", + description: "Apply a patch", + format: { type: "grammar", syntax: "lark" }, + }, + { + type: "function", + name: "ordinary", + description: "Ordinary function", + parameters: { type: "object" }, + }, + ], + tool_choice: stream + ? { + type: "allowed_tools", + mode: "required", + tools: [{ type: "function", name: "ordinary" }], + } + : { type: "function", name: "ordinary" }, + }), + }), config, { model: "", provider: "" }); + + if (!stream) { + const body = await response.json() as { output: Array> }; + expect(body.output[0]).toEqual(upstreamItem); + continue; + } + + const blocks = (await response.text()).split("\n\n").filter(block => block.includes("data: {")); + const payloads = blocks.map(dataPayload); + const inputDone = payloads.find(payload => payload.type === "response.custom_tool_call_input.done"); + expect(inputDone).toMatchObject({ input: DECORATED_PATCH }); + const itemDone = payloads.find(payload => payload.type === "response.output_item.done") as { + item?: Record; + } | undefined; + expect(itemDone?.item).toMatchObject({ type: "custom_tool_call", input: DECORATED_PATCH }); + const completed = payloads.find(payload => payload.type === "response.completed") as { + response?: { output?: Array> }; + } | undefined; + expect(completed?.response?.output?.[0]).toMatchObject({ input: DECORATED_PATCH }); + } + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses repairs authorized native apply_patch calls in JSON and SSE", async () => { + const savedFetch = globalThis.fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + }, + }, + } as OcxConfig; + const upstreamItem = { + type: "custom_tool_call", + id: "ctc_patch", + call_id: "call_patch", + name: "apply_patch", + input: DECORATED_PATCH, + status: "completed", + }; + + globalThis.fetch = (async (_input, init) => { + const outbound = JSON.parse(String(init?.body)) as { stream?: boolean }; + if (outbound.stream === true) { + const upstream = [ + frame("response.output_item.added", { + output_index: 0, + item: { ...upstreamItem, input: "", status: "in_progress" }, + }), + frame("response.custom_tool_call_input.done", { + output_index: 0, + item_id: "ctc_patch", + input: DECORATED_PATCH, + }), + frame("response.output_item.done", { output_index: 0, item: upstreamItem }), + frame("response.completed", { + response: { id: "resp_patch_stream", status: "completed", output: [upstreamItem] }, + }), + "data: [DONE]", + ].join("\n\n") + "\n\n"; + return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); + } + return Response.json({ id: "resp_patch_json", status: "completed", output: [upstreamItem] }); + }) as typeof fetch; + + try { + for (const stream of [false, true]) { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream, + input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], + tools: [{ + type: "custom", + name: "apply_patch", + description: "Apply a patch", + format: { type: "grammar", syntax: "lark" }, + }], + }), + }), config, { model: "", provider: "" }); + + if (!stream) { + const body = await response.json() as { output: Array> }; + expect(body.output[0]).toMatchObject({ + type: "custom_tool_call", + name: "apply_patch", + input: CANONICAL_PATCH, + }); + continue; + } + + const blocks = (await response.text()).split("\n\n").filter(block => block.includes("data: {")); + const payloads = blocks.map(dataPayload); + const inputDone = payloads.find(payload => payload.type === "response.custom_tool_call_input.done"); + expect(inputDone).toMatchObject({ input: CANONICAL_PATCH }); + const itemDone = payloads.find(payload => payload.type === "response.output_item.done") as { + item?: Record; + } | undefined; + expect(itemDone?.item).toMatchObject({ type: "custom_tool_call", input: CANONICAL_PATCH }); + const completed = payloads.find(payload => payload.type === "response.completed") as { + response?: { output?: Array> }; + } | undefined; + expect(completed?.response?.output?.[0]).toMatchObject({ input: CANONICAL_PATCH }); + } + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses does not restore a custom image tool replaced by hosted preference", async () => { + const savedFetch = globalThis.fetch; + let outboundBody: Record | undefined; + const upstreamItem = { + type: "function_call", + id: "fc_image", + call_id: "call_image", + name: "image_gen.generate", + arguments: "{}", + status: "completed", + }; + globalThis.fetch = (async (_input, init) => { + outboundBody = JSON.parse(String(init?.body)) as Record; + return new Response(JSON.stringify({ + id: "resp_image", + status: "completed", + output: [upstreamItem], + }), { headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://fixture.test/v1", + authMode: "key", + apiKey: "fixture-key", + modelPreferHostedTools: { "deepseek-v4-flash": ["image_generation"] }, + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "fixture/deepseek-v4-flash", + stream: false, + input: [{ role: "user", content: [{ type: "input_text", text: "draw" }] }], + tools: [{ + type: "custom", + name: "image_gen.generate", + description: "Generate an image", + format: { type: "grammar", syntax: "lark" }, + }], + }), + }), config, { model: "", provider: "" }); + const body = await response.json() as { output: Array> }; + const outboundTools = outboundBody?.tools as Array> | undefined; + + expect(outboundTools).toEqual([{ type: "image_generation" }]); + expect(body.output[0]).toEqual(upstreamItem); + } finally { + globalThis.fetch = savedFetch; + } + }); + + test("handleResponses leaves custom tools native for forward-auth passthrough", async () => { + const savedFetch = globalThis.fetch; + let outboundBody: Record | undefined; + let outboundAuthorization: string | null = null; + let outboundUrl = ""; + const upstreamItem = { + type: "function_call", + id: "fc_exec", + call_id: "call_exec", + name: "exec", + arguments: "{\"input\":\"native\"}", + status: "completed", + }; + globalThis.fetch = (async (input, init) => { + outboundUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + outboundBody = JSON.parse(String(init?.body)) as Record; + outboundAuthorization = new Headers(init?.headers).get("authorization"); + return new Response(JSON.stringify({ id: "resp_forward", status: "completed", output: [upstreamItem] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + const config = { + port: 0, + defaultProvider: "fixture", + providers: { + fixture: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + }, + }, + } as OcxConfig; + + try { + takeSpendHome(); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer caller-token" }, + body: JSON.stringify({ + model: "fixture/native-model", + stream: false, + input: "run", + tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], + }), + }), config, { model: "", provider: "" }); + const clientBody = await response.json() as { output: Array> }; + const outboundTools = outboundBody?.tools as Array> | undefined; + + expect(outboundUrl).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(outboundAuthorization).toBe("Bearer caller-token"); + expect(outboundTools?.[0]).toMatchObject({ type: "custom", name: "exec" }); + expect(clientBody.output[0]).toMatchObject({ type: "function_call", name: "exec" }); + expect(clientBody.output[0]).not.toHaveProperty("input"); + } finally { + globalThis.fetch = savedFetch; + } + }); +}); diff --git a/tests/responses/responses-custom-tool-repair.test.ts b/tests/responses/responses-custom-tool-repair.test.ts index 98336d0241..3011f7c307 100644 --- a/tests/responses/responses-custom-tool-repair.test.ts +++ b/tests/responses/responses-custom-tool-repair.test.ts @@ -6,23 +6,8 @@ import { } from "../../src/responses/custom-tool-compat"; import { compileCodeModeHelperInput } from "../../src/responses/code-mode-helper-compat"; import { createRoutedCustomToolRestoreBlockRewrite } from "../../src/server/responses-custom-tool-repair"; -import { handleResponses } from "../../src/server/responses"; -import type { OcxConfig } from "../../src/types"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; - -function dataPayload(block: string): Record { - const line = block.split(/\r?\n/).find(entry => entry.startsWith("data:")); - if (!line) throw new Error("missing SSE data line"); - return JSON.parse(line.slice(5).trim()) as Record; -} - -function frame(event: string, payload: Record): string { - return `event: ${event}\ndata: ${JSON.stringify({ type: event, ...payload })}`; -} - -const DECORATED_PATCH = "*** Begin Patch ***\n*** Update File: README.md\n@@\n-old\n+new\n*** End Patch ***"; -const CANONICAL_PATCH = "*** Begin Patch\n*** Update File: README.md\n@@\n-old\n+new\n*** End Patch"; -const WRAPPED_DECORATED_PATCH = JSON.stringify({ input: DECORATED_PATCH }); +import { CANONICAL_PATCH, DECORATED_PATCH, WRAPPED_DECORATED_PATCH, dataPayload, frame } from "../helpers/custom-tool-repair-fixtures"; describe("routed Responses custom-tool compatibility", () => { test("restores legacy structured shell aliases as executable unified-exec input", () => { @@ -1195,949 +1180,4 @@ describe("routed Responses custom-tool compatibility", () => { rewrite.dispose?.(); }); - test("handleResponses sends an upstream-safe exec function and restores client SSE", async () => { - const savedFetch = globalThis.fetch; - let outboundBody: Record | undefined; - const upstreamItem = { - type: "function_call", - id: "fc_exec", - call_id: "call_exec", - name: "exec", - arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", - status: "completed", - }; - const upstream = [ - frame("response.output_item.added", { output_index: 0, item: { ...upstreamItem, arguments: "", status: "in_progress" } }), - frame("response.function_call_arguments.done", { output_index: 0, item_id: "fc_exec", arguments: upstreamItem.arguments }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { response: { id: "resp_1", status: "completed", output: [upstreamItem] } }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - globalThis.fetch = (async (_input, init) => { - outboundBody = JSON.parse(String(init?.body)) as Record; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: true, - input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], - tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], - }), - }), config, { model: "", provider: "" }); - const clientSse = await response.text(); - const outboundTools = outboundBody?.tools as Array> | undefined; - - expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "exec" }); - expect(clientSse).toContain('"type":"custom_tool_call"'); - expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); - expect(clientSse).toContain('"input":"const apps = await sky.list_apps();"'); - expect(clientSse).not.toContain("response.function_call_arguments.done"); - expect(clientSse).not.toContain('"type":"function_call"'); - expect(clientSse).toContain("data: [DONE]"); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses lowers and restores apply_patch when the destination denies custom tools", async () => { - const savedFetch = globalThis.fetch; - let outboundBody: Record | undefined; - const upstreamItem = { - type: "function_call", - id: "fc_patch_next", - call_id: "call_patch_next", - name: "apply_patch", - arguments: JSON.stringify({ input: "*** Begin Patch\n*** End Patch" }), - status: "completed", - }; - const upstream = [ - frame("response.output_item.added", { - output_index: 0, - item: { ...upstreamItem, arguments: "", status: "in_progress" }, - }), - frame("response.function_call_arguments.done", { - output_index: 0, - item_id: upstreamItem.id, - arguments: upstreamItem.arguments, - }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { - response: { id: "resp_patch", status: "completed", output: [upstreamItem] }, - }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - globalThis.fetch = (async (_input, init) => { - outboundBody = JSON.parse(String(init?.body)) as Record; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - supportsResponsesCustomTools: false, - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/grok-4.6", - stream: true, - input: [ - { - type: "custom_tool_call", - id: "ctc_patch_prior", - call_id: "call_patch_prior", - name: "apply_patch", - input: "noop", - }, - { type: "custom_tool_call_output", call_id: "call_patch_prior", output: "done" }, - ], - tools: [{ - type: "custom", - name: "apply_patch", - description: "Apply a patch", - format: { type: "grammar", syntax: "lark" }, - }], - }), - }), config, { model: "", provider: "" }); - const clientSse = await response.text(); - const outboundTools = outboundBody?.tools as Array> | undefined; - const outboundInput = outboundBody?.input as Array> | undefined; - - expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "apply_patch" }); - expect(outboundInput?.[0]).toMatchObject({ - type: "function_call", - call_id: "call_patch_prior", - name: "apply_patch", - arguments: JSON.stringify({ input: "noop" }), - }); - expect(outboundInput?.[1]).toMatchObject({ - type: "function_call_output", - call_id: "call_patch_prior", - output: "done", - }); - expect(clientSse).toContain('"type":"custom_tool_call"'); - expect(clientSse).toContain('"id":"ctc_patch_next"'); - expect(clientSse).toContain('"call_id":"call_patch_next"'); - expect(clientSse).toContain('"name":"apply_patch"'); - expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); - expect(clientSse).toContain("data: [DONE]"); - expect(clientSse).not.toContain('"type":"function_call"'); - expect(clientSse).not.toContain("response.function_call_arguments.done"); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses lowers apply_patch for a noncanonical forward destination that denies custom tools", async () => { - const savedFetch = globalThis.fetch; - let outboundBody: Record | undefined; - let outboundAuthorization: string | null = null; - let outboundUrl = ""; - const upstreamItem = { - type: "function_call", - id: "fc_patch_next", - call_id: "call_patch_next", - name: "apply_patch", - arguments: JSON.stringify({ input: "*** Begin Patch\n*** End Patch" }), - status: "completed", - }; - const upstream = [ - frame("response.output_item.added", { - output_index: 0, - item: { ...upstreamItem, arguments: "", status: "in_progress" }, - }), - frame("response.function_call_arguments.done", { - output_index: 0, - item_id: upstreamItem.id, - arguments: upstreamItem.arguments, - }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { - response: { id: "resp_patch", status: "completed", output: [upstreamItem] }, - }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - globalThis.fetch = (async (input, init) => { - outboundUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; - outboundBody = JSON.parse(String(init?.body)) as Record; - outboundAuthorization = new Headers(init?.headers).get("authorization"); - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://provider.example/v1", - authMode: "forward", - headers: { authorization: "Bearer provider-static" }, - supportsResponsesCustomTools: false, - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json", authorization: "Bearer caller-secret" }, - body: JSON.stringify({ - model: "fixture/grok-4.6", - stream: true, - input: [ - { - type: "custom_tool_call", - id: "ctc_patch_prior", - call_id: "call_patch_prior", - name: "apply_patch", - input: "noop", - }, - { type: "custom_tool_call_output", call_id: "call_patch_prior", output: "done" }, - ], - tools: [{ - type: "custom", - name: "apply_patch", - description: "Apply a patch", - format: { type: "grammar", syntax: "lark" }, - }], - }), - }), config, { model: "", provider: "" }); - const clientSse = await response.text(); - const outboundTools = outboundBody?.tools as Array> | undefined; - const outboundInput = outboundBody?.input as Array> | undefined; - - expect(outboundUrl).toBe("https://provider.example/v1/responses"); - expect(outboundAuthorization).toBe("Bearer provider-static"); - expect(outboundTools?.[0]).toMatchObject({ type: "function", name: "apply_patch" }); - expect(outboundInput?.[0]).toMatchObject({ - type: "function_call", - call_id: "call_patch_prior", - name: "apply_patch", - arguments: JSON.stringify({ input: "noop" }), - }); - expect(outboundInput?.[1]).toMatchObject({ - type: "function_call_output", - call_id: "call_patch_prior", - output: "done", - }); - expect(clientSse).toContain('"type":"custom_tool_call"'); - expect(clientSse).toContain('"id":"ctc_patch_next"'); - expect(clientSse).toContain('"call_id":"call_patch_next"'); - expect(clientSse).toContain('"name":"apply_patch"'); - expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"'); - expect(clientSse).toContain("data: [DONE]"); - expect(clientSse).not.toContain('"type":"function_call"'); - expect(clientSse).not.toContain("response.function_call_arguments.done"); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses continuation rewrites custom_tool_call_output and keeps call_id ordered", async () => { - const savedFetch = globalThis.fetch; - const outboundBodies: Array> = []; - const firstUpstreamItem = { - type: "function_call", - id: "fc_exec", - call_id: "call_exec", - name: "exec", - arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", - status: "completed", - }; - const secondUpstreamMessage = { - type: "message", - id: "msg_2", - role: "assistant", - content: [{ type: "output_text", text: "27 apps" }], - status: "completed", - }; - let turn = 0; - globalThis.fetch = (async (_input, init) => { - outboundBodies.push(JSON.parse(String(init?.body)) as Record); - turn += 1; - if (turn === 1) { - const upstream = [ - frame("response.output_item.added", { output_index: 0, item: { ...firstUpstreamItem, arguments: "", status: "in_progress" } }), - frame("response.function_call_arguments.done", { output_index: 0, item_id: "fc_exec", arguments: firstUpstreamItem.arguments }), - frame("response.output_item.done", { output_index: 0, item: firstUpstreamItem }), - frame("response.completed", { response: { id: "resp_1", status: "completed", output: [firstUpstreamItem] } }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - } - const upstream = [ - frame("response.output_item.added", { output_index: 0, item: { ...secondUpstreamMessage, content: [], status: "in_progress" } }), - frame("response.output_item.done", { output_index: 0, item: secondUpstreamMessage }), - frame("response.completed", { response: { id: "resp_2", status: "completed", output: [secondUpstreamMessage] } }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - const tools = [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }]; - - try { - const first = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: true, - input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], - tools, - }), - }), config, { model: "", provider: "" }); - const firstSse = await first.text(); - expect(firstSse).toContain('"type":"custom_tool_call"'); - expect(firstSse).toContain('"call_id":"call_exec"'); - expect(firstSse).not.toContain('"type":"function_call"'); - - const second = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: true, - input: [ - { role: "user", content: [{ type: "input_text", text: "list apps" }] }, - { - type: "custom_tool_call", - id: "ctc_exec", - call_id: "call_exec", - name: "exec", - input: "const apps = await sky.list_apps();", - }, - { type: "custom_tool_call_output", call_id: "call_exec", output: "27 apps" }, - { type: "custom_tool_call_output", call_id: "call_other", output: "wrong pairing must stay distinct" }, - ], - tools, - }), - }), config, { model: "", provider: "" }); - const secondSse = await second.text(); - const continuationInput = outboundBodies[1]?.input as Array>; - expect(outboundBodies).toHaveLength(2); - expect(continuationInput).toEqual(expect.arrayContaining([ - expect.objectContaining({ - type: "function_call", - call_id: "call_exec", - name: "exec", - arguments: JSON.stringify({ input: "const apps = await sky.list_apps();" }), - }), - expect.objectContaining({ - type: "function_call_output", - call_id: "call_exec", - output: "27 apps", - }), - ])); - const execOutput = continuationInput.find(item => item.type === "function_call_output" && item.call_id === "call_exec"); - const otherOutput = continuationInput.find(item => item.call_id === "call_other"); - expect(execOutput).toMatchObject({ type: "function_call_output", output: "27 apps" }); - expect(otherOutput).toMatchObject({ type: "custom_tool_call_output", call_id: "call_other" }); - expect(continuationInput.filter(item => item.type === "function_call_output")).toHaveLength(1); - expect(secondSse).toContain('"text":"27 apps"'); - expect(secondSse).toContain('"id":"resp_2"'); - expect(secondSse).not.toContain('"type":"function_call"'); - expect(secondSse.indexOf("resp_2")).toBeLessThan(secondSse.indexOf("data: [DONE]")); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses restores routed custom calls in non-streaming JSON", async () => { - const savedFetch = globalThis.fetch; - const upstreamItem = { - type: "function_call", - id: "fc_exec", - call_id: "call_exec", - name: "exec", - arguments: "{\"input\":\"const apps = await sky.list_apps();\"}", - status: "completed", - }; - globalThis.fetch = (async () => new Response(JSON.stringify({ - id: "resp_json", - status: "completed", - output: [upstreamItem], - }), { headers: { "content-type": "application/json" } })) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: false, - input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }], - tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], - }), - }), config, { model: "", provider: "" }); - const body = await response.json() as { output: Array> }; - - expect(body.output[0]).toMatchObject({ - type: "custom_tool_call", - id: "ctc_exec", - name: "exec", - input: "const apps = await sky.list_apps();", - }); - expect(body.output[0]).not.toHaveProperty("arguments"); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses does not restore routed custom calls excluded by request policy", async () => { - const savedFetch = globalThis.fetch; - const upstreamItem = { - type: "function_call", - id: "fc_exec", - call_id: "call_exec", - name: "exec", - arguments: "{\"input\":\"ignored policy\"}", - status: "completed", - }; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - const execTool = { - type: "custom", - name: "exec", - description: "Run JavaScript", - format: { type: "grammar", syntax: "lark" }, - }; - const ordinaryTool = { - type: "function", - name: "ordinary", - description: "Ordinary function", - parameters: { type: "object" }, - }; - const cases: Array<{ - name: string; - stream: boolean; - tools: Array>; - toolChoice?: unknown; - metadata?: unknown; - /** The upstream call names a tool this request never declared at all (#1700). */ - undeclared?: boolean; - }> = [ - { - name: "streaming none", - stream: true, - tools: [execTool], - toolChoice: "none", - }, - { - name: "streaming allowlist", - stream: true, - tools: [execTool, ordinaryTool], - toolChoice: { - type: "allowed_tools", - mode: "required", - tools: [{ type: "function", name: "ordinary" }], - }, - }, - { - name: "named ordinary function", - stream: false, - tools: [execTool, ordinaryTool], - toolChoice: { type: "function", name: "ordinary" }, - }, - { - name: "custom-looking metadata without a declared tool", - stream: false, - tools: [ordinaryTool], - metadata: { nested: { type: "custom", name: "exec" } }, - undeclared: true, - }, - ]; - - globalThis.fetch = (async (_input, init) => { - const outboundBody = JSON.parse(String(init?.body)) as { stream?: boolean }; - if (outboundBody.stream === true) { - const upstream = [ - frame("response.output_item.added", { - output_index: 0, - item: { ...upstreamItem, arguments: "", status: "in_progress" }, - }), - frame("response.function_call_arguments.done", { - output_index: 0, - item_id: "fc_exec", - arguments: upstreamItem.arguments, - }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { - response: { id: "resp_policy", status: "completed", output: [upstreamItem] }, - }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - } - return new Response(JSON.stringify({ - id: "resp_policy", - status: "completed", - output: [upstreamItem], - }), { headers: { "content-type": "application/json" } }); - }) as typeof fetch; - - try { - for (const policyCase of cases) { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: policyCase.stream, - input: [{ role: "user", content: [{ type: "input_text", text: policyCase.name }] }], - tools: policyCase.tools, - ...(policyCase.toolChoice !== undefined ? { tool_choice: policyCase.toolChoice } : {}), - ...(policyCase.metadata !== undefined ? { metadata: policyCase.metadata } : {}), - }), - }), config, { model: "", provider: "" }); - - if (policyCase.stream) { - const clientSse = await response.text(); - expect(clientSse).toContain('"type":"function_call"'); - expect(clientSse).toContain('"id":"fc_exec"'); - expect(clientSse).toContain("response.function_call_arguments.done"); - expect(clientSse).not.toContain("custom_tool_call"); - expect(clientSse).not.toContain("ctc_exec"); - } else if (policyCase.undeclared) { - // #1700: this request's catalog holds only `ordinary` — a metadata blob that merely - // looks like a tool declaration declares nothing — so a call to `exec` is refused - // instead of relayed. The restore contract still holds either way: it never became - // a custom_tool_call. - expect(response.status).toBe(502); - const body = await response.json() as { error: { message: string } }; - expect(body.error.message).toContain('undeclared client tool "exec"'); - } else { - const body = await response.json() as { output: Array> }; - expect(body.output[0]).toEqual(upstreamItem); - } - } - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses preserves native apply_patch calls that were never converted", async () => { - const savedFetch = globalThis.fetch; - const upstreamItem = { - type: "function_call", - id: "fc_patch", - call_id: "call_patch", - name: "apply_patch", - arguments: "{\"patch\":\"*** Begin Patch\"}", - status: "completed", - }; - globalThis.fetch = (async () => new Response(JSON.stringify({ - id: "resp_patch", - status: "completed", - output: [upstreamItem], - }), { headers: { "content-type": "application/json" } })) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: false, - input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], - tools: [{ - type: "custom", - name: "apply_patch", - description: "Apply a patch", - format: { type: "grammar", syntax: "lark" }, - }], - }), - }), config, { model: "", provider: "" }); - const body = await response.json() as { output: Array> }; - - expect(body.output[0]).toEqual(upstreamItem); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses preserves disallowed native apply_patch input in JSON and SSE", async () => { - const savedFetch = globalThis.fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - const upstreamItem = { - type: "custom_tool_call", - id: "ctc_patch", - call_id: "call_patch", - name: "apply_patch", - input: DECORATED_PATCH, - status: "completed", - }; - - globalThis.fetch = (async (_input, init) => { - const outbound = JSON.parse(String(init?.body)) as { stream?: boolean }; - if (outbound.stream === true) { - const upstream = [ - frame("response.output_item.added", { - output_index: 0, - item: { ...upstreamItem, input: "", status: "in_progress" }, - }), - frame("response.custom_tool_call_input.done", { - output_index: 0, - item_id: "ctc_patch", - input: DECORATED_PATCH, - }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { - response: { id: "resp_patch_stream", status: "completed", output: [upstreamItem] }, - }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - } - return Response.json({ id: "resp_patch_json", status: "completed", output: [upstreamItem] }); - }) as typeof fetch; - - try { - for (const stream of [false, true]) { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream, - input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], - tools: [ - { - type: "custom", - name: "apply_patch", - description: "Apply a patch", - format: { type: "grammar", syntax: "lark" }, - }, - { - type: "function", - name: "ordinary", - description: "Ordinary function", - parameters: { type: "object" }, - }, - ], - tool_choice: stream - ? { - type: "allowed_tools", - mode: "required", - tools: [{ type: "function", name: "ordinary" }], - } - : { type: "function", name: "ordinary" }, - }), - }), config, { model: "", provider: "" }); - - if (!stream) { - const body = await response.json() as { output: Array> }; - expect(body.output[0]).toEqual(upstreamItem); - continue; - } - - const blocks = (await response.text()).split("\n\n").filter(block => block.includes("data: {")); - const payloads = blocks.map(dataPayload); - const inputDone = payloads.find(payload => payload.type === "response.custom_tool_call_input.done"); - expect(inputDone).toMatchObject({ input: DECORATED_PATCH }); - const itemDone = payloads.find(payload => payload.type === "response.output_item.done") as { - item?: Record; - } | undefined; - expect(itemDone?.item).toMatchObject({ type: "custom_tool_call", input: DECORATED_PATCH }); - const completed = payloads.find(payload => payload.type === "response.completed") as { - response?: { output?: Array> }; - } | undefined; - expect(completed?.response?.output?.[0]).toMatchObject({ input: DECORATED_PATCH }); - } - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses repairs authorized native apply_patch calls in JSON and SSE", async () => { - const savedFetch = globalThis.fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - }, - }, - } as OcxConfig; - const upstreamItem = { - type: "custom_tool_call", - id: "ctc_patch", - call_id: "call_patch", - name: "apply_patch", - input: DECORATED_PATCH, - status: "completed", - }; - - globalThis.fetch = (async (_input, init) => { - const outbound = JSON.parse(String(init?.body)) as { stream?: boolean }; - if (outbound.stream === true) { - const upstream = [ - frame("response.output_item.added", { - output_index: 0, - item: { ...upstreamItem, input: "", status: "in_progress" }, - }), - frame("response.custom_tool_call_input.done", { - output_index: 0, - item_id: "ctc_patch", - input: DECORATED_PATCH, - }), - frame("response.output_item.done", { output_index: 0, item: upstreamItem }), - frame("response.completed", { - response: { id: "resp_patch_stream", status: "completed", output: [upstreamItem] }, - }), - "data: [DONE]", - ].join("\n\n") + "\n\n"; - return new Response(upstream, { headers: { "content-type": "text/event-stream" } }); - } - return Response.json({ id: "resp_patch_json", status: "completed", output: [upstreamItem] }); - }) as typeof fetch; - - try { - for (const stream of [false, true]) { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream, - input: [{ role: "user", content: [{ type: "input_text", text: "patch" }] }], - tools: [{ - type: "custom", - name: "apply_patch", - description: "Apply a patch", - format: { type: "grammar", syntax: "lark" }, - }], - }), - }), config, { model: "", provider: "" }); - - if (!stream) { - const body = await response.json() as { output: Array> }; - expect(body.output[0]).toMatchObject({ - type: "custom_tool_call", - name: "apply_patch", - input: CANONICAL_PATCH, - }); - continue; - } - - const blocks = (await response.text()).split("\n\n").filter(block => block.includes("data: {")); - const payloads = blocks.map(dataPayload); - const inputDone = payloads.find(payload => payload.type === "response.custom_tool_call_input.done"); - expect(inputDone).toMatchObject({ input: CANONICAL_PATCH }); - const itemDone = payloads.find(payload => payload.type === "response.output_item.done") as { - item?: Record; - } | undefined; - expect(itemDone?.item).toMatchObject({ type: "custom_tool_call", input: CANONICAL_PATCH }); - const completed = payloads.find(payload => payload.type === "response.completed") as { - response?: { output?: Array> }; - } | undefined; - expect(completed?.response?.output?.[0]).toMatchObject({ input: CANONICAL_PATCH }); - } - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses does not restore a custom image tool replaced by hosted preference", async () => { - const savedFetch = globalThis.fetch; - let outboundBody: Record | undefined; - const upstreamItem = { - type: "function_call", - id: "fc_image", - call_id: "call_image", - name: "image_gen.generate", - arguments: "{}", - status: "completed", - }; - globalThis.fetch = (async (_input, init) => { - outboundBody = JSON.parse(String(init?.body)) as Record; - return new Response(JSON.stringify({ - id: "resp_image", - status: "completed", - output: [upstreamItem], - }), { headers: { "content-type": "application/json" } }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://fixture.test/v1", - authMode: "key", - apiKey: "fixture-key", - modelPreferHostedTools: { "deepseek-v4-flash": ["image_generation"] }, - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "fixture/deepseek-v4-flash", - stream: false, - input: [{ role: "user", content: [{ type: "input_text", text: "draw" }] }], - tools: [{ - type: "custom", - name: "image_gen.generate", - description: "Generate an image", - format: { type: "grammar", syntax: "lark" }, - }], - }), - }), config, { model: "", provider: "" }); - const body = await response.json() as { output: Array> }; - const outboundTools = outboundBody?.tools as Array> | undefined; - - expect(outboundTools).toEqual([{ type: "image_generation" }]); - expect(body.output[0]).toEqual(upstreamItem); - } finally { - globalThis.fetch = savedFetch; - } - }); - - test("handleResponses leaves custom tools native for forward-auth passthrough", async () => { - const savedFetch = globalThis.fetch; - let outboundBody: Record | undefined; - let outboundAuthorization: string | null = null; - let outboundUrl = ""; - const upstreamItem = { - type: "function_call", - id: "fc_exec", - call_id: "call_exec", - name: "exec", - arguments: "{\"input\":\"native\"}", - status: "completed", - }; - globalThis.fetch = (async (input, init) => { - outboundUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; - outboundBody = JSON.parse(String(init?.body)) as Record; - outboundAuthorization = new Headers(init?.headers).get("authorization"); - return new Response(JSON.stringify({ id: "resp_forward", status: "completed", output: [upstreamItem] }), { - headers: { "content-type": "application/json" }, - }); - }) as typeof fetch; - const config = { - port: 0, - defaultProvider: "fixture", - providers: { - fixture: { - adapter: "openai-responses", - baseUrl: "https://chatgpt.com/backend-api/codex", - authMode: "forward", - }, - }, - } as OcxConfig; - - try { - const response = await handleResponses(new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json", authorization: "Bearer caller-token" }, - body: JSON.stringify({ - model: "fixture/native-model", - stream: false, - input: "run", - tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }], - }), - }), config, { model: "", provider: "" }); - const clientBody = await response.json() as { output: Array> }; - const outboundTools = outboundBody?.tools as Array> | undefined; - - expect(outboundUrl).toBe("https://chatgpt.com/backend-api/codex/responses"); - expect(outboundAuthorization).toBe("Bearer caller-token"); - expect(outboundTools?.[0]).toMatchObject({ type: "custom", name: "exec" }); - expect(clientBody.output[0]).toMatchObject({ type: "function_call", name: "exec" }); - expect(clientBody.output[0]).not.toHaveProperty("input"); - } finally { - globalThis.fetch = savedFetch; - } - }); }); diff --git a/tests/responses/responses-undeclared-tool-guard.test.ts b/tests/responses/responses-undeclared-tool-guard.test.ts index d0e4000bde..99a50b2b1a 100644 --- a/tests/responses/responses-undeclared-tool-guard.test.ts +++ b/tests/responses/responses-undeclared-tool-guard.test.ts @@ -4,7 +4,7 @@ * with the target file untouched. The bridged paths already fail closed on the same condition * (`declaredToolNames`, src/bridge/sse.ts); these pin the passthrough's equivalent. */ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { collectDeclaredNamelessClientCallTypes, collectDeclaredBareWireToolNames, @@ -25,6 +25,15 @@ import { handleResponses } from "../../src/server/responses"; import { expandPreviousResponseInput } from "../../src/responses/state"; import type { OcxConfig } from "../../src/types"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { readAll, streamFromText } from "../helpers/sse-stream"; + +// A case that calls handleResponses directly never runs startServer, so it never takes the +// spend-journal writer lease and its dispatch is refused before it reaches its own contract. +// Dropped in teardown so a case that throws mid-assertion cannot leave the lease behind. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +afterEach(() => { releaseSpendHome?.(); releaseSpendHome = undefined; }); /** One SSE event block without its blank-line delimiter. */ function frame(type: string, payload: Record): string { @@ -36,33 +45,6 @@ function sse(type: string, payload: Record): string { return `${frame(type, payload)}\n\n`; } -function streamFromText(text: string): ReadableStream { - const chunk = new TextEncoder().encode(text); - let sent = false; - return new ReadableStream({ - pull(controller) { - if (sent) { - controller.close(); - return; - } - sent = true; - controller.enqueue(chunk); - }, - }); -} - -async function readAll(stream: ReadableStream): Promise { - const reader = stream.getReader(); - const decoder = new TextDecoder(); - let text = ""; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - text += decoder.decode(value, { stream: true }); - } - return text; -} - async function relay( upstream: string, declared: Iterable, @@ -810,6 +792,7 @@ describe("the reported turn, end to end through handleResponses", () => { const savedFetch = globalThis.fetch; globalThis.fetch = (async () => upstream()) as typeof fetch; try { + takeSpendHome(); return await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -949,6 +932,7 @@ describe("a refused turn does not become continuation state", () => { { headers: { "content-type": "application/json" } }, )) as typeof fetch; try { + takeSpendHome(); return await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -1056,6 +1040,7 @@ describe("a refused turn does not become continuation state", () => { })) as typeof fetch; let response: Response; try { + takeSpendHome(); response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -1119,6 +1104,7 @@ describe("a refused turn does not become continuation state", () => { })) as typeof fetch; let response: Response; try { + takeSpendHome(); response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -1208,6 +1194,7 @@ describe("real relay and continuation caller normalization (#4176 / #4181)", () }), }); + takeSpendHome(); const turn1Res = await handleResponses(turn1Req, config, { model: "", provider: "" }); expect(turn1Res.status).toBe(200); const clientStreamText = await turn1Res.text(); @@ -1240,6 +1227,7 @@ describe("real relay and continuation caller normalization (#4176 / #4181)", () }), }); + takeSpendHome(); const turn2Res = await handleResponses(turn2Req, config, { model: "", provider: "" }); expect(turn2Res.status).toBe(200); await turn2Res.json(); @@ -1278,6 +1266,7 @@ describe("real relay and continuation caller normalization (#4176 / #4181)", () })) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -1315,6 +1304,7 @@ describe("real relay and continuation caller normalization (#4176 / #4181)", () })) as typeof fetch; try { + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -1389,6 +1379,7 @@ describe("empty and absent tool catalogs", () => { const savedFetch = globalThis.fetch; globalThis.fetch = (async () => upstream()) as typeof fetch; try { + takeSpendHome(); return await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -2324,6 +2315,7 @@ describe("xAI hosted-call authorization through handleResponses", () => { }); }) as typeof fetch; try { + takeSpendHome(); return await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, diff --git a/tests/usage/request-log.test.ts b/tests/usage/request-log.test.ts index f27426480a..0d591e46d4 100644 --- a/tests/usage/request-log.test.ts +++ b/tests/usage/request-log.test.ts @@ -41,25 +41,14 @@ import { mkdtempSync} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { log } from "../helpers/request-log-entry"; import { decodeRequestLogCursor, selectRequestLogPoll } from "../../src/server/request-log-cursor"; async function* replayAdapterEvents(events: AdapterEvent[]): AsyncGenerator { for (const event of events) yield event; } -function log(overrides: Partial): RequestLogEntry { - return { - requestId: "ocx-test", - timestamp: 1, - model: "gpt-test", - provider: "openai", - status: 200, - durationMs: 10, - usageStatus: "unreported", - ...overrides, - }; -} - describe("request log metadata", () => { test("Claude evidence is normalized before direct ring ingress and cannot be mutated afterwards", () => { const previousHome = process.env.OPENCODEX_HOME; @@ -240,6 +229,9 @@ describe("request log metadata", () => { }, } as OcxConfig; + // This row calls the handler directly, so it takes the spend-journal writer lease that + // startServer would have taken. Released in the finally, before the fetch stub is restored. + const releaseSpendHome = acquireOwnedSpendHome(); try { const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", @@ -257,6 +249,7 @@ describe("request log metadata", () => { sendCount: 1, })]); } finally { + releaseSpendHome(); globalThis.fetch = originalFetch; } }); From 4de35b4305bc09363b9ab3dc81e8d610eaf07233 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:44:23 +0900 Subject: [PATCH 16/34] test(spend): release each turn's body before the lease that served it Review found cases that dispatched, asserted on headers or on what the upstream stub recorded, and then discarded the Response without touching its body. The body is a live stream, so the case finished with a reader still attached and the lease was dropped underneath it. That is the state the single-writer rule exists to keep from happening, and it is invisible until something downstream reports a pending handle instead of the assertion that actually failed. Each of these now consumes or cancels the body after its assertions and before it returns: the annotation and DeepSeek inbound drive helpers, the two FastWire characterization paths, both service-tier paths, and the six image-bridge cases that only ever read a header. The 400 case is untouched because it already reads its body. No assertion, expected value, mock, fixture or timeout changes. Local checks: NOT RUN. --- tests/adapters/empty-tool-output-annotation.test.ts | 5 ++++- tests/images/z-handler-activation.test.ts | 8 ++++++++ tests/providers/deepseek-inbound-wire.test.ts | 5 ++++- tests/routing/fastwire-characterization-wire.test.ts | 8 ++++++-- tests/service/service-tier-capability.test.ts | 8 ++++++-- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/adapters/empty-tool-output-annotation.test.ts b/tests/adapters/empty-tool-output-annotation.test.ts index 8538834f7f..7114836bd4 100644 --- a/tests/adapters/empty-tool-output-annotation.test.ts +++ b/tests/adapters/empty-tool-output-annotation.test.ts @@ -165,7 +165,7 @@ describe("openai-responses empty tool output annotation", () => { }) as typeof fetch; // Direct dispatch needs the writer lease to prevent spend-ledger ownership failures. releaseSpendHome = acquireOwnedSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -174,6 +174,9 @@ describe("openai-responses empty tool output annotation", () => { config, { model: "", provider: "" }, ); + // The turn's body is a live stream. Releasing it here means no reader is still attached + // when the lease is dropped, which is what turns a finished case into a pending one. + await turn.body?.cancel(); return requests[0] ?? { body: {} }; } diff --git a/tests/images/z-handler-activation.test.ts b/tests/images/z-handler-activation.test.ts index 6c07f2a5d3..a2227dc8b1 100644 --- a/tests/images/z-handler-activation.test.ts +++ b/tests/images/z-handler-activation.test.ts @@ -170,6 +170,9 @@ describe("image bridge dispatch priority (handler activation)", () => { const res = await post(true, [{ type: "image_generation" }]); expect(imageBridgeRun).toBe(true); expect(res.headers.get("content-type")).toBe("text/event-stream"); + // The bridge answers with a live SSE stream. Releasing it here means no reader is + // still attached when this suite drops its lease in afterAll. + await res.body?.cancel(); }); test("alias-only image tool_choice keeps canonical bridge interception armed", async () => { @@ -191,6 +194,7 @@ describe("image bridge dispatch priority (handler activation)", () => { expect(imageBridgeToolNames).toContain("generate_image"); expect(imageBridgeToolNames).toContain("image_gen"); expect(res.headers.get("content-type")).toBe("text/event-stream"); + await res.body?.cancel(); }); test("stream=false + image_generation tool → 400 (bridge requires stream=true)", async () => { @@ -208,6 +212,7 @@ describe("image bridge dispatch priority (handler activation)", () => { expect(webSearchRun).toBe(true); expect(imageBridgeRun).toBe(false); expect(res.headers.get("content-type")).toBe("text/event-stream"); + await res.body?.cancel(); }); test("routed compaction with image_generation tool → image bridge does NOT hijack compaction (#424)", async () => { @@ -229,6 +234,7 @@ describe("image bridge dispatch priority (handler activation)", () => { ); expect(imageBridgeRun).toBe(false); expect(res.headers.get("content-type")).toBe("text/event-stream"); + await res.body?.cancel(); }); test("dual-tool on a runTurn adapter → image bridge wins (web-search loop has no runTurn support)", async () => { @@ -241,6 +247,7 @@ describe("image bridge dispatch priority (handler activation)", () => { expect(imageBridgeRun).toBe(true); expect(runTurnCalled).toBe(false); expect(res.headers.get("content-type")).toBe("text/event-stream"); + await res.body?.cancel(); } finally { useRunTurnAdapter = false; } @@ -256,6 +263,7 @@ describe("image bridge dispatch priority (handler activation)", () => { expect(webSearchRun).toBe(false); expect(runTurnCalled).toBe(false); expect(res.headers.get("content-type")).toBe("text/event-stream"); + await res.body?.cancel(); } finally { useRunTurnAdapter = false; } diff --git a/tests/providers/deepseek-inbound-wire.test.ts b/tests/providers/deepseek-inbound-wire.test.ts index a9bf4730b7..0db58c6a6f 100644 --- a/tests/providers/deepseek-inbound-wire.test.ts +++ b/tests/providers/deepseek-inbound-wire.test.ts @@ -202,7 +202,7 @@ describe("the inbound scope survives the handleResponses replay", () => { const requests = captureUpstreamRequests(); const config = { providers: { deepseek: deepseekProvider() } } as unknown as OcxConfig; takeSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -215,6 +215,9 @@ describe("the inbound scope survives the handleResponses replay", () => { ...(inboundTransport === undefined ? {} : { inboundTransport }), }, ); + // The turn's body is a live stream. Releasing it here means no reader is still attached + // when the lease is dropped, which is what turns a finished case into a pending one. + await turn.body?.cancel(); return requests[0] ?? { url: "", body: {} }; } diff --git a/tests/routing/fastwire-characterization-wire.test.ts b/tests/routing/fastwire-characterization-wire.test.ts index 1ee18ef05f..5f4d3f6380 100644 --- a/tests/routing/fastwire-characterization-wire.test.ts +++ b/tests/routing/fastwire-characterization-wire.test.ts @@ -53,7 +53,7 @@ async function driveResponses(args: { }; takeSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -64,6 +64,9 @@ async function driveResponses(args: { {}, ); + // The turn's body is a live stream. Releasing it here means no reader is still attached when + // the lease is dropped, which is what turns a finished case into a pending one. + await turn.body?.cancel(); expect(bodies).toHaveLength(1); return { outboundBody: bodies[0]!, logCtx }; } @@ -364,7 +367,8 @@ describe("FastWire characterization: rawBody observation point", () => { }); takeSpendHome(); - await handleResponses(request, config, { model: "", provider: "" }, {}); + const turn = await handleResponses(request, config, { model: "", provider: "" }, {}); + await turn.body?.cancel(); expect(outboundBody?.service_tier).toBe("priority"); expect(adapterRawBody?.service_tier).toBe("flex"); } finally { diff --git a/tests/service/service-tier-capability.test.ts b/tests/service/service-tier-capability.test.ts index 20b3dde0bd..6ab92047eb 100644 --- a/tests/service/service-tier-capability.test.ts +++ b/tests/service/service-tier-capability.test.ts @@ -420,7 +420,7 @@ describe("the gate fires on the live handleResponses path", () => { const { bodies } = captureBody(); const config = { providers: { [providerName]: provider }, ...(fastMode === undefined ? {} : { fastMode }) } as unknown as OcxConfig; takeSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -430,6 +430,9 @@ describe("the gate fires on the live handleResponses path", () => { { model: "", provider: "" }, {}, ); + // The turn's body is a live stream. Releasing it here means no reader is still attached + // when the lease is dropped, which is what turns a finished case into a pending one. + await turn.body?.cancel(); return bodies[0] ?? {}; } @@ -470,7 +473,7 @@ describe("the gate fires on the live handleResponses path", () => { const { bodies } = captureBody(); const logCtx: RequestLogContext = { model: "", provider: "" }; takeSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -485,6 +488,7 @@ describe("the gate fires on the live handleResponses path", () => { logCtx, {}, ); + await turn.body?.cancel(); const upstreamBody = bodies[0]; expect(upstreamBody).toBeDefined(); From 4231b275c1e8dddc5ea5b96dd1d86351fdd41ba7 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:50:24 +0900 Subject: [PATCH 17/34] test(spend): read the gateway attempt row's body before its lease is released The row asserts on the request-log metadata only, so the turn's body was still attached when the finally released the lease that served it. Reading it keeps the case's own assertions untouched and leaves nothing pending behind them. Bounded fixture hygiene. Nothing about the production body lifetime changes, and no claim is made that the static buffer leaks on its own. Local checks: NOT RUN. --- tests/usage/request-log.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/usage/request-log.test.ts b/tests/usage/request-log.test.ts index 0d591e46d4..f4842c2780 100644 --- a/tests/usage/request-log.test.ts +++ b/tests/usage/request-log.test.ts @@ -248,6 +248,9 @@ describe("request log metadata", () => { adapter: "openai-responses", sendCount: 1, })]); + // Read before the lease is released: the row asserts on metadata only, so without this it + // finishes with the turn's body still attached and the lease dropped underneath it. + await response.text(); } finally { releaseSpendHome(); globalThis.fetch = originalFetch; From d2bc0d81bb0137e8105fd47695e2c9119e32c2b6 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 20:59:30 +0900 Subject: [PATCH 18/34] test(spend): repair the three failures the first hosted run found Hosted CI at 4de35b43 reported three distinct problems, and only one of them was a missing lease. The Lab activation guard was the serious one. Its SERVE_ANCHOR is the literal text of the statement that creates the listener, and this branch assigns that listener from spendLedgerLifecycle.track(Bun.serve(...)) so the lease can roll the listener back on a failed start. The anchor stopped matching, indexOf returned -1, and the guard failed loudly rather than measuring an empty string, which is exactly what it was written to do. The anchor now names the real statement, so it starts the same window it always did, and the new body-level call is registered for review rather than skipped: track binds the listener's stop, records a rollback closure and returns the same server, with no await. The lifecycle's release() is not registered because it is called inside the async stop wrapper, which this scan skips as a nested function. Nothing in the guard is relaxed. The second was a regression this branch introduced. main-account-hard-lock-auth took the lease in a file-level beforeEach, and two of its describes spawn a child that runs startServer against the same home. The child then failed with SPEND_LEDGER_OWNER_BUSY, which is the ownership contract working correctly and the case failing for a reason it is not about. The lease is now taken only by the two cases that dispatch in-process. No other fixture in this batch spawns a child; that was checked rather than assumed. The third was an inventory gap. The first pass searched for handleResponses and its compact and policy-fallback siblings, and missed handleChatCompletions and handleNativeChatCompletions, so the chat fixtures were never leased. Eight more files take it now, and the search is by the full handler set rather than by one of them. ws-upstream needed room first: its runtime pin and the two wrappers that bind it move to a sibling helper, verbatim, because the file was at its cap. Files whose handler calls stop before a physical dispatch are deliberately untouched: OAuth admission refusals, unknown-provider routing, the policy fallback rows with an injected runCore, and the reasoning-envelope rows that answer 404 for a deliberately absent model. Assertion counts are unchanged in every file; the deletions in this diff are re-indentation where a body was wrapped in try/finally. No cap moves. Local checks: NOT RUN. --- .../openai/openai-chat-native-policy.test.ts | 62 ++++++++++++++----- .../main-account-hard-lock-auth.test.ts | 14 +++-- tests/helpers/ws-upstream-fixtures.ts | 34 ++++++++++ tests/lab/core-lab-boundary.test.ts | 7 ++- .../cyber-policy-error-fidelity.test.ts | 6 +- .../upstream-transient-retry.test.ts | 42 ++++++++----- .../chat-conversation-affinity.test.ts | 7 +++ .../responses/chat-json-sse-fallback.test.ts | 16 ++++- tests/responses/chat-refusal.test.ts | 7 +++ ...esponses-compact-handoff-admission.test.ts | 39 +++++++++--- tests/responses/ws-upstream.test.ts | 39 ++++++------ tests/routing/combo-management-api.test.ts | 47 +++++++++++--- 12 files changed, 240 insertions(+), 80 deletions(-) create mode 100644 tests/helpers/ws-upstream-fixtures.ts diff --git a/tests/adapters/openai/openai-chat-native-policy.test.ts b/tests/adapters/openai/openai-chat-native-policy.test.ts index 9357cf2215..d5ef745abe 100644 --- a/tests/adapters/openai/openai-chat-native-policy.test.ts +++ b/tests/adapters/openai/openai-chat-native-policy.test.ts @@ -15,13 +15,25 @@ import { clearKeyCooldowns } from "../../../src/providers/key-failover"; import { fastPolicyForModel } from "../../../src/providers/service-tier"; import { handleChatCompletions } from "../../../src/server/chat-completions"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; const PROVIDER_NAME = "native-tier-fixture"; const MODEL_ID = "model"; const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { + // Taken only for direct Chat dispatches so pure policy cases do not open the journal. + releaseSpendHome ??= acquireOwnedSpendHome(); +}; +const dropSpendHome = (): void => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}; afterEach(() => { + // Released first so a failed assertion cannot leave the active home lease live. + dropSpendHome(); globalThis.fetch = originalFetch; clearKeyCooldowns(PROVIDER_NAME); }); @@ -219,6 +231,7 @@ describe("native Chat passthrough service-tier policy", () => { providers: { [PROVIDER_NAME]: target }, } as OcxConfig; + takeSpendHome(); const response = await handleChatCompletions( new Request("http://localhost/v1/chat/completions", { method: "POST", @@ -233,9 +246,13 @@ describe("native Chat passthrough service-tier policy", () => { { model: "", provider: "" }, ); - expect(response.status).toBe(200); - expect(captured).toHaveLength(1); - expect(captured[0]).not.toHaveProperty("service_tier"); + try { + expect(response.status).toBe(200); + expect(captured).toHaveLength(1); + expect(captured[0]).not.toHaveProperty("service_tier"); + } finally { + await response.body?.cancel(); + } }); test("forced Fast injects the policy wire value and forced default drops the caller tier", () => { @@ -250,6 +267,8 @@ describe("native Chat passthrough service-tier policy", () => { const previousHome = process.env.OPENCODEX_HOME; const home = mkdtempSync(join(tmpdir(), "ocx-native-tier-failover-")); process.env.OPENCODEX_HOME = home; + // Taken after this case installs its home so key-failover dispatch owns that journal. + takeSpendHome(); const captured: Array<{ authorization: string | null; body: Record }> = []; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { captured.push({ @@ -301,11 +320,17 @@ describe("native Chat passthrough service-tier policy", () => { { model: "", provider: "" }, ); - expect(response.status).toBe(200); - expect(captured.map(entry => entry.authorization)).toEqual(["Bearer key-one", "Bearer key-two"]); - expect(captured).toHaveLength(2); - for (const entry of captured) expect(entry.body).not.toHaveProperty("service_tier"); + try { + expect(response.status).toBe(200); + expect(captured.map(entry => entry.authorization)).toEqual(["Bearer key-one", "Bearer key-two"]); + expect(captured).toHaveLength(2); + for (const entry of captured) expect(entry.body).not.toHaveProperty("service_tier"); + } finally { + await response.body?.cancel(); + } } finally { + // Released before this case restores and removes its home so no live database is unlinked. + dropSpendHome(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; removeTreeWithRetry(home); @@ -420,6 +445,7 @@ describe("main and native Chat tier authorization parity", () => { const target = provider({ modelCapabilities: { [MODEL_ID]: { inputModalities: ["text", "image"] } }, }); + takeSpendHome(); const response = await handleChatCompletions( new Request("http://localhost/v1/chat/completions", { method: "POST", @@ -439,15 +465,19 @@ describe("main and native Chat tier authorization parity", () => { { model: "", provider: "" }, ); - expect(response.status).toBe(200); - expect(captured).toHaveLength(1); - const parts = (JSON.parse(captured[0]!) as { messages: Array<{ content: unknown }> }) - .messages.flatMap(m => (Array.isArray(m.content) ? m.content : [])) - .filter((p): p is { type: string; image_url: { url: string } } => - typeof p === "object" && p !== null && (p as { type?: unknown }).type === "image_url"); - // Well over the 3.5MiB image budget, and still byte-identical on the wire. - expect(parts).toHaveLength(4); - for (const part of parts) expect(part.image_url.url).toBe(url); + try { + expect(response.status).toBe(200); + expect(captured).toHaveLength(1); + const parts = (JSON.parse(captured[0]!) as { messages: Array<{ content: unknown }> }) + .messages.flatMap(m => (Array.isArray(m.content) ? m.content : [])) + .filter((p): p is { type: string; image_url: { url: string } } => + typeof p === "object" && p !== null && (p as { type?: unknown }).type === "image_url"); + // Well over the 3.5MiB image budget, and still byte-identical on the wire. + expect(parts).toHaveLength(4); + for (const part of parts) expect(part.image_url.url).toBe(url); + } finally { + await response.body?.cancel(); + } }); }); diff --git a/tests/codex-integration/main-account-hard-lock-auth.test.ts b/tests/codex-integration/main-account-hard-lock-auth.test.ts index 3645380c93..3e0bb6a799 100644 --- a/tests/codex-integration/main-account-hard-lock-auth.test.ts +++ b/tests/codex-integration/main-account-hard-lock-auth.test.ts @@ -108,6 +108,12 @@ function addAlternative(cfg: OcxConfig): void { } let releaseSpendHome: (() => void) | undefined; +// Taken by the two cases that dispatch in-process, never for the whole file. Two describes here +// spawn a child that runs startServer against this same home, and that child takes the real +// lease: a parent holding one turns the child's startup into SPEND_LEDGER_OWNER_BUSY, which is +// the contract working and the case failing for a reason it is not about. +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +const dropSpendHome = (): void => { releaseSpendHome?.(); releaseSpendHome = undefined; }; beforeEach(() => { tokenExpiry = Math.floor(Date.now() / 1000) + 86_400; @@ -125,16 +131,12 @@ beforeEach(() => { clearAccountNeedsReauth("hard-lock-pool"); mainAccount.setMainAccountPlan(null); writeMain(); - // Dispatches without starting a server, so it takes the spend-journal lease itself. Taken - // last because the lease binds the home in effect at the moment it is taken. - releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { // Released before this case's home is removed: an open lease inside a directory being // deleted fails the removal on Windows and leaves an unlinked live database on POSIX. - releaseSpendHome?.(); - releaseSpendHome = undefined; + dropSpendHome(); mock.restore(); clearAccountQuota(); clearThreadAccountMap(); @@ -503,6 +505,7 @@ describe("main quota policy at native admission", () => { model: "fixture-model", output: [], usage: { input_tokens: 1, output_tokens: 0, total_tokens: 1 }, }); }, { preconnect() {} })); + takeSpendHome(); const post = (model: string) => handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { ...Object.fromEntries(caller()), "content-type": "application/json" }, @@ -542,6 +545,7 @@ describe("main quota policy at native admission", () => { sends.push({ url: request.url, authorization: request.headers.get("authorization") }); return Response.json({ id: "cmp_policy_transport", object: "response.compaction", output: [] }); }, { preconnect() {} })); + takeSpendHome(); const post = (model: string) => handleResponsesCompact(new Request("http://localhost/v1/responses/compact", { method: "POST", headers: { ...Object.fromEntries(caller()), "content-type": "application/json" }, diff --git a/tests/helpers/ws-upstream-fixtures.ts b/tests/helpers/ws-upstream-fixtures.ts new file mode 100644 index 0000000000..8fd6c71955 --- /dev/null +++ b/tests/helpers/ws-upstream-fixtures.ts @@ -0,0 +1,34 @@ +import { + codexWsUpstreamFetch as rawCodexWsUpstreamFetch, + shouldUseCodexWsUpstream as rawShouldUseCodexWsUpstream, +} from "../../src/server/responses/ws-upstream"; + +/** + * Runtime pin and request shapes for the Codex WebSocket upstream suite. + * + * Moved verbatim out of tests/responses/ws-upstream.test.ts: that file sits at its file-size + * cap, and the repository answer to a cap is a sibling helper rather than compressed control + * flow. The two wrappers exist only to bind the pinned runtime identity, which is what makes + * the suite read the bounded-relay path instead of whatever the host Bun reports. + */ +export const BOUNDED_WS_RUNTIME = "1.4.0"; + +export function shouldUseCodexWsUpstream(url: string, init?: RequestInit, upstreamWebsocket = false): boolean { + return rawShouldUseCodexWsUpstream(url, init, BOUNDED_WS_RUNTIME, upstreamWebsocket); +} + +export function codexWsUpstreamFetch( + url: string, + init: RequestInit, + fallback: typeof fetch, +): Promise { + return rawCodexWsUpstreamFetch(url, init, fallback, BOUNDED_WS_RUNTIME); +} + +export function streamingInit(body: Record = {}): RequestInit { + return { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer test" }, + body: JSON.stringify({ model: "gpt-5.5", stream: true, ...body }), + }; +} diff --git a/tests/lab/core-lab-boundary.test.ts b/tests/lab/core-lab-boundary.test.ts index 4e54e4ec5c..32a5a9758f 100644 --- a/tests/lab/core-lab-boundary.test.ts +++ b/tests/lab/core-lab-boundary.test.ts @@ -138,7 +138,11 @@ export function namesLabDirectly(source: string): boolean { * routes subagents to a different model than the operator configured. Nothing goes red; * the wrong model simply answers. */ -const SERVE_ANCHOR = "server = Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost });"; +// The anchor carries the spend-ledger wrapper because the listener is registered for rollback +// at the moment it is created. It is still the same statement and still the start of the same +// window; what changed is the expression the listener is assigned from. An anchor that no +// longer matches makes every scan below measure an empty string, which is why they assert on it. +const SERVE_ANCHOR = "server = spendLedgerLifecycle.track(Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost }));"; const ACTIVATION_ANCHOR = "if (labActivationRequired(config, labConfigDir)) {"; /** * The window ends at the RETURN, not at the activation check. @@ -1017,6 +1021,7 @@ describe("activation window stays synchronous", () => { "(...).then()": "Promise.then on the fire-and-forget `import('../codex/plan-from-token')` chain. then() registers a callback and returns immediately; the callback is a nested function this scan skips. Awaiting the import would already fail Guard 3.", "(...).catch()": "Promise.catch on that same dynamic-import chain. Same fire-and-forget: it cannot suspend startServer.", "backgroundLifecycle.scheduleStartupRun()": "src/server/background-lifecycle.ts owns this object method. The call site cannot resolve the declaration statically; scheduleStartupRun is declared `(): void` and is documented as never blocking listen.", + "spendLedgerLifecycle.track()": "Instance method on the lifecycle from acquireSpendLedgerServerLifecycle in src/server/index/spend-ledger-lifecycle.ts, called on each listener as it is created. It binds the listener's stop, records a rollback closure and returns the same server; it is declared `(server: T): T` and contains no await. An `await spendLedgerLifecycle.track(...)` would already fail Guard 3. The lifecycle's release() is not here because it is called inside the async stop wrapper, which this scan skips as a nested function.", }; /** diff --git a/tests/providers/cyber-policy-error-fidelity.test.ts b/tests/providers/cyber-policy-error-fidelity.test.ts index bff86a295a..7adc39f3f2 100644 --- a/tests/providers/cyber-policy-error-fidelity.test.ts +++ b/tests/providers/cyber-policy-error-fidelity.test.ts @@ -17,6 +17,7 @@ import { formatPassthroughUpstreamError } from "../../src/server/responses/passt import { consumeComboFailure } from "../../src/server/responses/core"; import { handleResponses } from "../../src/server/responses"; import type { AdapterEvent, OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { createTestTranslatorBudget, withTestTranslatorBudget } from "../helpers/translator-budget"; const createOpenAIChatAdapter = (...args: Parameters) => @@ -219,6 +220,8 @@ describe("cyber_policy error fidelity", () => { }, }, } as OcxConfig; + // Taken after the inherited test home is in effect so physical dispatch can open its ledger. + const releaseSpendHome = acquireOwnedSpendHome(); try { const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", @@ -235,6 +238,8 @@ describe("cyber_policy error fidelity", () => { }, }); } finally { + // Released before surrounding teardown can replace the home and strand its live ledger. + releaseSpendHome(); upstream.stop(true); } }); @@ -475,4 +480,3 @@ describe("#2488 nested policy identity is not hidden by an outer envelope", () = expect(failure.response.status).toBe(502); }); }); - diff --git a/tests/providers/upstream-transient-retry.test.ts b/tests/providers/upstream-transient-retry.test.ts index 956496e44e..516f31850a 100644 --- a/tests/providers/upstream-transient-retry.test.ts +++ b/tests/providers/upstream-transient-retry.test.ts @@ -9,6 +9,7 @@ import { import { transientRetryPolicyFor } from "../../src/providers/key-failover"; import { handleChatCompletions } from "../../src/server/chat-completions"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; function bodyResponse(status: number, headers?: Record): Response { // ReadableStream body so cancel() is observable. @@ -290,24 +291,31 @@ describe("native Chat completions and the replay refusal", () => { }, } as unknown as OcxConfig; - const response = await handleChatCompletions( - new Request("http://localhost/v1/chat/completions", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "replay-refusal-fixture/model", - messages: [{ role: "user", content: "ping" }], + // Taken after the inherited test home is in effect so native Chat can record its send. + const releaseSpendHome = acquireOwnedSpendHome(); + try { + const response = await handleChatCompletions( + new Request("http://localhost/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "replay-refusal-fixture/model", + messages: [{ role: "user", content: "ping" }], + }), }), - }), - config, - { model: "", provider: "" }, - ); + config, + { model: "", provider: "" }, + ); - expect(sends).toBe(1); - expect(response.status).toBe(429); - expect(response.headers.get("Retry-After")).toBeNull(); - expect(await response.json()).toMatchObject({ - error: { code: "upstream_reset_replay_refused" }, - }); + expect(sends).toBe(1); + expect(response.status).toBe(429); + expect(response.headers.get("Retry-After")).toBeNull(); + expect(await response.json()).toMatchObject({ + error: { code: "upstream_reset_replay_refused" }, + }); + } finally { + // Released after the response body is consumed so no stream retains the ledger owner. + releaseSpendHome(); + } }); }); diff --git a/tests/responses/chat-conversation-affinity.test.ts b/tests/responses/chat-conversation-affinity.test.ts index 97c599b308..3254c70904 100644 --- a/tests/responses/chat-conversation-affinity.test.ts +++ b/tests/responses/chat-conversation-affinity.test.ts @@ -6,6 +6,7 @@ import { handleChatCompletions } from "../../src/server/chat-completions"; import type { OcxConfig } from "../../src/types"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; // #3433 transport contract only: these client-assigned fixture IDs are not a capture of Hermes. @@ -14,14 +15,20 @@ const identityHeaders = ["session_id", "session-id", "thread-id", "x-codex-paren let isolated: IsolatedCodexHome; let home: string; let previousHome: string | undefined; +let releaseSpendHome: (() => void) | undefined; beforeEach(() => { isolated = installIsolatedCodexHome("ocx-chat-identity-"); previousHome = process.env.OPENCODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-chat-identity-config-")); process.env.OPENCODEX_HOME = home; + // Taken after this case installs its home so direct Chat dispatch owns that journal. + releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + // Released before this case restores and removes its home so no live database is unlinked. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; isolated.restore(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/responses/chat-json-sse-fallback.test.ts b/tests/responses/chat-json-sse-fallback.test.ts index e3fc1bb871..bc357fa540 100644 --- a/tests/responses/chat-json-sse-fallback.test.ts +++ b/tests/responses/chat-json-sse-fallback.test.ts @@ -6,9 +6,21 @@ import { responsesJsonToChatCompletion, collectChatCompletion, responsesSseToCha import { jsonCompletionSse } from "../../src/server/chat-native-sse"; import { getRequestLogEntries } from "../../src/server/request-log"; import { readUsageEntries } from "../../src/usage/log"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; let upstream: ReturnType | undefined; -afterEach(async () => { await upstream?.stop(true); upstream = undefined; }); +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { + // Taken only for direct Chat dispatches so mixed pure-converter cases do not open the journal. + releaseSpendHome ??= acquireOwnedSpendHome(); +}; +afterEach(async () => { + // Released first so a failed assertion cannot leave the inherited sandbox lease live. + releaseSpendHome?.(); + releaseSpendHome = undefined; + await upstream?.stop(true); + upstream = undefined; +}); interface Chunk { choices: Array<{ index: number; delta: { @@ -34,6 +46,7 @@ async function streamFixture(output: unknown[], status = "completed", cancel = f adapter: "openai-responses", baseUrl: `http://127.0.0.1:${upstream.port}/v1`, authMode: "key", apiKey: "fixture-key", allowPrivateNetwork: true, models: ["model"], } } }; + takeSpendHome(); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "fixture/model", stream: !delivery.jsonFinish, messages: [{ role: "user", content: "fixture" }], @@ -327,6 +340,7 @@ describe("service_tier echo relay", () => { body: JSON.stringify({ model: "fixture/model", stream, messages: [{ role: "user", content: "ping" }] }), }), config, { model: "", provider: "" }, { requestId: `tier-relay-${stream}`, start: Date.now() }); + takeSpendHome(); const jsonResponse = await post(false); expect(jsonResponse.status).toBe(200); expect(await jsonResponse.json()).toMatchObject({ service_tier: "priority", choices: [{ finish_reason: "stop" }] }); diff --git a/tests/responses/chat-refusal.test.ts b/tests/responses/chat-refusal.test.ts index 27f6ff86ef..9e2a81eff7 100644 --- a/tests/responses/chat-refusal.test.ts +++ b/tests/responses/chat-refusal.test.ts @@ -9,6 +9,7 @@ import { jsonCompletionSse, nativeChatSse } from "../../src/server/chat-native-s import type { OcxConfig } from "../../src/types"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; type Rec = Record; @@ -400,13 +401,19 @@ describe("refusal handler delivery matrix", () => { const originalFetch = globalThis.fetch; let isolatedHome: IsolatedCodexHome | undefined; let previousOcxHome: string | undefined; + let releaseSpendHome: (() => void) | undefined; beforeEach(() => { previousOcxHome = process.env.OPENCODEX_HOME; isolatedHome = installIsolatedCodexHome("ocx-refusal-fixture-"); process.env.OPENCODEX_HOME = isolatedHome.path; + // Taken after this matrix installs its home so direct Chat dispatch owns that journal. + releaseSpendHome = acquireOwnedSpendHome(); globalThis.fetch = (async () => { throw new Error("unstubbed external transport"); }) as typeof fetch; }); afterEach(() => { + // Released before the matrix restores its home so no live database survives teardown. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; resetProviderRequestPacingForTest(); if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; diff --git a/tests/responses/responses-compact-handoff-admission.test.ts b/tests/responses/responses-compact-handoff-admission.test.ts index 5fcb2aa67e..3ebe8c7870 100644 --- a/tests/responses/responses-compact-handoff-admission.test.ts +++ b/tests/responses/responses-compact-handoff-admission.test.ts @@ -8,6 +8,7 @@ import { clearCodexUpstreamHealth } from "../../src/codex/routing"; import { clearUpstreamHostHealth } from "../../src/codex/upstream-host-health"; import { handleResponsesCompact } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const originalFetch = globalThis.fetch; @@ -44,7 +45,7 @@ describe("compact handoff route admission namespacing", () => { } as OcxConfig; } - function withPoolEnv(run: (config: OcxConfig) => Promise): Promise { + async function withPoolEnv(run: (config: OcxConfig) => Promise): Promise { const testDir = mkdtempSync(join(tmpdir(), "ocx-compact-handoff-admission-")); const previousOpencodexHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; @@ -60,7 +61,13 @@ describe("compact handoff route admission namespacing", () => { chatgptAccountId: "pool_acc", }); updateAccountQuota("pool-a", 10); - return run(poolConfig()).finally(() => { + // Taken after this helper installs its home so direct compact dispatch owns that journal. + const releaseSpendHome = acquireOwnedSpendHome(); + try { + return await run(poolConfig()); + } finally { + // Released before this helper restores and removes its home so no live database is unlinked. + releaseSpendHome(); globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); clearUpstreamHostHealth(); @@ -70,7 +77,7 @@ describe("compact handoff route admission namespacing", () => { else process.env.OPENCODEX_HOME = previousOpencodexHome; if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; - }); + } } function compactionRequest( @@ -155,8 +162,12 @@ describe("compact handoff route admission namespacing", () => { // The owner stores the deepseek route under (its principal, this lane). const stored = await compact("deepseek/deepseek-v4-flash", owner); - expect(stored.status).toBe(200); - expect(calls).toEqual([{ model: "deepseek-v4-flash", nativeCompact: false }]); + try { + expect(stored.status).toBe(200); + expect(calls).toEqual([{ model: "deepseek-v4-flash", nativeCompact: false }]); + } finally { + await stored.body?.cancel(); + } // A different admitted principal re-sending the same lane header must not // claim it: every attempt stays on the requested model's native compact. @@ -171,16 +182,24 @@ describe("compact handoff route admission namespacing", () => { ] as const) { calls.length = 0; const res = await compact("openai-apikey/gpt-5.6-sol", intruder); - expect(res.status).toBe(502); - expect(calls.length).toBeGreaterThan(0); - expect(calls.every(call => call.model === "gpt-5.6-sol" && call.nativeCompact)).toBe(true); + try { + expect(res.status).toBe(502); + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(call => call.model === "gpt-5.6-sol" && call.nativeCompact)).toBe(true); + } finally { + await res.body?.cancel(); + } } // The owner's own quota-blocked retry still finds the route and hands off. calls.length = 0; const handoff = await compact("openai-apikey/gpt-5.6-sol", owner); - expect(handoff.status).toBe(200); - expect(calls.at(-1)).toEqual({ model: "deepseek-v4-flash", nativeCompact: false }); + try { + expect(handoff.status).toBe(200); + expect(calls.at(-1)).toEqual({ model: "deepseek-v4-flash", nativeCompact: false }); + } finally { + await handoff.body?.cancel(); + } }); // Five request sequences ride the transient-502 retry ladder; the default // 5s budget is not enough on a contended host. diff --git a/tests/responses/ws-upstream.test.ts b/tests/responses/ws-upstream.test.ts index c9afb0cd7d..3cb3312fab 100644 --- a/tests/responses/ws-upstream.test.ts +++ b/tests/responses/ws-upstream.test.ts @@ -30,9 +30,10 @@ import { } from "../../src/server/responses/ws-upstream"; import type { OcxProviderConfig } from "../../src/types"; import type { OcxConfig } from "../../src/types"; +import { BOUNDED_WS_RUNTIME, codexWsUpstreamFetch, shouldUseCodexWsUpstream, streamingInit } from "../helpers/ws-upstream-fixtures"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const CODEX_URL = "https://chatgpt.com/backend-api/codex/responses"; -const BOUNDED_WS_RUNTIME = "1.4.0"; // #864 keeps win32 rewrite traffic out of the tee()+JS-pull chain, so // `isWin32EagerRewrite(platform, needsClientRewrite)` sends it through the eager @@ -45,26 +46,6 @@ const BOUNDED_WS_RUNTIME = "1.4.0"; // constant that only held before the backfill landed. const EAGER_RELAY_FORCED_BY_PLATFORM = isWin32EagerRewrite(process.platform, true); -function shouldUseCodexWsUpstream(url: string, init?: RequestInit, upstreamWebsocket = false): boolean { - return rawShouldUseCodexWsUpstream(url, init, BOUNDED_WS_RUNTIME, upstreamWebsocket); -} - -function codexWsUpstreamFetch( - url: string, - init: RequestInit, - fallback: typeof fetch, -): Promise { - return rawCodexWsUpstreamFetch(url, init, fallback, BOUNDED_WS_RUNTIME); -} - -function streamingInit(body: Record = {}): RequestInit { - return { - method: "POST", - headers: { "content-type": "application/json", authorization: "Bearer test" }, - body: JSON.stringify({ model: "gpt-5.5", stream: true, ...body }), - }; -} - describe("shouldUseCodexWsUpstream", () => { test("uses HTTP SSE on runtimes without a bounded response sink", async () => { expect(bunSupportsBoundedCodexWsRelay("1.3.14")).toBe(false); @@ -230,7 +211,14 @@ beforeEach(() => { for (const key of PROXY_ENV_KEYS) delete process.env[key]; }); +// A case that calls handleResponses directly never takes the writer lease startServer takes, +// so its dispatch is refused. Dropped in teardown so a throwing case cannot leave it behind. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.WebSocket = RealWebSocket; globalThis.fetch = RealFetch; FakeWebSocket.instances = []; @@ -468,6 +456,7 @@ describe("handleResponses Codex WS relay selection", () => { const config = { ...forwardConfig(), plaintextV2AgentMessages: true } as OcxConfig; const request = plaintextV2CollaborationRequest(); + takeSpendHome(); const response = await handleResponses(request, config, { model: "", provider: "" }, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, }); @@ -533,6 +522,7 @@ describe("handleResponses Codex WS relay selection", () => { }); const config = { ...forwardConfig(), plaintextV2AgentMessages: true } as OcxConfig; + takeSpendHome(); const response = await handleResponses( plaintextV2CollaborationRequest(), config, @@ -559,6 +549,7 @@ describe("handleResponses Codex WS relay selection", () => { }); }); + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, }); @@ -581,6 +572,7 @@ describe("handleResponses Codex WS relay selection", () => { { status: 200, headers: { "content-type": "text/event-stream" } }, )) as typeof fetch; + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, }); @@ -602,6 +594,7 @@ describe("handleResponses Codex WS relay selection", () => { }); const logCtx = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, }); @@ -621,6 +614,7 @@ describe("handleResponses Codex WS relay selection", () => { }); const logCtx = { model: "", provider: "" }; + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, }); @@ -645,6 +639,7 @@ describe("handleResponses Codex WS relay selection", () => { { status: 200, headers: { "content-type": "text/event-stream" } }, )) as typeof fetch; + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); expect(FakeWebSocket.instances).toHaveLength(0); @@ -682,6 +677,7 @@ describe("handleResponses Codex WS relay selection", () => { { status: 200, headers: { "content-type": "text/event-stream" } }, )) as typeof fetch; + takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); const text = await response.text(); @@ -730,6 +726,7 @@ describe("codexWsUpstreamFetch", () => { ws.emit("message", { data: JSON.stringify({ type: "response.completed", response: { id: "r1", status: "completed", output: [] } }) }); }; globalThis.WebSocket = CapturingSocket as unknown as typeof WebSocket; + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { authorization: "Bearer fixture", "content-type": "application/json", "x-openai-internal-codex-responses-lite": "true" }, diff --git a/tests/routing/combo-management-api.test.ts b/tests/routing/combo-management-api.test.ts index ba6bd7e2d0..7bdd2fb69e 100644 --- a/tests/routing/combo-management-api.test.ts +++ b/tests/routing/combo-management-api.test.ts @@ -41,6 +41,7 @@ import type { OcxConfig } from "../../src/types"; import { syncCatalogModels } from "../../src/codex/catalog"; import { injectClaudeAgentDefs } from "../../src/claude/agents-inject"; import { catalogConvergenceFactory } from "../helpers/catalog-convergence"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const VALID_COMBO = { targets: [{ provider: "a", model: "m1" }] }; @@ -1250,6 +1251,8 @@ describe("supported disabled-provider activation", () => { return Response.json({ error: { message: "default provider must not be reached" } }, { status: 500 }); }, }); + // Taken after withTempHome installs this case's home so physical dispatch owns its ledger. + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = baseConfig({ defaultProvider: "c", @@ -1283,9 +1286,13 @@ describe("supported disabled-provider activation", () => { headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: false }), }), config, { model: "", provider: "" }); - expect(routed.status).toBe(200); - expect(bHits).toBe(1); - expect(cHits).toBe(0); + try { + expect(routed.status).toBe(200); + expect(bHits).toBe(1); + expect(cHits).toBe(0); + } finally { + await routed.body?.cancel(); + } expect((await comboApi(config, "PATCH", "/api/providers?name=b", { disabled: true }))?.status).toBe(200); const diagnostics = readConfigDiagnostics(); @@ -1305,6 +1312,8 @@ describe("supported disabled-provider activation", () => { expect(bHits).toBe(1); expect(cHits).toBe(0); } finally { + // Released before withTempHome removes the directory, preventing a live database there. + releaseSpendHome(); await upstreamB.stop(true); await upstreamC.stop(true); } @@ -1335,6 +1344,8 @@ describe("combo response-path strategy accounting", () => { let bHits = 0; const upstreamA = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { aHits += 1; return completion("a"); } }); const upstreamB = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { bHits += 1; return completion("b"); } }); + // Taken after the inherited test home is in effect so both physical dispatches can record. + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = baseConfig({ providers: { @@ -1343,10 +1354,22 @@ describe("combo response-path strategy accounting", () => { }, combos: { free: { strategy: "least-used", targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] } }, }); - expect((await handleResponses(responseRequest(), config, { model: "", provider: "" })).status).toBe(200); - expect((await handleResponses(responseRequest(), config, { model: "", provider: "" })).status).toBe(200); + const first = await handleResponses(responseRequest(), config, { model: "", provider: "" }); + try { + expect(first.status).toBe(200); + } finally { + await first.body?.cancel(); + } + const second = await handleResponses(responseRequest(), config, { model: "", provider: "" }); + try { + expect(second.status).toBe(200); + } finally { + await second.body?.cancel(); + } expect({ aHits, bHits }).toEqual({ aHits: 1, bHits: 1 }); } finally { + // Released after both bodies are cancelled so no stream retains the ledger owner. + releaseSpendHome(); await upstreamA.stop(true); await upstreamB.stop(true); } @@ -1361,6 +1384,8 @@ describe("combo response-path strategy accounting", () => { fetch() { aHits += 1; return Response.json({ error: { message: "busy" } }, { status: 429, headers: { "retry-after": "60" } }); }, }); const upstreamB = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { bHits += 1; return completion("b"); } }); + // Taken after the inherited test home is in effect so failover dispatch can record its sends. + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = baseConfig({ providers: { @@ -1370,10 +1395,16 @@ describe("combo response-path strategy accounting", () => { combos: { free: { strategy: "reset-window", targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] } }, }); const response = await handleResponses(responseRequest(), config, { model: "", provider: "" }); - expect(response.status).toBe(200); - expect({ aHits, bHits }).toEqual({ aHits: 1, bHits: 1 }); - expect(isComboTargetInCooldown("free", { provider: "a", model: "m1" })).toBe(true); + try { + expect(response.status).toBe(200); + expect({ aHits, bHits }).toEqual({ aHits: 1, bHits: 1 }); + expect(isComboTargetInCooldown("free", { provider: "a", model: "m1" })).toBe(true); + } finally { + await response.body?.cancel(); + } } finally { + // Released after the response body is cancelled so no stream retains the ledger owner. + releaseSpendHome(); await upstreamA.stop(true); await upstreamB.stop(true); } From 92c2dae321daca6653be060353414f571f34109e Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:08:27 +0900 Subject: [PATCH 19/34] test(spend): lease the four mixed fixtures the transitive audit found All four run a real server in most of their cases and dispatch directly in a few, so none of them can take a file-wide lease: the lease is taken by the cases that dispatch in-process and by nothing else. cursor-effort-rows takes it per case, inside the try that already stops the mock upstream, and releases it first in that finally. claude-messages-endpoint takes it inside invokeMessages, beside the turn-admission lease it already holds, and releases it in the same finally. chat-completions-endpoint and server-combo-failover-e2e take it at each direct dispatch and drop it at the top of the afterEach they already have, ahead of the home restore and removal. Where a case here also starts a server, the acquire is a second reference on the same directory rather than a competing owner, so the server keeps serving and the release only drops the reference this case added. That is different from a CHILD process, which cannot acquire at all while the parent holds the lease; the one fixture in that shape was repaired separately. server-combo-failover-e2e needed room first: it was one line under its cap, so its five upstream response builders move to a sibling helper, verbatim. Assertion counts are unchanged in all four files. No cap moves. Local checks: NOT RUN. --- .../claude-messages-endpoint.test.ts | 5 ++ tests/helpers/combo-failover-upstream.ts | 60 +++++++++++++++ .../cursor/cursor-effort-rows.test.ts | 11 +++ .../chat-completions-endpoint.test.ts | 20 +++++ .../server/server-combo-failover-e2e.test.ts | 75 ++++++------------- 5 files changed, 117 insertions(+), 54 deletions(-) create mode 100644 tests/helpers/combo-failover-upstream.ts diff --git a/tests/claude-integration/claude-messages-endpoint.test.ts b/tests/claude-integration/claude-messages-endpoint.test.ts index a84e09a878..461397f174 100644 --- a/tests/claude-integration/claude-messages-endpoint.test.ts +++ b/tests/claude-integration/claude-messages-endpoint.test.ts @@ -31,6 +31,7 @@ import { estimateTokens } from "../../src/lib/token-estimate"; import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { SERVER_BUDGET_MS } from "../helpers/test-budget"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; import { @@ -1110,6 +1111,9 @@ test("routed Claude requests give OpenAI sidecars main auth without leaking it t const invokeMessages = async (): Promise => { const turnAdmissionLease = tryAdmitTurn(); if (!turnAdmissionLease) throw new Error("test turn admission unavailable"); + // Held for exactly this dispatch, beside the turn lease it already takes. File-wide would be + // wrong: most cases in this file start a real server, and that server takes the same lease. + const releaseSpendHome = acquireOwnedSpendHome(); const start = Date.now(); try { const response = await handleClaudeMessages( @@ -1129,6 +1133,7 @@ test("routed Claude requests give OpenAI sidecars main auth without leaking it t await response.text(); return response.status; } finally { + releaseSpendHome(); turnAdmissionLease.release(); } }; diff --git a/tests/helpers/combo-failover-upstream.ts b/tests/helpers/combo-failover-upstream.ts new file mode 100644 index 0000000000..ff9d0ee62b --- /dev/null +++ b/tests/helpers/combo-failover-upstream.ts @@ -0,0 +1,60 @@ +/** + * Upstream response shapes for the combo failover suite. + * + * Moved verbatim out of tests/server/server-combo-failover-e2e.test.ts: that file is one line + * under its file-size cap and needed room to take the spend-journal lease. These builders + * decide nothing; each returns exactly the payload its callers were already constructing. + */ +export function chatSuccess(text: string, model = "model"): Response { + return Response.json({ + id: `chatcmpl-${model}`, + object: "chat.completion", + model, + choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }], + usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 }, + }); +} + +export function chatStream(text: string): Response { + const frames = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: text }, finish_reason: null }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + return new Response(frames, { headers: { "content-type": "text/event-stream" } }); +} + +export function chatTruncatedZeroOutputStream(): Response { + const frames = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: null }] })}\n\n`, + ].join(""); + return new Response(frames, { headers: { "content-type": "text/event-stream" } }); +} + +export function chatErrorStream(message: string, prefix?: string): Response { + const frames = [ + ...(prefix + ? [`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: prefix }, finish_reason: null }] })}\n\n`] + : []), + `data: ${JSON.stringify({ error: { type: "server_error", code: "upstream_server_error", message } })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + return new Response(frames, { headers: { "content-type": "text/event-stream" } }); +} + +export function responsesSuccess(text: string, model = "responses-model"): Record { + return { + id: `resp-${model}`, + object: "response", + status: "completed", + model, + output: [{ + id: "msg_backup", + type: "message", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text, annotations: [] }], + }], + usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, + }; +} diff --git a/tests/providers/cursor/cursor-effort-rows.test.ts b/tests/providers/cursor/cursor-effort-rows.test.ts index e6296b9de8..235d6e0256 100644 --- a/tests/providers/cursor/cursor-effort-rows.test.ts +++ b/tests/providers/cursor/cursor-effort-rows.test.ts @@ -21,6 +21,7 @@ import { startServer } from "../../../src/server"; import type { RequestLogContext } from "../../../src/server/request-log"; import type { OcxConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { SERVER_BUDGET_MS } from "../../helpers/test-budget"; setDefaultTimeout(SERVER_BUDGET_MS); @@ -206,6 +207,9 @@ describe("Cursor effort variant rows", () => { test("Responses effort rows route the base model and pass through the existing cap", async () => { const upstream = mockChatUpstream(); + // Case-local, never file-wide: this file also starts a real server elsewhere, and that + // server takes the same lease. A lease held across those cases would refuse their startup. + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = ingressConfig(`${upstream.server.url.toString().replace(/\/$/u, "")}/v1`); const response = await handleResponses(new Request("http://localhost/v1/responses", { @@ -224,12 +228,14 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { + releaseSpendHome(); upstream.server.stop(true); } }); test("Chat effort rows use Responses normalization instead of the native-chat shortcut", async () => { const upstream = mockChatUpstream(); + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = ingressConfig(`${upstream.server.url.toString().replace(/\/$/u, "")}/v1`); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { @@ -248,12 +254,14 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { + releaseSpendHome(); upstream.server.stop(true); } }); test("Messages effort rows resolve after route directives and before native passthrough", async () => { const upstream = mockChatUpstream(); + const releaseSpendHome = acquireOwnedSpendHome(); try { const config = ingressConfig(`${upstream.server.url.toString().replace(/\/$/u, "")}/v1`); const response = await handleClaudeMessages(new Request("http://localhost/v1/messages", { @@ -278,6 +286,7 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { + releaseSpendHome(); upstream.server.stop(true); } }); @@ -285,6 +294,7 @@ describe("Cursor effort variant rows", () => { test("mixed-case aliases ending in an effort stay on their configured provider", async () => { const intended = mockChatUpstream(); const fallback = mockChatUpstream(); + const releaseSpendHome = acquireOwnedSpendHome(); try { const config: OcxConfig = { port: 0, @@ -352,6 +362,7 @@ describe("Cursor effort variant rows", () => { expect(intended.captured.every(body => body.model === "private-model")).toBe(true); expect(fallback.captured).toHaveLength(0); } finally { + releaseSpendHome(); intended.server.stop(true); fallback.server.stop(true); } diff --git a/tests/responses/chat-completions-endpoint.test.ts b/tests/responses/chat-completions-endpoint.test.ts index d23a857b8b..b87b317777 100644 --- a/tests/responses/chat-completions-endpoint.test.ts +++ b/tests/responses/chat-completions-endpoint.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadConfig, saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { ownedServiceHomeInspection } from "../helpers/owned-service-home-inspection"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; @@ -60,6 +61,11 @@ let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; const originalFetch = globalThis.fetch; +// A case that calls a handler directly never takes the writer lease startServer takes, so its +// dispatch is refused. Never file-wide: most cases here start a real server that takes it too. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-chat-completions-"); @@ -69,6 +75,8 @@ beforeEach(() => { }); afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; resetProviderRequestPacingForTest(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; @@ -367,6 +375,7 @@ async function driveChatFallbackServiceTier( }, }, } as OcxConfig; + takeSpendHome(); const response = await handleChatCompletions( new Request("http://localhost/v1/chat/completions", { method: "POST", @@ -973,8 +982,10 @@ test("chat-native consumes pacing before the response-header timeout starts", as }), }); + takeSpendHome(); const first = await handleChatCompletions(request(), config, {} as Parameters[2]); expect(first.status).toBe(200); + takeSpendHome(); const secondPending = handleChatCompletions(request(), config, {} as Parameters[2]); await Bun.sleep(5); expect(starts).toBe(1); @@ -1004,6 +1015,7 @@ test("chat-native stays outside the Responses empty-completion retry guard", asy } as Partial & { fetch: typeof globalThis.fetch }); config.emptyCompletionRetry = true; + takeSpendHome(); const response = await handleChatCompletions( new Request("http://localhost/v1/chat/completions", { method: "POST", @@ -1632,6 +1644,7 @@ test("chat-native client cancellation cancels the upstream stream and logs 499", body: JSON.stringify({ model: "mock/test-model", stream: true, messages: [{ role: "user", content: "hi" }] }), }); const logCtx = {} as Parameters[2]; + takeSpendHome(); const response = await handleChatCompletions( request, mockConfig("https://provider.example/v1"), @@ -1683,6 +1696,7 @@ test("chat-native request abort releases its active-turn lease and logs 499", as body: JSON.stringify({ model: "mock/test-model", stream: true, messages: [{ role: "user", content: "hi" }] }), signal: clientAbort.signal, }); + takeSpendHome(); const response = await handleChatCompletions( request, mockConfig("https://provider.example/v1"), @@ -1722,6 +1736,7 @@ test("chat-native cancelled non-streaming SSE returns 499 instead of partial suc readStarted(); }, }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + takeSpendHome(); const result = handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", signal: clientAbort.signal, headers: { "content-type": "application/json" }, @@ -1755,6 +1770,7 @@ test("chat-native SSE enforces configured stall timeout despite non-progress fra cancel() { cancels += 1; clearInterval(timer); }, }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; try { + takeSpendHome(); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", signal: clientAbort.signal, headers: { "content-type": "application/json" }, @@ -1854,6 +1870,7 @@ test("chat-native valid terminal retains precedence over a late non-streaming ab }, cancel() { clientAbort.abort("late cancellation after terminal"); }, }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + takeSpendHome(); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", signal: clientAbort.signal, headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), @@ -1872,6 +1889,7 @@ test("chat-native non-streaming SSE reports a typed stall failure instead of par }, cancel() { cancels += 1; }, }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + takeSpendHome(); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), @@ -1918,6 +1936,7 @@ test("chat-native non-streaming SSE collects CRLF multiline and split UTF-8 fram controller.close(); }, }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + takeSpendHome(); const response = await handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "mock/test-model", stream: false, messages: [{ role: "user", content: "hi" }] }), @@ -1942,6 +1961,7 @@ test("chat-native direct streaming without an admission lease does not record a headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "mock/test-model", stream: true, messages: [{ role: "user", content: "hi" }] }), }); + takeSpendHome(); const response = await handleChatCompletions( request, mockConfig("https://provider.example/v1"), diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index afbfc5ecac..c5dcae6637 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -38,6 +38,8 @@ import { startServer } from "../../src/server"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; import { catalogConvergenceFactory } from "../helpers/catalog-convergence"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { chatErrorStream, chatStream, chatSuccess, chatTruncatedZeroOutputStream, responsesSuccess } from "../helpers/combo-failover-upstream"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { heldResponse } from "../helpers/held-response"; import { clearResponseStateForTests, @@ -142,6 +144,12 @@ let originalFetch: typeof fetch; let originalNow: () => number; const servers: Array> = []; +// A case that calls a handler directly never takes the writer lease startServer takes, so its +// dispatch is refused. Taken at the dispatch helpers rather than file-wide: the cases that do +// start a real server already own it, and this only ever adds a reference on the same home. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + beforeEach(() => { originalFetch = globalThis.fetch; originalNow = Date.now; @@ -167,6 +175,8 @@ beforeEach(() => { }); afterEach(async () => { + releaseSpendHome?.(); + releaseSpendHome = undefined; let responseStatePending = true; try { for (const server of servers.splice(0)) await server.stop(true); @@ -204,60 +214,6 @@ function baseUrl(server: ReturnType): string { return `${server.url.toString().replace(/\/$/, "")}/v1`; } -function chatSuccess(text: string, model = "model"): Response { - return Response.json({ - id: `chatcmpl-${model}`, - object: "chat.completion", - model, - choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }], - usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 }, - }); -} - -function chatStream(text: string): Response { - const frames = [ - `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: text }, finish_reason: null }] })}\n\n`, - `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } })}\n\n`, - "data: [DONE]\n\n", - ].join(""); - return new Response(frames, { headers: { "content-type": "text/event-stream" } }); -} - -function chatTruncatedZeroOutputStream(): Response { - const frames = [ - `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: null }] })}\n\n`, - ].join(""); - return new Response(frames, { headers: { "content-type": "text/event-stream" } }); -} - -function chatErrorStream(message: string, prefix?: string): Response { - const frames = [ - ...(prefix - ? [`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: prefix }, finish_reason: null }] })}\n\n`] - : []), - `data: ${JSON.stringify({ error: { type: "server_error", code: "upstream_server_error", message } })}\n\n`, - "data: [DONE]\n\n", - ].join(""); - return new Response(frames, { headers: { "content-type": "text/event-stream" } }); -} - -function responsesSuccess(text: string, model = "responses-model"): Record { - return { - id: `resp-${model}`, - object: "response", - status: "completed", - model, - output: [{ - id: "msg_backup", - type: "message", - role: "assistant", - status: "completed", - content: [{ type: "output_text", text, annotations: [] }], - }], - usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, - }; -} - function comboConfig( providers: OcxConfig["providers"], targets = Object.keys(providers).map((name, index) => ({ provider: name, model: `m${index + 1}` })), @@ -277,6 +233,7 @@ async function post( options: HandleOptions = {}, headers: Record = {}, ): Promise { + takeSpendHome(); return handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, @@ -294,6 +251,7 @@ async function postLogged( ): Promise { const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, @@ -317,6 +275,7 @@ async function postModelLogged( ): Promise { const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, @@ -2064,6 +2023,7 @@ describe("server combo failover 030 activation matrix", () => { : authKind === "org-only-jwt" ? "org-foreign" : "acct-scoped-sidecar", }; const response = authKind === "chat-valid" + takeSpendHome(); ? await (await import("../../src/server/chat-completions")).handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model: "combo/free", messages: [{ role: "user", content: "search" }], stream: true, tools: [{ type: "web_search" }] }), @@ -3383,6 +3343,7 @@ describe("server combo failover 030 activation matrix", () => { let cancels = 0; const parent: RequestLogContext = { model: "", provider: "" }; const snapshots: RequestLogContext[] = []; + takeSpendHome(); const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", session_id: "hop-recall" }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: true }), @@ -3426,6 +3387,7 @@ describe("server combo failover 030 activation matrix", () => { const finalized = deferred(); const observed: Array<{ status: number; log: RequestLogContext }> = []; try { + takeSpendHome(); const response = await within(handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: true }), @@ -3688,6 +3650,7 @@ describe("cursor conversation continuity across store:false chains", () => { } async function postCursor(config: OcxConfig, raw: Record): Promise { + takeSpendHome(); return handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -3784,6 +3747,7 @@ describe("cursor conversation continuity across store:false chains", () => { const seen: string[] = []; customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); + takeSpendHome(); const postThreadTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { @@ -3814,6 +3778,7 @@ describe("cursor conversation continuity across store:false chains", () => { const seen: string[] = []; customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); + takeSpendHome(); const postDesktopTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { @@ -3844,6 +3809,7 @@ describe("cursor conversation continuity across store:false chains", () => { const seen: string[] = []; customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); + takeSpendHome(); const postThreadTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { @@ -3883,6 +3849,7 @@ describe("combo compact failover", () => { async function postCompactLogged(config: OcxConfig): Promise { const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); + takeSpendHome(); const response = await handleResponsesCompact(compactRequest({ model: "combo/free", stream: false, From a6251415a35668d06db85575cfbf52b3850a4e5a Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:10:52 +0900 Subject: [PATCH 20/34] test(spend): lease the websocket steering path at its shared fixture The steering rows did not fail with an ownership message. They failed with 'fixture condition timed out' after roughly a second, because the websocket handler dispatches through the real request path: the turn was refused, the response.created frame never arrived, and the wait expired naming nothing. Two dozen rows across three files reported it that way. The lease is taken where the turn begins, not row by row. beginInjection in the shared native-injection fixture covers ws-native-result-continuations and ws-steering-stability, and installInjectionFixture drops it in the afterEach it already registers. ws-native-steering has its own begin() and hooks and gets the same treatment there. responses-snapshot-repair-server has one row that calls the handler directly while every other row starts a real server, so that row takes the lease itself and the file drops it in teardown. Assertion counts unchanged. No cap moves. Local checks: NOT RUN. --- tests/helpers/native-injection-fixture.ts | 9 +++++++++ .../responses/responses-snapshot-repair-server.test.ts | 10 ++++++++++ tests/responses/ws-native-steering.test.ts | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index 07f2703334..97e4875469 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -6,6 +6,12 @@ import type { ServeOptionsContext } from "../../src/server/index/serve-options"; import type { WsData } from "../../src/server/ws-bridge"; import { clearRequestLogsForTests } from "../../src/server/request-log"; import { runOptionalShutdownHooks } from "../../src/lib/optional-shutdown-hooks"; +import { acquireOwnedSpendHome } from "./owned-spend-home"; + +// The websocket handler dispatches through the real request path, so it reaches the shared spend +// journal and needs the writer lease startServer would have taken. Without it the turn is refused +// and the symptom is the fixture's own wait timing out, which names nothing. +let releaseSpendHome: (() => void) | undefined; export type Frame = Record; const realSocket = globalThis.WebSocket; @@ -61,6 +67,7 @@ export function injectionClient(fields: Frame = {}, settings = injectionConfig() return { ws, sent, send, handler }; } export async function beginInjection(fields: Frame = {}, settings = injectionConfig(), credential = "test") { + releaseSpendHome ??= acquireOwnedSpendHome(); const client = injectionClient(fields, settings, credential); await waitForInjection(() => client.sent.some(frame => frame.type === "response.created")); const socket = InjectionSocket.all.at(-1)!; @@ -99,6 +106,8 @@ export function installInjectionFixture() { clearRequestLogsForTests(); }); afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; for (const socket of InjectionSocket.all) socket.close(); InjectionSocket.all = []; runOptionalShutdownHooks(); globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; diff --git a/tests/responses/responses-snapshot-repair-server.test.ts b/tests/responses/responses-snapshot-repair-server.test.ts index f6e4ac0e92..301b47e751 100644 --- a/tests/responses/responses-snapshot-repair-server.test.ts +++ b/tests/responses/responses-snapshot-repair-server.test.ts @@ -10,6 +10,11 @@ import { createGrokResponsesControlFrameBlockRewrite } from "../../src/server/gr import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +// Case-local, never file-wide: the other rows start a real server that takes the same lease. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; setDefaultTimeout(30_000); @@ -114,6 +119,8 @@ beforeEach(() => { }); afterEach(async () => { + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.fetch = originalFetch; await isolated.restore(); removeTreeWithRetry(TEST_DIR); @@ -195,6 +202,9 @@ describe("responsesSnapshotRepair through /v1/responses", () => { }, } as OcxConfig; + // This row calls the handler directly instead of going through the server the other rows + // start, so it takes the writer lease itself. Dropped in the file's own teardown. + takeSpendHome(); const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index e2e9983109..5b787a05e1 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -10,6 +10,12 @@ import { getRequestLogEntries, clearRequestLogsForTests } from "../../src/server import { runOptionalShutdownHooks } from "../../src/lib/optional-shutdown-hooks"; import { MAX_ACTIVE_TURNS, tryAdmitTurn } from "../../src/server/lifecycle"; import { configSchema } from "../../src/config/schema/config-schema"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +// The websocket handler dispatches through the real request path, so it reaches the shared spend +// journal and needs the writer lease startServer would have taken. Without it the turn is refused +// and the symptom is this file's own waitFor timing out, which names nothing. +let releaseSpendHome: (() => void) | undefined; type Frame = Record; const realSocket = globalThis.WebSocket; @@ -58,6 +64,7 @@ function downstream(fields: Frame = {}, settings = config(), credential = "test" return { ws, sent, send, handler }; } async function begin(fields: Frame = {}, credential = "test") { + releaseSpendHome ??= acquireOwnedSpendHome(); const client = downstream(fields, config(), credential); await waitFor(() => client.sent.some(frame => frame.type === "response.created")); const socket = Socket.all.find(s => s.options.headers.authorization === `Bearer ${credential}`)!; @@ -79,6 +86,8 @@ beforeEach(() => { clearRequestLogsForTests(); }); afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; for (const socket of Socket.all) socket.close(); Socket.all = []; runOptionalShutdownHooks(); From b6328e893f08ceefc36b1a7165188f0a861746c0 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:25:04 +0900 Subject: [PATCH 21/34] test(spend): fix the audit batch and the two ownership-identity traps The critical one first: a scripted insertion put takeSpendHome() between a ternary's condition and its question mark in server-combo-failover-e2e, which is not valid TypeScript. It now sits above the statement. An AST syntax screen over every changed file finds no remaining parse error. Two failures had the same root and neither announced itself as an ownership problem. The websocket fixtures took the lease in begin() and beginInjection(), but the failing rows call downstream() and injectionClient() directly, so the lease moved down to the seam where the client actually dispatches. And the subagent streaming rows force process.platform to win32 to reach the eager-relay path; ownership identity lowercases the state directory on win32, so the lease taken under the real platform stopped matching the directory the dispatch checks the moment the override landed, and the turn was refused with no terminal at all. Those rows now retake the lease under the platform they are pretending to run on and give it back before restoring the descriptor. Release ordering is corrected wherever teardown can still settle a turn. The steering fixtures close their synthetic clients, then the upstream sockets, then run the shutdown hooks, and only then release. The cursor rows stop their fake upstreams first. The combo suite stops its listeners and flushes response state first, and still releases before its home is removed. The Claude endpoint case now holds one lease across both of its turns instead of taking a fresh one per invocation, and drops it after both upstreams are down. Six files that dispatch through the shared agent-task-recovery post() helper take the lease too. One of them installs its own OPENCODEX_HOME inside a describe, so its lease is taken there rather than at file level, for the same reason as the platform case: a lease binds the directory in effect when it was taken. The sparse-JSON snapshot-repair row was still missing one. No assertion, expected value, mock, fixture or timeout changes; no cap moves. Local checks: NOT RUN. The syntax screen is a pure AST read and is not a substitute for hosted typecheck or tests. --- .../claude-messages-endpoint.test.ts | 10 ++++++---- tests/helpers/native-injection-fixture.ts | 13 ++++++++++--- tests/providers/cursor/cursor-effort-rows.test.ts | 10 ++++++---- .../responses-snapshot-repair-server.test.ts | 2 ++ tests/responses/ws-native-steering.test.ts | 13 ++++++++++--- .../subagent-fallback-handle-responses.test.ts | 8 ++++++++ tests/server/agent-task-recovery-cache.test.ts | 11 +++++++++++ tests/server/agent-task-recovery-combo.test.ts | 11 +++++++++++ tests/server/agent-task-recovery-fallback.test.ts | 11 +++++++++++ tests/server/agent-task-recovery-security.test.ts | 11 +++++++++++ tests/server/agent-task-recovery.test.ts | 11 +++++++++++ .../server-agent-task-recovery-replay.test.ts | 13 ++++++++++++- tests/server/server-combo-failover-e2e.test.ts | 8 +++++--- 13 files changed, 114 insertions(+), 18 deletions(-) diff --git a/tests/claude-integration/claude-messages-endpoint.test.ts b/tests/claude-integration/claude-messages-endpoint.test.ts index 461397f174..ad5fadf5ab 100644 --- a/tests/claude-integration/claude-messages-endpoint.test.ts +++ b/tests/claude-integration/claude-messages-endpoint.test.ts @@ -1111,9 +1111,6 @@ test("routed Claude requests give OpenAI sidecars main auth without leaking it t const invokeMessages = async (): Promise => { const turnAdmissionLease = tryAdmitTurn(); if (!turnAdmissionLease) throw new Error("test turn admission unavailable"); - // Held for exactly this dispatch, beside the turn lease it already takes. File-wide would be - // wrong: most cases in this file start a real server, and that server takes the same lease. - const releaseSpendHome = acquireOwnedSpendHome(); const start = Date.now(); try { const response = await handleClaudeMessages( @@ -1133,10 +1130,13 @@ test("routed Claude requests give OpenAI sidecars main auth without leaking it t await response.text(); return response.status; } finally { - releaseSpendHome(); turnAdmissionLease.release(); } }; + // One lease for the whole case rather than one per invocation: both turns run against the same + // home, and re-taking it between them would discard the ledger this case is still accounting + // into. File-wide would be wrong the other way, since most cases here start a real server. + const releaseSpendHome = acquireOwnedSpendHome(); try { expect(await invokeMessages()).toBe(200); @@ -1166,6 +1166,8 @@ test("routed Claude requests give OpenAI sidecars main auth without leaking it t } finally { await forward.stop(true); await routed.stop(true); + // After both upstreams are down, so nothing is still settling against the journal. + releaseSpendHome(); } }); diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index 97e4875469..a6a157e6f5 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -12,6 +12,9 @@ import { acquireOwnedSpendHome } from "./owned-spend-home"; // journal and needs the writer lease startServer would have taken. Without it the turn is refused // and the symptom is the fixture's own wait timing out, which names nothing. let releaseSpendHome: (() => void) | undefined; +// Every synthetic client this fixture opens, so teardown can close them before the lease is +// given back rather than leaving a handler mid-turn against a journal nobody owns. +const clients: Array<{ close(): void }> = []; export type Frame = Record; const realSocket = globalThis.WebSocket; @@ -56,18 +59,19 @@ export const waitForInjection = async (condition: () => boolean) => { throw new Error("injection fixture condition timed out"); }; export function injectionClient(fields: Frame = {}, settings = injectionConfig(), credential = "test") { + releaseSpendHome ??= acquireOwnedSpendHome(); const handler = createWebsocketHandler({ config: settings, deps: {} } as ServeOptionsContext); const sent: Frame[] = []; const ws = { readyState: 1, data: { headers: new Headers({ authorization: `Bearer ${credential}`, "thread-id": `injection-fixture-${++nextId}`, "openai-beta": "fixture_beta=v1" }) } as WsData, send: (text: string) => { sent.push(JSON.parse(text)); return 1; }, close() { handler.close(ws, 1000, "fixture close"); }, } as unknown as ServerWebSocket; const send = (frame: Frame) => handler.message(ws, JSON.stringify(frame)); + clients.push(ws); send({ type: "response.create", model: settings.defaultProvider === "api" ? "api/gpt-5.6-sol" : "gpt-5.6-sol", input: "initial", multi_agent: { enabled: true }, tools: [{ type: "function", name: "get_value", parameters: { type: "object", properties: {} } }], ...fields }); return { ws, sent, send, handler }; } export async function beginInjection(fields: Frame = {}, settings = injectionConfig(), credential = "test") { - releaseSpendHome ??= acquireOwnedSpendHome(); const client = injectionClient(fields, settings, credential); await waitForInjection(() => client.sent.some(frame => frame.type === "response.created")); const socket = InjectionSocket.all.at(-1)!; @@ -106,10 +110,13 @@ export function installInjectionFixture() { clearRequestLogsForTests(); }); afterEach(() => { - releaseSpendHome?.(); - releaseSpendHome = undefined; + for (const client of clients.splice(0)) client.close(); for (const socket of InjectionSocket.all) socket.close(); InjectionSocket.all = []; runOptionalShutdownHooks(); + // Released only after the clients, the upstream sockets and the shutdown hooks: each can + // still settle a turn that accounts against the journal this lease owns. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } }); diff --git a/tests/providers/cursor/cursor-effort-rows.test.ts b/tests/providers/cursor/cursor-effort-rows.test.ts index 235d6e0256..d3f4f92236 100644 --- a/tests/providers/cursor/cursor-effort-rows.test.ts +++ b/tests/providers/cursor/cursor-effort-rows.test.ts @@ -228,8 +228,10 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { - releaseSpendHome(); upstream.server.stop(true); + // The fake upstreams stop first: a turn still settling against the journal must not + // outlive the lease that owns it. + releaseSpendHome(); } }); @@ -254,8 +256,8 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { - releaseSpendHome(); upstream.server.stop(true); + releaseSpendHome(); } }); @@ -286,8 +288,8 @@ describe("Cursor effort variant rows", () => { reasoning_effort: "high", }); } finally { - releaseSpendHome(); upstream.server.stop(true); + releaseSpendHome(); } }); @@ -362,9 +364,9 @@ describe("Cursor effort variant rows", () => { expect(intended.captured.every(body => body.model === "private-model")).toBe(true); expect(fallback.captured).toHaveLength(0); } finally { - releaseSpendHome(); intended.server.stop(true); fallback.server.stop(true); + releaseSpendHome(); } }); diff --git a/tests/responses/responses-snapshot-repair-server.test.ts b/tests/responses/responses-snapshot-repair-server.test.ts index 301b47e751..fc3c098c52 100644 --- a/tests/responses/responses-snapshot-repair-server.test.ts +++ b/tests/responses/responses-snapshot-repair-server.test.ts @@ -454,6 +454,8 @@ describe("responsesSnapshotRepair through /v1/responses", () => { test("sparse JSON completion inference precedes function repair in client output and replay", async () => { const expected = '{"cell_id":"4","yield_time_ms":120000}'; + // Dispatches directly rather than through a server, so it takes the lease itself. + takeSpendHome(); const item = { type: "function_call", id: "fc_sparse_wait", call_id: "call_sparse_wait", name: "wait", arguments: '{"cell_id":4,"yield_time_ms":120000.0}' }; let responseId = `resp_sparse_${crypto.randomUUID()}`; let capturedInput: Array> = []; diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index 5b787a05e1..bb9b3e88da 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -16,6 +16,9 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; // journal and needs the writer lease startServer would have taken. Without it the turn is refused // and the symptom is this file's own waitFor timing out, which names nothing. let releaseSpendHome: (() => void) | undefined; +// Every synthetic client this file opens, so teardown can close them before the lease is given +// back rather than leaving a handler mid-turn against a journal nobody owns. +const clients: Array<{ close(): void }> = []; type Frame = Record; const realSocket = globalThis.WebSocket; @@ -54,17 +57,18 @@ const waitFor = async (condition: () => boolean) => { throw new Error("fixture condition timed out"); }; function downstream(fields: Frame = {}, settings = config(), credential = "test") { + releaseSpendHome ??= acquireOwnedSpendHome(); const handler = createWebsocketHandler({ config: settings, deps: {} } as ServeOptionsContext); const sent: Frame[] = []; const ws = { readyState: 1, data: { headers: new Headers({ authorization: `Bearer ${credential}`, "thread-id": `fixture-${credential}`, session_id: `fixture-${credential}` }) } as WsData, send: (text: string) => { sent.push(JSON.parse(text)); return 1; }, close() { handler.close(ws); }, } as unknown as ServerWebSocket; const send = (frame: Frame) => handler.message(ws, JSON.stringify(frame)); + clients.push(ws); send({ type: "response.create", model: "gpt-5.5", input: "initial", ...fields }); return { ws, sent, send, handler }; } async function begin(fields: Frame = {}, credential = "test") { - releaseSpendHome ??= acquireOwnedSpendHome(); const client = downstream(fields, config(), credential); await waitFor(() => client.sent.some(frame => frame.type === "response.created")); const socket = Socket.all.find(s => s.options.headers.authorization === `Bearer ${credential}`)!; @@ -86,11 +90,14 @@ beforeEach(() => { clearRequestLogsForTests(); }); afterEach(() => { - releaseSpendHome?.(); - releaseSpendHome = undefined; + for (const client of clients.splice(0)) client.close(); for (const socket of Socket.all) socket.close(); Socket.all = []; runOptionalShutdownHooks(); + // Released only after the clients, the upstream sockets and the shutdown hooks, because each + // of those can still settle a turn that accounts against the journal this lease owns. + releaseSpendHome?.(); + releaseSpendHome = undefined; globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } diff --git a/tests/routing/subagent-fallback-handle-responses.test.ts b/tests/routing/subagent-fallback-handle-responses.test.ts index 3b042433c1..98243dc09e 100644 --- a/tests/routing/subagent-fallback-handle-responses.test.ts +++ b/tests/routing/subagent-fallback-handle-responses.test.ts @@ -2160,6 +2160,12 @@ describe("native passthrough terminal finalization", () => { const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); // Force win32 so eager-relay decision path is reachable via streamMode override. Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + // Ownership identity lowercases the state directory on win32, so the lease this file took + // under the real platform stops matching the directory the dispatch checks the moment the + // override lands, and the turn is refused with no terminal at all. Retake it under the + // platform this case is pretending to run on, and give it back before restoring. + releaseSpendHome?.(); + releaseSpendHome = acquireOwnedSpendHome(); try { const response = await postSpawn( cfg, @@ -2177,6 +2183,8 @@ describe("native passthrough terminal finalization", () => { responseText, }; } finally { + releaseSpendHome?.(); + releaseSpendHome = undefined; if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor); } } diff --git a/tests/server/agent-task-recovery-cache.test.ts b/tests/server/agent-task-recovery-cache.test.ts index 35b5ba7050..1b5c5567fe 100644 --- a/tests/server/agent-task-recovery-cache.test.ts +++ b/tests/server/agent-task-recovery-cache.test.ts @@ -17,9 +17,20 @@ import { recoverySse, routedConfig, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const realDateNow = Date.now; +// Direct handler dispatch never takes the writer lease that startServer would take, so it is refused. +let releaseSpendHome: (() => void) | undefined; +beforeEach(() => { + releaseSpendHome = acquireOwnedSpendHome(); +}); +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); + describe("agent task recovery cache", () => { beforeEach(() => resetAgentTaskRecoveryCache()); diff --git a/tests/server/agent-task-recovery-combo.test.ts b/tests/server/agent-task-recovery-combo.test.ts index 8823270ff2..7df943326b 100644 --- a/tests/server/agent-task-recovery-combo.test.ts +++ b/tests/server/agent-task-recovery-combo.test.ts @@ -25,8 +25,15 @@ import { recoverySse, routedConfig, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +// Direct handler dispatch never takes the writer lease that startServer would take, so it is +// refused. Taken inside the block below rather than here, because that block installs its own +// OPENCODEX_HOME after this hook would have run, and a lease binds the directory in effect when +// it was taken. +let releaseSpendHome: (() => void) | undefined; + function providerCompletion(): Response { return Response.json({ id: "chatcmpl_combo_recovery", @@ -58,6 +65,7 @@ describe("combo path encrypted agent task recovery", () => { beforeEach(() => { home = mkdtempSync(join(tmpdir(), "ocx-agent-task-combo-")); process.env["OPENCODEX_HOME"] = home; + releaseSpendHome = acquireOwnedSpendHome(); clearResponseStateMemoryForTests(); resetAgentTaskRecoveryState(); clearCachedProviderQuotas(); @@ -70,6 +78,9 @@ describe("combo path encrypted agent task recovery", () => { clearCachedProviderQuotas(); clearComboTargetCooldowns(); clearResponseStateForTests(); + // Released after the state flush and before the directory holding it is removed. + releaseSpendHome?.(); + releaseSpendHome = undefined; removeTreeWithRetry(home); if (priorHome === undefined) delete process.env["OPENCODEX_HOME"]; else process.env["OPENCODEX_HOME"] = priorHome; diff --git a/tests/server/agent-task-recovery-fallback.test.ts b/tests/server/agent-task-recovery-fallback.test.ts index 19495add89..1bb983c13e 100644 --- a/tests/server/agent-task-recovery-fallback.test.ts +++ b/tests/server/agent-task-recovery-fallback.test.ts @@ -13,6 +13,17 @@ import { recoverySse, routedConfig, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +// Direct handler dispatch never takes the writer lease that startServer would take, so it is refused. +let releaseSpendHome: (() => void) | undefined; +beforeEach(() => { + releaseSpendHome = acquireOwnedSpendHome(); +}); +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); describe("agent task recovery fallback routing", () => { beforeEach(() => { diff --git a/tests/server/agent-task-recovery-security.test.ts b/tests/server/agent-task-recovery-security.test.ts index c7a5ede55d..e478a160c8 100644 --- a/tests/server/agent-task-recovery-security.test.ts +++ b/tests/server/agent-task-recovery-security.test.ts @@ -19,9 +19,20 @@ import { routedConfig, ROUTING_ENVELOPE, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const realDateNow = Date.now; +// Direct handler dispatch never takes the writer lease that startServer would take, so it is refused. +let releaseSpendHome: (() => void) | undefined; +beforeEach(() => { + releaseSpendHome = acquireOwnedSpendHome(); +}); +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); + describe("agent task recovery security", () => { beforeEach(() => resetAgentTaskRecoveryState()); diff --git a/tests/server/agent-task-recovery.test.ts b/tests/server/agent-task-recovery.test.ts index faf117f1a7..ffbeb29627 100644 --- a/tests/server/agent-task-recovery.test.ts +++ b/tests/server/agent-task-recovery.test.ts @@ -26,6 +26,17 @@ import { ROUTING_ENVELOPE, SECOND_FERNET_TASK, } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +// Direct handler dispatch never takes the writer lease that startServer would take, so it is refused. +let releaseSpendHome: (() => void) | undefined; +beforeEach(() => { + releaseSpendHome = acquireOwnedSpendHome(); +}); +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); describe("agent task recovery (opt-in, default off)", () => { beforeEach(() => { diff --git a/tests/server/server-agent-task-recovery-replay.test.ts b/tests/server/server-agent-task-recovery-replay.test.ts index caaad660aa..9c660b2e29 100644 --- a/tests/server/server-agent-task-recovery-replay.test.ts +++ b/tests/server/server-agent-task-recovery-replay.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, spyOn, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; import { createKiroAdapter } from "../../src/adapters/kiro"; import { ADAPTER_REGISTRY } from "../../src/adapters/registry"; import { parseRequest } from "../../src/responses/parser"; @@ -7,8 +7,19 @@ import { conversationIdFromResponsesRequest } from "../../src/server/request-log import type { OcxParsedRequest } from "../../src/types"; import { recoverEncryptedAgentTask, resetAgentTaskRecoveryState, restoreCachedEncryptedAgentTasks } from "../../src/server/responses/agent-task-recovery"; import { codexHeaders, encryptedInput, fakeChatGptJwt, FERNET_TASK, SECOND_FERNET_TASK, originalFetch, recoverySse, routedConfig } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; afterEach(() => { globalThis.fetch = originalFetch; resetAgentTaskRecoveryState(); }); +// Direct handler dispatch never takes the writer lease that startServer would take, so it is refused. +let releaseSpendHome: (() => void) | undefined; +beforeEach(() => { + releaseSpendHome = acquireOwnedSpendHome(); +}); +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; +}); + test("replay reuses admitted recovery after a tool result without another network call", async () => { let calls = 0; globalThis.fetch = (async () => { calls++; return new Response(recoverySse("Read nonce.txt exactly.")); }) as typeof fetch; diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index c5dcae6637..a70987bbf1 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -175,14 +175,16 @@ beforeEach(() => { }); afterEach(async () => { - releaseSpendHome?.(); - releaseSpendHome = undefined; let responseStatePending = true; try { for (const server of servers.splice(0)) await server.stop(true); await flushResponseState(); responseStatePending = responseStatePersistPendingForTests(); } finally { + // After the listeners are stopped and the response state is flushed, both of which can + // still account against the journal, and before the home below is removed. + releaseSpendHome?.(); + releaseSpendHome = undefined; clearResponseStateForTests(); clearCursorThreadContinuityForTests(); globalThis.fetch = originalFetch; @@ -2022,8 +2024,8 @@ describe("server combo failover 030 activation matrix", () => { "chatgpt-account-id": authKind === "mismatched-account" ? "other-account" : authKind === "org-only-jwt" ? "org-foreign" : "acct-scoped-sidecar", }; + takeSpendHome(); const response = authKind === "chat-valid" - takeSpendHome(); ? await (await import("../../src/server/chat-completions")).handleChatCompletions(new Request("http://localhost/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model: "combo/free", messages: [{ role: "user", content: "search" }], stream: true, tools: [{ type: "web_search" }] }), From 6b5132b0550440e0db37fa27a41852640961fdf3 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:26:17 +0900 Subject: [PATCH 22/34] test(codex): give the overlapping successor its own state directory Both hard-kill cases launch the successor while the owner is still listening, on purpose: that overlap is what proves the contended snapshot, the denied main admission, the takeover after the kill and the auth-temp scrub. They shared one OPENCODEX_HOME, and the spend journal allows one writer per state directory, so the successor was refused before it ever bound and the case reported a startup failure instead of the transition it is about. The successor now gets its own config directory under the same fixture root, written with the same helper and the same account shape. CODEX_HOME is unchanged, and the native-main lock, the recovery journal and the vault all derive from that, so every assertion still observes the same shared native state. This is the pattern the file already uses for its other deliberately overlapping child a few cases earlier; it is now a named helper that also restores the parent's OPENCODEX_HOME after writing. The launch stays before the kill and the lease stays required. Nothing about the cross-process ownership contract changes. Local checks: NOT RUN. --- .../native-main-owner-lifetime.test.ts | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/codex-integration/native-main-owner-lifetime.test.ts b/tests/codex-integration/native-main-owner-lifetime.test.ts index 1bca364f8b..96fe8fb424 100644 --- a/tests/codex-integration/native-main-owner-lifetime.test.ts +++ b/tests/codex-integration/native-main-owner-lifetime.test.ts @@ -133,6 +133,26 @@ function fixture(configName = "opencodex", includePool = true): Fixture { return { root, codexHome, configDir, key, manager }; } +/** + * The same fixture with its own state directory, for a successor launched while the owner still + * listens. + * + * These cases deliberately overlap two live proxies to prove the contended, admission-denied, + * hard-kill and takeover sequence. They cannot share one OPENCODEX_HOME: the spend journal + * allows one writer per state directory, so the successor would be refused before it ever bound + * and the case would report a startup failure instead of the transition it is about. CODEX_HOME + * is unchanged, and the native-main lock, recovery journal and vault all derive from that, so + * every assertion in these cases still observes the same shared native state. + */ +function successorConfig(f: Fixture, configName: string, includePool: boolean): Fixture { + const configDir = join(f.root, configName); + const parentConfigDir = process.env.OPENCODEX_HOME; + writeConfig(f.codexHome, configDir, MAIN_CODEX_ACCOUNT_ID, includePool); + if (parentConfigDir === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = parentConfigDir; + return { ...f, configDir }; +} + // Each wait bounds a real child proxy doing real work: spawning Bun, opening the // owner SQLite database, and acquiring or releasing the lease. On the Windows // shards four Bun pools share one runner, so the fixed 10s bounds were reporting @@ -465,7 +485,7 @@ describe("native-main process owner lease", () => { const target = await f.manager.finishStage(stage.stageId, stage.writerToken, "target"); expect(source.profile.state).toBe("active"); - const successorFixture = { ...f }; + const successorFixture = successorConfig(f, "crash-a-successor", false); const owner = new ChildHarness(f, { NATIVE_OWNER_HOLD_SWITCH_BOUNDARY: "auth-replaced" }); let successor: ChildHarness | undefined; try { @@ -532,7 +552,7 @@ describe("native-main process owner lease", () => { expect(readFileSync(tempPath, "utf8")).toContain("access-target"); expect(probeNativeProfileRecoveryState(f.manager.context)).toBe("journal"); - successor = new ChildHarness(f, { NATIVE_OWNER_HOLD_RECOVERY: "1" }); + successor = new ChildHarness(successorConfig(f, "temp-crash-successor", false), { NATIVE_OWNER_HOLD_RECOVERY: "1" }); await successor.waitFor(event => event.event === "listening"); await successor.snapshot(isContended); await owner.hardKill(); From cf496e39b484dcce1a0831fa6edd695d5ed3326c Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:38:14 +0900 Subject: [PATCH 23/34] test(spend): wait for the turns to finish before giving the lease back Two lifetime gaps the review found, both about teardown racing a turn that is still accounting. handler.close only STARTS the websocket pump cancellation. The fixtures closed their synthetic clients and released immediately, so a reader could still be settling against a journal nobody owned. Both fixtures now wait, on the bounded seam they already use, until the socket has dropped its stream cancel and its native control, then run the shutdown hooks, then release, then restore. The teardowns are async for that reason. The combo suite leases every dispatch but its status-only rows never read the transformed body they get back. The shared helpers now hand each turn through a tracker, and teardown cancels every body that is still unread and unlocked before the listeners stop, the response state is flushed and the lease is given back. Rows that do read their body are unaffected, because a consumed or locked body is skipped. Pure in-memory rows that never dispatch are deliberately still lease-free. Assertion counts unchanged, no cap moves, and an AST syntax screen over every changed file is clean. Local checks: NOT RUN. --- tests/helpers/native-injection-fixture.ts | 15 ++++-- tests/responses/ws-native-steering.test.ts | 15 ++++-- .../server/server-combo-failover-e2e.test.ts | 46 +++++++++++-------- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index a6a157e6f5..f56401b7d6 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -14,7 +14,7 @@ import { acquireOwnedSpendHome } from "./owned-spend-home"; let releaseSpendHome: (() => void) | undefined; // Every synthetic client this fixture opens, so teardown can close them before the lease is // given back rather than leaving a handler mid-turn against a journal nobody owns. -const clients: Array<{ close(): void }> = []; +const clients: Array> = []; export type Frame = Record; const realSocket = globalThis.WebSocket; @@ -109,12 +109,17 @@ export function installInjectionFixture() { globalThis.fetch = (async () => { fallbackCalls++; throw new Error("network disabled in injection fixture"); }) as typeof fetch; clearRequestLogsForTests(); }); - afterEach(() => { - for (const client of clients.splice(0)) client.close(); + afterEach(async () => { + // handler.close only STARTS the pump cancellation. Waiting for the socket to drop its stream + // cancel and its native control is what proves the turn finished accounting; releasing the + // lease before that leaves a reader settling against a journal nobody owns. + for (const client of clients.splice(0)) { + client.close(); + await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } for (const socket of InjectionSocket.all) socket.close(); InjectionSocket.all = []; runOptionalShutdownHooks(); - // Released only after the clients, the upstream sockets and the shutdown hooks: each can - // still settle a turn that accounts against the journal this lease owns. + // Released only once those have all settled. releaseSpendHome?.(); releaseSpendHome = undefined; globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index bb9b3e88da..0646f73b5f 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -18,7 +18,7 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; let releaseSpendHome: (() => void) | undefined; // Every synthetic client this file opens, so teardown can close them before the lease is given // back rather than leaving a handler mid-turn against a journal nobody owns. -const clients: Array<{ close(): void }> = []; +const clients: Array> = []; type Frame = Record; const realSocket = globalThis.WebSocket; @@ -89,13 +89,18 @@ beforeEach(() => { globalThis.fetch = (async () => { fallbackCalls++; throw new Error("unexpected network/fallback in native steering fixture"); }) as typeof fetch; clearRequestLogsForTests(); }); -afterEach(() => { - for (const client of clients.splice(0)) client.close(); +afterEach(async () => { + // handler.close only STARTS the pump cancellation. Waiting for the socket to drop its stream + // cancel and its native control is what proves the turn finished accounting; releasing the + // lease before that leaves a reader settling against a journal nobody owns. + for (const client of clients.splice(0)) { + client.close(); + await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } for (const socket of Socket.all) socket.close(); Socket.all = []; runOptionalShutdownHooks(); - // Released only after the clients, the upstream sockets and the shutdown hooks, because each - // of those can still settle a turn that accounts against the journal this lease owns. + // Released only once those have all settled. releaseSpendHome?.(); releaseSpendHome = undefined; globalThis.WebSocket = realSocket; diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index a70987bbf1..5af3b16f7f 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -149,6 +149,11 @@ const servers: Array> = []; // start a real server already own it, and this only ever adds a reference on the same home. let releaseSpendHome: (() => void) | undefined; const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; +// Every turn these helpers hand back. A row that asserts only on a status leaves a +// transformed body unread, and cancelling those before the listeners stop is what keeps a +// reader from settling against the journal after its lease is gone. +const pendingTurns: Response[] = []; +function trackTurn(response: Response): Response { pendingTurns.push(response); return response; } beforeEach(() => { originalFetch = globalThis.fetch; @@ -177,6 +182,11 @@ beforeEach(() => { afterEach(async () => { let responseStatePending = true; try { + // Rows that assert only on a status leave a transformed body unread. Cancelling those first + // means no reader is still attached when the listeners stop and the lease is given back. + for (const turn of pendingTurns.splice(0)) { + if (!turn.bodyUsed && turn.body && !turn.body.locked) await turn.body.cancel().catch(() => {}); + } for (const server of servers.splice(0)) await server.stop(true); await flushResponseState(); responseStatePending = responseStatePersistPendingForTests(); @@ -236,11 +246,11 @@ async function post( headers: Record = {}, ): Promise { takeSpendHome(); - return handleResponses(new Request("http://localhost/v1/responses", { + return trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), - }), config, { model: "", provider: "" }, options); + }), config, { model: "", provider: "" }, options)); } let loggedRequestSequence = 0; @@ -254,11 +264,11 @@ async function postLogged( const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = await handleResponses(new Request("http://localhost/v1/responses", { + const response = trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), - }), config, logCtx, options); + }), config, logCtx, options)); loggedRequestSequence += 1; return responseWithDeferredRequestLog( response, @@ -278,11 +288,11 @@ async function postModelLogged( const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = await handleResponses(new Request("http://localhost/v1/responses", { + const response = trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model, input: "hello", stream: false, ...raw }), - }), config, logCtx, options); + }), config, logCtx, options)); loggedRequestSequence += 1; return responseWithDeferredRequestLog( response, @@ -3346,7 +3356,7 @@ describe("server combo failover 030 activation matrix", () => { const parent: RequestLogContext = { model: "", provider: "" }; const snapshots: RequestLogContext[] = []; takeSpendHome(); - const response = await handleResponses(new Request("http://localhost/v1/responses", { + const response = trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", session_id: "hop-recall" }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: true }), }), config, parent, { @@ -3357,7 +3367,7 @@ describe("server combo failover 030 activation matrix", () => { finalized.resolve(); }, onNativePassthroughCancel: () => { cancels += 1; }, - }); + })); expect(response.status).toBe(200); await response.text(); await within(finalized.promise); @@ -3653,11 +3663,11 @@ describe("cursor conversation continuity across store:false chains", () => { async function postCursor(config: OcxConfig, raw: Record): Promise { takeSpendHome(); - return handleResponses(new Request("http://localhost/v1/responses", { + return trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ stream: false, store: false, ...raw }), - }), config, { model: "", provider: "" }, {}); + }), config, { model: "", provider: "" }, {})); } function cursorConfig(): OcxConfig { @@ -3750,7 +3760,7 @@ describe("cursor conversation continuity across store:false chains", () => { customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); takeSpendHome(); - const postThreadTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { + const postThreadTurn = async (input: unknown) => trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", @@ -3763,7 +3773,7 @@ describe("cursor conversation continuity across store:false chains", () => { store: false, prompt_cache_key: "shared-cache-key", }), - }), config, { model: "", provider: "" }, {}); + }), config, { model: "", provider: "" }, {})); expect((await postThreadTurn("start")).status).toBe(200); expect((await postThreadTurn([ @@ -3781,7 +3791,7 @@ describe("cursor conversation continuity across store:false chains", () => { customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); takeSpendHome(); - const postDesktopTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { + const postDesktopTurn = async (input: unknown) => trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", @@ -3794,7 +3804,7 @@ describe("cursor conversation continuity across store:false chains", () => { stream: false, store: false, }), - }), config, { model: "", provider: "" }, {}); + }), config, { model: "", provider: "" }, {})); expect((await postDesktopTurn("start")).status).toBe(200); expect((await postDesktopTurn([ @@ -3812,7 +3822,7 @@ describe("cursor conversation continuity across store:false chains", () => { customCursorTransportFactory = fakeCursorTransportFactory(seen); const config = cursorConfig(); takeSpendHome(); - const postThreadTurn = (input: unknown) => handleResponses(new Request("http://localhost/v1/responses", { + const postThreadTurn = async (input: unknown) => trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", @@ -3825,7 +3835,7 @@ describe("cursor conversation continuity across store:false chains", () => { store: false, prompt_cache_key: "shared-cache-key", }), - }), config, { model: "", provider: "" }, {}); + }), config, { model: "", provider: "" }, {})); expect((await postThreadTurn("hello")).status).toBe(200); expect((await postThreadTurn([ @@ -3852,11 +3862,11 @@ describe("combo compact failover", () => { const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = await handleResponsesCompact(compactRequest({ + const response = trackTurn(await handleResponsesCompact(compactRequest({ model: "combo/free", stream: false, input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "earlier turn" }] }], - }), config, logCtx); + }), config, logCtx)); loggedRequestSequence += 1; return responseWithDeferredRequestLog(response, `combo-compact-${loggedRequestSequence}`, start, logCtx); } From 55912860fd67cc041f1c26bebb376a86dd865075 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:45:57 +0900 Subject: [PATCH 24/34] test(spend): track the wrapper each logged helper returns, not the raw turn The three logged helpers tracked the response they got from the handler and then returned a deferred-request-log wrapper built around it. The wrapper locks the raw body, so teardown skipped the raw as locked and never saw the wrapper at all: the body that rows actually hold was the one left unread. Each helper now tracks what it returns. The cancellation no longer swallows its error. A body that refuses to cancel is a real defect and fails the case, but only after every other turn and every listener has still been given its chance to close, so one bad body cannot leave the rest of the suite holding ports or a lease. Assertion counts unchanged, no cap moves, AST syntax screen clean. Local checks: NOT RUN. --- .../server/server-combo-failover-e2e.test.ts | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/server/server-combo-failover-e2e.test.ts b/tests/server/server-combo-failover-e2e.test.ts index 5af3b16f7f..483772891d 100644 --- a/tests/server/server-combo-failover-e2e.test.ts +++ b/tests/server/server-combo-failover-e2e.test.ts @@ -181,11 +181,16 @@ beforeEach(() => { afterEach(async () => { let responseStatePending = true; + let cancelFailure: unknown; try { // Rows that assert only on a status leave a transformed body unread. Cancelling those first - // means no reader is still attached when the listeners stop and the lease is given back. + // means no reader is still attached when the listeners stop and the lease is given back. A + // cancel that throws fails the case rather than being swallowed, but not before every other + // turn and the listeners below have had their chance to close. for (const turn of pendingTurns.splice(0)) { - if (!turn.bodyUsed && turn.body && !turn.body.locked) await turn.body.cancel().catch(() => {}); + if (turn.bodyUsed || !turn.body || turn.body.locked) continue; + try { await turn.body.cancel(); } + catch (error) { cancelFailure ??= error; } } for (const server of servers.splice(0)) await server.stop(true); await flushResponseState(); @@ -213,6 +218,7 @@ afterEach(async () => { clearCodexUpstreamHealth(); clearRequestLogsForTests(); } + if (cancelFailure !== undefined) throw cancelFailure; expect(responseStatePending).toBe(false); }); @@ -264,18 +270,18 @@ async function postLogged( const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { + const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model: "combo/free", input: "hello", stream: false, ...raw }), - }), config, logCtx, options)); + }), config, logCtx, options); loggedRequestSequence += 1; - return responseWithDeferredRequestLog( + return trackTurn(responseWithDeferredRequestLog( response, `combo-test-${loggedRequestSequence}`, start, logCtx, - ); + )); } async function postModelLogged( @@ -288,18 +294,18 @@ async function postModelLogged( const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = trackTurn(await handleResponses(new Request("http://localhost/v1/responses", { + const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ model, input: "hello", stream: false, ...raw }), - }), config, logCtx, options)); + }), config, logCtx, options); loggedRequestSequence += 1; - return responseWithDeferredRequestLog( + return trackTurn(responseWithDeferredRequestLog( response, `direct-test-${loggedRequestSequence}`, start, logCtx, - ); + )); } async function latestAttemptReceipts(config: OcxConfig) { @@ -3862,13 +3868,13 @@ describe("combo compact failover", () => { const logCtx: RequestLogContext = { model: "", provider: "" }; const start = Date.now(); takeSpendHome(); - const response = trackTurn(await handleResponsesCompact(compactRequest({ + const response = await handleResponsesCompact(compactRequest({ model: "combo/free", stream: false, input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "earlier turn" }] }], - }), config, logCtx)); + }), config, logCtx); loggedRequestSequence += 1; - return responseWithDeferredRequestLog(response, `combo-compact-${loggedRequestSequence}`, start, logCtx); + return trackTurn(responseWithDeferredRequestLog(response, `combo-compact-${loggedRequestSequence}`, start, logCtx)); } function canonicalPoolConfig( From 70a86c75ea4278662754ffc1887364af3ee8e732 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 21:57:36 +0900 Subject: [PATCH 25/34] test(routing): narrow the eager-relay platform claim to the relay decision The four streaming rows reported an empty terminal list and read as a relay defect. They were not. The eager relay is reachable only on win32 and darwin, so the rows overwrote process.platform globally to reach it, and that redirects far more than the relay: every filesystem, ACL and state-directory decision in the process follows it. The spend-ledger owner lowercases its home on win32, and on a case-sensitive filesystem the lowercased temp directory is a DIFFERENT directory, so the send could not be reserved and the turn delivered no terminal at all. Retaking the lease under the fake platform could not fix that, because the directory it then owned was not the one the case was using. The claim is now narrowed to the two calls that actually choose the relay path, through an internal test seam in the delivery module. No config key reaches it, and ownership, home casing and the win32 policy itself are untouched. The rows also say more than they did. The status, the event-stream content type and the relay path itself are asserted before the callback is inspected, so a turn that never delivered now says so instead of presenting as a missing callback, and the legacy-tee and eager-relay variants each prove which path they actually took rather than assuming the streamMode was honoured. Assertion coverage grows; nothing is relaxed. AST syntax screen clean, no cap moves. Local checks: NOT RUN. --- src/server/responses/passthrough-delivery.ts | 23 ++++++++++++++-- ...subagent-fallback-handle-responses.test.ts | 26 ++++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index a625273356..ab792894b0 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -99,6 +99,24 @@ import { normalizeDefaultNamespaceInJson, } from "../responses-undeclared-tool-guard"; import { isWin32EagerRewrite, selectEagerPath } from "../../lib/bun-stream-caps"; + +/** + * Platform override for the two relay-path policy calls below. Tests only. + * + * The eager relay is reachable only on win32 and darwin, so a Linux shard cannot exercise it + * without claiming to be one of them. Overwriting `process.platform` globally does that, and a + * great deal more: every filesystem, ACL and state-directory decision in the process follows it, + * and the spend-ledger owner lowercases its home on win32, which on a case-sensitive filesystem + * names a DIFFERENT directory. A row that did that stopped being able to reserve its send and + * delivered no terminal at all, reporting as a relay defect. This narrows the claim to the two + * calls that actually choose the relay path. + */ +let relayPlatformForTests: NodeJS.Platform | undefined; + +/** Internal test contract, not operator configuration: no config key reaches this. */ +export function setRelayPlatformForTests(platform: NodeJS.Platform | undefined): void { + relayPlatformForTests = platform; +} import { linkAbortSignal, UPSTREAM_JSON_BODY_READ_OPTIONS } from "./core-lifetime"; import { registerTurn, unregisterTurn, trackStreamLifetime } from "../lifecycle"; import { relaySseEagerBounded } from "../relay-eager"; @@ -529,12 +547,13 @@ export async function deliverPassthroughResponse( ? composeSseBlockRewrites(...blockRewrites) : undefined; const needsClientRewrite = clientBlockRewrite !== undefined; + const relayPlatform = relayPlatformForTests ?? process.platform; // #864: win32 rewrite traffic must never enter the tee()+JS-pull chain // (Bun#32111 JS-sink segfault — text frames pass, the terminal block is // lost). The eager single reader applies the same rewrites inline. - const win32EagerRewrite = isWin32EagerRewrite(process.platform, needsClientRewrite); + const win32EagerRewrite = isWin32EagerRewrite(relayPlatform, needsClientRewrite); const eagerPath = selectEagerPath( - process.platform, + relayPlatform, needsClientRewrite, config.streamMode ?? "auto", ); diff --git a/tests/routing/subagent-fallback-handle-responses.test.ts b/tests/routing/subagent-fallback-handle-responses.test.ts index 98243dc09e..314ea7191b 100644 --- a/tests/routing/subagent-fallback-handle-responses.test.ts +++ b/tests/routing/subagent-fallback-handle-responses.test.ts @@ -9,6 +9,7 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { setRelayPlatformForTests } from "../../src/server/responses/passthrough-delivery"; import { clearAccountQuota, updateAccountQuota, @@ -2157,15 +2158,12 @@ describe("native passthrough terminal finalization", () => { const terminals: ResponsesTerminalStatus[] = []; mockSseUpstream(sseBody); - const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - // Force win32 so eager-relay decision path is reachable via streamMode override. - Object.defineProperty(process, "platform", { value: "win32", configurable: true }); - // Ownership identity lowercases the state directory on win32, so the lease this file took - // under the real platform stops matching the directory the dispatch checks the moment the - // override lands, and the turn is refused with no terminal at all. Retake it under the - // platform this case is pretending to run on, and give it back before restoring. - releaseSpendHome?.(); - releaseSpendHome = acquireOwnedSpendHome(); + // The eager relay is only reachable on win32 and darwin, and this shard is neither. The + // claim is narrowed to the relay decision itself: overwriting process.platform globally + // also redirects filesystem, ACL and state-directory identity, and the spend-ledger owner + // lowercases its home on win32, which on a case-sensitive filesystem is a different + // directory. That made the send unreservable and the turn delivered no terminal at all. + setRelayPlatformForTests("win32"); try { const response = await postSpawn( cfg, @@ -2174,6 +2172,12 @@ describe("native passthrough terminal finalization", () => { onNativePassthroughTerminal: (status) => terminals.push(status), }, ); + // Asserted before the callback is inspected, so a turn that never delivered says so + // instead of presenting as a missing callback. + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + // The relay path this case exists to exercise, proven rather than assumed. + expect(isEagerRelaySseResponse(response)).toBe(streamMode === "eager-relay"); const responseText = await response.text(); // Allow inspection consumer microtasks to settle. await Bun.sleep(20); @@ -2183,9 +2187,7 @@ describe("native passthrough terminal finalization", () => { responseText, }; } finally { - releaseSpendHome?.(); - releaseSpendHome = undefined; - if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor); + setRelayPlatformForTests(undefined); } } From 58fb3bb8ed3c83b465d744b2649707245e57d427 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:08:29 +0900 Subject: [PATCH 26/34] fix(spend): close three failure paths in the ledger's own storage All three are on paths the happy case never reaches, which is why a green suite said nothing about them. A fresh state directory was never claimed. Config ownership refuses to claim a directory that already has contents, and the owner database lives inside that directory, so creating the database first left a new home with no owner marker and no manifest at all. Nothing recorded the database or its sidecars, and a later uninstall could not remove them. The paths are now registered while the directory is still empty, and the regression proves an uninstall takes the whole directory back afterwards. An entry that could not be inspected read as absent. Only ENOENT means absent; a permission denial or an I/O error means we do not know, and answering no file skipped the file-safety assertion and appended to whatever was actually there. It now refuses, and it refuses in the module's own vocabulary rather than handing a client the errno and path of a state file. The ledger already treats a journal it cannot make durable as a degradation rather than an outage, so this surfaces as reserve-not-durable and does not fail the request. A failed compaction left its temp behind. The name carries random bytes, so a validate, harden or rename that threw left a uniquely named file and the next attempt made another: repeated failures accumulated rather than overwriting one fixed name. Only that exact temp is removed, only on the failure path, so the original journal and the primary error both survive. The failed-start rollback also stopped waiting for its own listeners. Bun's Server.stop(true) resolves once connections are closed, and the rollback discarded that promise, so the state directory was handed back while a listener could still be serving. It now holds the lease until every stop has settled, while staying synchronous and returning void, because startServer must not become async. Rollback failures stay contained so the startup error that caused them is still the one reported. Not covered by a test: the compaction failure path has no deterministic lever without an injection seam, so the cleanup is asserted by inspection only. Local checks: NOT RUN. AST syntax screen clean, no cap moves. --- src/lib/spend-ledger-owner.ts | 8 +++- src/lib/spend-reservation-ledger.ts | 37 ++++++++++++++++--- src/server/index/spend-ledger-lifecycle.ts | 25 +++++++++++-- tests/lib/spend-ledger-file-journal.test.ts | 33 +++++++++++++++++ tests/lib/spend-ledger-owner.test.ts | 25 ++++++++++++- ...plaintext-v2-agent-messages-server.test.ts | 6 ++- 6 files changed, 120 insertions(+), 14 deletions(-) diff --git a/src/lib/spend-ledger-owner.ts b/src/lib/spend-ledger-owner.ts index 21654d38cc..40cf6dd07b 100644 --- a/src/lib/spend-ledger-owner.ts +++ b/src/lib/spend-ledger-owner.ts @@ -89,14 +89,18 @@ function prepareOwnerPath(configDir: string): { home: string; path: string } { if (process.platform !== "win32") chmodSync(home, 0o700); hardenSecretDir(home, { required: true }); const path = join(home, SPEND_LEDGER_OWNER_FILENAME); + // Registered while the directory is still EMPTY. Config ownership refuses to claim a + // directory that already has contents, so creating the database first meant a fresh home + // never got an ownership marker at all: the manifest was never written, and the database and + // its sidecars were left behind by a later uninstall because nothing recorded them. + recordOwnedConfigPath(home, path); + for (const suffix of OWNER_SIDECARS) recordOwnedConfigPath(home, `${path}${suffix}`); try { closeSync(openSync(path, "wx", 0o600)); } catch (error) { if (errorCode(error) !== "EEXIST") throw error; } assertPrivateFile(path); if (process.platform !== "win32") chmodSync(path, 0o600); hardenSecretPath(path, { required: true }); assertPrivateFile(path); - recordOwnedConfigPath(home, path); - for (const suffix of OWNER_SIDECARS) recordOwnedConfigPath(home, `${path}${suffix}`); return { home: process.platform === "win32" ? home.toLowerCase() : home, path }; } diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 3c71d4dd6b..758cee3ec6 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -53,7 +53,7 @@ * that are re-applied to an EXISTING file rather than trusted from its creation. */ -import { appendFileSync, chmodSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; +import { appendFileSync, chmodSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; // Definition-site import, not the ../config barrel -- same reasoning as @@ -407,8 +407,20 @@ function ledgerEntryExists(path: string): boolean { try { lstatSync(path); return true; - } catch { - return false; + } catch (error) { + // Only a genuinely absent entry is absent. Treating every failure as "no file" meant a + // permission denial or an I/O error skipped assertSafeLedgerFile entirely and let the + // append proceed against whatever is actually there, which is the case that check exists + // for. An entry we cannot inspect is a refusal, not an empty slot. + // + // Raised as the module's typed error rather than the raw fs error: the path and errno of a + // state file are not something a client should be handed. + if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return false; + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger storage could not be inspected safely.", + { cause: error }, + ); } } @@ -468,9 +480,22 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ mode: 0o600, flag: "wx", }); - assertSafeLedgerFile(temp); - hardenLedgerFile(temp, { force: true }); - renameSync(temp, path); + // Everything after the temp exists is failure-cleaned. The name carries random bytes, so + // a validate, harden or rename that throws used to leave a uniquely named file behind and + // the next attempt made another: repeated failures accumulated instead of overwriting one + // fixed name. Only this exact temp is removed, and only on the failure path, so the + // original journal and the primary error both survive. + let renamed = false; + try { + assertSafeLedgerFile(temp); + hardenLedgerFile(temp, { force: true }); + renameSync(temp, path); + renamed = true; + } finally { + if (!renamed) { + try { unlinkSync(temp); } catch { /* the compaction failure is the one to report */ } + } + } assertSafeLedgerFile(path); hardenLedgerFile(path, { force: true }); }, diff --git a/src/server/index/spend-ledger-lifecycle.ts b/src/server/index/spend-ledger-lifecycle.ts index 1b5e592166..a0678e74fb 100644 --- a/src/server/index/spend-ledger-lifecycle.ts +++ b/src/server/index/spend-ledger-lifecycle.ts @@ -32,13 +32,32 @@ export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedge track }>(server: T): T { // Capture the raw stop before startServer replaces the public method with full teardown. const stop = server.stop.bind(server); - failedStartStops.push(() => { try { void stop(true); } catch { /* preserve startup failure */ } }); + failedStartStops.push(() => stop(true)); return server; }, release, releaseAfterFailedStart(): void { - for (const stop of failedStartStops.reverse()) stop(); - release(); + // Every listener that came up is stopped, newest first, and the lease is held until those + // stops have actually SETTLED. Bun's Server.stop(true) returns a promise that resolves + // once connections are closed, so discarding it handed the state directory back while a + // listener could still be serving, which is the one thing single-writer ownership exists + // to prevent. + // + // This stays synchronous and returns void on purpose: startServer must not become async, + // so the wait is a continuation rather than an await. Rollback failures are contained + // because the startup error that brought us here is the one worth reporting. + const settling: Promise[] = []; + for (const stop of failedStartStops.splice(0).reverse()) { + try { + const pending = stop(); + if (pending !== undefined) settling.push(Promise.resolve(pending)); + } catch { /* a rollback failure must not replace the startup error that caused it */ } + } + const finish = (): void => { + try { release(); } catch { /* same: the startup error is the one that matters */ } + }; + if (settling.length === 0) { finish(); return; } + void Promise.allSettled(settling).then(finish); }, }; } diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index f3e358e6dc..90bbbb748b 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -12,6 +12,7 @@ import { import { acquireSpendLedgerOwner, mintSpendLedgerStorage, + SpendLedgerOwnerError, type SpendLedgerOwnerLease, } from "../../src/lib/spend-ledger-owner"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -54,6 +55,38 @@ afterEach(() => { }); describe("spend ledger file journal", () => { + test.skipIf(!posixModes || process.getuid?.() === 0)( + "an entry that cannot be read is refused rather than reported absent", () => { + const dir = ownedHome("ocx-spend-journal-unreadable-"); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); + journal.append(line("alias-one")); + + const before = readdirSync(dir).sort(); + // Deny traversal of the owned directory, so the entry's lstat fails with EACCES instead of + // ENOENT. Answering "absent" for that is what this pins: the file-safety assertion is + // skipped for an entry that reads as missing, so a journal nobody can inspect would have + // been appended to as though the slot were empty. + chmodSync(dir, 0o000); + try { + expect(() => journal.read()).toThrowError(SpendLedgerOwnerError); + // The refusal carries the module's own vocabulary, not the errno and path of a state file. + expect(() => journal.read()).toThrow(/could not be inspected safely/); + } finally { + chmodSync(dir, 0o700); + } + // Nothing was created or reset while the directory was unreadable. + expect(readdirSync(dir).sort()).toEqual(before); + // The same entry is readable again once the directory is, so the refusal was about the + // failed inspection and not about the journal's own contents. + expect(journal.read()).toHaveLength(1); + }); + + test("a genuinely absent entry still reads as empty", () => { + ownedHome("ocx-spend-journal-absent-"); + // The ENOENT control for the case above: absent is still absent, and only absent is. + expect(createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)).read()).toEqual([]); + }); + test.skipIf(!posixModes)("a journal that already exists is re-hardened, not trusted", () => { const dir = ownedHome("ocx-spend-journal-"); const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index de7d64190c..c18848d453 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -1,6 +1,6 @@ /** Cross-process ownership for the process-wide spend journal (#5123). */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, linkSync, mkdirSync, mkdtempSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, linkSync, mkdirSync, mkdtempSync, readFileSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -23,6 +23,7 @@ import { spendLedgerDiagnosticsSnapshot, } from "../../src/lib/spend-reservation-ledger"; import { helperPath } from "../helpers/repo-root"; +import { CONFIG_OWNER_FILE, CONFIG_UNINSTALL_MANIFEST, removeOwnedConfigState } from "../../src/lib/config-ownership"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS } from "../helpers/test-budget"; @@ -377,6 +378,28 @@ describe("in-process references and privacy", () => { }); describe("backing file identity", () => { + test("a fresh state directory is claimed before its database exists, so it stays removable", () => { + // Config ownership refuses to claim a directory that already has contents, and the owner + // database lives inside that directory. Creating the database first therefore left a fresh + // home with no owner marker and no manifest at all, so nothing recorded the database or its + // sidecars and a later uninstall could not remove them. + expect(existsSync(home)).toBe(false); + acquireSpendLedgerOwner().release(); + + expect(existsSync(join(home, CONFIG_OWNER_FILE))).toBe(true); + const manifest = JSON.parse( + readFileSync(join(home, CONFIG_UNINSTALL_MANIFEST), "utf8"), + ) as { paths: string[] }; + expect(manifest.paths).toContain(SPEND_LEDGER_OWNER_FILENAME); + for (const sidecar of ["-journal", "-wal", "-shm"]) { + expect(manifest.paths).toContain(`${SPEND_LEDGER_OWNER_FILENAME}${sidecar}`); + } + + // The point of recording them: an uninstall can now take the whole directory back. + expect(removeOwnedConfigState(home).status).toBe("removed"); + expect(existsSync(home)).toBe(false); + }); + const expectBackingAliasesRefused = (kind: "hardlink" | "symlink"): void => { const first = acquireSpendLedgerOwner(home); sharedSpendLedger().reserve({ diff --git a/tests/server/plaintext-v2-agent-messages-server.test.ts b/tests/server/plaintext-v2-agent-messages-server.test.ts index c94cb9404c..46829c129f 100644 --- a/tests/server/plaintext-v2-agent-messages-server.test.ts +++ b/tests/server/plaintext-v2-agent-messages-server.test.ts @@ -19,8 +19,10 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; const originalFetch = globalThis.fetch; let releaseInheritedSpendHome: (() => void) | undefined; -// Taken per inherited-home dispatch because the pool retry row installs a different home. -const takeInheritedSpendHome = (): void => { releaseInheritedSpendHome = acquireOwnedSpendHome(); }; +// Taken per inherited-home dispatch because the pool retry row installs a different home. The +// ??= keeps a second call inside one case idempotent rather than replacing the release callback +// it would need; no row here calls it twice today, so this is defence, not a fixed regression. +const takeInheritedSpendHome = (): void => { releaseInheritedSpendHome ??= acquireOwnedSpendHome(); }; beforeEach(() => { clearResponseStateForTests(); }); afterEach(() => { // Released first so a failed row cannot leak its writer lease into the next case. From 6842dcb51a33fde4f34cd9dba436987d36e308a9 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:16:12 +0900 Subject: [PATCH 27/34] test(spend): prove the storage and rollback failure paths instead of asserting them The previous commit fixed three failure paths and shipped one of them with no test, which is how a failure path stays broken. These are the regressions. Compaction now has a narrow fault seam for its own filesystem steps, because there is no portable way to make a validate, harden or rename fail on demand. It is an internal test contract: no config key reaches it and it defaults to absent. Four cases drive it. A failure at validate, harden or rename, repeated three times, leaves no compaction residue and an unchanged journal; a write that stops partway leaves neither residue nor a truncated journal; and a candidate name that already belongs to something else is left exactly as it was. That last case drove a real change. The temp was created by a combined write, so whether the entry was ours had to be inferred from which error the write threw. It is now an exclusive open first, so ownership is a fact: EEXIST means the name is not ours and is never removed, and every failure after the open is cleaned because the entry is provably ours, including a short write. The descriptor is closed on the failure path too. The rollback has its own file now. Two listeners whose stops stay pending prove the lease is still held after both are asked, still held when only one has settled, and returned once both have; a listener whose stop rejects proves the others are still stopped and the directory is still returned. Both also pin the newest-first order. failedStartStops is typed as returning void or a promise, which is what the rollback awaits. The old annotation compiled while statically erasing the promise; the runtime closure returned it either way, so this corrects the contract rather than a behaviour. New file registered in both layout maps. Local checks: NOT RUN. --- scripts/test-layout/layout.json | 1 + src/lib/spend-reservation-ledger.ts | 57 ++++++++--- src/server/index/spend-ledger-lifecycle.ts | 5 +- tests/fixtures/test-layout-expected.json | 1 + tests/lib/spend-ledger-file-journal.test.ts | 75 ++++++++++++++- tests/server/spend-ledger-lifecycle.test.ts | 101 ++++++++++++++++++++ 6 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 tests/server/spend-ledger-lifecycle.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 7db22d3c25..1ad2292413 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1346,6 +1346,7 @@ "sponsor-presets.test.ts": "providers", "spend-ceiling-enforcement.test.ts": "lib", "spend-ledger-file-journal.test.ts": "lib", + "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 758cee3ec6..e90bb44f34 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -53,7 +53,7 @@ * that are re-applied to an EXISTING file rather than trusted from its creation. */ -import { appendFileSync, chmodSync, lstatSync, mkdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { appendFileSync, chmodSync, closeSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; // Definition-site import, not the ../config barrel -- same reasoning as @@ -396,6 +396,23 @@ function hardenLedgerFile(path: string, options: { readonly force?: boolean } = } catch { /* best-effort: a non-owner cannot chmod */ } } +/** + * Fault injection for the journal's own filesystem steps. Internal test contract, not config. + * + * The compaction cleanup only runs when a step after the exclusive create fails, and there is no + * portable way to make a validate, harden or rename fail on demand. Without a seam the cleanup + * would ship asserted by reading alone, which is how a failure path stays broken. + */ +export type SpendJournalFaultStep = "create" | "write" | "validate" | "harden" | "rename"; + +let journalFaultForTests: ((step: SpendJournalFaultStep, temp: string) => void) | undefined; + +export function setSpendJournalFaultForTests( + fault: ((step: SpendJournalFaultStep, temp: string) => void) | undefined, +): void { + journalFaultForTests = fault; +} + /** * Does a directory entry exist here, whatever it points at? * @@ -475,25 +492,39 @@ export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJ // leaves either the old journal or the new one, never a half-written ledger. if (ledgerEntryExists(path)) assertSafeLedgerFile(path); const temp = `${path}.compact-${process.pid}-${randomBytes(6).toString("hex")}`; - writeFileSync(temp, lines.map((line) => line + "\n").join(""), { - encoding: "utf8", - mode: 0o600, - flag: "wx", - }); - // Everything after the temp exists is failure-cleaned. The name carries random bytes, so - // a validate, harden or rename that throws used to leave a uniquely named file behind and - // the next attempt made another: repeated failures accumulated instead of overwriting one - // fixed name. Only this exact temp is removed, and only on the failure path, so the - // original journal and the primary error both survive. + // The creation is INSIDE the cleanup, not before it. The name carries random bytes, so a + // failure anywhere after the entry exists used to leave a uniquely named file and the next + // attempt made another: repeated failures accumulated instead of overwriting one fixed + // name. Only an entry this call created is removed, so a name that turned out to belong to + // something else is left alone, and the original journal and the primary error survive. + let fd: number | undefined; + let created = false; let renamed = false; try { + // Exclusive create FIRST, so "this entry is ours" is a fact rather than a guess about + // which error a combined write threw. EEXIST leaves created false and the name is left + // alone; every failure after this point is cleaned because the entry is provably ours, + // including a write that stopped partway through. + journalFaultForTests?.("create", temp); + fd = openSync(temp, "wx", 0o600); + created = true; + journalFaultForTests?.("write", temp); + writeFileSync(fd, lines.map((line) => line + "\n").join(""), { encoding: "utf8" }); + closeSync(fd); + fd = undefined; + journalFaultForTests?.("validate", temp); assertSafeLedgerFile(temp); + journalFaultForTests?.("harden", temp); hardenLedgerFile(temp, { force: true }); + journalFaultForTests?.("rename", temp); renameSync(temp, path); renamed = true; } finally { - if (!renamed) { - try { unlinkSync(temp); } catch { /* the compaction failure is the one to report */ } + if (fd !== undefined) { + try { closeSync(fd); } catch { /* the compaction failure is the one to report */ } + } + if (created && !renamed) { + try { unlinkSync(temp); } catch { /* same */ } } } assertSafeLedgerFile(path); diff --git a/src/server/index/spend-ledger-lifecycle.ts b/src/server/index/spend-ledger-lifecycle.ts index a0678e74fb..d725064247 100644 --- a/src/server/index/spend-ledger-lifecycle.ts +++ b/src/server/index/spend-ledger-lifecycle.ts @@ -18,7 +18,10 @@ export interface SpendLedgerServerLifecycle { /** Acquire before config loading so every later startup failure has one rollback owner. */ export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedgerServerLifecycle { const owner: SpendLedgerOwnerLease = acquireSpendLedgerOwner(configDir); - const failedStartStops: Array<() => void> = []; + // Each entry returns whatever the listener's own stop returned. Typed as void-or-promise + // because the rollback below has to WAIT on it: declaring it `() => void` let the call site + // compile while statically erasing the promise it needs to await. + const failedStartStops: Array<() => void | Promise> = []; let released = false; const release = (): void => { if (released) return; diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index b88224e089..00dd94ad81 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1174,6 +1174,7 @@ "sponsor-presets.test.ts": "providers", "spend-ceiling-enforcement.test.ts": "lib", "spend-ledger-file-journal.test.ts": "lib", + "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index 90bbbb748b..f292562bc0 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { chmodSync, mkdtempSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -8,6 +8,7 @@ import { SPEND_LEDGER_JOURNAL_FILENAME, SPEND_LEDGER_SALT_FILENAME, resetSharedSpendLedgerForTest, + setSpendJournalFaultForTests, } from "../../src/lib/spend-reservation-ledger"; import { acquireSpendLedgerOwner, @@ -44,6 +45,7 @@ function ownedHome(prefix: string): string { } afterEach(() => { + setSpendJournalFaultForTests(undefined); for (const lease of leases.splice(0)) { try { lease.release(); } catch { /* a failed release must not mask the case's result */ } } @@ -128,6 +130,77 @@ describe("spend ledger file journal", () => { if (posixModes) expect(modeOf(path)).toBe(0o600); }); + /** + * Compaction failure paths, driven through the journal's own fault seam. + * + * The temp name carries random bytes, so a failure that leaves it behind is not one stale file + * but one per attempt. Each case below drives the same compaction repeatedly and asserts the + * directory holds no compaction residue and the original journal is untouched. + */ + for (const step of ["validate", "harden", "rename"] as const) { + test(`a compaction that fails at ${step} leaves no temp behind`, () => { + const dir = ownedHome(`ocx-spend-compact-${step}-`); + const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); + journal.append(line("alias-one")); + const original = readFileSync(path, "utf8"); + + setSpendJournalFaultForTests((actual) => { + if (actual === step) throw new Error(`fault injected at ${step}`); + }); + for (let attempt = 0; attempt < 3; attempt += 1) { + expect(() => journal.rewrite?.call(journal, [line("checkpoint")])).toThrow(/fault injected/); + } + setSpendJournalFaultForTests(undefined); + + expect(readdirSync(dir).filter(name => name.includes(".compact-"))).toEqual([]); + expect(readFileSync(path, "utf8")).toBe(original); + }); + } + + test("a compaction whose write stops partway leaves neither residue nor a truncated journal", () => { + const dir = ownedHome("ocx-spend-compact-partial-"); + const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); + journal.append(line("alias-one")); + const original = readFileSync(path, "utf8"); + + // The entry already exists by the time the write runs, which is the case the exclusive + // create exists to make unambiguous: a short write still leaves a file that is ours. + setSpendJournalFaultForTests((actual, temp) => { + if (actual !== "write") return; + expect(existsSync(temp)).toBe(true); + throw Object.assign(new Error("no space left on device"), { code: "ENOSPC" }); + }); + for (let attempt = 0; attempt < 3; attempt += 1) { + expect(() => journal.rewrite?.call(journal, [line("checkpoint")])).toThrow(/no space left/); + } + setSpendJournalFaultForTests(undefined); + + expect(readdirSync(dir).filter(name => name.includes(".compact-"))).toEqual([]); + expect(readFileSync(path, "utf8")).toBe(original); + }); + + test("a compaction candidate that already exists is left exactly as it was", () => { + const dir = ownedHome("ocx-spend-compact-eexist-"); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); + journal.append(line("alias-one")); + + // Occupy the exact candidate name before the exclusive create reaches it. The create then + // fails EEXIST, and because this call never created the entry it must not remove it. + let occupied: string | undefined; + setSpendJournalFaultForTests((actual, temp) => { + if (actual !== "create" || occupied !== undefined) return; + occupied = temp; + writeFileSync(temp, "not ours\n", { encoding: "utf8", mode: 0o600, flag: "wx" }); + }); + expect(() => journal.rewrite?.call(journal, [line("checkpoint")])).toThrow(); + setSpendJournalFaultForTests(undefined); + + expect(occupied).toBeDefined(); + expect(readFileSync(occupied!, "utf8")).toBe("not ours\n"); + }); + test("the alias salt is minted once and reused, so replay still matches live requests", () => { const dir = ownedHome("ocx-spend-salt-"); const path = join(dir, SPEND_LEDGER_SALT_FILENAME); diff --git a/tests/server/spend-ledger-lifecycle.test.ts b/tests/server/spend-ledger-lifecycle.test.ts new file mode 100644 index 0000000000..17154d46b8 --- /dev/null +++ b/tests/server/spend-ledger-lifecycle.test.ts @@ -0,0 +1,101 @@ +/** + * The failed-start rollback owns two things at once: stopping whatever came up, and giving the + * state directory back. Doing the second before the first has finished is what single-writer + * ownership exists to prevent, and it cannot be observed from a passing startup. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; +import { spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; +import { resetSharedSpendLedgerForTest } from "../../src/lib/spend-reservation-ledger"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +let root = ""; +let home = ""; +let previousHome: string | undefined; +const stopOrder: string[] = []; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "ocx-spend-lifecycle-")); + home = join(root, "state"); + previousHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + stopOrder.length = 0; + resetSharedSpendLedgerForTest(); +}); + +afterEach(() => { + resetSharedSpendLedgerForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(root); +}); + +/** A listener whose stop stays pending until the case says otherwise, like a real drain. */ +function controlledListener(name: string, mode: "resolve" | "reject" = "resolve") { + let settle: () => void = () => {}; + const stopped = new Promise((resolve, reject) => { + settle = () => { if (mode === "reject") reject(new Error(`${name} stop failed`)); else resolve(); }; + }); + stopped.catch(() => { /* the rollback owns this rejection; the case must not be unhandled */ }); + return { + settle, + server: { + stop(_closeActiveConnections?: boolean): Promise { + stopOrder.push(name); + return stopped; + }, + }, + }; +} + +/** Let the rollback's allSettled continuation run without inventing a duration. */ +async function drainContinuations(): Promise { + for (let turn = 0; turn < 8; turn += 1) await Promise.resolve(); +} + +test("a failed start keeps the lease until every listener has actually stopped", async () => { + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const first = controlledListener("first"); + const second = controlledListener("second"); + lifecycle.track(first.server); + lifecycle.track(second.server); + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + + lifecycle.releaseAfterFailedStart(); + // Newest first, and both asked before anything is awaited. + expect(stopOrder).toEqual(["second", "first"]); + await drainContinuations(); + // Still held: neither stop has settled, so a listener could still be serving. + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + + first.settle(); + await drainContinuations(); + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + + second.settle(); + await drainContinuations(); + expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); +}); + +test("a listener whose stop rejects still stops the rest and still returns the directory", async () => { + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const failing = controlledListener("failing", "reject"); + const healthy = controlledListener("healthy"); + lifecycle.track(healthy.server); + lifecycle.track(failing.server); + + lifecycle.releaseAfterFailedStart(); + expect(stopOrder).toEqual(["failing", "healthy"]); + + failing.settle(); + await drainContinuations(); + // One refusal does not strand the directory, and it does not skip the other listener either. + expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + + healthy.settle(); + await drainContinuations(); + expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); +}); From c1644a2771fc2eea4c068fc9a41151de2bf8709a Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:18:26 +0900 Subject: [PATCH 28/34] test: repair the two source-anchor and platform assumptions CI found Both are test-side. No production guard moves. The F4 bind oracle pinned the public serve call by its exact old text, and the listener is now handed to a lifecycle registrar as it is created. The assertion is wrapper-aware without giving anything up: the serve call's argument object is still pinned exactly, so the public bind takes bindHost and nothing else, and server is still what that call is assigned to, through at most one registrar call. Both negative assertions about a hardcoded loopback host are unchanged. The streaming pair claimed win32, and on win32 a turn needing a client rewrite takes the eager relay unconditionally under #864. So the legacy half could never be legacy there, and the marker assertion I added last round was reporting that honestly: status and content type passed, the path was eager either way. The pair now claims darwin, which is the platform where the configured mode actually decides: legacy-tee resolves to tee, eager-relay to the eager relay, and both halves keep their health and terminal-callback coverage. The alternative would have been to weaken the marker assertion, which would have hidden exactly the thing it was added to prove. AST syntax screen clean; the new anchor was checked against the current source and still rejects a hardcoded 127.0.0.1 bind. Local checks: NOT RUN. --- .../subagent-fallback-handle-responses.test.ts | 14 +++++++++----- .../windows-deploy-close-regressions.test.ts | 9 ++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/routing/subagent-fallback-handle-responses.test.ts b/tests/routing/subagent-fallback-handle-responses.test.ts index 314ea7191b..cc7eca13c0 100644 --- a/tests/routing/subagent-fallback-handle-responses.test.ts +++ b/tests/routing/subagent-fallback-handle-responses.test.ts @@ -2158,12 +2158,16 @@ describe("native passthrough terminal finalization", () => { const terminals: ResponsesTerminalStatus[] = []; mockSseUpstream(sseBody); - // The eager relay is only reachable on win32 and darwin, and this shard is neither. The - // claim is narrowed to the relay decision itself: overwriting process.platform globally - // also redirects filesystem, ACL and state-directory identity, and the spend-ledger owner - // lowercases its home on win32, which on a case-sensitive filesystem is a different + // darwin, not win32, because this pair is about streamMode choosing the path. On win32 a + // turn that needs a client rewrite takes the eager relay unconditionally (#864), so the + // legacy half could never be legacy there and the marker assertion below would be a lie. + // darwin is the platform where the configured mode actually decides. + // + // The claim is also narrowed to the relay decision itself: overwriting process.platform + // globally redirects filesystem, ACL and state-directory identity too, and the spend-ledger + // owner lowercases its home on win32, which on a case-sensitive filesystem is a different // directory. That made the send unreservable and the turn delivered no terminal at all. - setRelayPlatformForTests("win32"); + setRelayPlatformForTests("darwin"); try { const response = await postSpawn( cfg, diff --git a/tests/windows/windows-deploy-close-regressions.test.ts b/tests/windows/windows-deploy-close-regressions.test.ts index 5d4504fea6..b65925b29b 100644 --- a/tests/windows/windows-deploy-close-regressions.test.ts +++ b/tests/windows/windows-deploy-close-regressions.test.ts @@ -90,7 +90,14 @@ describe("server bind canonicalizes explicit localhost but preserves wildcards ( // loopback-only, so a bare substring ban would forbid the fix rather than the defect. // Pin the assertion to the public serve call instead: it must take bindHost and nothing // else. - expect(src).toContain("server = Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost });"); + // Wrapper-aware, because the public listener is now handed to a lifecycle registrar as it is + // created. Both halves of the original claim are kept: the serve call's argument object is + // pinned exactly, so the public bind still takes bindHost and nothing else, and `server` is + // still what that call is assigned to, through at most one registrar call. + expect(src).toContain("Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost })"); + expect(src).toMatch( + /\bserver = (?:[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\()?Bun\.serve\(\{ \.\.\.serveOptions, port: listenPort, hostname: bindHost \}\)/, + ); expect(src).not.toMatch(/port: listenPort,\s*\n\s*hostname: "127\.0\.0\.1"/); expect(src).not.toContain("port: listenPort, hostname: \"127.0.0.1\""); }); From be2563f78455454f41d58d5a62fcef1d722095be Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:24:14 +0900 Subject: [PATCH 29/34] test(spend): close the three fixture gaps the review named The startup rollback case asserted ownership was returned in the same turn as the throw, which only held while the rollback discarded its stop promise. It now waits for the directory to come back, then reacquires, and awaits the blocking listener's own stop in its teardown. The assertion that the start actually threw is unchanged. The partial-write case was not partial. The fault threw before any byte landed, so it proved cleanup of an empty file rather than of a short write. The fault now writes a real prefix into the entry the exclusive create already made, and asserts the prefix is there, before failing with ENOSPC. That is the residue the cleanup has to remove, and the journal and the absence of residue are still asserted across repeated attempts. The unreadable-entry contract had only a POSIX chmod case, which is skipped on Windows and proves nothing as root. A narrow stat step on the same internal fault seam now proves it everywhere, for both EACCES and EIO, with only the journal's own inspection failing: the salt stays readable and every other filesystem step is real. It checks the refusal is the module's typed error and that nothing was reset - no truncation, no new entries, and the salt still mints. The chmod case stays as real-filesystem evidence where it can run. Also worth recording: the enforce-target failure on the previous head was a concurrency cancellation, not a gate refusal. AST syntax screen clean. Local checks: NOT RUN. --- src/lib/spend-reservation-ledger.ts | 3 +- tests/lib/spend-ledger-file-journal.test.ts | 35 +++++++++++++++++-- .../server/spend-ledger-owner-startup.test.ts | 18 ++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index e90bb44f34..1a7ff2b4c6 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -403,7 +403,7 @@ function hardenLedgerFile(path: string, options: { readonly force?: boolean } = * portable way to make a validate, harden or rename fail on demand. Without a seam the cleanup * would ship asserted by reading alone, which is how a failure path stays broken. */ -export type SpendJournalFaultStep = "create" | "write" | "validate" | "harden" | "rename"; +export type SpendJournalFaultStep = "stat" | "create" | "write" | "validate" | "harden" | "rename"; let journalFaultForTests: ((step: SpendJournalFaultStep, temp: string) => void) | undefined; @@ -422,6 +422,7 @@ export function setSpendJournalFaultForTests( */ function ledgerEntryExists(path: string): boolean { try { + journalFaultForTests?.("stat", path); lstatSync(path); return true; } catch (error) { diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index f292562bc0..c0ec49dd48 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { appendFileSync, chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -89,6 +89,33 @@ describe("spend ledger file journal", () => { expect(createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)).read()).toEqual([]); }); + test("a journal whose entry cannot be inspected refuses on every platform", () => { + const dir = ownedHome("ocx-spend-journal-stat-fault-"); + const journalPath = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); + const journal = createOwnedFileSpendJournal(mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME)); + journal.append(line("alias-one")); + const original = readFileSync(journalPath, "utf8"); + const before = readdirSync(dir).sort(); + + // The chmod case above is real but POSIX-only and meaningless as root. This one is the same + // contract proved everywhere: only the JOURNAL's own inspection fails, the salt stays + // readable, and every other filesystem step is real. + for (const code of ["EACCES", "EIO"] as const) { + setSpendJournalFaultForTests((step, target) => { + if (step !== "stat" || target !== journalPath) return; + throw Object.assign(new Error(`injected ${code}`), { code }); + }); + expect(() => journal.read()).toThrowError(SpendLedgerOwnerError); + expect(() => journal.append(line("alias-two"))).toThrow(/could not be inspected safely/); + setSpendJournalFaultForTests(undefined); + } + + // Refused, not reset: no truncation, no new entries, and the salt is still mintable. + expect(readFileSync(journalPath, "utf8")).toBe(original); + expect(readdirSync(dir).sort()).toEqual(before); + expect(loadOrCreateSpendLedgerSalt(mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME))).toMatch(/^[0-9a-f]{32,}$/); + }); + test.skipIf(!posixModes)("a journal that already exists is re-hardened, not trusted", () => { const dir = ownedHome("ocx-spend-journal-"); const path = join(dir, SPEND_LEDGER_JOURNAL_FILENAME); @@ -166,10 +193,14 @@ describe("spend ledger file journal", () => { const original = readFileSync(path, "utf8"); // The entry already exists by the time the write runs, which is the case the exclusive - // create exists to make unambiguous: a short write still leaves a file that is ours. + // create exists to make unambiguous. A real prefix goes in first, so this is a SHORT write + // rather than a failure before any byte landed: the residue that must be cleaned is a file + // with content in it. setSpendJournalFaultForTests((actual, temp) => { if (actual !== "write") return; expect(existsSync(temp)).toBe(true); + appendFileSync(temp, line("half-written").slice(0, 12), { encoding: "utf8" }); + expect(readFileSync(temp, "utf8").length).toBeGreaterThan(0); throw Object.assign(new Error("no space left on device"), { code: "ENOSPC" }); }); for (let attempt = 0; attempt < 3; attempt += 1) { diff --git a/tests/server/spend-ledger-owner-startup.test.ts b/tests/server/spend-ledger-owner-startup.test.ts index d93baf5c52..f8649f7e2f 100644 --- a/tests/server/spend-ledger-owner-startup.test.ts +++ b/tests/server/spend-ledger-owner-startup.test.ts @@ -78,17 +78,29 @@ test("startServer acquires before serving and final stop releases", async () => expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); }); -test("a partial start that bound public before an auxiliary failure releases ownership", () => { +/** Bounded wait: the rollback returns the directory on a continuation, not in the throw's turn. */ +async function waitForOwnership(expected: "held" | "unheld"): Promise { + for (let turn = 0; turn < 200; turn += 1) { + if (spendLedgerOwnerSnapshot().ownership === expected) return; + await Bun.sleep(5); + } + expect(spendLedgerOwnerSnapshot().ownership).toBe(expected); +} + +test("a partial start that bound public before an auxiliary failure releases ownership", async () => { const blocker = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("blocked") }); try { const candidate = config(); candidate.unauthenticatedLoopbackListener = { enabled: true, port: blocker.port }; saveConfig(candidate); expect(() => startServer(0)).toThrow(); - expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); + // The rollback stops the listener it already bound BEFORE it gives the directory back, so + // ownership returns once that stop settles rather than in the same turn as the throw. + // Asserting it synchronously passed only while the rollback discarded the stop promise. + await waitForOwnership("unheld"); const next = acquireSpendLedgerOwner(); next.release(); } finally { - blocker.stop(true); + await blocker.stop(true); } }); From 4d8493f9094bbf3acc996ebef26497b0941673f4 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:30:14 +0900 Subject: [PATCH 30/34] test: keep websocket teardown failure-safe and drain the store-default turns Two public review findings, both correct against the current source. The websocket fixtures awaited a bounded completion wait before the rest of their teardown. When that wait gave up, the hook stopped there: remaining sockets stayed open, the shutdown hooks never ran, the writer lease was never returned and the replaced globals were never restored, so one slow turn poisoned every case after it. The wait is now inside a try and everything else is in the finally, in the same order as before. The failure still propagates, and the comment says why that matters: a wait that expired is not evidence the turn settled, only that the fixture could not prove it settled. No timeout moved. The store-default rows ask for a stream and then read only the captured upstream request, so the turn's own body was left live while the teardown handed back the lease. They drain it now. An earlier review of mine reported every body as consumed; that was wrong, and the source is what settles it. AST syntax screen clean. Local checks: NOT RUN. --- tests/helpers/native-injection-fixture.ts | 26 +++++++++------- .../responses-inbound-store-default.test.ts | 6 +++- tests/responses/ws-native-steering.test.ts | 30 +++++++++++-------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index f56401b7d6..98712c8daa 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -113,16 +113,22 @@ export function installInjectionFixture() { // handler.close only STARTS the pump cancellation. Waiting for the socket to drop its stream // cancel and its native control is what proves the turn finished accounting; releasing the // lease before that leaves a reader settling against a journal nobody owns. - for (const client of clients.splice(0)) { - client.close(); - await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + // + // The rest runs even when that wait gives up, and the failure still propagates. A wait that + // expired is NOT evidence the turn settled: it means this fixture could not prove it, and + // the case should say so while still handing back the lease and the globals it replaced. + try { + for (const client of clients.splice(0)) { + client.close(); + await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } + } finally { + for (const socket of InjectionSocket.all) socket.close(); + InjectionSocket.all = []; runOptionalShutdownHooks(); + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; + for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } } - for (const socket of InjectionSocket.all) socket.close(); - InjectionSocket.all = []; runOptionalShutdownHooks(); - // Released only once those have all settled. - releaseSpendHome?.(); - releaseSpendHome = undefined; - globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; - for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } }); } diff --git a/tests/responses/responses-inbound-store-default.test.ts b/tests/responses/responses-inbound-store-default.test.ts index 73e6890b2b..7f370de68c 100644 --- a/tests/responses/responses-inbound-store-default.test.ts +++ b/tests/responses/responses-inbound-store-default.test.ts @@ -70,7 +70,7 @@ describe("/v1/responses defaults store:false only for the canonical forward Code const { urls, bodies } = captureUpstream(); // Direct dispatch needs the writer lease to prevent spend-ledger ownership failures. releaseSpendHome = acquireOwnedSpendHome(); - await handleResponses( + const turn = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, @@ -84,6 +84,10 @@ describe("/v1/responses defaults store:false only for the canonical forward Code config, { model: "", provider: "" }, ); + // Every row here asks for a stream and then reads only the captured upstream REQUEST, so + // the turn's own body was left live. Draining it lets the parser, the completion callbacks + // and the lifetime cleanup finish before the teardown below hands back the writer lease. + await turn.text(); let parsed: Record | null = null; try { parsed = bodies[0] ? (JSON.parse(bodies[0]) as Record) : null; } catch { parsed = null; } return { url: urls[0] ?? "", body: parsed }; diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index 0646f73b5f..9a95d3fc6e 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -93,19 +93,25 @@ afterEach(async () => { // handler.close only STARTS the pump cancellation. Waiting for the socket to drop its stream // cancel and its native control is what proves the turn finished accounting; releasing the // lease before that leaves a reader settling against a journal nobody owns. - for (const client of clients.splice(0)) { - client.close(); - await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + // + // The rest runs even when that wait gives up, and the failure still propagates. A wait that + // expired is NOT evidence the turn settled: it means this fixture could not prove it, and the + // case should say so while still handing back the lease and the globals it replaced. + try { + for (const client of clients.splice(0)) { + client.close(); + await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } + } finally { + for (const socket of Socket.all) socket.close(); + Socket.all = []; + runOptionalShutdownHooks(); + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.WebSocket = realSocket; + globalThis.fetch = realFetch; + for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } } - for (const socket of Socket.all) socket.close(); - Socket.all = []; - runOptionalShutdownHooks(); - // Released only once those have all settled. - releaseSpendHome?.(); - releaseSpendHome = undefined; - globalThis.WebSocket = realSocket; - globalThis.fetch = realFetch; - for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } }); test("configuration is explicit opt-in and malformed values fail closed", () => { From 271139883a552f1aac9ce3f847eb94e3327e5efc Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:35:47 +0900 Subject: [PATCH 31/34] test: finish the websocket teardown so one failure cannot strand the rest My previous attempt wrapped the whole client loop in one try, which left two gaps the review caught. A wait that gave up aborted the loop, so every client after it kept its socket open for the next case to inherit. And the lease release can itself throw, which took the global restore down with it. Each step is attempted now and the first failure is kept: every client is closed and waited on, every upstream socket is closed, the shutdown hooks run, and the lease is released, each guarded so a failure in one does not skip the others. Restoring the replaced globals sits in an outer finally, so it happens whatever else went wrong. The collected failure is thrown afterwards. That last part is the point: a collected failure still fails the case. It means the fixture could not prove the turn settled, not that it settled. No timeout moved and the bounded completion assertion is unchanged. AST syntax screen clean. Local checks: NOT RUN. --- tests/helpers/native-injection-fixture.ts | 22 +++++++++++++++------ tests/responses/ws-native-steering.test.ts | 23 +++++++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index 98712c8daa..1b4a233876 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -117,18 +117,28 @@ export function installInjectionFixture() { // The rest runs even when that wait gives up, and the failure still propagates. A wait that // expired is NOT evidence the turn settled: it means this fixture could not prove it, and // the case should say so while still handing back the lease and the globals it replaced. + let failure: unknown; + const note = (error: unknown): void => { failure ??= error; }; try { + // Every client gets its close and its wait even after an earlier one gave up. Stopping at + // the first failure left the rest open for the next case to inherit. for (const client of clients.splice(0)) { - client.close(); - await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + try { + client.close(); + await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } catch (error) { note(error); } } + for (const socket of InjectionSocket.all) { + try { socket.close(); } catch (error) { note(error); } + } + InjectionSocket.all = []; + try { runOptionalShutdownHooks(); } catch (error) { note(error); } + // The release itself can throw, and it used to take the global restore down with it. + try { releaseSpendHome?.(); } catch (error) { note(error); } finally { releaseSpendHome = undefined; } } finally { - for (const socket of InjectionSocket.all) socket.close(); - InjectionSocket.all = []; runOptionalShutdownHooks(); - releaseSpendHome?.(); - releaseSpendHome = undefined; globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } } + if (failure !== undefined) throw failure; }); } diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index 9a95d3fc6e..9877229099 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -97,21 +97,30 @@ afterEach(async () => { // The rest runs even when that wait gives up, and the failure still propagates. A wait that // expired is NOT evidence the turn settled: it means this fixture could not prove it, and the // case should say so while still handing back the lease and the globals it replaced. + let failure: unknown; + const note = (error: unknown): void => { failure ??= error; }; try { + // Every client gets its close and its wait even after an earlier one gave up. Stopping at + // the first failure left the rest open for the next case to inherit. for (const client of clients.splice(0)) { - client.close(); - await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + try { + client.close(); + await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); + } catch (error) { note(error); } + } + for (const socket of Socket.all) { + try { socket.close(); } catch (error) { note(error); } } - } finally { - for (const socket of Socket.all) socket.close(); Socket.all = []; - runOptionalShutdownHooks(); - releaseSpendHome?.(); - releaseSpendHome = undefined; + try { runOptionalShutdownHooks(); } catch (error) { note(error); } + // The release itself can throw, and it used to take the global restore down with it. + try { releaseSpendHome?.(); } catch (error) { note(error); } finally { releaseSpendHome = undefined; } + } finally { globalThis.WebSocket = realSocket; globalThis.fetch = realFetch; for (const key of proxyKeys) { delete process.env[key]; if (savedProxy[key] !== undefined) process.env[key] = savedProxy[key]; } } + if (failure !== undefined) throw failure; }); test("configuration is explicit opt-in and malformed values fail closed", () => { From 9e789c1613b22109dd0398e9619be8902bff9662 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 22:47:41 +0900 Subject: [PATCH 32/34] test(server): fix the three late-batch failures the shard finally reached These were not new. A shard stops at its first failed batch, so batch 26 only became visible once the earlier batches passed. All three are fixture-side. The management-auth ACL case timed out icacls for the state directory as well as the management token file. Only the token file was ever load-bearing for its claim, and the assertion that the state's source is "environment" is what proves which path answered. The directory is hardened with required: true by the spend-journal owner during startServer, which refuses rather than soft-failing, because an unverified ACL on a directory holding a secret is not something to proceed past. The stub is narrowed to the token file and the claim is unchanged. Activation E overwrote process.platform globally to reach the relay path. That also changes state-directory identity, which is lowercased on win32 and so names a different directory on a case-sensitive filesystem, and the harness server's own writer lease stopped matching. The retry was refused before it could reach the second account, which is why the case saw acct-pool-a alone. It uses the narrow relay seam now, so the platform claim reaches the relay decision and nothing else. Same root cause as the subagent rows, different file. The combo failover row dispatches through the handler rather than the harness server, so it takes the lease itself and drains its response before releasing. No production policy moved. Assertions are unchanged; the drain is the only addition. Local checks: NOT RUN. --- tests/server/server-auth.test.ts | 14 +++++++++++--- tests/server/server-management-auth.test.ts | 14 +++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index ad8d5b48bf..aec00ee0be 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -36,6 +36,8 @@ import { startServer, } from "../../src/server"; import { clearRequestLogsForTests, getRequestLogEntries } from "../../src/server/request-log"; +import { setRelayPlatformForTests } from "../../src/server/responses/passthrough-delivery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { readUsageEntries } from "../../src/usage/log"; import { handleManagementAPI } from "../../src/server/management-api"; import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; @@ -753,6 +755,7 @@ describe("server local API auth", () => { }; let acceptedCount = 0; + const releaseSpendHome = acquireOwnedSpendHome(); try { const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", @@ -767,8 +770,10 @@ describe("server local API auth", () => { expect(response.status).toBe(429); expect(acceptedCount).toBe(1); expect(upstreamModels).toEqual(["first-model", "second-model"]); + await response.text(); } finally { await upstream.stop(true); + releaseSpendHome(); } }); @@ -3479,8 +3484,11 @@ describe("server local API auth", () => { }); test("Activation E: both stream modes retry only before response relay construction", async () => { - const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + // The relay decision only, not the whole process. A global platform override also changes + // state-directory identity, which is lowercased on win32 and so names a different directory + // on a case-sensitive filesystem: the server's own writer lease stopped matching and the + // retry was refused before it could reach the second account. + setRelayPlatformForTests("win32"); try { for (const streamMode of ["legacy-tee", "eager-relay"] as const) { const positive = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" @@ -3512,7 +3520,7 @@ describe("server local API auth", () => { } } } finally { - if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor); + setRelayPlatformForTests(undefined); } }, { timeout: SERVER_BUDGET_MS }); diff --git a/tests/server/server-management-auth.test.ts b/tests/server/server-management-auth.test.ts index d5dae24041..a055499381 100644 --- a/tests/server/server-management-auth.test.ts +++ b/tests/server/server-management-auth.test.ts @@ -1794,13 +1794,17 @@ describe("management and data-plane credential separation", () => { saveConfig(remoteConfig()); process.env.USERNAME ??= "tester"; setPlatformForTests("win32"); - // Env-token init never needs file ACL. Time out management-token paths so a - // broken file-backed ACL cannot be what made management available; allow - // other file hardens so startServer → saveConfig works on real win32 - // (config-mutation directory harden soft-fails home timeouts). + // Env-token init never needs file ACL. Time out the management TOKEN FILE so a broken + // file-backed ACL cannot be what made management available; the assertion that the state's + // source is "environment" is what proves which path answered. + // + // The state directory itself is no longer timed out. It was never load-bearing for this + // claim, and it is hardened with required: true by the spend-journal owner during + // startServer, which correctly refuses rather than soft-failing: an unverified ACL on the + // directory holding a secret is not something to proceed past. setIcaclsRunnerForTests(args => { const target = args[0] ?? ""; - if (target === testHome || target.endsWith("admin-api-token")) { + if (target.endsWith("admin-api-token")) { return { success: false, exitCode: null, timedOut: true, stdout: "" }; } return { success: true, exitCode: 0, timedOut: false, stdout: "" }; From 647dc675019a995b97c73c7b7ce02fe4cebaf5f4 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 23:02:36 +0900 Subject: [PATCH 33/34] test(spend): fix two macOS-only failures in my own storage regressions All four Linux shards passed; macOS found two things Linux could not. The seam-driven case compared the faulted entry against a path it built itself. The owned home is the REAL path of the directory, and on macOS the temp root is a symlink, so the equality never held and the fault silently did nothing: the read succeeded and the case failed asking why nothing threw. It matches on the entry name now, which is path-shape independent and still leaves the salt alone. The chmod case asserted the exact refusal message. Which gate notices first is platform-dependent: where the directory cannot be traversed at all, the ownership check cannot resolve it before the entry is ever inspected, so the refusal arrives from there instead. Both are the same module's refusal, so the case pins the type and the absence of a leaked path, and the seam-driven case below it pins the exact message on every platform. That is what the seam was added for. Also batched, since the review asked for it conditionally and the condition holds: client close and the completion wait now have separate guards in both websocket teardowns. close() runs the production handler, so a throw there would have skipped that client's wait as well as reporting its own failure. Local checks: NOT RUN. --- tests/helpers/native-injection-fixture.ts | 4 +++- tests/lib/spend-ledger-file-journal.test.ts | 19 ++++++++++++++----- tests/responses/ws-native-steering.test.ts | 4 +++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/helpers/native-injection-fixture.ts b/tests/helpers/native-injection-fixture.ts index 1b4a233876..c7d3a10b31 100644 --- a/tests/helpers/native-injection-fixture.ts +++ b/tests/helpers/native-injection-fixture.ts @@ -123,8 +123,10 @@ export function installInjectionFixture() { // Every client gets its close and its wait even after an earlier one gave up. Stopping at // the first failure left the rest open for the next case to inherit. for (const client of clients.splice(0)) { + // Separate guards: close() runs the production handler, so a throw there would otherwise + // skip this client's completion wait as well as its own failure. + try { client.close(); } catch (error) { note(error); } try { - client.close(); await waitForInjection(() => client.data.cancel === undefined && client.data.nativeControl === undefined); } catch (error) { note(error); } } diff --git a/tests/lib/spend-ledger-file-journal.test.ts b/tests/lib/spend-ledger-file-journal.test.ts index c0ec49dd48..ea45233c3a 100644 --- a/tests/lib/spend-ledger-file-journal.test.ts +++ b/tests/lib/spend-ledger-file-journal.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { appendFileSync, chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { createOwnedFileSpendJournal, loadOrCreateSpendLedgerSalt, @@ -70,9 +70,15 @@ describe("spend ledger file journal", () => { // been appended to as though the slot were empty. chmodSync(dir, 0o000); try { - expect(() => journal.read()).toThrowError(SpendLedgerOwnerError); - // The refusal carries the module's own vocabulary, not the errno and path of a state file. - expect(() => journal.read()).toThrow(/could not be inspected safely/); + // Refused, and refused in the module's own vocabulary rather than with the errno and path + // of a state file. WHICH gate notices first is platform-dependent: where the directory + // cannot be traversed at all, the ownership check cannot resolve it before the entry is + // ever inspected. Both are the same refusal, so the type and the absence of a leaked path + // are what this pins; the seam-driven case below pins the exact message everywhere. + let refusal: unknown; + try { journal.read(); } catch (error) { refusal = error; } + expect(refusal).toBeInstanceOf(SpendLedgerOwnerError); + expect((refusal as Error).message).not.toContain(dir); } finally { chmodSync(dir, 0o700); } @@ -101,8 +107,11 @@ describe("spend ledger file journal", () => { // contract proved everywhere: only the JOURNAL's own inspection fails, the salt stays // readable, and every other filesystem step is real. for (const code of ["EACCES", "EIO"] as const) { + // Matched by entry name, not by full path: the owned home is the REAL path of the + // directory, and on macOS the temp root is a symlink, so an equality check against the + // path this case built never fired and the fault silently did nothing. setSpendJournalFaultForTests((step, target) => { - if (step !== "stat" || target !== journalPath) return; + if (step !== "stat" || basename(target) !== SPEND_LEDGER_JOURNAL_FILENAME) return; throw Object.assign(new Error(`injected ${code}`), { code }); }); expect(() => journal.read()).toThrowError(SpendLedgerOwnerError); diff --git a/tests/responses/ws-native-steering.test.ts b/tests/responses/ws-native-steering.test.ts index 9877229099..dfa89bed85 100644 --- a/tests/responses/ws-native-steering.test.ts +++ b/tests/responses/ws-native-steering.test.ts @@ -103,8 +103,10 @@ afterEach(async () => { // Every client gets its close and its wait even after an earlier one gave up. Stopping at // the first failure left the rest open for the next case to inherit. for (const client of clients.splice(0)) { + // Separate guards: close() runs the production handler, so a throw there would otherwise + // skip this client's completion wait as well as its own failure. + try { client.close(); } catch (error) { note(error); } try { - client.close(); await waitFor(() => client.data.cancel === undefined && client.data.nativeControl === undefined); } catch (error) { note(error); } } From a67cd9b3be6965f3b7435aa0702b6b9b5bf68fc3 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 19 Sep 2026 23:13:57 +0900 Subject: [PATCH 34/34] test(spend): make the contention matrix test the modes it names The acceptance mapping found a labelling gap and it was real. The matrix iterated holder/contender mode pairs, but contenderMode reached only the test name and the marker suffix: busyError() acquired with nothing configured, so the contender had no mode at all. Both rows varied only the holder. No other fixture covered it either; every other spawned holder in this suite is observe and none configures a contender. The contender now records its own policy before it tries to acquire, which is exactly the state a second instance starts in: configured one way or the other and not yet a writer. Recording a policy touches no journal while this process owns nothing, so this adds configuration without adding a second writer. The matrix is all four combinations. The rule under test is that ownership does not depend on either side's ceiling, and a matrix missing observe/observe and enforced/enforced was not testing the claim its names made. One case is new rather than restored: a ceiling turned on while another process still owns the directory. It proves the transition changes what would be refused, never who may write, and that the ownership refusal is identical on both sides of it. Every real-process and timeout assertion is unchanged, and no production code moves. The gate was already unconditional; what was missing was the proof. Local checks: NOT RUN. --- tests/lib/spend-ledger-owner.test.ts | 46 +++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/lib/spend-ledger-owner.test.ts b/tests/lib/spend-ledger-owner.test.ts index c18848d453..25584deb8b 100644 --- a/tests/lib/spend-ledger-owner.test.ts +++ b/tests/lib/spend-ledger-owner.test.ts @@ -16,11 +16,13 @@ import { import { SPEND_LEDGER_JOURNAL_FILENAME, SPEND_LEDGER_SALT_FILENAME, + configureSharedSpendLedger, createOwnedFileSpendJournal, loadOrCreateSpendLedgerSalt, resetSharedSpendLedgerForTest, sharedSpendLedger, spendLedgerDiagnosticsSnapshot, + type SpendReservationPolicy, } from "../../src/lib/spend-reservation-ledger"; import { helperPath } from "../helpers/repo-root"; import { CONFIG_OWNER_FILE, CONFIG_UNINSTALL_MANIFEST, removeOwnedConfigState } from "../../src/lib/config-ownership"; @@ -113,6 +115,22 @@ function busyError(): SpendLedgerOwnerError { throw new Error("expected spend-ledger ownership refusal"); } +/** + * The contender's own spend configuration, in this process, before it tries to acquire. + * + * Recording a policy touches no journal while this process owns nothing, so this is exactly the + * state a second instance is in when it starts: configured one way or the other, and not yet a + * writer. Without it the matrix below named a contender mode it never activated. + */ +function contenderPolicy(mode: "observe" | "enforced"): SpendReservationPolicy { + return { + root: mode === "enforced" ? { maxTokens: 1_000 } : {}, + identity: {}, + pool: {}, + retentionMs: 60_000, + }; +} + describe("real process ownership", () => { test("only a parent-exit restart environment carries the handoff marker", () => { const source = { OCX_SPEND_LEDGER_RESTART_PARENT_PID: "stale", KEEP_ME: "yes" }; @@ -123,10 +141,19 @@ describe("real process ownership", () => { }); }); - for (const [holderMode, contenderMode] of [["observe", "enforced"], ["enforced", "observe"]] as const) { + // All four combinations, not the two mixed ones. The rule under test is that ownership does + // not depend on either side's ceiling, so a matrix missing observe/observe and + // enforced/enforced was not testing the claim its names made. + for (const [holderMode, contenderMode] of [ + ["observe", "observe"], + ["observe", "enforced"], + ["enforced", "observe"], + ["enforced", "enforced"], + ] as const) { test(`${holderMode} and ${contenderMode} configurations contend identically`, async () => { const holder = spawnHolder(home, holderMode, `${holderMode}-${contenderMode}`); await waitForMarker(holder.holdMarker, holder.child); + configureSharedSpendLedger(contenderPolicy(contenderMode)); const refusal = busyError(); expect(refusal.code).toBe("SPEND_LEDGER_OWNER_BUSY"); writeFileSync(holder.releaseMarker, "release"); @@ -134,6 +161,23 @@ describe("real process ownership", () => { }, SPAWN_BUDGET_MS); } + test("turning a ceiling on after an observe-only start does not change contention", async () => { + const holder = spawnHolder(home, "observe", "ceiling-transition"); + await waitForMarker(holder.holdMarker, holder.child); + + configureSharedSpendLedger(contenderPolicy("observe")); + expect(busyError().code).toBe("SPEND_LEDGER_OWNER_BUSY"); + + // The operator enables a ceiling while another process still owns the directory. A policy + // is a recorded value until a ledger exists, so this changes what WOULD be refused, never + // who may write, and the ownership refusal is identical either side of the transition. + configureSharedSpendLedger(contenderPolicy("enforced")); + expect(busyError().code).toBe("SPEND_LEDGER_OWNER_BUSY"); + + writeFileSync(holder.releaseMarker, "release"); + expect((await childResult(holder.child)).status).toBe("acquired"); + }, SPAWN_BUDGET_MS); + test("independent state directories are independent", async () => { const holder = spawnHolder(home, "observe", "independent"); await waitForMarker(holder.holdMarker, holder.child);