From c67e36a6fdbe7e68f5a6710a426140a1cffa15e0 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:30:24 +0900 Subject: [PATCH 1/9] docs: lock combo recovery composition and cancellation guards --- .../050_combo_recovery.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md index 2dc1be9ba1..940a2f2197 100644 --- a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md +++ b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md @@ -17,3 +17,29 @@ Before: a merely configured native target suppresses recovery even when not usab Remote tests cover native disabled/cooldown, native 401 exhaustion, canonical summary exhausted with eligible account, noncanonical quota veto, caller eligibility, cooldown waiting, all targets unavailable skips recovery, recovery failure never dispatches plaintext/ciphertext, aborted recovery at both sites returns cancellation, no retry after client output. Preserve 32-inflight and no-persist safeguards where owned by recovery helper. CodeRabbit HTTPS-only suggestion is assessed against existing http provider policy: do not invent combo-only URL permission changes. Record evidence-backed rebuttal or a narrowly necessary fix during P/security audit. This carry does not change provider URL policy or credentials. Exact-head CI + independent security review required; no live Kiro or local suites. + +## Current composition and cancellation amendment + +The lower stack PR is #3753 at b73809f7e (full CI34018758265 and independent +security/final reviews passed). Source #3706 remains c311e9598; its source-only +patch applies cleanly to this foundation. Preserve every opaque preflight and +client-reader repair; only handleComboResponses changes in core. + +At the initial unreadable-task recovery site, a false helper result returns 499 +when the caller signal is aborted, otherwise the existing unreadable-task 400. +At native exhaustion, recheck caller cancellation after routed-target waiting and +recovery, before adopting the last native failure. A successful helper remains +one-shot; normal failed recovery preserves the prior failure and never dispatches +unreadable ciphertext or persists recovered plaintext. Add deterministic abort +fixtures at both recovery sites using the existing fake upstream boundary. + +Canonical forward providers defer account/model quota admission to the existing +native selector; caller eligibility, target cooldowns and attempted exclusions +still apply. Noncanonical hosts and third-party cached quota remain filtered. + +No combo-only HTTPS restriction is added: this routes recovered content through +the same operator-configured provider transport as the already-supported all-routed +recovery case. Recovery credentials still go only to its existing fixed backend, +and explicit opt-in, loopback/caller guards and no-persist policy remain unchanged. +Introducing a new URL policy only for this combo branch would contradict the +existing configured-provider contract without evidence of a distinct boundary. From 1b30050258f1895cec5be3f79adcc6010c0e04f7 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:31:26 +0900 Subject: [PATCH 2/9] docs: include combo recovery guide consistency --- .../_plan/260906_release_244_followups/050_combo_recovery.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md index 940a2f2197..df522fa3ad 100644 --- a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md +++ b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md @@ -43,3 +43,7 @@ recovery case. Recovery credentials still go only to its existing fixed backend, and explicit opt-in, loopback/caller guards and no-persist policy remain unchanged. Introducing a new URL policy only for this combo branch would contradict the existing configured-provider contract without evidence of a distinct boundary. + +Also update the English guides/sub-agent-surface.md paragraph that currently says +combo routing is unchanged and native-only. The configuration pages alone would +leave that guide contradicting the newly reachable opt-in routed recovery path. From fd5e90f1b4cb01906444ee26d6b642322be4b0d1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:33:01 +0900 Subject: [PATCH 3/9] fix(combos): recover unavailable native tasks without losing cancellation Carry #3706 onto the verified opaque recovery foundation, keep canonical quota ownership and preserve caller cancellation at both recovery sites. Co-authored-by: yxr1995-maker <257504378+yxr1995-maker@users.noreply.github.com> --- .../docs/fr/reference/configuration/agents.md | 4 +- .../content/docs/guides/sub-agent-surface.md | 7 +- .../docs/ja/reference/configuration/agents.md | 2 +- .../docs/ko/reference/configuration/agents.md | 2 +- .../docs/reference/configuration/agents.md | 13 +-- .../docs/ru/reference/configuration/agents.md | 3 +- .../docs/tr/reference/configuration/agents.md | 16 ++-- .../zh-cn/reference/configuration/agents.md | 2 +- .../zh-tw/reference/configuration/agents.md | 2 +- src/combos/resolve.ts | 17 ++-- src/server/responses/core.ts | 86 ++++++++++++------- 11 files changed, 95 insertions(+), 59 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/configuration/agents.md b/docs-site/src/content/docs/fr/reference/configuration/agents.md index dace4ce567..2dfdd19a90 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/agents.md +++ b/docs-site/src/content/docs/fr/reference/configuration/agents.md @@ -58,7 +58,7 @@ Pour un tour enfant créé, l’ordre de repli est le suivant : Les chaînes de repli propres à un rôle doivent résider dans la configuration d’opencodex. L’ajout de `model_fallback` dans `$CODEX_HOME/agents/*.toml` amène Codex 0.146+ à rejeter le fichier de rôle entier à cause de ce champ inconnu, puis à ignorer le rôle (#1190). Une ancienne ligne `model_fallback` dans le fichier TOML reste lue par souci de rétrocompatibilité, mais `ocx doctor` la signale. -opencodex ignore les candidats désactivés, non routables, en mauvais état, en période de temporisation ou ayant atteint le seuil de quota. L’instantané de disponibilité est mis en cache pendant `subagentModelFallbackPollMs`. Les tâches enfants chiffrées limitent la chaîne aux cibles ChatGPT natives canoniques et aux routes Responses directes avec authentification par clé explicitement approuvées via `allowEncryptedV2AgentTasks: true` ; si aucune ne peut consommer la charge chiffrée, la requête échoue au lieu d’envoyer un texte chiffré illisible à une autre destination. Les combos restent limités aux cibles natives canoniques. +opencodex ignore les candidats désactivés, non routables, en mauvais état, en période de temporisation ou ayant atteint le seuil de quota. L’instantané de disponibilité est mis en cache pendant `subagentModelFallbackPollMs`. Les tâches enfants chiffrées limitent la chaîne aux cibles ChatGPT natives canoniques et aux routes Responses directes avec authentification par clé explicitement approuvées via `allowEncryptedV2AgentTasks: true` ; si aucune ne peut consommer la charge chiffrée, la requête échoue au lieu d’envoyer un texte chiffré illisible à une autre destination. Un combo essaie d’abord une cible native canonique disponible ; si aucune n’est sélectionnable et que `agentTaskRecovery` est activé, un `NEW_TASK` chiffré est récupéré une fois avant l’envoi routé du combo. ```json { @@ -111,7 +111,7 @@ Ce mécanisme ne protège pas contre un autre processus exécuté sous le même N’activez cette option que si la requête authentifiée supplémentaire, la consommation de quota, la présence de texte en clair dans le processus et la dépendance à un service privé sont acceptables. Dans le cas contraire, privilégiez un enfant ChatGPT natif ou une délégation hétérogène v1. -Ce mécanisme de récupération s’applique aux enfants routés directement. Au maximum 32 requêtes de récupération peuvent être actives simultanément ; toute absence supplémentaire dans le cache échoue de manière sûre. Pour les tâches chiffrées, le routage par combinaison conserve son filtre existant limité aux cibles natives et n’utilise pas la récupération. +Ce mécanisme de récupération s’applique aux enfants routés directement et aux `NEW_TASK` chiffrés d’un combo. Au maximum 32 requêtes de récupération peuvent être actives simultanément ; toute absence supplémentaire dans le cache échoue de manière sûre. Un combo disposant d’une cible native canonique disponible continue d’envoyer directement le texte chiffré ; la récupération ne s’exécute que si aucune cible native n’est sélectionnable. Un échec de récupération, l’épuisement des cibles ou leur indisponibilité conserve l’échec fermé sans transmettre le texte chiffré à un fournisseur routé. ## Plafonds d’effort diff --git a/docs-site/src/content/docs/guides/sub-agent-surface.md b/docs-site/src/content/docs/guides/sub-agent-surface.md index da33f30d4f..e0706a5638 100644 --- a/docs-site/src/content/docs/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/guides/sub-agent-surface.md @@ -169,8 +169,11 @@ byte-for-byte fidelity is not guaranteed. It rejects generic/API-key proxy calle `unreadable_encrypted_agent_task` on any failure. See [Agent configuration: Encrypted v2 task recovery](/reference/configuration/agents/#encrypted-v2-task-recovery) for the full trust boundary and configuration. -Combo routing remains unchanged and continues to consider only canonical native ChatGPT targets for -encrypted tasks. +Combo routing prefers a selectable canonical native ChatGPT target for encrypted tasks. If none +is usable, or native authorization attempts are exhausted, an explicitly enabled recovery may +make the task readable for one available routed target. All recovery trust and no-persistence +guards above still apply; a configured but disabled or cooling native target does not block this +fallback, and cancellation never becomes an unreadable-task error. ## Rejected encrypted history diff --git a/docs-site/src/content/docs/ja/reference/configuration/agents.md b/docs-site/src/content/docs/ja/reference/configuration/agents.md index f0453461a4..2b185b81c4 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ja/reference/configuration/agents.md @@ -53,7 +53,7 @@ V1 ガイダンスは、`max` または `ultra` でのみプロアクティブ 拒否し、ロールをスキップします(#1190)。TOML 内のレガシー `model_fallback` 行は後方互換性の ために引き続き読み取られますが、`ocx doctor` がそれをフラグ付けします。 -opencodex は、無効、ルーティング不能、異常、冷却期間、またはクォータしきい値の候補をスキップします。可用性スナップショットは `subagentModelFallbackPollMs` に対してキャッシュされます。暗号化された子タスクでは、チェーンを正規のネイティブ ChatGPT ターゲットと、`allowEncryptedV2AgentTasks: true` で明示的に信頼された直接のキー認証 Responses ルートに制限します。暗号化されたペイロードを処理できる対象がない場合、読み取り不可能な暗号文を別の場所へ送らず、リクエストは失敗します。コンボは引き続き正規のネイティブ対象だけを使用します。 +opencodex は、無効、ルーティング不能、異常、冷却期間、またはクォータしきい値の候補をスキップします。可用性スナップショットは `subagentModelFallbackPollMs` に対してキャッシュされます。暗号化された子タスクでは、チェーンを正規のネイティブ ChatGPT ターゲットと、`allowEncryptedV2AgentTasks: true` で明示的に信頼された直接のキー認証 Responses ルートに制限します。暗号化されたペイロードを処理できる対象がない場合、読み取り不可能な暗号文を別の場所へ送らず、リクエストは失敗します。コンボはまず利用可能な正規ネイティブ対象を試し、選択できるネイティブ対象がなく `agentTaskRecovery` が有効な場合、暗号化された `NEW_TASK` をルーティングされたコンボ送信の前に一度だけ復旧します。 ```json { diff --git a/docs-site/src/content/docs/ko/reference/configuration/agents.md b/docs-site/src/content/docs/ko/reference/configuration/agents.md index ec5764bbd2..1c999536f2 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ko/reference/configuration/agents.md @@ -53,7 +53,7 @@ V1 안내는 `max` 또는 `ultra`에서만 선제 텍스트로 제공됩니다. 거부하고 역할을 건너뜁니다 (#1190). TOML의 기존 `model_fallback` 줄은 하위 호환성을 위해 계속 읽히지만 `ocx doctor`가 이를 표시합니다. -opencodex는 비활성, 라우팅 불가, 비정상, 쿨다운 중, 또는 할당량 임계값에 걸린 후보를 건너뜁니다. 사용 가능성 스냅샷은 `subagentModelFallbackPollMs` 동안 캐시됩니다. 암호화된 하위 작업은 정규 네이티브 ChatGPT 대상과 `allowEncryptedV2AgentTasks: true`로 명시적으로 신뢰한 직접 키 인증 Responses 라우트만 후보로 사용합니다. 암호화된 페이로드를 처리할 수 있는 대상이 없으면 읽을 수 없는 암호문을 다른 곳으로 보내지 않고 요청이 실패합니다. 콤보는 계속 정규 네이티브 대상만 사용합니다. +opencodex는 비활성, 라우팅 불가, 비정상, 쿨다운 중, 또는 할당량 임계값에 걸린 후보를 건너뜁니다. 사용 가능성 스냅샷은 `subagentModelFallbackPollMs` 동안 캐시됩니다. 암호화된 하위 작업은 정규 네이티브 ChatGPT 대상과 `allowEncryptedV2AgentTasks: true`로 명시적으로 신뢰한 직접 키 인증 Responses 라우트만 후보로 사용합니다. 암호화된 페이로드를 처리할 수 있는 대상이 없으면 읽을 수 없는 암호문을 다른 곳으로 보내지 않고 요청이 실패합니다. 콤보는 먼저 사용 가능한 정규 네이티브 대상을 시도하고, 선택 가능한 네이티브 대상이 없으며 `agentTaskRecovery`가 켜져 있으면 암호화된 `NEW_TASK`를 라우팅된 콤보 전송 전에 한 번 복구합니다. ```json { diff --git a/docs-site/src/content/docs/reference/configuration/agents.md b/docs-site/src/content/docs/reference/configuration/agents.md index 8b1c536032..03741e013f 100644 --- a/docs-site/src/content/docs/reference/configuration/agents.md +++ b/docs-site/src/content/docs/reference/configuration/agents.md @@ -117,8 +117,9 @@ opencodex skips disabled, unroutable, unhealthy, cooling-down, or quota-threshol availability snapshot is cached for `subagentModelFallbackPollMs`. Encrypted child tasks restrict the chain to canonical native ChatGPT targets plus direct key-auth Responses routes explicitly trusted with `allowEncryptedV2AgentTasks: true`; if none can consume the encrypted payload, the -request fails instead of routing unreadable ciphertext elsewhere. Combo routing remains -canonical-native-only. +request fails instead of routing unreadable ciphertext elsewhere. Combo routing first tries an +available canonical native target; when none is selectable and `agentTaskRecovery` is enabled, +an encrypted `NEW_TASK` is recovered once before routed combo dispatch. ```json { @@ -203,9 +204,11 @@ Enable this only when the additional authenticated request, quota use, plaintext and private-backend dependency are acceptable. Prefer a native ChatGPT child or v1 heterogeneous delegation when they are not. -This recovery path applies to direct-routed children. At most 32 recovery requests can be active at -once; additional misses fail closed. Combo routing keeps its existing native-only filter for -encrypted tasks and does not invoke recovery. +This recovery path applies to direct-routed children and encrypted combo `NEW_TASK` spawns. At +most 32 recovery requests can be active at once; additional misses fail closed. A combo with an +available canonical native target still sends ciphertext directly; recovery runs only when no +native target is selectable. Failed recovery, exhausted targets, or unavailable targets still fail +closed without forwarding ciphertext to a routed provider. ## Effort caps diff --git a/docs-site/src/content/docs/ru/reference/configuration/agents.md b/docs-site/src/content/docs/ru/reference/configuration/agents.md index 1b8def013e..a3a0bc434b 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ru/reference/configuration/agents.md @@ -84,7 +84,8 @@ cooldown либо уже достигли порога quota. Availability-сн native ChatGPT-target'ами и прямыми key-auth Responses-маршрутами, явно доверенными через `allowEncryptedV2AgentTasks: true`. Если ни один из них не может обработать encrypted payload, запрос завершается ошибкой вместо отправки нечитаемого ciphertext наружу. Combo по-прежнему -использует только канонические native-цели. +сначала выбирает доступную каноническую native-цель; если её нельзя выбрать и включён +`agentTaskRecovery`, encrypted `NEW_TASK` восстанавливается один раз перед routed combo dispatch. ```json { diff --git a/docs-site/src/content/docs/tr/reference/configuration/agents.md b/docs-site/src/content/docs/tr/reference/configuration/agents.md index 7b07247a73..3bf154f14b 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/agents.md +++ b/docs-site/src/content/docs/tr/reference/configuration/agents.md @@ -122,7 +122,9 @@ görevlerinde zincir, kurallı yerel ChatGPT hedefleriyle ve `allowEncryptedV2AgentTasks: true` kullanılarak açıkça güvenilen doğrudan anahtar kimlik doğrulamalı Responses rotalarıyla sınırlıdır. Hiçbiri şifrelenmiş yükü işleyemezse istek, okunamayan şifreli metni başka bir yere yönlendirmek yerine -başarısız olur. Kombolar yalnızca kurallı yerel hedefleri kullanmaya devam eder. +başarısız olur. Kombo önce kullanılabilir kurallı yerel hedefi dener; seçilebilir +yerel hedef kalmazsa ve `agentTaskRecovery` etkinse, şifrelenmiş `NEW_TASK` yönlendirilen +kombo gönderiminden önce bir kez kurtarılır. ```json { @@ -226,10 +228,13 @@ sınırı ve özel arka uç bağımlılığı kabul edilebilir olduğunda etkinl Olmadıklarında yerel bir ChatGPT çocuğunu veya v1 heterojen yetkilendirmesini tercih edin. -Bu kurtarma yolu doğrudan yönlendirilen çocuklara uygulanır. Aynı anda en fazla -32 kurtarma isteği etkin olabilir; ek ıskalamalar kapalı olarak başarısız olur. -Kombo yönlendirmesi şifrelenmiş görevler için mevcut yalnızca yerel filtresini -korur ve kurtarmayı çağırmaz. +Bu kurtarma yolu doğrudan yönlendirilen çocuklara ve bir kombodaki şifrelenmiş +`NEW_TASK` oluşturma isteklerine uygulanır. Aynı anda en fazla 32 kurtarma isteği +etkin olabilir; ek ıskalamalar kapalı olarak başarısız olur. Kullanılabilir kanonik +yerel hedefi olan bir kombo şifreli metni yine doğrudan gönderir; kurtarma yalnızca +seçilebilir yerel hedef kalmadığında çalışır. Kurtarma hatası, tükenen hedefler veya +kullanılamayan hedefler, şifreli metin yönlendirilen sağlayıcıya gönderilmeden yine +kapalı biçimde başarısız olur. ## Çaba sınırları @@ -248,4 +253,3 @@ ile `xhigh` arasını sunar. v1, varsayılan ve v2 davranışının yeni başlayanlara yönelik açıklaması için [Alt ajan yüzeyleri](/tr/guides/sub-agent-surface/) sayfasına bakın. - diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md index bcdb51cf05..238f13ac8a 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md @@ -52,7 +52,7 @@ per-role fallback 链必须放在 opencodex 配置里。把 `model_fallback` 写 `$CODEX_HOME/agents/*.toml` 会让 Codex 0.146+ 把整个角色文件当作未知字段拒绝并跳过该角色 (#1190)。TOML 中的旧版 `model_fallback` 仍会被读取以保持向后兼容,但 `ocx doctor` 会标记它。 -opencodex 会跳过已禁用、不可路由、不健康、处于冷却中,或已达到配额阈值的候选项。可用性快照会在 `subagentModelFallbackPollMs` 期间缓存。对于加密的子任务,候选链只包含规范的原生 ChatGPT 目标,以及通过 `allowEncryptedV2AgentTasks: true` 明确信任的直接密钥认证 Responses 路由。如果没有任何目标能处理加密载荷,请求就会失败,而不是把不可读的密文路由到别处。combo 仍然只使用规范的原生目标。 +opencodex 会跳过已禁用、不可路由、不健康、处于冷却中,或已达到配额阈值的候选项。可用性快照会在 `subagentModelFallbackPollMs` 期间缓存。对于加密的子任务,候选链只包含规范的原生 ChatGPT 目标,以及通过 `allowEncryptedV2AgentTasks: true` 明确信任的直接密钥认证 Responses 路由。如果没有任何目标能处理加密载荷,请求就会失败,而不是把不可读的密文路由到别处。combo 会先尝试可用的规范原生目标;如果没有可选择的原生目标且已启用 `agentTaskRecovery`,会在路由到 combo 目标前对加密的 `NEW_TASK` 恢复一次。 ```json { diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md b/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md index 87141789a1..15545db7ed 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md @@ -50,7 +50,7 @@ V1 指引僅在 `max` 或 `ultra` 時為主動文字。V2 僅在存在偏好模 Codex 0.146+ 會將角色檔案中的 `model_fallback` 視為未知欄位並略過整個角色;`ocx doctor` 也會對此發出警告。因此新的角色級 fallback 應設定在 opencodex,而不是角色 TOML 中。 -opencodex 會跳過已停用、不可路由、不健康、冷卻中或達到配額閾值的候選項。可用性快取保存 `subagentModelFallbackPollMs`。對於加密的子任務,候選鏈僅包含規範的原生 ChatGPT 目標,以及透過 `allowEncryptedV2AgentTasks: true` 明確信任的直接金鑰驗證 Responses 路由。若無任何目標可處理加密 payload,請求會失敗,而不會將無法讀取的密文路由到別處。組合仍只使用規範的原生目標。 +opencodex 會跳過已停用、不可路由、不健康、冷卻中或達到配額閾值的候選項。可用性快取保存 `subagentModelFallbackPollMs`。對於加密的子任務,候選鏈僅包含規範的原生 ChatGPT 目標,以及透過 `allowEncryptedV2AgentTasks: true` 明確信任的直接金鑰驗證 Responses 路由。若無任何目標可處理加密 payload,請求會失敗,而不會將無法讀取的密文路由到別處。組合會先嘗試可用的規範原生目標;若沒有可選擇的原生目標且已啟用 `agentTaskRecovery`,會在路由到組合目標前對加密的 `NEW_TASK` 恢復一次。 ```json { diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index ae48650b0d..bd88e82244 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -1,6 +1,7 @@ import type { OcxComboTarget, OcxConfig } from "../types"; import { getCachedProviderQuota } from "../providers/quota-routing-cache"; import type { ProviderQuota } from "../providers/quota-types"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; import { sleepWithAbort } from "../lib/upstream-retry"; import { coolComboTarget, @@ -60,9 +61,13 @@ export class NoAvailableComboTargetsError extends Error { } } -function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget): boolean { - return Object.hasOwn(config.providers, target.provider) - && config.providers[target.provider]?.disabled !== true; +function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean { + if (!Object.hasOwn(config.providers, target.provider)) return false; + const provider = config.providers[target.provider]; + if (!provider || provider.disabled === true) return false; + // Native account selection owns model-scoped quota; a provider summary cannot veto it. + return isCanonicalOpenAiForwardProvider(provider) + || !cachedProviderQuotaIsExhausted(getCachedProviderQuota(target.provider, now), now); } function quotaWindowExhausted(percent: number | undefined, resetAt: number | undefined, now: number): boolean { @@ -159,8 +164,7 @@ export function pickComboTarget( const excluded = new Set(options.exclude ?? []); const now = options.now ?? Date.now(); const eligible = (target: Required): boolean => - targetProviderIsUsable(config, target) - && !cachedProviderQuotaIsExhausted(getCachedProviderQuota(target.provider, now), now) + targetProviderIsUsable(config, target, now) && !isComboTargetInCooldown(comboId, target, now) && !excluded.has(targetKey(target)) && (options.eligible?.(target) ?? true); @@ -338,8 +342,7 @@ export async function pickComboTargetWithWait( const combo = getCombo(config, comboId); if (!combo) throw new UnknownComboError(comboId); const waitingTargets = combo.targets.filter(target => - targetProviderIsUsable(config, target) - && !cachedProviderQuotaIsExhausted(getCachedProviderQuota(target.provider, now), now) + targetProviderIsUsable(config, target, now) && !excluded.has(targetKey(target)) && isComboTargetInCooldown(comboId, target, now) && (customEligible?.(target) ?? true), diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 25fcc4df0f..db0499a418 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2524,19 +2524,10 @@ export async function handleComboResponses( let comboPayloadReadable = false; const payloadEligible = (target: (typeof combo.targets)[number]): boolean => comboPayloadReadable || !unreadableEncryptedAgentTask || canDecryptUnreadableAgentTask(target); - const initialNow = Date.now(); - let pick: ReturnType = null; - const pickWithWait = (pickOptions: { - exclude?: Iterable; - eligible?: (target: NonNullable["targets"][number]) => boolean; - now?: number; - }) => pickComboTargetWithWait(config, comboId, { - ...pickOptions, - waitForCooldownMs: combo.waitForCooldownMs, - abortSignal: options.abortSignal, - }); - - if (unreadableEncryptedAgentTask && !combo.targets.some(canDecryptUnreadableAgentTask)) { + let encryptedTaskRecoveryAttempted = false; + const recoverUnreadableEncryptedTask = async (): Promise => { + if (encryptedTaskRecoveryAttempted) return false; + encryptedTaskRecoveryAttempted = true; const recovery = agentTaskRecoveryConfig(config); if ( (options.inboundWire ?? "responses") !== "responses" @@ -2550,19 +2541,7 @@ export async function handleComboResponses( config, { parentThreadId: inboundClientThreadId }, ); - return unreadableEncryptedAgentTaskResponse(); - } - pick = await pickWithWait({ now: initialNow }); - if (!pick) { - discardEncryptedAgentTaskRecovery( - req, - (body as { input?: unknown } | undefined)?.input, - config, - { parentThreadId: inboundClientThreadId }, - ); - return options.abortSignal?.aborted - ? clientCancelledResponse() - : comboUnavailable(comboId); + return false; } let recovered = false; try { @@ -2587,15 +2566,45 @@ export async function handleComboResponses( config, { parentThreadId: inboundClientThreadId }, ); - return unreadableEncryptedAgentTaskResponse(); + return false; } comboPayloadReadable = true; comboReplaySnapshot.recoveredPlaintext = true; - } else { - pick = await pickWithWait({ - eligible: payloadEligible, - now: initialNow, - }); + return true; + }; + const initialNow = Date.now(); + const pickWithWait = (pickOptions: { + exclude?: Iterable; + eligible?: (target: NonNullable["targets"][number]) => boolean; + now?: number; + }) => pickComboTargetWithWait(config, comboId, { + ...pickOptions, + waitForCooldownMs: combo.waitForCooldownMs, + abortSignal: options.abortSignal, + }); + let pick = await pickWithWait({ + eligible: payloadEligible, + now: initialNow, + }); + + if (unreadableEncryptedAgentTask && !pick) { + pick = await pickWithWait({ now: initialNow }); + if (!pick) { + discardEncryptedAgentTaskRecovery( + req, + (body as { input?: unknown } | undefined)?.input, + config, + { parentThreadId: inboundClientThreadId }, + ); + return options.abortSignal?.aborted + ? clientCancelledResponse() + : comboUnavailable(comboId); + } + if (!(await recoverUnreadableEncryptedTask())) { + return options.abortSignal?.aborted + ? clientCancelledResponse() + : unreadableEncryptedAgentTaskResponse(); + } } if (!pick) { @@ -2806,6 +2815,7 @@ export async function handleComboResponses( `[combo] ${comboId}: ${targetKey(pick.target)} failed with ${failure.response.status} after ${Date.now() - started}ms`, ); const failureNow = Date.now(); + const attemptedTargets = pick.attempted; const nextPick = advanceComboAfterFailure(config, pick, { retryAfter: failure.retryAfter, resetAt: failure.resetAt, @@ -2829,6 +2839,18 @@ export async function handleComboResponses( }); } if (!pick) { + if (options.abortSignal?.aborted) return clientCancelledResponse(); + if (unreadableEncryptedAgentTask && !comboPayloadReadable) { + const recoveredTarget = await pickWithWait({ + exclude: attemptedTargets, + now: failureNow, + }); + if (recoveredTarget && await recoverUnreadableEncryptedTask()) { + pick = recoveredTarget; + continue; + } + } + // Waiting or recovery may have observed cancellation after the check above. if (options.abortSignal?.aborted) return clientCancelledResponse(); adoptFailedChildLog(childLog); } From 0c97836c8148b28fdd18032e709b199b00af4189 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:33:01 +0900 Subject: [PATCH 4/9] test(combos): preserve canonical and third-party quota selection --- tests/codex-integration/combos.test.ts | 95 ++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/codex-integration/combos.test.ts b/tests/codex-integration/combos.test.ts index 1c4924d3de..98174c3848 100644 --- a/tests/codex-integration/combos.test.ts +++ b/tests/codex-integration/combos.test.ts @@ -858,6 +858,101 @@ describe("combo failure policy and advancement", () => { expect(pick?.target.provider).toBe("b"); }); + test.each(["pool", "direct"] as const)("defers native %s quota decisions to account and model scoped authentication", mode => { + const now = 50_000; + const config = baseConfig({ + providers: { + a: { + adapter: "openai-responses", + authMode: "forward", + codexAccountMode: mode, + baseUrl: "https://chatgpt.com/backend-api/codex", + }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb" }, + }, + }); + setCachedProviderQuotaForTests("a", { weeklyPercent: 100, updatedAt: now }); + + const pick = pickComboTarget(config, "free", { now }); + + expect(pick?.target.provider).toBe("a"); + }); + + test("native provider summary quota does not suppress a bounded cooldown wait", async () => { + const now = 50_000; + const config = baseConfig({ + providers: { + a: { + adapter: "openai-responses", + authMode: "forward", + codexAccountMode: "pool", + baseUrl: "https://chatgpt.com/backend-api/codex", + }, + }, + combos: { + free: { + targets: [{ provider: "a", model: "m1" }], + waitForCooldownMs: 2_000, + }, + }, + }); + setCachedProviderQuotaForTests("a", { weeklyPercent: 100, updatedAt: now }); + coolComboTarget("free", { provider: "a", model: "m1" }, { now, cooldownMs: 1_000 }); + const sleeps: number[] = []; + + const pick = await pickComboTargetWithWait(config, "free", { + now, + waitForCooldownMs: 2_000, + sleep: async ms => { sleeps.push(ms); }, + }); + + expect(pick?.target.provider).toBe("a"); + expect(sleeps).toEqual([1_000]); + }); + + test("still filters exhausted quota on a noncanonical forward destination", () => { + const now = 50_000; + const config = baseConfig({ + providers: { + a: { + adapter: "openai-responses", + authMode: "forward", + codexAccountMode: "pool", + baseUrl: "https://chatgpt.com.example/backend-api/codex", + }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb" }, + }, + }); + setCachedProviderQuotaForTests("a", { weeklyPercent: 100, updatedAt: now }); + + const pick = pickComboTarget(config, "free", { now }); + + expect(pick?.target.provider).toBe("b"); + }); + + test("retains caller eligibility restrictions for native targets", () => { + const now = 50_000; + const config = baseConfig({ + providers: { + a: { + adapter: "openai-responses", + authMode: "forward", + codexAccountMode: "pool", + baseUrl: "https://chatgpt.com/backend-api/codex", + }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb" }, + }, + }); + setCachedProviderQuotaForTests("a", { weeklyPercent: 100, updatedAt: now }); + + const pick = pickComboTarget(config, "free", { + now, + eligible: target => target.provider !== "a", + }); + + expect(pick?.target.provider).toBe("b"); + }); + test("elapsed quota reset does not permanently blacklist a provider", () => { const now = 50_000; const config = baseConfig(); From cd054d92641a8bc3583abee1ee7e942c9c4d2bfb Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:34:59 +0900 Subject: [PATCH 5/9] test(combos): cover unavailable native recovery and abort at both sites --- .../server/agent-task-recovery-combo.test.ts | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/tests/server/agent-task-recovery-combo.test.ts b/tests/server/agent-task-recovery-combo.test.ts index 0e6e22a590..8823270ff2 100644 --- a/tests/server/agent-task-recovery-combo.test.ts +++ b/tests/server/agent-task-recovery-combo.test.ts @@ -10,6 +10,11 @@ import { } from "../../src/responses/state"; import { resetAgentTaskRecoveryState } from "../../src/server/responses/agent-task-recovery"; import { agentTaskRecoveryCacheSnapshotForTests } from "../../src/server/responses/agent-task-recovery-cache"; +import { clearComboTargetCooldowns, coolComboTarget } from "../../src/combos/failover"; +import { + clearCachedProviderQuotas, + setCachedProviderQuotaForTests, +} from "../../src/providers/quota-routing-cache"; import { codexHeaders, encryptedInput, @@ -55,11 +60,15 @@ describe("combo path encrypted agent task recovery", () => { process.env["OPENCODEX_HOME"] = home; clearResponseStateMemoryForTests(); resetAgentTaskRecoveryState(); + clearCachedProviderQuotas(); + clearComboTargetCooldowns(); }); afterEach(() => { globalThis.fetch = originalFetch; resetAgentTaskRecoveryState(); + clearCachedProviderQuotas(); + clearComboTargetCooldowns(); clearResponseStateForTests(); removeTreeWithRetry(home); if (priorHome === undefined) delete process.env["OPENCODEX_HOME"]; @@ -172,6 +181,122 @@ describe("combo path encrypted agent task recovery", () => { expect(providerFetches).toBe(1); }); + test.each(["disabled", "cooldown"] as const)("recovers a mixed combo when the native target is blocked by %s", async (reason) => { + const config = comboConfig([ + { provider: "xai", model: "grok-4.5" }, + { provider: "openai", model: "gpt-5.5" }, + ]); + if (reason === "disabled") { + config.providers.openai!.disabled = true; + } else { + coolComboTarget("routed", { provider: "openai", model: "gpt-5.5" }, { cooldownMs: 60_000 }); + } + const assignment = "MIXED-RECOVERY-PRIVATE-ASSIGNMENT"; + const recoveryBodies: string[] = []; + const forwardedBodies: string[] = []; + globalThis.fetch = (async (input, init) => { + const body = typeof init?.body === "string" ? init.body : ""; + if (String(input).includes("chatgpt.com")) { + recoveryBodies.push(body); + return new Response(recoverySse(assignment), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + forwardedBodies.push(body); + return providerCompletion(); + }) as typeof fetch; + + const response = await post(config, "combo/routed", encryptedInput(), codexHeaders()); + await response.text(); + + expect(response.status).toBe(200); + expect(recoveryBodies).toHaveLength(1); + expect(forwardedBodies).toHaveLength(1); + expect(forwardedBodies[0]).toContain(assignment); + expect(forwardedBodies[0]).not.toContain(FERNET_TASK); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); + + test("fails closed without routed dispatch when mixed-combo recovery fails", async () => { + const config = comboConfig([ + { provider: "xai", model: "grok-4.5" }, + { provider: "openai", model: "gpt-5.5" }, + ]); + coolComboTarget("routed", { provider: "openai", model: "gpt-5.5" }, { cooldownMs: 60_000 }); + const urls: string[] = []; + globalThis.fetch = (async (input) => { + urls.push(String(input)); + return new Response("unavailable", { status: 503 }); + }) as typeof fetch; + + const response = await post(config, "combo/routed", encryptedInput(), codexHeaders()); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "unreadable_encrypted_agent_task" } }); + expect(urls).toHaveLength(1); + expect(urls[0]).toContain("chatgpt.com/backend-api/codex/responses"); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); + + test("recovers once when the selected native target fails model authorization", async () => { + const config = comboConfig([ + { provider: "openai", model: "gpt-5.5" }, + { provider: "xai", model: "grok-4.5" }, + ]); + const assignment = "RECOVERED-AFTER-NATIVE-401"; + const chatgptBodies: string[] = []; + const forwardedBodies: string[] = []; + globalThis.fetch = (async (input, init) => { + const body = typeof init?.body === "string" ? init.body : ""; + if (String(input).includes("chatgpt.com")) { + chatgptBodies.push(body); + if (body.includes("capture_assignment")) { + return new Response(recoverySse(assignment), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return Response.json( + { error: { message: "model is not enabled for this account", code: "model_not_found" } }, + { status: 401 }, + ); + } + forwardedBodies.push(body); + return providerCompletion(); + }) as typeof fetch; + + const response = await post(config, "combo/routed", encryptedInput(), codexHeaders()); + await response.text(); + + expect(response.status).toBe(200); + expect(chatgptBodies).toHaveLength(2); + expect(chatgptBodies[0]).not.toContain("capture_assignment"); + expect(chatgptBodies[1]).toContain("capture_assignment"); + expect(forwardedBodies).toHaveLength(1); + expect(forwardedBodies[0]).toContain(assignment); + expect(forwardedBodies[0]).not.toContain(FERNET_TASK); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); + + test("does not recover when every mixed-combo target is unavailable", async () => { + const config = comboConfig([ + { provider: "xai", model: "grok-4.5" }, + { provider: "openai", model: "gpt-5.5" }, + ]); + coolComboTarget("routed", { provider: "openai", model: "gpt-5.5" }, { cooldownMs: 60_000 }); + setCachedProviderQuotaForTests("xai", { updatedAt: Date.now(), weeklyPercent: 100 }); + globalThis.fetch = (async () => { + throw new Error("No network call is permitted without an eligible execution target"); + }) as typeof fetch; + + const response = await post(config, "combo/routed", encryptedInput(), codexHeaders()); + + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ error: { code: "combo_unavailable" } }); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); + test("keeps an opted-in Responses target out of encrypted combo dispatch", async () => { const config = comboConfig([ { provider: "relay", model: "relay-model" }, @@ -251,4 +376,68 @@ describe("combo path encrypted agent task recovery", () => { expect(forwardedBodies[0]).toContain(FERNET_TASK); expect(forwardedBodies[0]).not.toContain("capture_assignment"); }); + + test.each([ + { site: "native-disabled", expectedNative: 0 }, + { site: "native-401", expectedNative: 1 }, + ] as const)("cancels $site recovery before routed dispatch or plaintext cache", async ({ site, expectedNative }) => { + const config = comboConfig([ + { provider: "openai", model: "gpt-5.5" }, + { provider: "xai", model: "grok-4.5" }, + ]); + if (site === "native-disabled") { + config.providers.openai!.disabled = true; + } + const controller = new AbortController(); + let markRecoveryStarted: (() => void) | undefined; + const recoveryStarted = new Promise((resolve) => { + markRecoveryStarted = resolve; + }); + let nativeFetches = 0; + let recoveryFetches = 0; + let routedFetches = 0; + globalThis.fetch = ((input, init) => { + const body = typeof init?.body === "string" ? init.body : ""; + if (!String(input).includes("chatgpt.com")) { + routedFetches += 1; + return Promise.resolve(providerCompletion()); + } + if (body.includes("capture_assignment")) { + recoveryFetches += 1; + markRecoveryStarted?.(); + return new Promise((_resolve, reject) => { + const signal = init?.signal; + const rejectAbort = () => reject(signal?.reason ?? new DOMException("aborted", "AbortError")); + if (signal?.aborted) rejectAbort(); + else signal?.addEventListener("abort", rejectAbort, { once: true }); + }); + } + nativeFetches += 1; + return Promise.resolve(Response.json( + { error: { message: "model is not enabled for this account", code: "model_not_found" } }, + { status: 401 }, + )); + }) as typeof fetch; + + const pending = post( + config, + "combo/routed", + encryptedInput(), + codexHeaders(), + controller.signal, + ); + await recoveryStarted; + controller.abort(new DOMException("client disconnected", "AbortError")); + const response = await pending; + await runPendingResponseStatePersistForTests(); + const payload = await response.json() as { error?: { code?: string } }; + + expect(response.status).toBe(499); + expect(payload).toMatchObject({ error: { code: "client_cancelled" } }); + expect(nativeFetches).toBe(expectedNative); + expect(recoveryFetches).toBe(1); + expect(routedFetches).toBe(0); + expect(agentTaskRecoveryCacheSnapshotForTests()).toEqual({ entries: 0, bytes: 0 }); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); }); From f5c88beb9e11555d2acc7db942cf99d24b70f481 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 16:35:22 +0900 Subject: [PATCH 6/9] docs: record mixed combo recovery verification scope --- .../051_combo_recovery_implementation.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md diff --git a/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md b/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md new file mode 100644 index 0000000000..672ed62464 --- /dev/null +++ b/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md @@ -0,0 +1,19 @@ +# Mixed combo recovery implementation + +The carry changes only combo selection in core and provider usability in the +combo resolver. A selectable native target keeps priority. If native candidates +are unavailable or exhausted, an available routed target may be selected after +one explicitly enabled encrypted-task recovery. Existing caller admission, +fixed recovery backend, attempt exclusions and plaintext no-persistence remain. + +Canonical native quota belongs to account/model selection; cached summaries keep +filtering third-party and noncanonical providers. Both initial and late recovery +failures recheck caller cancellation, including cancellation during target waiting, +before returning an unreadable-task or prior native error. + +Original contributor tests cover disabled/cooldown/native-401, failed recovery, +unavailable targets, canonical/noncanonical quota and eligibility. The new paired +abort fixture waits for the recovery fetch to start, then cancels its actual signal; +499/client_cancelled, no routed call and empty cache/continuation stores are asserted. +No local suites/typecheck/build or live Kiro request are used. Hosted exact-head CI +and independent source/security/final reviews supply integration evidence. From b668dc8ddb5d9d684e2c58a3db938723581544e2 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 17:15:27 +0900 Subject: [PATCH 7/9] docs: finalize verified combo recovery composition record --- .../050_combo_recovery.md | 9 +++++++-- .../051_combo_recovery_implementation.md | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md index df522fa3ad..33e4a905c4 100644 --- a/devlog/_plan/260906_release_244_followups/050_combo_recovery.md +++ b/devlog/_plan/260906_release_244_followups/050_combo_recovery.md @@ -20,8 +20,8 @@ CodeRabbit HTTPS-only suggestion is assessed against existing http provider poli ## Current composition and cancellation amendment -The lower stack PR is #3753 at b73809f7e (full CI34018758265 and independent -security/final reviews passed). Source #3706 remains c311e9598; its source-only +The lower stack PR #3753 is merged as b9f2acc82 from cd6d4d346 (full +CI34020474748 and independent security/final reviews passed). Source #3706 remains c311e9598; its source-only patch applies cleanly to this foundation. Preserve every opaque preflight and client-reader repair; only handleComboResponses changes in core. @@ -47,3 +47,8 @@ existing configured-provider contract without evidence of a distinct boundary. Also update the English guides/sub-agent-surface.md paragraph that currently says combo routing is unchanged and native-only. The configuration pages alone would leave that guide contradicting the newly reachable opt-in routed recovery path. + +The parent now also preserves native preflight read resets/cancellation and +tee/eager failed terminal accounting, including semantic streamAborted parity. +The combo delta remains unchanged through that cascade; a fresh composition +review confirmed the same patch and the complete child runtime passed CI34020475627. diff --git a/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md b/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md index 672ed62464..a3ec651598 100644 --- a/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md +++ b/devlog/_plan/260906_release_244_followups/051_combo_recovery_implementation.md @@ -17,3 +17,19 @@ abort fixture waits for the recovery fetch to start, then cancels its actual sig 499/client_cancelled, no routed call and empty cache/continuation stores are asserted. No local suites/typecheck/build or live Kiro request are used. Hosted exact-head CI and independent source/security/final reviews supply integration evidence. + +## Verified composition + +- Source fd5e90f1b and regressions cd054d926 passed independent source/security + and final reviews. The initial full hosted run was CI34019564577. +- Parent #3753 required a separate repair cycle for preflight read failures and + tee EOF account outcomes. That repair is merged on dev as b9f2acc82; source + cd6d4d346 passed CI34020474748 and its two review threads are resolved. +- The resulting child e1f5a5b8d passed full CI34020475627. Stable patch ID + 8b62ad9ebb675f63a6dd4933e22663b48e1d95f2 matches the original combo delta, + and a fresh composition review passed. This documentation closeout changes + no runtime or tests. Final PR-head checks remain visible on #3754. +- #3706 remains open until #3754 actually merges. Closure requires a fresh + merged-state and dev-ancestry check; a successful merge command is not assumed. + +No local suite, typecheck, build or live Kiro call was used for these results. From c7450268cf245ebf56e95178c624cee36e236df7 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 18:45:35 +0900 Subject: [PATCH 8/9] fix(combos): recover stored Pool failures without reopening account hops --- src/server/responses/core.ts | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 89ab0ab549..deb7b1062c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -2525,6 +2525,7 @@ export async function handleComboResponses( const payloadEligible = (target: (typeof combo.targets)[number]): boolean => comboPayloadReadable || !unreadableEncryptedAgentTask || canDecryptUnreadableAgentTask(target); let encryptedTaskRecoveryAttempted = false; + let storedPool401ReplayDispatched = false; const recoverUnreadableEncryptedTask = async (): Promise => { if (encryptedTaskRecoveryAttempted) return false; encryptedTaskRecoveryAttempted = true; @@ -2662,7 +2663,6 @@ export async function handleComboResponses( attemptRetained = true; }; let consumedChildFailure: ConsumedComboFailure | undefined; - let storedPool401ReplayDispatched = false; const callbackGate = createChildPassthroughCallbackGate(options); let response: Response; try { @@ -2795,13 +2795,36 @@ export async function handleComboResponses( (logCtx.attempts ??= []).push(attempt); attemptRetained = true; lastFailure = failure.response; + const failureDecision = comboFailureDecision(failure.response.status, failure.classificationText, { + code: failure.upstreamCode, + }); if (storedPool401ReplayDispatched) { + if (failureDecision === "hop" && unreadableEncryptedAgentTask && !comboPayloadReadable) { + const recoveredTarget = await pickWithWait({ + exclude: pick.attempted, + eligible: target => { + try { + const route = routeConcreteModel(config, `${target.provider}/${target.model}`); + return route.codexAccountMode === undefined + && !isCanonicalOpenAiForwardProvider(route.provider); + } catch { + return false; + } + }, + }); + if (options.abortSignal?.aborted) return clientCancelledResponse(); + if (recoveredTarget && await recoverUnreadableEncryptedTask()) { + pick = recoveredTarget; + continue; + } + if (options.abortSignal?.aborted) return clientCancelledResponse(); + } + // Keep the spent Pool budget sticky even after a recovered routed child: + // no later failure may reopen ordinary combo/native account hopping. adoptFailedChildLog(childLog); return lastFailure; } - if (comboFailureDecision(failure.response.status, failure.classificationText, { - code: failure.upstreamCode, - }) === "stop") { + if (failureDecision === "stop") { adoptFailedChildLog(childLog); if ( failure.response.status === 413 From 8de12699820ec88fec995e650f2bb0d127be1f65 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 18:53:15 +0900 Subject: [PATCH 9/9] test(combos): cover stored Pool recovery budget and cancellation --- .../docs/reference/configuration/agents.md | 6 +- .../responses-pool-401-refresh.test.ts | 351 +++++++++++++++++- 2 files changed, 347 insertions(+), 10 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/agents.md b/docs-site/src/content/docs/reference/configuration/agents.md index 03741e013f..54affc94ad 100644 --- a/docs-site/src/content/docs/reference/configuration/agents.md +++ b/docs-site/src/content/docs/reference/configuration/agents.md @@ -207,8 +207,10 @@ delegation when they are not. This recovery path applies to direct-routed children and encrypted combo `NEW_TASK` spawns. At most 32 recovery requests can be active at once; additional misses fail closed. A combo with an available canonical native target still sends ciphertext directly; recovery runs only when no -native target is selectable. Failed recovery, exhausted targets, or unavailable targets still fail -closed without forwarding ciphertext to a routed provider. +native target is selectable. After a stored Pool account's refresh and same-account replay are +exhausted, recovery can use the incoming caller credential for one available routed target without +trying another native account. Policy refusals remain terminal. Failed recovery, exhausted targets, +or unavailable targets still fail closed without forwarding ciphertext to a routed provider. ## Effort caps diff --git a/tests/responses/responses-pool-401-refresh.test.ts b/tests/responses/responses-pool-401-refresh.test.ts index 197191c92b..609b24d772 100644 --- a/tests/responses/responses-pool-401-refresh.test.ts +++ b/tests/responses/responses-pool-401-refresh.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createHash } from "node:crypto"; @@ -16,8 +16,22 @@ import { resetProviderRequestPacingForTest, setProviderRequestPacingLimitsForTest, } from "../../src/providers/request-pacing"; +import { + clearResponseStateForTests, + clearResponseStateMemoryForTests, + responseContinuationRetainedStoreSnapshot, + runPendingResponseStatePersistForTests, +} from "../../src/responses/state"; +import { resetAgentTaskRecoveryState } from "../../src/server/responses/agent-task-recovery"; +import { agentTaskRecoveryCacheSnapshotForTests } from "../../src/server/responses/agent-task-recovery-cache"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { + FERNET_TASK, + codexHeaders, + encryptedInput, + recoverySse, +} from "../helpers/agent-task-recovery"; import { removeTreeWithRetry } from "../helpers/remove-tree"; /** @@ -64,17 +78,25 @@ const THREAD_ID = "thread-2887"; function request( path: "/v1/responses" | "/v1/responses/compact", - options: { affined?: boolean; model?: string; headers?: HeadersInit; stream?: boolean } = {}, + options: { + affined?: boolean; + model?: string; + headers?: HeadersInit; + stream?: boolean; + input?: unknown; + } = {}, ): Request { const headers = new Headers(options.headers); headers.set("content-type", "application/json"); if (options.affined) headers.set("x-codex-parent-thread-id", THREAD_ID); + const compact = path.endsWith("compact"); + const input = options.input ?? (compact ? [] : "hello"); return new Request(`http://localhost${path}`, { method: "POST", headers, - body: JSON.stringify(path.endsWith("compact") - ? { model: options.model ?? "gpt-5.5", input: [] } - : { model: options.model ?? "gpt-5.5", input: "hello", stream: options.stream ?? false }), + body: JSON.stringify(compact + ? { model: options.model ?? "gpt-5.5", input } + : { model: options.model ?? "gpt-5.5", input, stream: options.stream ?? false }), }); } @@ -117,7 +139,14 @@ function readStoredGeneration(): number { return raw[ACCOUNT_ID]!.generation; } -type Harness = { sends: string[]; refreshes: string[] }; +type Harness = { + sends: string[]; + refreshes: string[]; + recoveryAuths: string[]; + backupAuths: string[]; + backupBodies: string[]; + canonicalAliasSends: number; +}; /** * Upstream rejects the old bearer once, the token endpoint rotates, and the replay with the @@ -126,11 +155,18 @@ type Harness = { sends: string[]; refreshes: string[] }; function installHarness(options: { refresh?: () => Response; responseForSend?: (authorization: string, sendNumber: number, url: URL) => Response | undefined; + recovery?: (authorization: string, init?: RequestInit) => Response | Promise; } = {}): Harness { const sends: string[] = []; const refreshes: string[] = []; + const recoveryAuths: string[] = []; + const backupAuths: string[] = []; + const backupBodies: string[] = []; + let canonicalAliasSends = 0; globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { const url = new URL(input instanceof Request ? input.url : String(input)); + const body = typeof init?.body === "string" ? init.body : ""; + const authorization = new Headers(init?.headers).get("authorization") ?? ""; if (url.hostname === "auth.openai.com") { refreshes.push(new URLSearchParams(String(init?.body)).get("refresh_token") ?? ""); if (options.refresh) return options.refresh(); @@ -140,10 +176,29 @@ function installHarness(options: { expires_in: 3600, }); } + if (body.includes("capture_assignment")) { + recoveryAuths.push(authorization); + if (options.recovery) return await options.recovery(authorization, init); + return new Response(recoverySse("RECOVERED-POOL-PLAINTEXT-SENTINEL"), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } if (!url.pathname.endsWith("/responses") && !url.pathname.endsWith("/responses/compact")) { return Response.json({ rate_limit: { primary_window: { used_percent: 10 } } }); } - const authorization = new Headers(init?.headers).get("authorization") ?? ""; + if (url.hostname === "backup.example" || url.hostname === "spare.example") { + backupAuths.push(authorization); + backupBodies.push(body); + } + if ( + url.hostname === "chatgpt.com" + && authorization !== "Bearer rejected-access" + && authorization !== "Bearer refreshed-access" + && authorization !== "Bearer other-access" + ) { + canonicalAliasSends += 1; + } sends.push(authorization); const customResponse = options.responseForSend?.(authorization, sends.length, url); if (customResponse) return customResponse; @@ -152,7 +207,7 @@ function installHarness(options: { } return Response.json({ id: "resp_replayed", object: "response", status: "completed", output: [] }); }) as typeof fetch; - return { sends, refreshes }; + return { sends, refreshes, recoveryAuths, backupAuths, backupBodies, get canonicalAliasSends() { return canonicalAliasSends; } }; } function recoveryComboConfig(): OcxConfig { @@ -175,6 +230,75 @@ function recoveryComboConfig(): OcxConfig { return cfg; } +function writeWorkAndOtherAccounts(): void { + writeStoredAccount({ + [OTHER_ACCOUNT_ID]: storedRecord({ + accessToken: "other-access", + refreshToken: "other-grant", + generation: 1, + chatgptAccountId: "acc-other", + }), + }); +} + +function encryptedRecoveryComboConfig(options: { + extraCanonical?: boolean; + extraSpare?: boolean; + includeBackup?: boolean; +} = {}): OcxConfig { + const cfg = recoveryComboConfig(); + cfg.agentTaskRecovery = { enabled: true }; + cfg.accountPoolStrategy = "fill-first"; + cfg.codexAccounts = [ + { id: ACCOUNT_ID, label: "work" }, + { id: OTHER_ACCOUNT_ID, label: "other" }, + ]; + if (options.extraCanonical) { + cfg.providers.chatgpt = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + }; + } + if (options.extraSpare) { + cfg.providers.spare = { + adapter: "openai-responses", + baseUrl: "https://spare.example/v1", + authMode: "key", + apiKey: "spare-test-key", + }; + } + const targets: Array<{ provider: string; model: string }> = [ + { provider: "openai", model: "gpt-5.5" }, + ]; + if (options.extraCanonical) targets.push({ provider: "chatgpt", model: "gpt-5.5" }); + if (options.includeBackup !== false) targets.push({ provider: "backup", model: "m2" }); + if (options.extraSpare) targets.push({ provider: "spare", model: "m3" }); + cfg.combos = { + recovery: { + strategy: "failover", + targets, + }, + }; + return cfg; +} + +function storedReplay401(authorization: string, url: URL): Response | undefined { + if (url.hostname === "spare.example") { + return Response.json({ id: "must-not-run-spare", object: "response", status: "completed", output: [] }); + } + if (authorization === "Bearer rejected-access") { + return Response.json({ error: { message: "rejected bearer" } }, { status: 401 }); + } + if (authorization === "Bearer refreshed-access") { + return Response.json({ error: { message: "replay rejected" } }, { status: 401 }); + } + if (authorization === "Bearer other-access") { + return Response.json({ id: "must-not-run-other", object: "response", status: "completed", output: [] }); + } + return undefined; +} + beforeEach(() => { home = mkdtempSync(join(tmpdir(), "ocx-responses-pool-401-")); previousOcxHome = process.env.OPENCODEX_HOME; @@ -185,6 +309,8 @@ beforeEach(() => { clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); clearThreadAccountMap(); + clearResponseStateMemoryForTests(); + resetAgentTaskRecoveryState(); writeStoredAccount(); }); @@ -195,6 +321,8 @@ afterEach(() => { clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); clearThreadAccountMap(); + resetAgentTaskRecoveryState(); + clearResponseStateForTests(); if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOcxHome; if (previousCodexHome === undefined) delete process.env.CODEX_HOME; @@ -892,3 +1020,210 @@ describe("ordinary pool 401 refresh and replay (#2887)", () => { expect(harness.refreshes).toEqual(["refresh-grant"]); }); }); + +describe("stored pool 401 replay then encrypted combo recovery", () => { + const assignment = "RECOVERED-POOL-PLAINTEXT-SENTINEL"; + + async function postEncryptedCombo( + cfg: OcxConfig, + headers: Headers, + abortSignal?: AbortSignal, + logCtx: RequestLogContext = { model: "", provider: "" } as RequestLogContext, + ): Promise { + return handleResponses( + request("/v1/responses", { + model: "combo/recovery", + headers, + input: encryptedInput(), + }), + cfg, + logCtx, + abortSignal ? { abortSignal } : {}, + ); + } + + test("refreshes once, recovers once with the caller bearer, and backups plaintext without storing it", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => { + if (url.hostname === "backup.example") { + return Response.json({ id: "resp_backup", object: "response", status: "completed", output: [] }); + } + return storedReplay401(authorization, url); + }, + }); + + const response = await postEncryptedCombo(cfg, headers); + await runPendingResponseStatePersistForTests(); + const payload = await response.clone().json() as { id?: string }; + + expect(response.status).toBe(200); + expect(typeof payload.id).toBe("string"); + expect(harness.refreshes).toEqual(["refresh-grant"]); + expect(harness.sends.filter(send => send === "Bearer rejected-access" || send === "Bearer refreshed-access")) + .toEqual(["Bearer rejected-access", "Bearer refreshed-access"]); + expect(harness.sends).not.toContain("Bearer other-access"); + expect(harness.recoveryAuths).toEqual([headers.get("authorization")]); + expect(harness.backupAuths).toEqual(["Bearer backup-test-key"]); + expect(harness.backupBodies).toHaveLength(1); + expect(harness.backupBodies[0]).toContain(assignment); + expect(harness.backupBodies[0]).not.toContain(FERNET_TASK); + expect(harness.canonicalAliasSends).toBe(0); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + const snapshotPath = join(home, "responses-state.json"); + const snapshot = existsSync(snapshotPath) ? readFileSync(snapshotPath, "utf8") : ""; + expect(snapshot).not.toContain(assignment); + expect(snapshot).not.toContain(payload.id!); + }); + + test("skips another canonical alias before the independently routed backup", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => { + if (url.hostname === "backup.example") { + return Response.json({ id: "resp_backup", object: "response", status: "completed", output: [] }); + } + return storedReplay401(authorization, url); + }, + }); + + const response = await postEncryptedCombo(cfg, headers); + expect(response.status).toBe(200); + expect(harness.canonicalAliasSends).toBe(0); + expect(harness.sends).not.toContain("Bearer other-access"); + expect(harness.backupBodies).toHaveLength(1); + expect(harness.backupBodies[0]).toContain(assignment); + }); + + test("abort during recovery returns 499 without backup, other-account spend, or cache", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true }); + const controller = new AbortController(); + let markRecoveryStarted: (() => void) | undefined; + const recoveryStarted = new Promise((resolve) => { + markRecoveryStarted = resolve; + }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => storedReplay401(authorization, url), + recovery: (_authorization, init) => { + markRecoveryStarted?.(); + return new Promise((_resolve, reject) => { + const signal = init?.signal; + const rejectAbort = () => reject(signal?.reason ?? new DOMException("aborted", "AbortError")); + if (signal?.aborted) rejectAbort(); + else signal?.addEventListener("abort", rejectAbort, { once: true }); + }); + }, + }); + + const pending = postEncryptedCombo(cfg, headers, controller.signal); + await recoveryStarted; + controller.abort(new DOMException("client disconnected", "AbortError")); + const response = await pending; + await runPendingResponseStatePersistForTests(); + const payload = await response.json() as { error?: { code?: string } }; + + expect(response.status).toBe(499); + expect(payload).toMatchObject({ error: { code: "client_cancelled" } }); + expect(harness.backupAuths).toEqual([]); + expect(harness.sends).not.toContain("Bearer other-access"); + expect(harness.canonicalAliasSends).toBe(0); + expect(agentTaskRecoveryCacheSnapshotForTests()).toEqual({ entries: 0, bytes: 0 }); + expect(responseContinuationRetainedStoreSnapshot().count).toBe(0); + }); + + test("recovery failure keeps the replay 401 and does not send backup", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => storedReplay401(authorization, url), + recovery: () => new Response("not-sse", { status: 500 }), + }); + + const response = await postEncryptedCombo(cfg, headers); + expect(response.status).toBe(401); + expect(harness.recoveryAuths).toHaveLength(1); + expect(harness.backupAuths).toEqual([]); + expect(harness.sends).toEqual(["Bearer rejected-access", "Bearer refreshed-access"]); + expect(harness.canonicalAliasSends).toBe(0); + }); + + test("no independently routed target retains the replay 401 without recovery", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true, includeBackup: false }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => storedReplay401(authorization, url), + }); + + const response = await postEncryptedCombo(cfg, headers); + expect(response.status).toBe(401); + expect(harness.recoveryAuths).toEqual([]); + expect(harness.backupAuths).toEqual([]); + expect(harness.sends).toEqual(["Bearer rejected-access", "Bearer refreshed-access"]); + expect(harness.canonicalAliasSends).toBe(0); + }); + + test("a hop-class backup failure cannot reopen later combo or native hops", async () => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true, extraSpare: true }); + const logCtx = { model: "", provider: "" } as RequestLogContext; + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => { + if (url.hostname === "backup.example") { + return Response.json({ error: { message: "backup overloaded" } }, { status: 503 }); + } + return storedReplay401(authorization, url); + }, + }); + + const response = await postEncryptedCombo(cfg, headers, undefined, logCtx); + expect(response.status).toBe(503); + expect(harness.recoveryAuths).toHaveLength(1); + expect(harness.sends.filter(send => send === "Bearer rejected-access" || send === "Bearer refreshed-access")) + .toEqual(["Bearer rejected-access", "Bearer refreshed-access"]); + expect(harness.sends).not.toContain("Bearer other-access"); + expect(harness.canonicalAliasSends).toBe(0); + expect(harness.backupAuths).toContain("Bearer backup-test-key"); + expect(harness.backupAuths).not.toContain("Bearer spare-test-key"); + expect((logCtx.attempts ?? []).filter(attempt => attempt.provider === "backup")).toHaveLength(1); + expect((logCtx.attempts ?? []).some(attempt => attempt.provider === "spare")).toBe(false); + expect((logCtx.attempts ?? []).filter(attempt => attempt.provider === "chatgpt")).toHaveLength(0); + }); + + test.each(["cyber_policy", "invalid_request_error"])("encrypted replay %s remains a terminal 400 without recovery or backup", async (code) => { + writeWorkAndOtherAccounts(); + const headers = codexHeaders(); + const cfg = encryptedRecoveryComboConfig({ extraCanonical: true }); + const harness = installHarness({ + responseForSend: (authorization, _sendNumber, url) => { + if (url.hostname === "backup.example") { + return Response.json({ id: "must-not-run", object: "response", status: "completed", output: [] }); + } + if (authorization === "Bearer rejected-access") { + return Response.json({ error: { message: "rejected bearer" } }, { status: 401 }); + } + if (authorization === "Bearer refreshed-access") { + return Response.json({ + error: { type: code, code, message: "blocked" }, + }, { status: 400 }); + } + return storedReplay401(authorization, url); + }, + }); + + const response = await postEncryptedCombo(cfg, headers); + expect(response.status).toBe(400); + expect(harness.recoveryAuths).toEqual([]); + expect(harness.backupAuths).toEqual([]); + expect(harness.sends).toEqual(["Bearer rejected-access", "Bearer refreshed-access"]); + expect(harness.canonicalAliasSends).toBe(0); + }); +});