From fb1320abbb8b0db78fbf836ab7d59fb27721c155 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 24 Jul 2026 08:45:36 +0200 Subject: [PATCH 01/12] feat: quota-aware subagent model fallback chain (#374) --- src/codex/paths.ts | 5 + src/codex/subagent-model-fallback.ts | 279 ++++++++++++++++++ .../management/agent-settings-routes.ts | 46 +++ src/server/responses/collaboration.ts | 5 +- src/server/responses/core.ts | 43 +++ src/types.ts | 10 + tests/subagent-model-fallback.test.ts | 148 ++++++++++ 7 files changed, 535 insertions(+), 1 deletion(-) create mode 100644 src/codex/subagent-model-fallback.ts create mode 100644 tests/subagent-model-fallback.test.ts diff --git a/src/codex/paths.ts b/src/codex/paths.ts index 1266db6db3..362bf09f1c 100644 --- a/src/codex/paths.ts +++ b/src/codex/paths.ts @@ -29,6 +29,11 @@ export const CODEX_PROFILE_PATH = join(CODEX_HOME, "opencodex.config.toml"); export const DEFAULT_CATALOG_PATH = join(CODEX_HOME, "opencodex-catalog.json"); export const CODEX_MODELS_CACHE_PATH = join(CODEX_HOME, "models_cache.json"); +/** Runtime CODEX_HOME lookup (honors CODEX_HOME env changes after import). */ +export function getCodexHome(): string { + return resolveCodexHome(); +} + export function tomlString(value: string): string { return JSON.stringify(value); } diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts new file mode 100644 index 0000000000..d3a98d9dcc --- /dev/null +++ b/src/codex/subagent-model-fallback.ts @@ -0,0 +1,279 @@ +/** + * Quota-aware subagent model fallback (issue #374). + * + * codex-rs spawns children with the agent-role TOML `model` pinned; when that model's + * provider quota is exhausted the child fails immediately. This module rewrites thread_spawn + * requests at the proxy choke point to the next healthy model in a configured fallback chain. + */ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { OcxParsedRequest, OcxConfig } from "../types"; +import { slugsEquivalent } from "../providers/slug-codec"; +import { CODEX_HOME, getCodexHome } from "./paths"; +import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; +import { computeCodexUsageScore } from "./routing"; +import { getMainAccountPlan } from "./main-account"; +import { isThreadSpawnRequest } from "../server/effort-policy"; + +export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; + +type ModelHealth = { + unavailableUntil: number; + reason: string; +}; + +const modelHealth = new Map(); +const quotaPrimedAt = new Map(); + +function pollIntervalMs(config: OcxConfig): number { + const configured = config.subagentModelFallbackPollMs; + if (typeof configured !== "number" || !Number.isFinite(configured) || configured < 1_000) { + return DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; + } + return configured; +} + +function normalizedChain(primary: string, config: OcxConfig, extra: readonly string[] = []): string[] { + const chain: string[] = []; + const seen = new Set(); + const push = (model: string | undefined) => { + if (!model || model.trim() === "") return; + const trimmed = model.trim(); + const key = trimmed.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + chain.push(trimmed); + }; + push(primary); + for (const model of extra) push(model); + for (const model of config.subagentModelFallback ?? []) push(model); + return chain; +} + +export function buildSubagentModelChain( + primary: string, + config: OcxConfig, + extraFallback: readonly string[] = [], +): string[] { + return normalizedChain(primary, config, extraFallback); +} + +function isNativeOpenAiSlug(model: string): boolean { + return !model.includes("/"); +} + +function quotaThreshold(config: OcxConfig): number { + const threshold = config.autoSwitchThreshold ?? 80; + return threshold > 0 ? threshold : Number.POSITIVE_INFINITY; +} + +function activeCodexAccountId(config: OcxConfig): string | null { + return config.activeCodexAccountId ?? null; +} + +export function isNativeModelQuotaExhausted(model: string, config: OcxConfig, now = Date.now()): boolean { + if (!isNativeOpenAiSlug(model)) return false; + const accountId = activeCodexAccountId(config); + if (!accountId) return false; + const quota = getAccountQuota(accountId); + const usage = computeCodexUsageScore(quota, getMainAccountPlan()); + if (usage >= CODEX_UNKNOWN_USAGE_SCORE) return false; + return usage >= quotaThreshold(config); +} + +export function isModelHealthBlocked(model: string, now = Date.now()): boolean { + const health = modelHealth.get(model.toLowerCase()); + return !!health && health.unavailableUntil > now; +} + +export function isSubagentModelUnavailable(model: string, config: OcxConfig, now = Date.now()): boolean { + if (isModelHealthBlocked(model, now)) return true; + if (isNativeOpenAiSlug(model)) return isNativeModelQuotaExhausted(model, config, now); + return false; +} + +export function selectAvailableSubagentModel( + primary: string, + config: OcxConfig, + extraFallback: readonly string[] = [], + now = Date.now(), +): { model: string; rewritten: boolean; skipped: string[] } { + const chain = normalizedChain(primary, config, extraFallback); + const skipped: string[] = []; + for (const candidate of chain) { + if (isSubagentModelUnavailable(candidate, config, now)) { + skipped.push(candidate); + continue; + } + return { model: candidate, rewritten: !slugsEquivalent(candidate, primary), skipped }; + } + return { model: primary, rewritten: false, skipped }; +} + +export function noteSubagentModelFailure(model: string, message: string, now = Date.now(), ttlMs?: number): void { + const interval = ttlMs ?? DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; + const lower = message.toLowerCase(); + const quotaLike = lower.includes("insufficient_quota") + || lower.includes("quota exhausted") + || lower.includes("usage limit") + || lower.includes("exceeded your current quota") + || lower.includes("account quota exceeded"); + if (!quotaLike) return; + modelHealth.set(model.toLowerCase(), { + unavailableUntil: now + interval, + reason: "quota_exhausted", + }); +} + +export function resetSubagentModelFallbackStateForTests(): void { + modelHealth.clear(); + quotaPrimedAt.clear(); +} + +function rewriteParsedModel(parsed: OcxParsedRequest, model: string): void { + parsed.modelId = model; + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = model; + } +} + +const TOML_MODEL = /^(model)\s*=\s*("(?:\\.|[^"\\])*")\s*$/; + +function parseTomlQuotedString(raw: string): string { + const trimmed = raw.trim(); + if ((trimmed.startsWith("\"") && trimmed.endsWith("\"")) + || (trimmed.startsWith("'") && trimmed.endsWith("'"))) { + return trimmed.slice(1, -1).replace(/\\"/g, "\""); + } + return trimmed; +} + +function readAgentModel(filePath: string): string | null { + try { + const content = readFileSync(filePath, "utf8"); + for (const line of content.split(/\r?\n/)) { + const match = line.match(TOML_MODEL); + if (!match) continue; + const model = parseTomlQuotedString(match[2] ?? ""); + return model.trim() === "" ? null : model.trim(); + } + } catch { + return null; + } + return null; +} + +export function readCodexAgentModel(role: string, codexHome = CODEX_HOME): string | null { + const file = join(codexHome, "agents", `${role}.toml`); + if (!existsSync(file)) return null; + return readAgentModel(file); +} + +export function resolveAgentModelFallbackForPrimary( + primary: string, + codexHome = CODEX_HOME, +): string[] { + const merged: string[] = []; + const seen = new Set(); + const push = (model: string | null | undefined) => { + if (!model || model.trim() === "") return; + const trimmed = model.trim(); + const key = trimmed.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + merged.push(trimmed); + }; + for (const role of listCodexAgentRoles(codexHome)) { + const model = readCodexAgentModel(role, codexHome); + if (!model || !slugsEquivalent(model, primary)) continue; + for (const fallback of readCodexAgentModelFallback(role, codexHome)) push(fallback); + } + return merged; +} + +export function maybePrimeSubagentQuota(config: OcxConfig, now = Date.now()): void { + if (!shouldPrimeSubagentQuota(config, now)) return; + void import("./auth-api") + .then(({ primeCodexPoolQuotas }) => primeCodexPoolQuotas(config, "subagent-spawn")) + .catch(() => {}); +} + +export function recordSubagentQuotaFailureForThreadSpawn( + headers: Headers, + model: string, + message: string | number, + config: OcxConfig, + now = Date.now(), +): void { + if (!isThreadSpawnRequest(headers)) return; + noteSubagentModelFailure(model, String(message), now, pollIntervalMs(config)); +} + +export function applySubagentModelFallback( + parsed: OcxParsedRequest, + headers: Headers, + config: OcxConfig, + now = Date.now(), +): { from?: string; to?: string; skipped?: string[] } | null { + if (!isThreadSpawnRequest(headers)) return null; + const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); + const globalFallback = config.subagentModelFallback ?? []; + if (globalFallback.length === 0 && roleFallback.length === 0) return null; + const selection = selectAvailableSubagentModel(parsed.modelId, config, roleFallback, now); + if (!selection.rewritten) return selection.skipped.length > 0 + ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } + : null; + const from = parsed.modelId; + rewriteParsedModel(parsed, selection.model); + return { from, to: selection.model, skipped: selection.skipped }; +} + +export function subagentFallbackGuidanceText(config: OcxConfig): string { + const chain = config.subagentModelFallback ?? []; + if (chain.length === 0) return ""; + const quoted = chain.map(model => `"${model}"`).join(", "); + return ` Subagent model fallback chain (priority order): ${quoted}. When the primary model is quota-exhausted, opencodex rewrites thread_spawn requests to the next available model automatically.`; +} + +const TOML_STRING_ARRAY = /^(model_fallback)\s*=\s*\[(.*)\]\s*$/; + +function parseTomlStringArray(raw: string): string[] { + const matches = [...raw.matchAll(/"((?:\\.|[^"\\])*)"/g)]; + return matches.map(match => match[1]!.replace(/\\"/g, "\"")); +} + +export function readAgentModelFallback(filePath: string): string[] | null { + try { + const content = readFileSync(filePath, "utf8"); + for (const line of content.split(/\r?\n/)) { + const match = line.match(TOML_STRING_ARRAY); + if (!match) continue; + return parseTomlStringArray(match[2] ?? ""); + } + } catch { + return null; + } + return null; +} + +export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME): string[] { + const file = join(codexHome, "agents", `${role}.toml`); + if (!existsSync(file)) return []; + return readAgentModelFallback(file) ?? []; +} + +export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { + const dir = join(codexHome, "agents"); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter(name => name.endsWith(".toml")) + .map(name => name.slice(0, -".toml".length)); +} + +export function shouldPrimeSubagentQuota(config: OcxConfig, now = Date.now()): boolean { + const key = "global"; + const last = quotaPrimedAt.get(key) ?? 0; + if (now - last < pollIntervalMs(config)) return false; + quotaPrimedAt.set(key, now); + return true; +} diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index c46b77dd04..c36f57ecd1 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -288,6 +288,52 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise return jsonResponse({ ok: true, applied: chosen }); } + // Priority-ordered subagent model fallback chain for quota-aware spawn routing. + if (url.pathname === "/api/subagent-model-fallback" && req.method === "GET") { + const models = await fetchAllModels(config); + const disabled = new Set(config.disabledModels ?? []); + const { listCatalogNativeSlugs } = await import("../../codex/catalog"); + const visibleRouted = [...new Set(models + .filter(m => ![...disabled].some(stored => + stored === catalogModelSlug(m) || slugEquals(stored, m.provider, m.id) + )) + .map(catalogModelSlug))]; + const available = [ + ...listCatalogNativeSlugs().filter(ns => !disabled.has(ns)), + ...visibleRouted, + ]; + return jsonResponse({ + models: config.subagentModelFallback ?? [], + pollMs: config.subagentModelFallbackPollMs ?? 60_000, + available, + }); + } + if (url.pathname === "/api/subagent-model-fallback" && req.method === "PUT") { + let body: { models?: unknown; pollMs?: unknown }; + try { body = await req.json(); } catch { return jsonResponse({ error: "invalid JSON body" }, 400); } + if ("models" in body) { + if (!Array.isArray(body.models)) return jsonResponse({ error: "models must be an array" }, 400); + const models = body.models.filter((m): m is string => typeof m === "string" && m.trim().length > 0); + if (models.length > 0) config.subagentModelFallback = models; + else delete config.subagentModelFallback; + } + if ("pollMs" in body) { + const pollMs = body.pollMs; + if (pollMs === null || pollMs === "") delete config.subagentModelFallbackPollMs; + else if (typeof pollMs === "number" && Number.isInteger(pollMs) && pollMs >= 5_000 && pollMs <= 600_000) { + config.subagentModelFallbackPollMs = pollMs; + } else { + return jsonResponse({ error: "pollMs must be an integer between 5000 and 600000" }, 400); + } + } + saveConfig(config); + return jsonResponse({ + ok: true, + models: config.subagentModelFallback ?? [], + pollMs: config.subagentModelFallbackPollMs ?? 60_000, + }); + } + // Claude Code inbound settings (GUI "Claude ON" toggle + Claude page). if (url.pathname === "/api/claude-code" && req.method === "GET") { const models = await fetchAllModels(config); diff --git a/src/server/responses/collaboration.ts b/src/server/responses/collaboration.ts index e0d37d54af..cecb88675e 100644 --- a/src/server/responses/collaboration.ts +++ b/src/server/responses/collaboration.ts @@ -61,6 +61,7 @@ import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } f import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { slugsEquivalent } from "../../providers/slug-codec"; +import { subagentFallbackGuidanceText } from "../../codex/subagent-model-fallback"; import { applyOpenAiVirtualModel, resolveOpenAiCompactModel } from "../../providers/openai-virtual-models"; import { isUsageDebugEnabled } from "../../usage/debug"; import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress"; @@ -160,6 +161,7 @@ export interface MultiAgentGuidanceOptions { injectionModel?: string; injectionEffort?: string; subagentModels?: string[]; + subagentModelFallback?: string[]; injectionPrompt?: string; } @@ -194,6 +196,7 @@ export async function multiAgentGuidanceText( injectionModel, injectionEffort, subagentModels, + subagentModelFallback, injectionPrompt, } = options; const surface = collabSurface(parsed); @@ -237,6 +240,7 @@ export async function multiAgentGuidanceText( + (injectionEffort ? `, reasoning_effort "${injectionEffort}"` : "") + " — use it unless the user names another."; } + text += subagentFallbackGuidanceText({ subagentModelFallback } as OcxConfig); text += roster; if (text.length > V2_GUIDANCE_CHAR_BUDGET) { // Roster is the only unbounded part — drop it before breaking the budget. @@ -314,4 +318,3 @@ export function injectDeveloperMessage(parsed: OcxParsedRequest, text: string): } } } - diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index e7d81f1104..5f26ea05bc 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -76,6 +76,12 @@ import { registerTurn, trackStreamLifetime, unregisterTurn } from "../lifecycle" import { redactSecretString } from "../../lib/redact"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { supportedLadderFor } from "../effort-policy"; +import { isThreadSpawnRequest } from "../effort-policy"; +import { + applySubagentModelFallback, + maybePrimeSubagentQuota, + recordSubagentQuotaFailureForThreadSpawn, +} from "../../codex/subagent-model-fallback"; import { beginRequestAttempt, catalogModelSupportsServiceTier, @@ -706,6 +712,18 @@ export async function handleResponses( } if (parsed._compactionRequest === true) parsed._cursorIsolateConversation = true; + if (isThreadSpawnRequest(req.headers)) { + await maybePrimeSubagentQuota(config); + const fallback = applySubagentModelFallback(parsed, req.headers, config); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } + } + } + let route; try { route = routeModel(config, parsed.modelId); @@ -768,6 +786,7 @@ export async function handleResponses( injectionModel: config.injectionModel, injectionEffort: config.injectionEffort, subagentModels: config.subagentModels, + subagentModelFallback: config.subagentModelFallback, injectionPrompt: config.injectionPrompt, }); if (guidance) { @@ -1178,6 +1197,14 @@ export async function handleResponses( if (terminalBodyWillRecord) { options.setTerminalOutcomeRecorder?.((status, httpStatusOverride) => { terminalRecorder(status, httpStatusOverride); + if (status === "failed") { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + parsed.modelId, + httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", + config, + ); + } options.onNativePassthroughTerminal?.(status); }); } else { @@ -1215,6 +1242,14 @@ export async function handleResponses( const reportNativeTerminal = recordTerminalOutcomes ? (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { terminalRecorder?.(status, httpStatusOverride); + if (status === "failed") { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + parsed.modelId, + httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", + config, + ); + } options.onNativePassthroughTerminal?.(status); } : undefined; @@ -1640,6 +1675,14 @@ export async function handleResponses( } const errorText = await upstreamResponse.text().catch(() => "unknown error"); cleanupUpstreamAbort(); + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + parsed.modelId, + upstreamResponse.status === 429 || upstreamResponse.status === 402 + ? upstreamResponse.status + : `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`, + config, + ); // Upstreams occasionally echo request details in error bodies — scrub token-shaped // material before it reaches the client-facing error surface. return formatErrorResponse(upstreamResponse.status, "upstream_error", `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`); diff --git a/src/types.ts b/src/types.ts index d4422034b0..cbe47d51d0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -454,6 +454,16 @@ export interface OcxConfig { * Codex's spawn_agent only advertises the first 5 routed models, so this picks which 5 appear. */ subagentModels?: string[]; + /** + * Priority-ordered fallback models for spawned sub-agents. When the requested + * model is quota-exhausted or recently failed, opencodex rewrites the child + * turn to the next available entry before routing. + */ + subagentModelFallback?: string[]; + /** + * TTL (ms) for cached sub-agent model availability probes. Default 60_000. + */ + subagentModelFallbackPollMs?: number; injectionModel?: string; /** * Optional reasoning effort the delegation prompt tells the agent to pass in spawn_agent calls diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts new file mode 100644 index 0000000000..a8dd695c22 --- /dev/null +++ b/tests/subagent-model-fallback.test.ts @@ -0,0 +1,148 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + applySubagentModelFallback, + buildSubagentModelChain, + isSubagentModelUnavailable, + noteSubagentModelFailure, + readCodexAgentModelFallback, + resetSubagentModelFallbackStateForTests, + resolveAgentModelFallbackForPrimary, + selectAvailableSubagentModel, + subagentFallbackGuidanceText, +} from "../src/codex/subagent-model-fallback"; +import { updateAccountQuota } from "../src/codex/quota"; +import type { OcxConfig } from "../src/types"; + +const savedCodexHome = process.env.CODEX_HOME; + +function cfg(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + providers: { openai: { adapter: "openai-responses" } }, + defaultProvider: "openai", + activeCodexAccountId: "main", + autoSwitchThreshold: 80, + subagentModelFallback: [ + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ], + ...overrides, + }; +} + +function codexHomeFixture(): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-subagent-fallback-")); + mkdirSync(join(dir, "agents"), { recursive: true }); + process.env.CODEX_HOME = dir; + return dir; +} + +afterEach(() => { + if (savedCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = savedCodexHome; + resetSubagentModelFallbackStateForTests(); +}); + +describe("subagent model fallback chain", () => { + test("buildSubagentModelChain dedupes and preserves order", () => { + expect(buildSubagentModelChain("gpt-5.6-sol", cfg())).toEqual([ + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ]); + expect(buildSubagentModelChain("kimi/k3", cfg())).toEqual([ + "kimi/k3", + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("selectAvailableSubagentModel skips quota-exhausted native models", () => { + updateAccountQuota("main", 95, undefined, 20); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); + expect(selected).toEqual({ + model: "alibaba-token-plan/qwen3.8-max-preview", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("selectAvailableSubagentModel skips cached routed failures", () => { + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted"); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); + expect(selected.model).toBe("kimi/k3"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); + }); + + test("applySubagentModelFallback rewrites parsed request model", () => { + updateAccountQuota("main", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + cfg(), + ); + expect(result).toEqual({ + from: "gpt-5.6-sol", + to: "alibaba-token-plan/qwen3.8-max-preview", + skipped: ["gpt-5.6-sol"], + }); + expect(parsed.modelId).toBe("alibaba-token-plan/qwen3.8-max-preview"); + expect((parsed._rawBody as { model?: string }).model).toBe("alibaba-token-plan/qwen3.8-max-preview"); + }); + + test("applySubagentModelFallback is a no-op for main turns", () => { + updateAccountQuota("main", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + expect(applySubagentModelFallback(parsed as never, new Headers(), cfg())).toBeNull(); + expect(parsed.modelId).toBe("gpt-5.6-sol"); + }); + + test("applySubagentModelFallback can use per-agent model_fallback without global config", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"alibaba-token-plan/qwen3.8-max-preview\"]", + "", + ].join("\n"), "utf8"); + updateAccountQuota("main", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + cfg({ subagentModelFallback: undefined }), + ); + expect(result?.to).toBe("alibaba-token-plan/qwen3.8-max-preview"); + expect(resolveAgentModelFallbackForPrimary("gpt-5.6-sol", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("subagentFallbackGuidanceText renders configured chain", () => { + expect(subagentFallbackGuidanceText(cfg())).toContain("gpt-5.6-sol"); + expect(subagentFallbackGuidanceText(cfg({ subagentModelFallback: undefined }))).toBe(""); + }); +}); From f5ac7adf4c7c632483f1c61e7d1d2483fcb6739c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:04:01 +0200 Subject: [PATCH 02/12] fix(subagents): harden quota-aware model fallback routing --- src/codex/routing.ts | 2 +- src/codex/subagent-model-fallback.ts | 63 +++++++++++++++---- .../management/agent-settings-routes.ts | 13 ++-- src/server/responses/collaboration.ts | 7 ++- src/server/responses/core.ts | 46 ++++++++++---- tests/subagent-model-fallback.test.ts | 27 +++++++- 6 files changed, 126 insertions(+), 32 deletions(-) diff --git a/src/codex/routing.ts b/src/codex/routing.ts index e67b6a44a5..bfa440d7c9 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -391,7 +391,7 @@ function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Da return ids; } -function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { +export function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { if (accountId === MAIN_CODEX_ACCOUNT_ID) return getMainAccountPlan(); return (config.codexAccounts ?? []).find(account => !account.isMain && account.id === accountId)?.plan; } diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index d3a98d9dcc..fd61ac3059 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -12,7 +12,8 @@ import { slugsEquivalent } from "../providers/slug-codec"; import { CODEX_HOME, getCodexHome } from "./paths"; import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; import { computeCodexUsageScore } from "./routing"; -import { getMainAccountPlan } from "./main-account"; +import { nativeOpenAiSlugs } from "./catalog"; +import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; @@ -24,6 +25,25 @@ type ModelHealth = { const modelHealth = new Map(); const quotaPrimedAt = new Map(); +const nativeSlugSet = () => new Set(nativeOpenAiSlugs().map(slug => slug.toLowerCase())); + +function healthKey(model: string, accountId: string | null): string { + return `${accountId ?? "none"}::${model.toLowerCase()}`; +} + +function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { + return (config.codexAccounts ?? []).find(account => account.id === accountId)?.plan; +} + +function isDisabledFallbackModel(model: string, config: OcxConfig): boolean { + const disabled = config.disabledModels ?? []; + if (disabled.length === 0) return false; + if (!model.includes("/")) return disabled.some(stored => slugEquals(stored, "openai", model)); + const slash = model.indexOf("/"); + const provider = model.slice(0, slash); + const modelId = model.slice(slash + 1); + return disabled.some(stored => stored === model || slugEquals(stored, provider, modelId)); +} function pollIntervalMs(config: OcxConfig): number { const configured = config.subagentModelFallbackPollMs; @@ -59,7 +79,7 @@ export function buildSubagentModelChain( } function isNativeOpenAiSlug(model: string): boolean { - return !model.includes("/"); + return nativeSlugSet().has(model.toLowerCase()); } function quotaThreshold(config: OcxConfig): number { @@ -76,18 +96,19 @@ export function isNativeModelQuotaExhausted(model: string, config: OcxConfig, no const accountId = activeCodexAccountId(config); if (!accountId) return false; const quota = getAccountQuota(accountId); - const usage = computeCodexUsageScore(quota, getMainAccountPlan()); + const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, accountId)); if (usage >= CODEX_UNKNOWN_USAGE_SCORE) return false; return usage >= quotaThreshold(config); } -export function isModelHealthBlocked(model: string, now = Date.now()): boolean { - const health = modelHealth.get(model.toLowerCase()); +export function isModelHealthBlocked(model: string, config: OcxConfig, now = Date.now()): boolean { + const health = modelHealth.get(healthKey(model, activeCodexAccountId(config))); return !!health && health.unavailableUntil > now; } export function isSubagentModelUnavailable(model: string, config: OcxConfig, now = Date.now()): boolean { - if (isModelHealthBlocked(model, now)) return true; + if (isDisabledFallbackModel(model, config)) return true; + if (isModelHealthBlocked(model, config, now)) return true; if (isNativeOpenAiSlug(model)) return isNativeModelQuotaExhausted(model, config, now); return false; } @@ -110,16 +131,26 @@ export function selectAvailableSubagentModel( return { model: primary, rewritten: false, skipped }; } -export function noteSubagentModelFailure(model: string, message: string, now = Date.now(), ttlMs?: number): void { +export function noteSubagentModelFailure( + model: string, + message: string, + config: OcxConfig, + now = Date.now(), + ttlMs?: number, +): void { const interval = ttlMs ?? DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; - const lower = message.toLowerCase(); + const normalized = String(message).trim(); + const lower = normalized.toLowerCase(); + const numericStatus = Number(normalized); const quotaLike = lower.includes("insufficient_quota") || lower.includes("quota exhausted") || lower.includes("usage limit") || lower.includes("exceeded your current quota") - || lower.includes("account quota exceeded"); + || lower.includes("account quota exceeded") + || numericStatus === 429 + || numericStatus === 402; if (!quotaLike) return; - modelHealth.set(model.toLowerCase(), { + modelHealth.set(healthKey(model, activeCodexAccountId(config)), { unavailableUntil: now + interval, reason: "quota_exhausted", }); @@ -206,7 +237,7 @@ export function recordSubagentQuotaFailureForThreadSpawn( now = Date.now(), ): void { if (!isThreadSpawnRequest(headers)) return; - noteSubagentModelFailure(model, String(message), now, pollIntervalMs(config)); + noteSubagentModelFailure(model, String(message), config, now, pollIntervalMs(config)); } export function applySubagentModelFallback( @@ -235,16 +266,24 @@ export function subagentFallbackGuidanceText(config: OcxConfig): string { return ` Subagent model fallback chain (priority order): ${quoted}. When the primary model is quota-exhausted, opencodex rewrites thread_spawn requests to the next available model automatically.`; } -const TOML_STRING_ARRAY = /^(model_fallback)\s*=\s*\[(.*)\]\s*$/; +const TOML_STRING_ARRAY = /^(model_fallback)\s*=\s*\[(.*)\]\s*$/s; function parseTomlStringArray(raw: string): string[] { const matches = [...raw.matchAll(/"((?:\\.|[^"\\])*)"/g)]; return matches.map(match => match[1]!.replace(/\\"/g, "\"")); } +function parseTomlModelFallback(content: string): string[] | null { + const match = content.match(/^\s*model_fallback\s*=\s*\[(.*)\]\s*$/ms); + if (!match) return null; + return parseTomlStringArray(match[1] ?? ""); +} + export function readAgentModelFallback(filePath: string): string[] | null { try { const content = readFileSync(filePath, "utf8"); + const multiline = parseTomlModelFallback(content); + if (multiline) return multiline; for (const line of content.split(/\r?\n/)) { const match = line.match(TOML_STRING_ARRAY); if (!match) continue; diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index c36f57ecd1..6eb95ec482 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -311,21 +311,26 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise if (url.pathname === "/api/subagent-model-fallback" && req.method === "PUT") { let body: { models?: unknown; pollMs?: unknown }; try { body = await req.json(); } catch { return jsonResponse({ error: "invalid JSON body" }, 400); } + let nextModels = config.subagentModelFallback; + let nextPollMs = config.subagentModelFallbackPollMs; if ("models" in body) { if (!Array.isArray(body.models)) return jsonResponse({ error: "models must be an array" }, 400); const models = body.models.filter((m): m is string => typeof m === "string" && m.trim().length > 0); - if (models.length > 0) config.subagentModelFallback = models; - else delete config.subagentModelFallback; + nextModels = models.length > 0 ? models : undefined; } if ("pollMs" in body) { const pollMs = body.pollMs; - if (pollMs === null || pollMs === "") delete config.subagentModelFallbackPollMs; + if (pollMs === null || pollMs === "") nextPollMs = undefined; else if (typeof pollMs === "number" && Number.isInteger(pollMs) && pollMs >= 5_000 && pollMs <= 600_000) { - config.subagentModelFallbackPollMs = pollMs; + nextPollMs = pollMs; } else { return jsonResponse({ error: "pollMs must be an integer between 5000 and 600000" }, 400); } } + if (nextModels !== undefined) config.subagentModelFallback = nextModels; + else delete config.subagentModelFallback; + if (nextPollMs !== undefined) config.subagentModelFallbackPollMs = nextPollMs; + else delete config.subagentModelFallbackPollMs; saveConfig(config); return jsonResponse({ ok: true, diff --git a/src/server/responses/collaboration.ts b/src/server/responses/collaboration.ts index cecb88675e..c4d0099b46 100644 --- a/src/server/responses/collaboration.ts +++ b/src/server/responses/collaboration.ts @@ -225,11 +225,12 @@ export async function multiAgentGuidanceText( .map(item => `${item.configured}:${item.reason}`) .join(", ")}`); } - if (!injectionModel && roster === "") return null; + const fallbackGuidance = subagentFallbackGuidanceText({ subagentModelFallback } as OcxConfig); + if (!injectionModel && roster === "" && fallbackGuidance === "") return null; if (injectionPrompt) { return `${applyInjectionPlaceholders(injectionPrompt, injectionModel, injectionEffort, roster)}`; } - if (!preferred && roster === "") return null; + if (!preferred && roster === "" && fallbackGuidance === "") return null; let text = "When the active spawn_agent tool supports optional \"model\" or \"reasoning_effort\" overrides, " + "use only models listed for this collaboration surface. " + "When setting either override, set fork_turns to \"none\" " @@ -240,7 +241,7 @@ export async function multiAgentGuidanceText( + (injectionEffort ? `, reasoning_effort "${injectionEffort}"` : "") + " — use it unless the user names another."; } - text += subagentFallbackGuidanceText({ subagentModelFallback } as OcxConfig); + text += fallbackGuidance; text += roster; if (text.length > V2_GUIDANCE_CHAR_BUDGET) { // Roster is the only unbounded part — drop it before breaking the budget. diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 5f26ea05bc..24df030c96 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -714,16 +714,10 @@ export async function handleResponses( if (isThreadSpawnRequest(req.headers)) { await maybePrimeSubagentQuota(config); - const fallback = applySubagentModelFallback(parsed, req.headers, config); - if (fallback) { - (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; - (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; - if (isInjectionDebugEnabled()) { - injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); - } - } } + const subagentQuotaFailureModel = parsed.modelId; + let route; try { route = routeModel(config, parsed.modelId); @@ -741,6 +735,36 @@ export async function handleResponses( return unreadableEncryptedAgentTaskResponse(); } + if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { + const fallback = applySubagentModelFallback(parsed, req.headers, config); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } + } + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + try { + route = routeModel(config, fallback.to); + } catch (err) { + if (err instanceof NoAvailableComboTargetsError) { + return comboUnavailableResponse(err.message); + } + return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); + } + if (route.modelId !== parsed.modelId) { + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = route.modelId; + } + parsed.modelId = route.modelId; + } + logCtx.model = route.modelId; + logCtx.provider = route.providerName; + logCtx.providerAdapter = route.provider.adapter; + } + } + // Apply the routed model id upstream: routing may strip a "/" namespace // (e.g. "opencode-go/deepseek-v4-pro" → "deepseek-v4-pro"). Adapters read parsed.modelId, // and the passthrough adapter serializes _rawBody, so rewrite both. @@ -1200,7 +1224,7 @@ export async function handleResponses( if (status === "failed") { recordSubagentQuotaFailureForThreadSpawn( req.headers, - parsed.modelId, + subagentQuotaFailureModel, httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", config, ); @@ -1245,7 +1269,7 @@ export async function handleResponses( if (status === "failed") { recordSubagentQuotaFailureForThreadSpawn( req.headers, - parsed.modelId, + subagentQuotaFailureModel, httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", config, ); @@ -1677,7 +1701,7 @@ export async function handleResponses( cleanupUpstreamAbort(); recordSubagentQuotaFailureForThreadSpawn( req.headers, - parsed.modelId, + subagentQuotaFailureModel, upstreamResponse.status === 429 || upstreamResponse.status === 402 ? upstreamResponse.status : `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`, diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index a8dd695c22..0f83ff0ab7 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -62,6 +62,7 @@ describe("subagent model fallback chain", () => { }); test("selectAvailableSubagentModel skips quota-exhausted native models", () => { + resetSubagentModelFallbackStateForTests(); updateAccountQuota("main", 95, undefined, 20); const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); expect(selected).toEqual({ @@ -72,12 +73,36 @@ describe("subagent model fallback chain", () => { }); test("selectAvailableSubagentModel skips cached routed failures", () => { - noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted"); + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg()); const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); expect(selected.model).toBe("kimi/k3"); expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); }); + test("noteSubagentModelFailure treats numeric 429 as quota-like", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "429", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + }); + + test("readCodexAgentModelFallback parses multiline TOML arrays", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [", + " \"alibaba-token-plan/qwen3.8-max-preview\",", + " \"kimi/k3\",", + "]", + "", + ].join("\n"), "utf8"); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + "kimi/k3", + ]); + }); + test("applySubagentModelFallback rewrites parsed request model", () => { updateAccountQuota("main", 95); const parsed = { From 449223e99c0c25de7664bd7d100b199cb8943f3e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:01:37 +0200 Subject: [PATCH 03/12] fix(subagents): defer fallback until auth and record tee failures --- src/codex/subagent-model-fallback.ts | 65 +++++++++++++++++++------ src/server/responses/core.ts | 69 +++++++++++++++------------ tests/subagent-model-fallback.test.ts | 36 +++++++++++++- 3 files changed, 124 insertions(+), 46 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index fd61ac3059..87c7189586 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -7,6 +7,7 @@ */ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; +import { hasOwnProvider } from "../config"; import type { OcxParsedRequest, OcxConfig } from "../types"; import { slugsEquivalent } from "../providers/slug-codec"; import { CODEX_HOME, getCodexHome } from "./paths"; @@ -15,7 +16,6 @@ import { computeCodexUsageScore } from "./routing"; import { nativeOpenAiSlugs } from "./catalog"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; - export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; type ModelHealth = { @@ -91,25 +91,56 @@ function activeCodexAccountId(config: OcxConfig): string | null { return config.activeCodexAccountId ?? null; } -export function isNativeModelQuotaExhausted(model: string, config: OcxConfig, now = Date.now()): boolean { +function resolveFallbackAccountId(config: OcxConfig, accountId?: string | null): string | null { + return accountId ?? activeCodexAccountId(config); +} + +function isRoutableFallbackModel(model: string, config: OcxConfig): boolean { + const slash = model.indexOf("/"); + if (slash > 0) { + const providerName = model.slice(0, slash); + if (!hasOwnProvider(config.providers, providerName)) return false; + const provider = config.providers[providerName]; + if (provider?.disabled === true) return false; + } + return true; +} + +export function isNativeModelQuotaExhausted( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { if (!isNativeOpenAiSlug(model)) return false; - const accountId = activeCodexAccountId(config); - if (!accountId) return false; - const quota = getAccountQuota(accountId); - const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, accountId)); + const resolvedAccountId = resolveFallbackAccountId(config, accountId); + if (!resolvedAccountId) return false; + const quota = getAccountQuota(resolvedAccountId); + const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, resolvedAccountId)); if (usage >= CODEX_UNKNOWN_USAGE_SCORE) return false; return usage >= quotaThreshold(config); } -export function isModelHealthBlocked(model: string, config: OcxConfig, now = Date.now()): boolean { - const health = modelHealth.get(healthKey(model, activeCodexAccountId(config))); +export function isModelHealthBlocked( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { + const health = modelHealth.get(healthKey(model, resolveFallbackAccountId(config, accountId))); return !!health && health.unavailableUntil > now; } -export function isSubagentModelUnavailable(model: string, config: OcxConfig, now = Date.now()): boolean { +export function isSubagentModelUnavailable( + model: string, + config: OcxConfig, + accountId?: string | null, + now = Date.now(), +): boolean { if (isDisabledFallbackModel(model, config)) return true; - if (isModelHealthBlocked(model, config, now)) return true; - if (isNativeOpenAiSlug(model)) return isNativeModelQuotaExhausted(model, config, now); + if (!isRoutableFallbackModel(model, config)) return true; + if (isModelHealthBlocked(model, config, accountId, now)) return true; + if (isNativeOpenAiSlug(model)) return isNativeModelQuotaExhausted(model, config, accountId, now); return false; } @@ -117,12 +148,13 @@ export function selectAvailableSubagentModel( primary: string, config: OcxConfig, extraFallback: readonly string[] = [], + accountId?: string | null, now = Date.now(), ): { model: string; rewritten: boolean; skipped: string[] } { const chain = normalizedChain(primary, config, extraFallback); const skipped: string[] = []; for (const candidate of chain) { - if (isSubagentModelUnavailable(candidate, config, now)) { + if (isSubagentModelUnavailable(candidate, config, accountId, now)) { skipped.push(candidate); continue; } @@ -135,6 +167,7 @@ export function noteSubagentModelFailure( model: string, message: string, config: OcxConfig, + accountId?: string | null, now = Date.now(), ttlMs?: number, ): void { @@ -150,7 +183,7 @@ export function noteSubagentModelFailure( || numericStatus === 429 || numericStatus === 402; if (!quotaLike) return; - modelHealth.set(healthKey(model, activeCodexAccountId(config)), { + modelHealth.set(healthKey(model, resolveFallbackAccountId(config, accountId)), { unavailableUntil: now + interval, reason: "quota_exhausted", }); @@ -234,23 +267,25 @@ export function recordSubagentQuotaFailureForThreadSpawn( model: string, message: string | number, config: OcxConfig, + accountId?: string | null, now = Date.now(), ): void { if (!isThreadSpawnRequest(headers)) return; - noteSubagentModelFailure(model, String(message), config, now, pollIntervalMs(config)); + noteSubagentModelFailure(model, String(message), config, accountId, now, pollIntervalMs(config)); } export function applySubagentModelFallback( parsed: OcxParsedRequest, headers: Headers, config: OcxConfig, + accountId?: string | null, now = Date.now(), ): { from?: string; to?: string; skipped?: string[] } | null { if (!isThreadSpawnRequest(headers)) return null; const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); const globalFallback = config.subagentModelFallback ?? []; if (globalFallback.length === 0 && roleFallback.length === 0) return null; - const selection = selectAvailableSubagentModel(parsed.modelId, config, roleFallback, now); + const selection = selectAvailableSubagentModel(parsed.modelId, config, roleFallback, accountId, now); if (!selection.rewritten) return selection.skipped.length > 0 ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } : null; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 24df030c96..66f7e1caf1 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -735,36 +735,6 @@ export async function handleResponses( return unreadableEncryptedAgentTaskResponse(); } - if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { - const fallback = applySubagentModelFallback(parsed, req.headers, config); - if (fallback) { - (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; - (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; - if (isInjectionDebugEnabled()) { - injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); - } - } - if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { - try { - route = routeModel(config, fallback.to); - } catch (err) { - if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailableResponse(err.message); - } - return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); - } - if (route.modelId !== parsed.modelId) { - if (parsed._rawBody && typeof parsed._rawBody === "object") { - (parsed._rawBody as { model?: string }).model = route.modelId; - } - parsed.modelId = route.modelId; - } - logCtx.model = route.modelId; - logCtx.provider = route.providerName; - logCtx.providerAdapter = route.provider.adapter; - } - } - // Apply the routed model id upstream: routing may strip a "/" namespace // (e.g. "opencode-go/deepseek-v4-pro" → "deepseek-v4-pro"). Adapters read parsed.modelId, // and the passthrough adapter serializes _rawBody, so rewrite both. @@ -921,6 +891,45 @@ export async function handleResponses( const identityScope = codexLogAccountId(authCtx); if (identityScope) parsed._cursorIdentityScope = identityScope; + const subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + ? authCtx.accountId + : config.activeCodexAccountId ?? null; + + if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { + const fallback = applySubagentModelFallback( + parsed, + req.headers, + config, + subagentFallbackAccountId, + ); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } + } + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + try { + route = routeModel(config, fallback.to); + } catch (err) { + if (err instanceof NoAvailableComboTargetsError) { + return comboUnavailableResponse(err.message); + } + return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); + } + if (route.modelId !== parsed.modelId) { + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = route.modelId; + } + parsed.modelId = route.modelId; + } + logCtx.model = route.modelId; + logCtx.provider = route.providerName; + logCtx.providerAdapter = route.provider.adapter; + } + } + // OAuth providers: swap in a fresh access token (auto-refreshed) as the Bearer key, so the // existing openai-chat / anthropic adapters authenticate with no change. const isOAuth401ReplayProvider = (route.providerName === "xai" || route.providerName === "github-copilot" || route.providerName === "kiro") diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 0f83ff0ab7..128c278322 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -21,7 +21,11 @@ const savedCodexHome = process.env.CODEX_HOME; function cfg(overrides: Partial = {}): OcxConfig { return { port: 10100, - providers: { openai: { adapter: "openai-responses" } }, + providers: { + openai: { adapter: "openai-responses" }, + "alibaba-token-plan": { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, defaultProvider: "openai", activeCodexAccountId: "main", autoSwitchThreshold: 80, @@ -72,6 +76,18 @@ describe("subagent model fallback chain", () => { }); }); + test("selectAvailableSubagentModel scopes quota exhaustion to the selected account", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("account-a", 95, undefined, 20); + updateAccountQuota("account-b", 10, undefined, 20); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg(), [], "account-b"); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + test("selectAvailableSubagentModel skips cached routed failures", () => { resetSubagentModelFallbackStateForTests(); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg()); @@ -80,6 +96,24 @@ describe("subagent model fallback chain", () => { expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); }); + test("selectAvailableSubagentModel skips stale fallback entries that cannot route", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + subagentModelFallback: [ + "missing-provider/does-not-exist", + "kimi/k3", + ], + }), + ); + expect(selected).toEqual({ + model: "kimi/k3", + rewritten: true, + skipped: ["gpt-5.6-sol", "missing-provider/does-not-exist"], + }); + }); + test("noteSubagentModelFailure treats numeric 429 as quota-like", () => { resetSubagentModelFallbackStateForTests(); noteSubagentModelFailure("kimi/k3", "429", cfg()); From e0eca6a746b8933fdaa95033306e95feba72eb5d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:56:18 +0200 Subject: [PATCH 04/12] fix(subagents): address PR #391 review feedback --- src/codex/subagent-model-fallback.ts | 15 +++- .../management/agent-settings-routes.ts | 9 ++- src/server/responses/collaboration.ts | 7 +- src/server/responses/core.ts | 81 +++++++++++++++++-- src/types.ts | 3 +- tests/multi-agent-compat.test.ts | 13 +++ tests/subagent-model-fallback.test.ts | 50 ++++++++++++ 7 files changed, 167 insertions(+), 11 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 87c7189586..7dbcbd66b5 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -150,10 +150,15 @@ export function selectAvailableSubagentModel( extraFallback: readonly string[] = [], accountId?: string | null, now = Date.now(), + nativeFallbackOnly = false, ): { model: string; rewritten: boolean; skipped: string[] } { const chain = normalizedChain(primary, config, extraFallback); const skipped: string[] = []; for (const candidate of chain) { + if (nativeFallbackOnly && !isNativeOpenAiSlug(candidate)) { + skipped.push(candidate); + continue; + } if (isSubagentModelUnavailable(candidate, config, accountId, now)) { skipped.push(candidate); continue; @@ -280,12 +285,20 @@ export function applySubagentModelFallback( config: OcxConfig, accountId?: string | null, now = Date.now(), + nativeFallbackOnly = false, ): { from?: string; to?: string; skipped?: string[] } | null { if (!isThreadSpawnRequest(headers)) return null; const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); const globalFallback = config.subagentModelFallback ?? []; if (globalFallback.length === 0 && roleFallback.length === 0) return null; - const selection = selectAvailableSubagentModel(parsed.modelId, config, roleFallback, accountId, now); + const selection = selectAvailableSubagentModel( + parsed.modelId, + config, + roleFallback, + accountId, + now, + nativeFallbackOnly, + ); if (!selection.rewritten) return selection.skipped.length > 0 ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } : null; diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index 6eb95ec482..f5d9d810fa 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -310,7 +310,14 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise } if (url.pathname === "/api/subagent-model-fallback" && req.method === "PUT") { let body: { models?: unknown; pollMs?: unknown }; - try { body = await req.json(); } catch { return jsonResponse({ error: "invalid JSON body" }, 400); } + try { + body = await req.json(); + } catch { + return jsonResponse({ error: "invalid JSON body" }, 400); + } + if (!body || typeof body !== "object" || Array.isArray(body)) { + return jsonResponse({ error: "invalid JSON body" }, 400); + } let nextModels = config.subagentModelFallback; let nextPollMs = config.subagentModelFallbackPollMs; if ("models" in body) { diff --git a/src/server/responses/collaboration.ts b/src/server/responses/collaboration.ts index c4d0099b46..907d1af533 100644 --- a/src/server/responses/collaboration.ts +++ b/src/server/responses/collaboration.ts @@ -228,7 +228,7 @@ export async function multiAgentGuidanceText( const fallbackGuidance = subagentFallbackGuidanceText({ subagentModelFallback } as OcxConfig); if (!injectionModel && roster === "" && fallbackGuidance === "") return null; if (injectionPrompt) { - return `${applyInjectionPlaceholders(injectionPrompt, injectionModel, injectionEffort, roster)}`; + return `${applyInjectionPlaceholders(injectionPrompt, injectionModel, injectionEffort, roster, fallbackGuidance)}`; } if (!preferred && roster === "" && fallbackGuidance === "") return null; let text = "When the active spawn_agent tool supports optional \"model\" or \"reasoning_effort\" overrides, " @@ -261,11 +261,12 @@ export async function multiAgentGuidanceText( export const V2_GUIDANCE_CHAR_BUDGET = 700; -export function applyInjectionPlaceholders(prompt: string, model?: string, effort?: string, roster?: string): string { +export function applyInjectionPlaceholders(prompt: string, model?: string, effort?: string, roster?: string, fallback?: string): string { return prompt .replaceAll("{{model}}", model ?? "") .replaceAll("{{effort}}", effort ?? "") - .replaceAll("{{roster}}", roster ?? ""); + .replaceAll("{{roster}}", roster ?? "") + .replaceAll("{{fallback}}", fallback ?? ""); } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 66f7e1caf1..a57262e4da 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -716,8 +716,6 @@ export async function handleResponses( await maybePrimeSubagentQuota(config); } - const subagentQuotaFailureModel = parsed.modelId; - let route; try { route = routeModel(config, parsed.modelId); @@ -891,16 +889,24 @@ export async function handleResponses( const identityScope = codexLogAccountId(authCtx); if (identityScope) parsed._cursorIdentityScope = identityScope; - const subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + let subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" ? authCtx.accountId : config.activeCodexAccountId ?? null; + let subagentQuotaFailureModel = parsed.modelId; if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { + const routeBeforeFallback = { + providerName: route.providerName, + modelId: route.modelId, + codexAccountMode: route.codexAccountMode, + }; const fallback = applySubagentModelFallback( parsed, req.headers, config, subagentFallbackAccountId, + Date.now(), + unreadableEncryptedAgentTask, ); if (fallback) { (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; @@ -909,6 +915,7 @@ export async function handleResponses( injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); } } + subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { try { route = routeModel(config, fallback.to); @@ -928,6 +935,55 @@ export async function handleResponses( logCtx.provider = route.providerName; logCtx.providerAdapter = route.provider.adapter; } + if ( + route.providerName !== routeBeforeFallback.providerName + || route.modelId !== routeBeforeFallback.modelId + || route.codexAccountMode !== routeBeforeFallback.codexAccountMode + ) { + try { + if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); + if (route.codexAccountMode) { + authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); + options.onCodexAuthContextResolved?.(authCtx); + } else { + authCtx = { kind: "main", accountId: null }; + options.onCodexAuthContextResolved?.(undefined); + } + selectedForwardHeaders = headersForCodexAuthContext(req.headers, authCtx); + } catch (err) { + if (err instanceof CodexAccountCooldownError) { + return formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down"); + } + if (err instanceof CodexThreadAffinityExpiredError) { + return formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"); + } + if (err instanceof CodexAuthContextError) { + const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); + console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); + return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); + } + if (err instanceof CodexPoolAuthenticationError) { + return formatErrorResponse(401, "authentication_error", err.message); + } + if (err instanceof CodexDirectAuthenticationError) { + return formatErrorResponse(401, "authentication_error", err.message); + } + if (err instanceof ForwardAdmissionCredentialError) { + return formatErrorResponse(401, "authentication_error", err.message); + } + throw err; + } + if (!isCodexAuthContextUsable(authCtx, config)) { + return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); + } + route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); + logCtx.provider = formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config); + const fallbackIdentityScope = codexLogAccountId(authCtx); + if (fallbackIdentityScope) parsed._cursorIdentityScope = fallbackIdentityScope; + subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + ? authCtx.accountId + : config.activeCodexAccountId ?? null; + } } // OAuth providers: swap in a fresh access token (auto-refreshed) as the Bearer key, so the @@ -1231,11 +1287,18 @@ export async function handleResponses( options.setTerminalOutcomeRecorder?.((status, httpStatusOverride) => { terminalRecorder(status, httpStatusOverride); if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage === undefined) return; recordSubagentQuotaFailureForThreadSpawn( req.headers, subagentQuotaFailureModel, - httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", + quotaFailureMessage, config, + subagentFallbackAccountId, ); } options.onNativePassthroughTerminal?.(status); @@ -1276,11 +1339,18 @@ export async function handleResponses( ? (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { terminalRecorder?.(status, httpStatusOverride); if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage === undefined) return; recordSubagentQuotaFailureForThreadSpawn( req.headers, subagentQuotaFailureModel, - httpStatusOverride ?? logCtx.terminalHttpStatus ?? "usage limit", + quotaFailureMessage, config, + subagentFallbackAccountId, ); } options.onNativePassthroughTerminal?.(status); @@ -1715,6 +1785,7 @@ export async function handleResponses( ? upstreamResponse.status : `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`, config, + subagentFallbackAccountId, ); // Upstreams occasionally echo request details in error bodies — scrub token-shaped // material before it reaches the client-facing error surface. diff --git a/src/types.ts b/src/types.ts index cbe47d51d0..3b36e35d70 100644 --- a/src/types.ts +++ b/src/types.ts @@ -491,7 +491,8 @@ export interface OcxConfig { * tags). When set, it replaces the built-in prompt on whichever * collab surface would have fired; firing gates are unchanged. Placeholders: * `{{model}}` -> injectionModel, `{{effort}}` -> injectionEffort, `{{roster}}` -> - * the resolved sub-agent roster block ("" when nothing resolves). + * the resolved sub-agent roster block ("" when nothing resolves), `{{fallback}}` -> + * the configured subagent model fallback guidance block ("" when unset). */ injectionPrompt?: string; /** diff --git a/tests/multi-agent-compat.test.ts b/tests/multi-agent-compat.test.ts index 5b55f6274b..e83c1e7cf1 100644 --- a/tests/multi-agent-compat.test.ts +++ b/tests/multi-agent-compat.test.ts @@ -382,6 +382,19 @@ describe("multiAgentGuidanceText", () => { expect(text).not.toContain("gpt-5.6-luna"); }); + test("injectionPrompt substitutes fallback guidance via {{fallback}}", async () => { + const text = await multiAgentGuidanceText( + parsedFixture({ tools: [{ name: "spawn_agent" }] }), + { + injectionPrompt: "FALLBACK={{fallback}}", + subagentModelFallback: ["alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }, + ); + expect(text).toContain("FALLBACK="); + expect(text).toContain("alibaba-token-plan/qwen3.8-max-preview"); + expect(text).toContain("kimi/k3"); + }); + test("v1 ignores injectionPrompt and custom prompt does not fire a bare v2 surface", async () => { codexHomeFixture(V2_ON); const custom = "CUSTOM RULES model={{model}} effort={{effort}}{{roster}}"; diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 128c278322..8c291dd4f9 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -120,6 +120,56 @@ describe("subagent model fallback chain", () => { expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); }); + test("noteSubagentModelFailure records the configured fallback slug", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg()); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); + expect(isSubagentModelUnavailable("qwen3.8-max-preview", cfg())).toBe(false); + }); + + test("selectAvailableSubagentModel can require native-only fallback for encrypted tasks", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("main", 95, undefined, 20); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg(), + [], + "main", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }); + }); + + test("selectAvailableSubagentModel can stay native-only for encrypted spawns", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("main", 95, undefined, 20); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg(), + [], + "main", + Date.now(), + true, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview", "kimi/k3"], + }); + }); + + test("noteSubagentModelFailure records failures under the configured fallback slug", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg(), "main"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "main")).toBe(true); + expect(isSubagentModelUnavailable("qwen3.8-max-preview", cfg(), "main")).toBe(false); + }); + test("readCodexAgentModelFallback parses multiline TOML arrays", () => { const dir = codexHomeFixture(); writeFileSync(join(dir, "agents", "executor.toml"), [ From 8cccf02c42505b858959dba163d1c5f33dfa160b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:11:51 +0200 Subject: [PATCH 05/12] fix(subagents): address follow-up PR #391 review feedback --- src/codex/subagent-model-fallback.ts | 17 ++++---- tests/subagent-model-fallback.test.ts | 59 ++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 7dbcbd66b5..9c58763c70 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -12,7 +12,7 @@ import type { OcxParsedRequest, OcxConfig } from "../types"; import { slugsEquivalent } from "../providers/slug-codec"; import { CODEX_HOME, getCodexHome } from "./paths"; import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; -import { computeCodexUsageScore } from "./routing"; +import { computeCodexUsageScore, getPoolAccountPlan } from "./routing"; import { nativeOpenAiSlugs } from "./catalog"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; @@ -28,17 +28,16 @@ const quotaPrimedAt = new Map(); const nativeSlugSet = () => new Set(nativeOpenAiSlugs().map(slug => slug.toLowerCase())); function healthKey(model: string, accountId: string | null): string { - return `${accountId ?? "none"}::${model.toLowerCase()}`; -} - -function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined { - return (config.codexAccounts ?? []).find(account => account.id === accountId)?.plan; + const scopedAccountId = nativeSlugSet().has(model.toLowerCase()) ? accountId : null; + return `${scopedAccountId ?? "none"}::${model.toLowerCase()}`; } function isDisabledFallbackModel(model: string, config: OcxConfig): boolean { const disabled = config.disabledModels ?? []; if (disabled.length === 0) return false; - if (!model.includes("/")) return disabled.some(stored => slugEquals(stored, "openai", model)); + if (!model.includes("/")) { + return disabled.some(stored => stored === model || slugEquals(stored, "openai", model)); + } const slash = model.indexOf("/"); const provider = model.slice(0, slash); const modelId = model.slice(slash + 1); @@ -99,7 +98,7 @@ function isRoutableFallbackModel(model: string, config: OcxConfig): boolean { const slash = model.indexOf("/"); if (slash > 0) { const providerName = model.slice(0, slash); - if (!hasOwnProvider(config.providers, providerName)) return false; + if (!hasOwnProvider(config.providers, providerName)) return true; const provider = config.providers[providerName]; if (provider?.disabled === true) return false; } @@ -322,7 +321,7 @@ function parseTomlStringArray(raw: string): string[] { } function parseTomlModelFallback(content: string): string[] | null { - const match = content.match(/^\s*model_fallback\s*=\s*\[(.*)\]\s*$/ms); + const match = content.match(/^\s*model_fallback\s*=\s*\[(.*?)\]\s*$/ms); if (!match) return null; return parseTomlStringArray(match[1] ?? ""); } diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 8c291dd4f9..32fe1a7534 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -13,7 +13,7 @@ import { selectAvailableSubagentModel, subagentFallbackGuidanceText, } from "../src/codex/subagent-model-fallback"; -import { updateAccountQuota } from "../src/codex/quota"; +import { clearAccountQuota, updateAccountQuota } from "../src/codex/quota"; import type { OcxConfig } from "../src/types"; const savedCodexHome = process.env.CODEX_HOME; @@ -48,6 +48,7 @@ function codexHomeFixture(): string { afterEach(() => { if (savedCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = savedCodexHome; + clearAccountQuota(); resetSubagentModelFallbackStateForTests(); }); @@ -187,6 +188,62 @@ describe("subagent model fallback chain", () => { ]); }); + test("readCodexAgentModelFallback stops at the model_fallback array terminator", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [", + " \"alibaba-token-plan/qwen3.8-max-preview\",", + "]", + "tools = [\"search\", \"edit\"]", + "", + ].join("\n"), "utf8"); + expect(readCodexAgentModelFallback("executor", dir)).toEqual([ + "alibaba-token-plan/qwen3.8-max-preview", + ]); + }); + + test("selectAvailableSubagentModel skips disabled bare native fallback entries", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + disabledModels: ["gpt-5.6-sol"], + subagentModelFallback: ["kimi/k3"], + }), + ); + expect(selected).toEqual({ + model: "kimi/k3", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("selectAvailableSubagentModel allows raw slash model ids without provider namespaces", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + subagentModelFallback: [ + "anthropic/claude-sonnet-4-6", + "kimi/k3", + ], + }), + ); + expect(selected).toEqual({ + model: "anthropic/claude-sonnet-4-6", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("noteSubagentModelFailure scopes routed-provider health globally", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg(), "account-a"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "account-b")).toBe(true); + }); + test("applySubagentModelFallback rewrites parsed request model", () => { updateAccountQuota("main", 95); const parsed = { From f7c8654cd0c15c5c83d2a34947fafefc737db526 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 05:41:54 +0200 Subject: [PATCH 06/12] fix(subagents): harden fallback routing for #391 Ensure fallback selection skips stale provider-prefixed entries while still allowing valid raw vendor/model slugs, and cache native slug lookups so selection no longer times out under test load. --- src/codex/subagent-model-fallback.ts | 19 +++++++++++++++++-- tests/subagent-model-fallback.test.ts | 3 +++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 9c58763c70..c14df6c13d 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -16,6 +16,7 @@ import { computeCodexUsageScore, getPoolAccountPlan } from "./routing"; import { nativeOpenAiSlugs } from "./catalog"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; +import { PROVIDER_REGISTRY } from "../providers/registry"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; type ModelHealth = { @@ -25,7 +26,17 @@ type ModelHealth = { const modelHealth = new Map(); const quotaPrimedAt = new Map(); -const nativeSlugSet = () => new Set(nativeOpenAiSlugs().map(slug => slug.toLowerCase())); +const NATIVE_SLUG_CACHE_MS = 60_000; +let nativeSlugCache: { cachedAt: number; slugs: Set } | null = null; +function nativeSlugSet(now = Date.now()): Set { + if (nativeSlugCache && (now - nativeSlugCache.cachedAt) < NATIVE_SLUG_CACHE_MS) { + return nativeSlugCache.slugs; + } + const slugs = new Set(nativeOpenAiSlugs().map(slug => slug.toLowerCase())); + nativeSlugCache = { cachedAt: now, slugs }; + return slugs; +} +const knownProviderIdSet = new Set(PROVIDER_REGISTRY.map(entry => entry.id.toLowerCase())); function healthKey(model: string, accountId: string | null): string { const scopedAccountId = nativeSlugSet().has(model.toLowerCase()) ? accountId : null; @@ -98,7 +109,11 @@ function isRoutableFallbackModel(model: string, config: OcxConfig): boolean { const slash = model.indexOf("/"); if (slash > 0) { const providerName = model.slice(0, slash); - if (!hasOwnProvider(config.providers, providerName)) return true; + if (!hasOwnProvider(config.providers, providerName)) { + // Allow well-known "vendor/model" ids (e.g. anthropic/claude-*) to flow as + // raw model ids through the default provider, but reject stale/typo prefixes. + return knownProviderIdSet.has(providerName.toLowerCase()); + } const provider = config.providers[providerName]; if (provider?.disabled === true) return false; } diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 32fe1a7534..cb6cda2010 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -91,6 +91,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel skips cached routed failures", () => { resetSubagentModelFallbackStateForTests(); + updateAccountQuota("main", 95, undefined, 20); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg()); const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); expect(selected.model).toBe("kimi/k3"); @@ -99,6 +100,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel skips stale fallback entries that cannot route", () => { resetSubagentModelFallbackStateForTests(); + updateAccountQuota("main", 95, undefined, 20); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg({ @@ -222,6 +224,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel allows raw slash model ids without provider namespaces", () => { resetSubagentModelFallbackStateForTests(); + updateAccountQuota("main", 95, undefined, 20); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg({ From 8f467d2905300375740607c2650eca9d734586ba Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:47:28 +0200 Subject: [PATCH 07/12] fix(subagents): await quota prime, atomic fallback PUT, shared rate-limit health Make maybePrimeSubagentQuota return an awaitable refresh promise, reject invalid fallback-chain entries without truncating config, and route health blocks through the shared rate-limit/quota classifier. --- src/codex/subagent-model-fallback.ts | 47 ++++++---- src/lib/errors.ts | 23 +++++ .../management/agent-settings-routes.ts | 13 ++- tests/subagent-model-fallback-api.test.ts | 91 +++++++++++++++++++ tests/subagent-model-fallback.test.ts | 46 ++++++++++ 5 files changed, 203 insertions(+), 17 deletions(-) create mode 100644 tests/subagent-model-fallback-api.test.ts diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index c14df6c13d..9fc6b00b54 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -8,6 +8,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { hasOwnProvider } from "../config"; +import { isRateLimitOrQuotaFailureMessage } from "../lib/errors"; import type { OcxParsedRequest, OcxConfig } from "../types"; import { slugsEquivalent } from "../providers/slug-codec"; import { CODEX_HOME, getCodexHome } from "./paths"; @@ -19,6 +20,9 @@ import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; +type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; +let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; + type ModelHealth = { unavailableUntil: number; reason: string; @@ -191,17 +195,7 @@ export function noteSubagentModelFailure( ttlMs?: number, ): void { const interval = ttlMs ?? DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; - const normalized = String(message).trim(); - const lower = normalized.toLowerCase(); - const numericStatus = Number(normalized); - const quotaLike = lower.includes("insufficient_quota") - || lower.includes("quota exhausted") - || lower.includes("usage limit") - || lower.includes("exceeded your current quota") - || lower.includes("account quota exceeded") - || numericStatus === 429 - || numericStatus === 402; - if (!quotaLike) return; + if (!isRateLimitOrQuotaFailureMessage(message)) return; modelHealth.set(healthKey(model, resolveFallbackAccountId(config, accountId)), { unavailableUntil: now + interval, reason: "quota_exhausted", @@ -211,6 +205,12 @@ export function noteSubagentModelFailure( export function resetSubagentModelFallbackStateForTests(): void { modelHealth.clear(); quotaPrimedAt.clear(); + subagentQuotaPrimeForTests = null; +} + +/** Test-only: inject the quota prime implementation used by {@link maybePrimeSubagentQuota}. */ +export function setSubagentQuotaPrimeForTests(fn: SubagentQuotaPrimeFn | null): void { + subagentQuotaPrimeForTests = fn; } function rewriteParsedModel(parsed: OcxParsedRequest, model: string): void { @@ -274,11 +274,26 @@ export function resolveAgentModelFallbackForPrimary( return merged; } -export function maybePrimeSubagentQuota(config: OcxConfig, now = Date.now()): void { - if (!shouldPrimeSubagentQuota(config, now)) return; - void import("./auth-api") - .then(({ primeCodexPoolQuotas }) => primeCodexPoolQuotas(config, "subagent-spawn")) - .catch(() => {}); +/** + * Best-effort quota refresh before subagent model selection. + * Returns a promise the caller must await so selection observes refreshed state. + * Refresh failures are swallowed here so spawn routing can continue. + */ +export function maybePrimeSubagentQuota(config: OcxConfig, now = Date.now()): Promise { + if (!shouldPrimeSubagentQuota(config, now)) return Promise.resolve(); + const run = async (): Promise => { + try { + if (subagentQuotaPrimeForTests) { + await subagentQuotaPrimeForTests(config, "subagent-spawn"); + return; + } + const { primeCodexPoolQuotas } = await import("./auth-api"); + await primeCodexPoolQuotas(config, "subagent-spawn"); + } catch { + // Owning boundary: do not fail the spawn path when priming is unavailable. + } + }; + return run(); } export function recordSubagentQuotaFailureForThreadSpawn( diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 6197df8229..a316e381bf 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -190,6 +190,29 @@ export function classifyError(status: number, type: string, message: string): Oc return { message, type, code: type || null }; } +/** + * True when a provider failure should participate in rate-limit / quota health blocking. + * Reuses {@link classifyError} so generic 429 wording and quota phrases stay aligned. + */ +export function isRateLimitOrQuotaFailureMessage(message: string): boolean { + const normalized = String(message ?? "").trim(); + if (!normalized) return false; + const numericStatus = Number(normalized); + if (numericStatus === 429 || numericStatus === 402) return true; + const statusHint = Number.isInteger(numericStatus) && numericStatus > 0 ? numericStatus : 0; + const classified = classifyError(statusHint, "", normalized); + if ( + classified.type === "rate_limit_error" + || classified.code === "rate_limit_exceeded" + || classified.type === "insufficient_quota" + || classified.code === "insufficient_quota" + ) { + return true; + } + // Retained quota cue used by subagent health before classifyError covered it. + return normalized.toLowerCase().includes("usage limit"); +} + /** Best-effort parse of a retry delay embedded in an upstream error message. */ export function parseRetryAfterFromMessage(message: string): number | undefined { const patterns = [ diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index f5d9d810fa..c41623ad7f 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -322,7 +322,18 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise let nextPollMs = config.subagentModelFallbackPollMs; if ("models" in body) { if (!Array.isArray(body.models)) return jsonResponse({ error: "models must be an array" }, 400); - const models = body.models.filter((m): m is string => typeof m === "string" && m.trim().length > 0); + const models: string[] = []; + for (let i = 0; i < body.models.length; i++) { + const entry = body.models[i]; + if (typeof entry !== "string" || entry.trim().length === 0) { + return jsonResponse({ + error: `models[${i}] must be a non-empty string`, + index: i, + value: entry, + }, 400); + } + models.push(entry.trim()); + } nextModels = models.length > 0 ? models : undefined; } if ("pollMs" in body) { diff --git a/tests/subagent-model-fallback-api.test.ts b/tests/subagent-model-fallback-api.test.ts new file mode 100644 index 0000000000..10fde15475 --- /dev/null +++ b/tests/subagent-model-fallback-api.test.ts @@ -0,0 +1,91 @@ +/** + * /api/subagent-model-fallback atomic validation (PR #391). + * Invalid chain entries must 400 without mutating the previous config. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import type { OcxConfig } from "../src/types"; + +const savedHome = process.env.OPENCODEX_HOME; +let tempHome: string | null = null; + +afterEach(() => { + if (savedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = savedHome; + if (tempHome) { + rmSync(tempHome, { recursive: true, force: true }); + tempHome = null; + } +}); + +function isolatedHome(): void { + tempHome = mkdtempSync(join(tmpdir(), "ocx-subagent-fallback-api-")); + process.env.OPENCODEX_HOME = tempHome; +} + +function makeConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + providers: {}, + defaultProvider: "openai", + subagentModelFallback: ["gpt-5.6-sol", "kimi/k3"], + ...overrides, + } as OcxConfig; +} + +async function put(config: OcxConfig, body: unknown): Promise { + const req = new Request("http://localhost/api/subagent-model-fallback", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const res = await handleManagementAPI(req, new URL(req.url), config); + expect(res).not.toBeNull(); + return res!; +} + +describe("/api/subagent-model-fallback atomic validation", () => { + test("rejects one invalid entry with 400 and leaves previous config unchanged", async () => { + isolatedHome(); + const previous = ["gpt-5.6-sol", "kimi/k3"]; + const config = makeConfig({ subagentModelFallback: [...previous] }); + + const res = await put(config, { + models: ["gpt-5.6-sol", 42, "alibaba-token-plan/qwen3.8-max-preview"], + }); + expect(res.status).toBe(400); + const body = await res.json() as { error: string; index: number; value: unknown }; + expect(body.error).toBe("models[1] must be a non-empty string"); + expect(body.index).toBe(1); + expect(body.value).toBe(42); + expect(config.subagentModelFallback).toEqual(previous); + }); + + test("rejects empty-string entries without truncating the chain", async () => { + isolatedHome(); + const previous = ["gpt-5.6-sol", "kimi/k3"]; + const config = makeConfig({ subagentModelFallback: [...previous] }); + + const res = await put(config, { + models: ["gpt-5.6-sol", " ", "kimi/k3"], + }); + expect(res.status).toBe(400); + const body = await res.json() as { error: string; index: number }; + expect(body.error).toBe("models[1] must be a non-empty string"); + expect(body.index).toBe(1); + expect(config.subagentModelFallback).toEqual(previous); + }); + + test("accepts a fully valid chain after validation", async () => { + isolatedHome(); + const config = makeConfig(); + const next = ["gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview"]; + const res = await put(config, { models: next }); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ ok: true, models: next }); + expect(config.subagentModelFallback).toEqual(next); + }); +}); diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index cb6cda2010..c329acac18 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -6,11 +6,13 @@ import { applySubagentModelFallback, buildSubagentModelChain, isSubagentModelUnavailable, + maybePrimeSubagentQuota, noteSubagentModelFailure, readCodexAgentModelFallback, resetSubagentModelFallbackStateForTests, resolveAgentModelFallbackForPrimary, selectAvailableSubagentModel, + setSubagentQuotaPrimeForTests, subagentFallbackGuidanceText, } from "../src/codex/subagent-model-fallback"; import { clearAccountQuota, updateAccountQuota } from "../src/codex/quota"; @@ -123,6 +125,50 @@ describe("subagent model fallback chain", () => { expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); }); + test("noteSubagentModelFailure records generic rate-limit wording as a health block", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "Rate limit exceeded. Please try again later.", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "Too Many Requests", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "provider temporarily rate limited", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + }); + + test("noteSubagentModelFailure ignores unrelated errors", () => { + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("kimi/k3", "connection refused", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + noteSubagentModelFailure("kimi/k3", "invalid_request_error: missing field", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + }); + + test("await maybePrimeSubagentQuota waits for deferred refresh before selection", async () => { + resetSubagentModelFallbackStateForTests(); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let midRefreshModel = ""; + + setSubagentQuotaPrimeForTests(async () => { + midRefreshModel = selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model; + await gate; + updateAccountQuota("main", 95, undefined, 20); + }); + + const priming = maybePrimeSubagentQuota(cfg()); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe("gpt-5.6-sol"); + release(); + await priming; + expect(midRefreshModel).toBe("gpt-5.6-sol"); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe( + "alibaba-token-plan/qwen3.8-max-preview", + ); + }); + test("noteSubagentModelFailure records the configured fallback slug", () => { resetSubagentModelFallbackStateForTests(); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg()); From 04d6a5bddb75f973b62213b723e9e4cea870f257 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:26:28 +0200 Subject: [PATCH 08/12] fix(subagents): settle fallback before normalize and share quota primes Release abandoned Codex probe leases on pre-upstream fallback reroutes, run virtual-model/effort/service-tier normalization only against the final route, allow encrypted children to reach native fallbacks, and coalesce concurrent quota primes with success-only TTL. --- src/codex/subagent-model-fallback.ts | 45 +- src/server/responses/core.ts | 450 ++++++++------- ...subagent-fallback-handle-responses.test.ts | 533 ++++++++++++++++++ tests/subagent-model-fallback.test.ts | 67 +++ 4 files changed, 852 insertions(+), 243 deletions(-) create mode 100644 tests/subagent-fallback-handle-responses.test.ts diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 9fc6b00b54..0af54e0a70 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -22,6 +22,7 @@ export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; +let quotaPrimeInFlight: Promise | null = null; type ModelHealth = { unavailableUntil: number; @@ -205,7 +206,9 @@ export function noteSubagentModelFailure( export function resetSubagentModelFallbackStateForTests(): void { modelHealth.clear(); quotaPrimedAt.clear(); + quotaPrimeInFlight = null; subagentQuotaPrimeForTests = null; + nativeSlugCache = null; } /** Test-only: inject the quota prime implementation used by {@link maybePrimeSubagentQuota}. */ @@ -213,6 +216,17 @@ export function setSubagentQuotaPrimeForTests(fn: SubagentQuotaPrimeFn | null): subagentQuotaPrimeForTests = fn; } +/** Test-only: inspect shared prime TTL / in-flight state. */ +export function getSubagentQuotaPrimeStateForTests(): { + primedAt: number; + inFlight: boolean; +} { + return { + primedAt: quotaPrimedAt.get("global") ?? 0, + inFlight: quotaPrimeInFlight !== null, + }; +} + function rewriteParsedModel(parsed: OcxParsedRequest, model: string): void { parsed.modelId = model; if (parsed._rawBody && typeof parsed._rawBody === "object") { @@ -276,24 +290,31 @@ export function resolveAgentModelFallbackForPrimary( /** * Best-effort quota refresh before subagent model selection. - * Returns a promise the caller must await so selection observes refreshed state. - * Refresh failures are swallowed here so spawn routing can continue. + * Concurrent callers share one in-flight promise. The success TTL is updated only + * after a successful refresh so failures remain retryable. Errors are swallowed so + * spawn routing can continue. */ export function maybePrimeSubagentQuota(config: OcxConfig, now = Date.now()): Promise { + if (quotaPrimeInFlight) return quotaPrimeInFlight; if (!shouldPrimeSubagentQuota(config, now)) return Promise.resolve(); - const run = async (): Promise => { + + quotaPrimeInFlight = (async () => { try { if (subagentQuotaPrimeForTests) { await subagentQuotaPrimeForTests(config, "subagent-spawn"); - return; + } else { + const { primeCodexPoolQuotas } = await import("./auth-api"); + await primeCodexPoolQuotas(config, "subagent-spawn"); } - const { primeCodexPoolQuotas } = await import("./auth-api"); - await primeCodexPoolQuotas(config, "subagent-spawn"); + quotaPrimedAt.set("global", Date.now()); } catch { // Owning boundary: do not fail the spawn path when priming is unavailable. + // Leave quotaPrimedAt untouched so a later spawn can retry. + } finally { + quotaPrimeInFlight = null; } - }; - return run(); + })(); + return quotaPrimeInFlight; } export function recordSubagentQuotaFailureForThreadSpawn( @@ -386,10 +407,8 @@ export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { .map(name => name.slice(0, -".toml".length)); } +/** True when a new quota prime should start (no success within the poll interval). */ export function shouldPrimeSubagentQuota(config: OcxConfig, now = Date.now()): boolean { - const key = "global"; - const last = quotaPrimedAt.get(key) ?? 0; - if (now - last < pollIntervalMs(config)) return false; - quotaPrimedAt.set(key, now); - return true; + const last = quotaPrimedAt.get("global") ?? 0; + return now - last >= pollIntervalMs(config); } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index a57262e4da..0dcc5b6e7f 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -9,7 +9,7 @@ import { parseRequest } from "../../responses/parser"; import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses"; import { expandPreviousResponseInput, previousResponseProviderState, rememberResponseState } from "../../responses/state"; -import { routeModel } from "../../router"; +import { routeModel, type RouteResult } from "../../router"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -420,6 +420,167 @@ function unreadableEncryptedAgentTaskResponse(): Response { ); } +type ResponsesAuthResolution = + | { ok: true; authCtx: CodexAuthContext; headers: Headers } + | { ok: false; response: Response }; + +/** + * Resolve Codex auth for a route. On unusable contexts, releases any probe lease + * before returning the 401 (nothing reaches upstream). + */ +async function resolveResponsesCodexAuth( + req: Request, + config: OcxConfig, + route: RouteResult, + options: HandleResponsesOptions, +): Promise { + try { + if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); + let authCtx: CodexAuthContext; + if (route.codexAccountMode) { + authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); + options.onCodexAuthContextResolved?.(authCtx); + } else { + authCtx = { kind: "main", accountId: null }; + options.onCodexAuthContextResolved?.(undefined); + } + if (!isCodexAuthContextUsable(authCtx, config)) { + releaseCodexAuthContextProbeLease(authCtx); + return { + ok: false, + response: formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"), + }; + } + return { + ok: true, + authCtx, + headers: headersForCodexAuthContext(req.headers, authCtx), + }; + } catch (err) { + if (err instanceof CodexAccountCooldownError) { + return { ok: false, response: formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down") }; + } + if (err instanceof CodexThreadAffinityExpiredError) { + return { + ok: false, + response: formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"), + }; + } + if (err instanceof CodexAuthContextError) { + const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); + console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); + return { + ok: false, + response: formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"), + }; + } + if (err instanceof CodexPoolAuthenticationError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + if (err instanceof CodexDirectAuthenticationError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + if (err instanceof ForwardAdmissionCredentialError) { + return { ok: false, response: formatErrorResponse(401, "authentication_error", err.message) }; + } + throw err; + } +} + +/** + * Apply every route-dependent request mutation against the final selected route. + * Must run only after subagent fallback has settled the model/provider. + */ +async function applyFinalRouteRequestNormalization(args: { + parsed: OcxParsedRequest; + route: RouteResult; + config: OcxConfig; + req: Request; + logCtx: RequestLogContext; +}): Promise { + const { parsed, route, config, req, logCtx } = args; + + // Apply the routed model id upstream: routing may strip a "/" namespace. + if (route.modelId !== parsed.modelId) { + if (parsed._rawBody && typeof parsed._rawBody === "object") { + (parsed._rawBody as { model?: string }).model = route.modelId; + } + parsed.modelId = route.modelId; + } + // Settle the wire once so logging, fast-mode, auth, and sidecars read the adapter + // this request will actually use (#404). + route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider); + logCtx.model = route.modelId; + logCtx.provider = route.providerName; + logCtx.providerAdapter = route.provider.adapter; + + // Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro". + applyOpenAiVirtualModel(parsed, route, logCtx); + + // Fast mode override for OpenAI-routed models. + if (config.fastMode !== undefined && route.provider.adapter === "openai-responses") { + const tier = config.fastMode ? "priority" : undefined; + if (parsed._rawBody && typeof parsed._rawBody === "object") { + if (tier) (parsed._rawBody as Record).service_tier = tier; + else delete (parsed._rawBody as Record).service_tier; + } + parsed.options.serviceTier = tier; + } + + { + const guidance = await multiAgentGuidanceText(parsed, { + multiAgentGuidanceEnabled: config.multiAgentGuidanceEnabled, + injectionModel: config.injectionModel, + injectionEffort: config.injectionEffort, + subagentModels: config.subagentModels, + subagentModelFallback: config.subagentModelFallback, + injectionPrompt: config.injectionPrompt, + }); + if (guidance) { + injectDeveloperMessage(parsed, guidance); + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] ${route.modelId}: multi-agent guidance injected (surface=${collabSurface(parsed)}, guidanceEnabled=${multiAgentGuidanceEnabled(config)}, ${guidance.length} chars)`); + } + } else if (isInjectionDebugEnabled() && collabSurface(parsed) !== null) { + injectionDebugLog(`[opencodex] ${route.modelId}: collab surface=${collabSurface(parsed)}, guidance silent (effort=${parsed.options.reasoning ?? "unset"}, injectionModel=${config.injectionModel ?? "unset"})`); + } + } + + { + const { applyEffortCap, effortCapAppliesTo, supportedLadderFor } = await import("../effort-policy"); + const surface = collabSurface(parsed); + if (effortCapAppliesTo(surface, req.headers, config, parsed._compactionRequest === true)) { + const capped = applyEffortCap(parsed, req.headers, config, supportedLadderFor(route)); + if (capped) { + logCtx.requestedEffort = `${capped.from}->${capped.to}`; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] ${route.modelId}: effort cap applied (${capped.from} -> ${capped.to}, ${capped.subagent ? "sub-agent" : "main"} turn)`); + } + } + } else if (isInjectionDebugEnabled() && (config.effortCap || config.subagentEffortCap)) { + injectionDebugLog(`[opencodex] ${route.modelId}: effort cap skipped (surface=${surface ?? "none"}, v2 feature only)`); + } + } + + { + const requestedModelId = logCtx.requestedModel ?? route.modelId; + const { nativeEffortClamp, shouldApplyNativeEffortClamp } = await import("../../codex/catalog"); + const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, requestedModelId) + ? nativeEffortClamp(route.modelId, parsed.options.reasoning) + : null; + if (clamped) { + parsed.options.reasoning = clamped; + const raw = parsed._rawBody as { reasoning?: { effort?: string } } | undefined; + if (raw?.reasoning && typeof raw.reasoning === "object") raw.reasoning.effort = clamped; + logCtx.requestedEffort = `${logCtx.requestedEffort ?? "max"}->${clamped}`; + } + } + logCtx.modelSupportsServiceTier = catalogModelSupportsServiceTier( + route.modelId, + logCtx.requestedServiceTier ?? logCtx.configuredServiceTier, + ); +} + export async function handleComboResponses( @@ -716,7 +877,7 @@ export async function handleResponses( await maybePrimeSubagentQuota(config); } - let route; + let route: RouteResult; try { route = routeModel(config, parsed.modelId); } catch (err) { @@ -726,180 +887,26 @@ export async function handleResponses( return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } - // The canonical ChatGPT backend can decrypt its V2 Fernet task tokens; routed - // providers cannot. Reject the raw-input classification before adapter construction - // or provider dispatch so an unreadable worker task cannot trigger a cost storm. - if (!isCanonicalOpenAiForwardProvider(route.provider) && unreadableEncryptedAgentTask) { - return unreadableEncryptedAgentTaskResponse(); - } - - // Apply the routed model id upstream: routing may strip a "/" namespace - // (e.g. "opencode-go/deepseek-v4-pro" → "deepseek-v4-pro"). Adapters read parsed.modelId, - // and the passthrough adapter serializes _rawBody, so rewrite both. - if (route.modelId !== parsed.modelId) { - if (parsed._rawBody && typeof parsed._rawBody === "object") { - (parsed._rawBody as { model?: string }).model = route.modelId; - } - parsed.modelId = route.modelId; - } - // Settle the wire once, right after the native model id is known, so logging, - // fast-mode injection, auth, and sidecar decisions all read the adapter this - // request will actually use rather than the provider-wide default (#404). - route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider); - logCtx.model = route.modelId; - logCtx.provider = route.providerName; - logCtx.providerAdapter = route.provider.adapter; - - // Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro". - // Must run before effort caps/native clamps so the base model gets correct limits. - applyOpenAiVirtualModel(parsed, route, logCtx); - - // Fast mode override: when config.fastMode is explicitly set, inject or strip - // service_tier for OpenAI-routed models. Undefined = passthrough (client decides). - if (config.fastMode !== undefined && route.provider.adapter === "openai-responses") { - const tier = config.fastMode ? "priority" : undefined; - if (parsed._rawBody && typeof parsed._rawBody === "object") { - if (tier) (parsed._rawBody as Record).service_tier = tier; - else delete (parsed._rawBody as Record).service_tier; - } - parsed.options.serviceTier = tier; - } - - // Multi-agent guidance shim: codex-rs emits its Proactive delegation developer - // message only on the v2 surface. The proxy fills the gaps: the Proactive text - // for v1 collab surfaces at the top tier (no model designation on v1), and the - // sub-agent model/roster designation plus fork_turns override rules on v2. - // The surface is judged from the request's own tool list. Runs BEFORE the - // mock-max clamp below so the synthetic top tier (ultra arrives as max on the - // codex wire) is still visible. Both request shapes are rewritten. - { - const guidance = await multiAgentGuidanceText(parsed, { - multiAgentGuidanceEnabled: config.multiAgentGuidanceEnabled, - injectionModel: config.injectionModel, - injectionEffort: config.injectionEffort, - subagentModels: config.subagentModels, - subagentModelFallback: config.subagentModelFallback, - injectionPrompt: config.injectionPrompt, - }); - if (guidance) { - injectDeveloperMessage(parsed, guidance); - if (isInjectionDebugEnabled()) injectionDebugLog(`[opencodex] ${route.modelId}: multi-agent guidance injected (surface=${collabSurface(parsed)}, guidanceEnabled=${multiAgentGuidanceEnabled(config)}, ${guidance.length} chars)`); - } else if (isInjectionDebugEnabled() && collabSurface(parsed) !== null) { - injectionDebugLog(`[opencodex] ${route.modelId}: collab surface=${collabSurface(parsed)}, guidance silent (effort=${parsed.options.reasoning ?? "unset"}, injectionModel=${config.injectionModel ?? "unset"})`); - } - } - - // Hard effort caps (effortCap / subagentEffortCap): enforcement companion to the advisory - // injection above — spawn-arg prompting cannot stop codex-rs from inheriting the parent's - // ultra-tier default on bare spawns (see src/server/effort-policy.ts). Runs BEFORE the - // mock-max clamp so a capped effort is what nativeness clamping then validates; rewrites - // both request shapes (same dual-write contract as the clamp below). - // GATE: v2 feature only (effortCapAppliesTo) — v2-surface main turns plus header-marked - // child turns admitted regardless of tool surface (depth-limited leaves carry no collab - // tools while shallower children do, so tool sniffing alone would cap siblings - // inconsistently); multiAgentMode "v1" disables caps entirely; compaction turns bypass - // caps so routed compaction matches native /v1/responses/compact (which never enters - // handleResponses). - { - const { applyEffortCap, effortCapAppliesTo, supportedLadderFor } = await import("../effort-policy"); - const surface = collabSurface(parsed); - if (effortCapAppliesTo(surface, req.headers, config, parsed._compactionRequest === true)) { - const capped = applyEffortCap(parsed, req.headers, config, supportedLadderFor(route)); - if (capped) { - logCtx.requestedEffort = `${capped.from}->${capped.to}`; - if (isInjectionDebugEnabled()) { - injectionDebugLog(`[opencodex] ${route.modelId}: effort cap applied (${capped.from} -> ${capped.to}, ${capped.subagent ? "sub-agent" : "main"} turn)`); - } - } - } else if (isInjectionDebugEnabled() && (config.effortCap || config.subagentEffortCap)) { - injectionDebugLog(`[opencodex] ${route.modelId}: effort cap skipped (surface=${surface ?? "none"}, v2 feature only)`); - } - } - - // Mock-max clamp: native models whose real ladder stops below max (gpt-5.5/5.4/…) - // receive `max` when the user picks Ultra (codex converts ultra->max client-side). - // Clamp to the model's highest real effort BEFORE any adapter — the ChatGPT - // passthrough serializes _rawBody verbatim, so both shapes must be rewritten. - // GUARD: judge nativeness by BOTH the originally requested id (logCtx.requestedModel) - // and the resolved provider identity. Routing strips the "/" namespace, and - // some third-party providers expose bare `defaultModel` selectors, so route.modelId - // alone can make a routed model masquerade as an off-snapshot native. Only the - // canonical built-in ChatGPT forward provider should receive the native clamp. - { - const requestedModelId = logCtx.requestedModel ?? route.modelId; - const { nativeEffortClamp, shouldApplyNativeEffortClamp } = await import("../../codex/catalog"); - const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, requestedModelId) - ? nativeEffortClamp(route.modelId, parsed.options.reasoning) - : null; - if (clamped) { - parsed.options.reasoning = clamped; - const raw = parsed._rawBody as { reasoning?: { effort?: string } } | undefined; - if (raw?.reasoning && typeof raw.reasoning === "object") raw.reasoning.effort = clamped; - logCtx.requestedEffort = `${logCtx.requestedEffort ?? "max"}->${clamped}`; - } - } - logCtx.modelSupportsServiceTier = catalogModelSupportsServiceTier( - route.modelId, - logCtx.requestedServiceTier ?? logCtx.configuredServiceTier, - ); - let authCtx: CodexAuthContext = { kind: "main", accountId: null }; - let selectedForwardHeaders: Headers; - try { - if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); - if (route.codexAccountMode) { - authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); - options.onCodexAuthContextResolved?.(authCtx); - } else { - options.onCodexAuthContextResolved?.(undefined); - } - selectedForwardHeaders = headersForCodexAuthContext(req.headers, authCtx); - } catch (err) { - if (err instanceof CodexAccountCooldownError) { - return formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down"); - } - if (err instanceof CodexThreadAffinityExpiredError) { - return formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"); - } - if (err instanceof CodexAuthContextError) { - const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); - console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); - } - if (err instanceof CodexPoolAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof CodexDirectAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof ForwardAdmissionCredentialError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - throw err; - } - if (!isCodexAuthContextUsable(authCtx, config)) { - // Nothing reaches upstream on this path, so give the probe back. - releaseCodexAuthContextProbeLease(authCtx); - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); - } - route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); - logCtx.provider = formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config); - // Prefer Codex pool account as the Cursor thread namespace when present. Cursor routes without - // codexAccountMode still get a credential-derived scope inside the Cursor adapter. - const identityScope = codexLogAccountId(authCtx); - if (identityScope) parsed._cursorIdentityScope = identityScope; - - let subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? authCtx.accountId - : config.activeCodexAccountId ?? null; + let selectedForwardHeaders = req.headers; + let authReady = false; + let subagentFallbackAccountId = config.activeCodexAccountId ?? null; let subagentQuotaFailureModel = parsed.modelId; + // Subagent fallback must settle the final model/provider BEFORE route-dependent + // normalization (virtual models, effort caps, service tier, wire protocol). + // Early auth is only for account-scoped quota selection; its probe lease is + // released whenever that context is abandoned before upstream. if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { - const routeBeforeFallback = { - providerName: route.providerName, - modelId: route.modelId, - codexAccountMode: route.codexAccountMode, - }; + const earlyAuth = await resolveResponsesCodexAuth(req, config, route, options); + if (!earlyAuth.ok) return earlyAuth.response; + authCtx = earlyAuth.authCtx; + selectedForwardHeaders = earlyAuth.headers; + authReady = true; + subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + ? authCtx.accountId + : config.activeCodexAccountId ?? null; + const fallback = applySubagentModelFallback( parsed, req.headers, @@ -916,76 +923,59 @@ export async function handleResponses( } } subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + let nextRoute: RouteResult; try { - route = routeModel(config, fallback.to); + nextRoute = routeModel(config, fallback.to); } catch (err) { + releaseCodexAuthContextProbeLease(authCtx); + authReady = false; if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); } return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } - if (route.modelId !== parsed.modelId) { - if (parsed._rawBody && typeof parsed._rawBody === "object") { - (parsed._rawBody as { model?: string }).model = route.modelId; - } - parsed.modelId = route.modelId; - } - logCtx.model = route.modelId; - logCtx.provider = route.providerName; - logCtx.providerAdapter = route.provider.adapter; - } - if ( - route.providerName !== routeBeforeFallback.providerName - || route.modelId !== routeBeforeFallback.modelId - || route.codexAccountMode !== routeBeforeFallback.codexAccountMode - ) { - try { - if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); - if (route.codexAccountMode) { - authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode); - options.onCodexAuthContextResolved?.(authCtx); - } else { - authCtx = { kind: "main", accountId: null }; - options.onCodexAuthContextResolved?.(undefined); - } - selectedForwardHeaders = headersForCodexAuthContext(req.headers, authCtx); - } catch (err) { - if (err instanceof CodexAccountCooldownError) { - return formatErrorResponse(429, "rate_limit_error", "Selected Codex account is cooling down"); - } - if (err instanceof CodexThreadAffinityExpiredError) { - return formatErrorResponse(409, "invalid_request_error", "Codex thread account affinity expired; start a new session"); - } - if (err instanceof CodexAuthContextError) { - const safeAccountLabel = formatCodexProviderForLog(route.providerName, err.accountId, config); - console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`); - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); - } - if (err instanceof CodexPoolAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof CodexDirectAuthenticationError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - if (err instanceof ForwardAdmissionCredentialError) { - return formatErrorResponse(401, "authentication_error", err.message); - } - throw err; - } - if (!isCodexAuthContextUsable(authCtx, config)) { - return formatErrorResponse(401, "authentication_error", "Selected Codex account needs reauthentication"); + const authReusable = route.providerName === nextRoute.providerName + && route.codexAccountMode === nextRoute.codexAccountMode; + if (!authReusable) { + // Abandon the pre-fallback auth context (and any probe lease) before + // resolving replacement auth — nothing has reached upstream yet. + releaseCodexAuthContextProbeLease(authCtx); + authCtx = { kind: "main", accountId: null }; + authReady = false; + selectedForwardHeaders = req.headers; } - route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); - logCtx.provider = formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config); - const fallbackIdentityScope = codexLogAccountId(authCtx); - if (fallbackIdentityScope) parsed._cursorIdentityScope = fallbackIdentityScope; - subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? authCtx.accountId - : config.activeCodexAccountId ?? null; + route = nextRoute; } } + // Encrypted child tasks may only reach the canonical native backend. This check + // runs against the FINAL route so native-only fallback can rescue a routed primary. + if (!isCanonicalOpenAiForwardProvider(route.provider) && unreadableEncryptedAgentTask) { + if (authReady) releaseCodexAuthContextProbeLease(authCtx); + return unreadableEncryptedAgentTaskResponse(); + } + + await applyFinalRouteRequestNormalization({ parsed, route, config, req, logCtx }); + + if (!authReady) { + const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); + if (!finalAuth.ok) return finalAuth.response; + authCtx = finalAuth.authCtx; + selectedForwardHeaders = finalAuth.headers; + } + + route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); + logCtx.provider = formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config); + // Prefer Codex pool account as the Cursor thread namespace when present. Cursor routes without + // codexAccountMode still get a credential-derived scope inside the Cursor adapter. + const identityScope = codexLogAccountId(authCtx); + if (identityScope) parsed._cursorIdentityScope = identityScope; + subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" + ? authCtx.accountId + : config.activeCodexAccountId ?? null; + // OAuth providers: swap in a fresh access token (auto-refreshed) as the Bearer key, so the // existing openai-chat / anthropic adapters authenticate with no change. const isOAuth401ReplayProvider = (route.providerName === "xai" || route.providerName === "github-copilot" || route.providerName === "kiro") diff --git a/tests/subagent-fallback-handle-responses.test.ts b/tests/subagent-fallback-handle-responses.test.ts new file mode 100644 index 0000000000..6313bbbed1 --- /dev/null +++ b/tests/subagent-fallback-handle-responses.test.ts @@ -0,0 +1,533 @@ +/** + * handleResponses integration coverage for PR #391 merge blockers: + * probe-lease release on fallback reroute, final-route normalization, + * encrypted native-only fallback, concurrent quota priming (unit-covered separately). + */ +import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { + clearAccountQuota, + updateAccountQuota, +} from "../src/codex/quota"; +import { + CODEX_QUOTA_PROBE_INTERVAL_MS, + clearCodexUpstreamHealth, + clearThreadAccountMap, + getCodexUpstreamHealth, + recordCodexUpstreamOutcome, +} from "../src/codex/routing"; +import { + resetSubagentModelFallbackStateForTests, +} from "../src/codex/subagent-model-fallback"; +import type { CodexAuthContext } from "../src/codex/auth-context"; +import { handleResponses } from "../src/server/responses"; +import type { OcxConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; + +setDefaultTimeout(30_000); + +const originalFetch = globalThis.fetch; +const originalNow = Date.now; +let testDir: string; +let previousOpencodexHome: string | undefined; +let previousCodexHome: string | undefined; + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-hr-")); + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + clearThreadAccountMap(); + clearCodexUpstreamHealth(); + clearAccountQuota(); + resetSubagentModelFallbackStateForTests(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + Date.now = originalNow; + clearThreadAccountMap(); + clearCodexUpstreamHealth(); + clearAccountQuota(); + resetSubagentModelFallbackStateForTests(); + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; +}); + +function fernetFixture(ciphertextBytes = 16): string { + const raw = Buffer.alloc(57 + ciphertextBytes, 0x5a); + raw[0] = 0x80; + raw.writeBigUInt64BE(1_720_000_000n, 1); + const unpadded = raw.toString("base64url"); + return `${unpadded}${"=".repeat((4 - (unpadded.length % 4)) % 4)}`; +} + +const FERNET_TASK = fernetFixture(); + +function encryptedAgentInput(): unknown[] { + return [{ + type: "agent_message", + author: "/root", + recipient: "/root/worker", + content: [{ type: "encrypted_content", encrypted_content: FERNET_TASK }], + }]; +} + +function readableAgentInput(): unknown[] { + return [{ + type: "agent_message", + author: "/root", + recipient: "/root/worker", + content: [{ type: "input_text", text: "do the work" }], + }]; +} + +function spawnHeaders(extra: HeadersInit = {}): Headers { + return new Headers({ + "content-type": "application/json", + "x-openai-subagent": "collab_spawn", + authorization: "Bearer caller-codex-token", + ...Object.fromEntries(new Headers(extra)), + }); +} + +function poolNativePlusRoutedConfig(overrides: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-apikey": { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool@example.test", isMain: false, chatgptAccountId: "pool_acc" }, + ], + ...overrides, + } as OcxConfig; +} + +function installPoolCredential(now: number): void { + saveCodexAccountCredential("pool-a", { + accessToken: "pool_token", + refreshToken: "pool_refresh", + expiresAt: now + 24 * 60 * 60_000, + chatgptAccountId: "pool_acc", + }); +} + +function mockUpstream(capture: { + urls: string[]; + bodies: string[]; + auths: Array; +}): void { + globalThis.fetch = (async (input, init) => { + capture.urls.push(String(input)); + capture.bodies.push(typeof init?.body === "string" ? init.body : ""); + const headers = new Headers(init?.headers); + capture.auths.push(headers.get("authorization")); + return Response.json({ + id: "resp_test", + object: "response", + status: "completed", + model: "gpt-5.6-sol", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; +} + +async function postSpawn( + config: OcxConfig, + body: Record, + options: Parameters[3] = {}, + logCtx: RequestLogContext = { model: "", provider: "" }, +): Promise { + return handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: spawnHeaders(), + body: JSON.stringify(body), + }), + config, + logCtx, + options, + ); +} + +describe("subagent fallback probe lease release", () => { + test("releases abandoned probe lease when fallback leaves the cooled pool account", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential(now); + const cfg = poolNativePlusRoutedConfig({ + subagentModelFallback: ["xai/grok-4.5"], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + Date.now = () => probeAt; + + const authPublications: Array = []; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: false }, + { onCodexAuthContextResolved: (ctx) => authPublications.push(ctx) }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + // Early pool auth published a probe-bearing context; final routed auth is undefined. + expect(authPublications.some((ctx) => ctx && "probeLeaseId" in ctx && ctx.probeLeaseId)).toBe(true); + expect(authPublications.at(-1)).toBeUndefined(); + }); + + test("same-provider model fallback reuses early probe auth without false cooldown", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential(now); + const cfg = poolNativePlusRoutedConfig({ + // Stay on the openai forward provider; only the model changes. + subagentModelFallback: ["gpt-5.5"], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + Date.now = () => probeAt; + + let finalAuth: CodexAuthContext | undefined; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: false }, + { onCodexAuthContextResolved: (ctx) => { finalAuth = ctx; } }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(finalAuth).toMatchObject({ kind: "pool", accountId: "pool-a" }); + // Early probe auth was reused (no false cooldown on same-account re-resolve). + expect((finalAuth as { probeLeaseId?: string }).probeLeaseId).toBeTruthy(); + // Terminal handling may clear the live health lease after the successful probe; + // the important contract is that the request completed with the probe-bearing context. + expect(response.status).not.toBe(429); + }); + + test("reroute auth failure releases the abandoned original probe lease", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential(now); + const cfg: OcxConfig = { + port: 0, + defaultProvider: "openai", + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + subagentModelFallback: ["openai-direct/gpt-5.5"], + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool@example.test", isMain: false, chatgptAccountId: "pool_acc" }, + ], + }; + updateAccountQuota("pool-a", 95, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + Date.now = () => now + CODEX_QUOTA_PROBE_INTERVAL_MS; + + // Omit authorization so direct-mode final auth fails after releasing pool probe. + const response = await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + "x-openai-subagent": "collab_spawn", + }, + body: JSON.stringify({ + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }), + }), + cfg, + { model: "", provider: "" }, + ); + + expect(response.status).toBe(401); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + }); +}); + +describe("subagent fallback final-route normalization", () => { + test("falls back to gpt-5.6-sol-pro and rewrites wire model + reasoning.mode", async () => { + const cfg = poolNativePlusRoutedConfig({ + activeCodexAccountId: undefined, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + "openai-apikey": { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + }, + subagentModelFallback: ["openai-apikey/gpt-5.6-sol-pro"], + fastMode: true, + }); + // Exhaust native primary via health block so fallback is chosen without pool quotas. + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await postSpawn( + cfg, + { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "high" }, + service_tier: "default", + }, + {}, + logCtx, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.openai.com"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { + model?: string; + reasoning?: { effort?: string; mode?: string }; + service_tier?: string; + }; + expect(body.model).toBe("gpt-5.6-sol"); + expect(body.reasoning?.mode).toBe("pro"); + expect(body.service_tier).toBe("priority"); + expect(logCtx.provider).toContain("openai-apikey"); + expect(logCtx.model).toBe("gpt-5.6-sol-pro"); + expect(logCtx.resolvedModel).toBe("gpt-5.6-sol"); + expect(logCtx.providerAdapter).toBe("openai-responses"); + }); + + test("routed primary falls back to native and preserves encrypted task passthrough", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(capture.bodies[0]).toContain(FERNET_TASK); + }); + + test("native primary falls back to routed for readable child tasks", async () => { + const cfg = poolNativePlusRoutedConfig({ + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "rate limit exceeded", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + }); +}); + +describe("encrypted child native-only fallback", () => { + test("rejects encrypted routed primary when only routed fallbacks exist", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["xai/grok-3"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + }); + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + throw new Error("must not dispatch"); + }) as typeof fetch; + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + const json = await response.json() as { error?: { code?: string } }; + expect(response.status).toBe(400); + expect(json.error?.code).toBe("unreadable_encrypted_agent_task"); + expect(fetchCalls).toBe(0); + }); + + test("skips exhausted native candidates before rejecting encrypted routed primary", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5", "xai/grok-3"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.5", "429", cfg); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }); + expect(response.status).toBe(400); + }); + + test("non-thread-spawn encrypted routed requests stay rejected without fallback", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5"], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const response = await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer caller-codex-token", + }, + body: JSON.stringify({ + model: "xai/grok-4.5", + input: encryptedAgentInput(), + stream: false, + }), + }), + cfg, + { model: "", provider: "" }, + ); + expect(response.status).toBe(400); + }); +}); diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index c329acac18..10af64a856 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { applySubagentModelFallback, buildSubagentModelChain, + getSubagentQuotaPrimeStateForTests, isSubagentModelUnavailable, maybePrimeSubagentQuota, noteSubagentModelFailure, @@ -167,6 +168,72 @@ describe("subagent model fallback chain", () => { expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe( "alibaba-token-plan/qwen3.8-max-preview", ); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBeGreaterThan(0); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(false); + }); + + test("concurrent maybePrimeSubagentQuota callers share one in-flight refresh", async () => { + resetSubagentModelFallbackStateForTests(); + let calls = 0; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + await gate; + updateAccountQuota("main", 95, undefined, 20); + }); + + const a = maybePrimeSubagentQuota(cfg()); + const b = maybePrimeSubagentQuota(cfg()); + const c = maybePrimeSubagentQuota(cfg()); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(true); + release(); + await Promise.all([a, b, c]); + expect(calls).toBe(1); + expect(selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model).toBe( + "alibaba-token-plan/qwen3.8-max-preview", + ); + }); + + test("failed quota prime does not mark success TTL and allows retry", async () => { + resetSubagentModelFallbackStateForTests(); + let calls = 0; + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + throw new Error("prime failed"); + }); + await maybePrimeSubagentQuota(cfg()); + expect(calls).toBe(1); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBe(0); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(false); + + setSubagentQuotaPrimeForTests(async () => { + calls += 1; + updateAccountQuota("main", 95, undefined, 20); + }); + await maybePrimeSubagentQuota(cfg()); + expect(calls).toBe(2); + expect(getSubagentQuotaPrimeStateForTests().primedAt).toBeGreaterThan(0); + }); + + test("reset clears timestamp, in-flight, health, and native slug cache", async () => { + resetSubagentModelFallbackStateForTests(); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + setSubagentQuotaPrimeForTests(async () => { + await gate; + throw new Error("cancelled after reset"); + }); + const priming = maybePrimeSubagentQuota(cfg()); + noteSubagentModelFailure("kimi/k3", "429", cfg()); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(true); + expect(getSubagentQuotaPrimeStateForTests().inFlight).toBe(true); + resetSubagentModelFallbackStateForTests(); + expect(getSubagentQuotaPrimeStateForTests()).toEqual({ primedAt: 0, inFlight: false }); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); + release(); + await priming; + expect(getSubagentQuotaPrimeStateForTests()).toEqual({ primedAt: 0, inFlight: false }); }); test("noteSubagentModelFailure records the configured fallback slug", () => { From b251999547331c580e101d90d15dfde5ac4675d3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:37:19 +0200 Subject: [PATCH 09/12] test(subagents): use documented native slugs in encrypted fallback coverage Avoid gpt-5.5 in native-only encrypted spawn tests: with an isolated CODEX_HOME, nativeOpenAiSlugs() can resolve to DOCUMENTED_NATIVE_OPENAI_ADDITIONS only, which omits gpt-5.5 and made CI reject before native fallback. --- tests/subagent-fallback-handle-responses.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/subagent-fallback-handle-responses.test.ts b/tests/subagent-fallback-handle-responses.test.ts index 6313bbbed1..b5a72f979b 100644 --- a/tests/subagent-fallback-handle-responses.test.ts +++ b/tests/subagent-fallback-handle-responses.test.ts @@ -364,9 +364,12 @@ describe("subagent fallback final-route normalization", () => { }); test("routed primary falls back to native and preserves encrypted task passthrough", async () => { + resetSubagentModelFallbackStateForTests(); const cfg = poolNativePlusRoutedConfig({ defaultProvider: "xai", - subagentModelFallback: ["gpt-5.5"], + // Prefer a documented native slug that other tests in this file do not health-block. + subagentModelFallback: ["gpt-5.6-terra"], + activeCodexAccountId: undefined, providers: { xai: { adapter: "openai-chat", @@ -392,7 +395,10 @@ describe("subagent fallback final-route normalization", () => { stream: false, }); - expect(response.status).toBe(200); + if (response.status !== 200) { + const body = await response.text(); + throw new Error(`expected 200, got ${response.status}: ${body}`); + } expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); expect(capture.bodies[0]).toContain(FERNET_TASK); }); @@ -464,9 +470,11 @@ describe("encrypted child native-only fallback", () => { }); test("skips exhausted native candidates before rejecting encrypted routed primary", async () => { + resetSubagentModelFallbackStateForTests(); const cfg = poolNativePlusRoutedConfig({ defaultProvider: "xai", - subagentModelFallback: ["gpt-5.5", "xai/grok-3"], + subagentModelFallback: ["gpt-5.6-terra", "xai/grok-3"], + activeCodexAccountId: undefined, providers: { xai: { adapter: "openai-chat", @@ -483,7 +491,7 @@ describe("encrypted child native-only fallback", () => { }, }); const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); - noteSubagentModelFailure("gpt-5.5", "429", cfg); + noteSubagentModelFailure("gpt-5.6-terra", "429", cfg); const response = await postSpawn(cfg, { model: "xai/grok-4.5", From 19a1d5557bdd93fd1902253691c2ed714659f636 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:16:24 +0200 Subject: [PATCH 10/12] fix(subagents): preview account before fallback and finalize terminals Avoid leasing primary Codex auth before routed fallback selection, clamp effort from the final native route, and always invoke passthrough terminal callbacks for non-quota stream failures. --- src/codex/routing.ts | 90 +++- src/codex/subagent-model-fallback.ts | 61 ++- src/server/responses/core.ts | 97 ++-- ...subagent-fallback-handle-responses.test.ts | 462 ++++++++++++++++-- 4 files changed, 616 insertions(+), 94 deletions(-) diff --git a/src/codex/routing.ts b/src/codex/routing.ts index bfa440d7c9..c2ab45a2ea 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -235,14 +235,8 @@ export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): * Returns the lease id, or null when no probe may go out right now. */ export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): string | null { - const health = upstreamHealth.get(accountId); - if (!health) return null; - const cooldownUntil = health.cooldownUntil; - if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null; - if (health.cooldownSource === "retry-after") return null; - if (health.probeLeaseId !== undefined) return null; - const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil; - if (now - origin < CODEX_QUOTA_PROBE_INTERVAL_MS) return null; + if (!canAcquireCodexQuotaProbeLease(accountId, now)) return null; + const health = upstreamHealth.get(accountId)!; const probeLeaseId = randomUUID(); upstreamHealth.set(accountId, { ...health, @@ -253,6 +247,18 @@ export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now return probeLeaseId; } +/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */ +export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean { + const health = upstreamHealth.get(accountId); + if (!health) return false; + const cooldownUntil = health.cooldownUntil; + if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false; + if (health.cooldownSource === "retry-after") return false; + if (health.probeLeaseId !== undefined) return false; + const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil; + return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS; +} + /** * Hand a probe lease back without recording an upstream outcome. Used by paths * that take a lease and then fail before any request reaches upstream. @@ -499,6 +505,74 @@ export function resolveCodexAccountForThread( return resolution.status === "selected" ? resolution.accountId : null; } +/** + * Side-effect-free preview of the Codex pool account native routing would prefer. + * Used for subagent fallback quota decisions before final auth. + * + * Does not mutate activeCodexAccountId, thread affinity, config on disk, or probe leases. + * Mirrors {@link resolveCodexAccountForThreadDetailed} account choice, including returning a + * configured cooled account so callers can evaluate probe/quota availability. + */ +export function previewCodexAccountForRequest( + threadId: string | null, + config: OcxConfig, + now = Date.now(), +): string | null { + if (threadId && threadAccountMap.has(threadId)) { + const entry = threadAccountMap.get(threadId)!; + if ( + !isThreadAffinityExpired(entry, now) + && isThreadAffinityGenerationLive(entry) + && isCodexAccountSelectable(config, entry.accountId, now) + && !shouldFailover(config, entry.accountId, now) + ) { + const threshold = config.autoSwitchThreshold ?? 80; + if (threshold > 0) { + const usage = computeCodexUsageScore( + getAccountQuota(entry.accountId), + getPoolAccountPlan(config, entry.accountId), + ); + if (usage >= threshold) { + const best = pickLowerUsageAccount(config, entry.accountId, usage, now); + if (best !== entry.accountId) return best; + } + } + return entry.accountId; + } + // Stale/unusable affinity is ignored for preview (no map mutation). + } + + let active = config.activeCodexAccountId ?? null; + if (!active) { + return pickLowestUsageCodexAccount(config, undefined, now); + } + if (!isCodexAccountSelectable(config, active, now)) { + const fallback = pickLowestUsageCodexAccount(config, active, now); + if (fallback) active = fallback; + else if (hasConfiguredPoolAccount(config, active)) return active; + else return null; + } + + const threshold = config.autoSwitchThreshold ?? 80; + if (threshold > 0) { + const usage = computeCodexUsageScore(getAccountQuota(active), getPoolAccountPlan(config, active)); + if (usage >= threshold) { + active = pickLowerUsageAccount(config, active, usage, now); + } + } + if (shouldFailover(config, active, now)) { + const best = pickLowestUsageCodexAccount(config, active, now); + if (best) active = best; + } + if (!isCodexAccountUsable(config, active)) { + return hasConfiguredPoolAccount(config, active) ? active : null; + } + if (isCodexAccountInCooldown(active, now)) { + return hasConfiguredPoolAccount(config, active) ? active : null; + } + return active; +} + export function resolveCodexAccountForThreadDetailed( threadId: string | null, config: OcxConfig, diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 0af54e0a70..cd02462b68 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -13,13 +13,26 @@ import type { OcxParsedRequest, OcxConfig } from "../types"; import { slugsEquivalent } from "../providers/slug-codec"; import { CODEX_HOME, getCodexHome } from "./paths"; import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; -import { computeCodexUsageScore, getPoolAccountPlan } from "./routing"; +import { + canAcquireCodexQuotaProbeLease, + computeCodexUsageScore, + getPoolAccountPlan, + isCodexAccountInCooldown, +} from "./routing"; import { nativeOpenAiSlugs } from "./catalog"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; +export type SubagentAvailabilityOptions = { + /** + * When true, native candidates require an explicit usable account id. + * Used for Codex pool configs where a null preview means no non-cooled account. + */ + requireNativeAccount?: boolean; +}; + type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; let quotaPrimeInFlight: Promise | null = null; @@ -106,8 +119,15 @@ function activeCodexAccountId(config: OcxConfig): string | null { return config.activeCodexAccountId ?? null; } -function resolveFallbackAccountId(config: OcxConfig, accountId?: string | null): string | null { - return accountId ?? activeCodexAccountId(config); +function resolveFallbackAccountId( + config: OcxConfig, + accountId?: string | null, + options: SubagentAvailabilityOptions = {}, +): string | null { + if (typeof accountId === "string") return accountId; + // With requireNativeAccount, null means the pool preview found no usable account. + if (accountId === null && options.requireNativeAccount) return null; + return activeCodexAccountId(config); } function isRoutableFallbackModel(model: string, config: OcxConfig): boolean { @@ -130,9 +150,10 @@ export function isNativeModelQuotaExhausted( config: OcxConfig, accountId?: string | null, now = Date.now(), + options: SubagentAvailabilityOptions = {}, ): boolean { if (!isNativeOpenAiSlug(model)) return false; - const resolvedAccountId = resolveFallbackAccountId(config, accountId); + const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); if (!resolvedAccountId) return false; const quota = getAccountQuota(resolvedAccountId); const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, resolvedAccountId)); @@ -145,8 +166,9 @@ export function isModelHealthBlocked( config: OcxConfig, accountId?: string | null, now = Date.now(), + options: SubagentAvailabilityOptions = {}, ): boolean { - const health = modelHealth.get(healthKey(model, resolveFallbackAccountId(config, accountId))); + const health = modelHealth.get(healthKey(model, resolveFallbackAccountId(config, accountId, options))); return !!health && health.unavailableUntil > now; } @@ -155,11 +177,23 @@ export function isSubagentModelUnavailable( config: OcxConfig, accountId?: string | null, now = Date.now(), + options: SubagentAvailabilityOptions = {}, ): boolean { if (isDisabledFallbackModel(model, config)) return true; if (!isRoutableFallbackModel(model, config)) return true; - if (isModelHealthBlocked(model, config, accountId, now)) return true; - if (isNativeOpenAiSlug(model)) return isNativeModelQuotaExhausted(model, config, accountId, now); + if (isModelHealthBlocked(model, config, accountId, now, options)) return true; + if (isNativeOpenAiSlug(model)) { + const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); + if (options.requireNativeAccount && !resolvedAccountId) return true; + if ( + resolvedAccountId + && isCodexAccountInCooldown(resolvedAccountId, now) + && !canAcquireCodexQuotaProbeLease(resolvedAccountId, now) + ) { + return true; + } + return isNativeModelQuotaExhausted(model, config, accountId, now, options); + } return false; } @@ -170,6 +204,7 @@ export function selectAvailableSubagentModel( accountId?: string | null, now = Date.now(), nativeFallbackOnly = false, + options: SubagentAvailabilityOptions = {}, ): { model: string; rewritten: boolean; skipped: string[] } { const chain = normalizedChain(primary, config, extraFallback); const skipped: string[] = []; @@ -178,7 +213,7 @@ export function selectAvailableSubagentModel( skipped.push(candidate); continue; } - if (isSubagentModelUnavailable(candidate, config, accountId, now)) { + if (isSubagentModelUnavailable(candidate, config, accountId, now, options)) { skipped.push(candidate); continue; } @@ -203,6 +238,14 @@ export function noteSubagentModelFailure( }); } +export function configUsesCodexAccountPool(config: OcxConfig): boolean { + return Object.values(config.providers).some( + provider => provider != null + && typeof provider === "object" + && (provider as { codexAccountMode?: string }).codexAccountMode === "pool", + ); +} + export function resetSubagentModelFallbackStateForTests(): void { modelHealth.clear(); quotaPrimedAt.clear(); @@ -336,6 +379,7 @@ export function applySubagentModelFallback( accountId?: string | null, now = Date.now(), nativeFallbackOnly = false, + options: SubagentAvailabilityOptions = {}, ): { from?: string; to?: string; skipped?: string[] } | null { if (!isThreadSpawnRequest(headers)) return null; const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); @@ -348,6 +392,7 @@ export function applySubagentModelFallback( accountId, now, nativeFallbackOnly, + options, ); if (!selection.rewritten) return selection.skipped.length > 0 ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 0dcc5b6e7f..26fb764fd9 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -56,6 +56,7 @@ import { } from "../../codex/auth-context"; import { formatCodexProviderForLog, + previewCodexAccountForRequest, recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; @@ -79,6 +80,7 @@ import { supportedLadderFor } from "../effort-policy"; import { isThreadSpawnRequest } from "../effort-policy"; import { applySubagentModelFallback, + configUsesCodexAccountPool, maybePrimeSubagentQuota, recordSubagentQuotaFailureForThreadSpawn, } from "../../codex/subagent-model-fallback"; @@ -514,6 +516,9 @@ async function applyFinalRouteRequestNormalization(args: { logCtx.provider = route.providerName; logCtx.providerAdapter = route.provider.adapter; + // Final selected model before virtual wire-model rewriting (Pro aliases). + const finalSelectedModelId = route.modelId; + // Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro". applyOpenAiVirtualModel(parsed, route, logCtx); @@ -563,9 +568,8 @@ async function applyFinalRouteRequestNormalization(args: { } { - const requestedModelId = logCtx.requestedModel ?? route.modelId; const { nativeEffortClamp, shouldApplyNativeEffortClamp } = await import("../../codex/catalog"); - const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, requestedModelId) + const clamped = shouldApplyNativeEffortClamp(route.providerName, route.provider, finalSelectedModelId) ? nativeEffortClamp(route.modelId, parsed.options.reasoning) : null; if (clamped) { @@ -889,31 +893,25 @@ export async function handleResponses( let authCtx: CodexAuthContext = { kind: "main", accountId: null }; let selectedForwardHeaders = req.headers; - let authReady = false; let subagentFallbackAccountId = config.activeCodexAccountId ?? null; let subagentQuotaFailureModel = parsed.modelId; // Subagent fallback must settle the final model/provider BEFORE route-dependent // normalization (virtual models, effort caps, service tier, wire protocol). - // Early auth is only for account-scoped quota selection; its probe lease is - // released whenever that context is abandoned before upstream. + // Preview the preferred Codex account without acquiring a probe lease or refreshing + // tokens — auth is resolved only after the final route is selected. if (isThreadSpawnRequest(req.headers) && !options.comboAttempt) { - const earlyAuth = await resolveResponsesCodexAuth(req, config, route, options); - if (!earlyAuth.ok) return earlyAuth.response; - authCtx = earlyAuth.authCtx; - selectedForwardHeaders = earlyAuth.headers; - authReady = true; - subagentFallbackAccountId = authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? authCtx.accountId - : config.activeCodexAccountId ?? null; - + const threadId = req.headers.get("x-codex-parent-thread-id"); + const previewAccountId = previewCodexAccountForRequest(threadId, config); + subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; const fallback = applySubagentModelFallback( parsed, req.headers, config, - subagentFallbackAccountId, + previewAccountId, Date.now(), unreadableEncryptedAgentTask, + { requireNativeAccount: configUsesCodexAccountPool(config) }, ); if (fallback) { (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; @@ -925,41 +923,26 @@ export async function handleResponses( subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { - let nextRoute: RouteResult; try { - nextRoute = routeModel(config, fallback.to); + route = routeModel(config, fallback.to); } catch (err) { - releaseCodexAuthContextProbeLease(authCtx); - authReady = false; if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); } return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } - const authReusable = route.providerName === nextRoute.providerName - && route.codexAccountMode === nextRoute.codexAccountMode; - if (!authReusable) { - // Abandon the pre-fallback auth context (and any probe lease) before - // resolving replacement auth — nothing has reached upstream yet. - releaseCodexAuthContextProbeLease(authCtx); - authCtx = { kind: "main", accountId: null }; - authReady = false; - selectedForwardHeaders = req.headers; - } - route = nextRoute; } } // Encrypted child tasks may only reach the canonical native backend. This check // runs against the FINAL route so native-only fallback can rescue a routed primary. if (!isCanonicalOpenAiForwardProvider(route.provider) && unreadableEncryptedAgentTask) { - if (authReady) releaseCodexAuthContextProbeLease(authCtx); return unreadableEncryptedAgentTaskResponse(); } await applyFinalRouteRequestNormalization({ parsed, route, config, req, logCtx }); - if (!authReady) { + { const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); if (!finalAuth.ok) return finalAuth.response; authCtx = finalAuth.authCtx; @@ -1282,14 +1265,15 @@ export async function handleResponses( || logCtx.terminalHttpStatus === 402 ? (httpStatusOverride ?? logCtx.terminalHttpStatus) : undefined; - if (quotaFailureMessage === undefined) return; - recordSubagentQuotaFailureForThreadSpawn( - req.headers, - subagentQuotaFailureModel, - quotaFailureMessage, - config, - subagentFallbackAccountId, - ); + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } } options.onNativePassthroughTerminal?.(status); }); @@ -1334,14 +1318,15 @@ export async function handleResponses( || logCtx.terminalHttpStatus === 402 ? (httpStatusOverride ?? logCtx.terminalHttpStatus) : undefined; - if (quotaFailureMessage === undefined) return; - recordSubagentQuotaFailureForThreadSpawn( - req.headers, - subagentQuotaFailureModel, - quotaFailureMessage, - config, - subagentFallbackAccountId, - ); + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } } options.onNativePassthroughTerminal?.(status); } @@ -1387,6 +1372,22 @@ export async function handleResponses( // client-cancel (no terminal seen) is finalized separately via consumeForInspection's onCancel. const reportNativeTerminal = (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { terminalRecorder?.(status, httpStatusOverride); + if (status === "failed") { + const quotaFailureMessage = httpStatusOverride === 429 || httpStatusOverride === 402 + || logCtx.terminalHttpStatus === 429 + || logCtx.terminalHttpStatus === 402 + ? (httpStatusOverride ?? logCtx.terminalHttpStatus) + : undefined; + if (quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + subagentFallbackAccountId, + ); + } + } options.onNativePassthroughTerminal?.(status); }; consumeForInspection( diff --git a/tests/subagent-fallback-handle-responses.test.ts b/tests/subagent-fallback-handle-responses.test.ts index b5a72f979b..afaf12ccba 100644 --- a/tests/subagent-fallback-handle-responses.test.ts +++ b/tests/subagent-fallback-handle-responses.test.ts @@ -1,7 +1,8 @@ /** * handleResponses integration coverage for PR #391 merge blockers: - * probe-lease release on fallback reroute, final-route normalization, - * encrypted native-only fallback, concurrent quota priming (unit-covered separately). + * pre-fallback account preview (no probe lease), final-route normalization, + * native effort clamp on final route, pool account preview for native fallback, + * encrypted native-only fallback, native passthrough terminal finalization. */ import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; import { mkdtempSync, rmSync } from "node:fs"; @@ -17,15 +18,19 @@ import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth, + previewCodexAccountForRequest, recordCodexUpstreamOutcome, + resolveCodexAccountForThreadDetailed, } from "../src/codex/routing"; import { + isModelHealthBlocked, resetSubagentModelFallbackStateForTests, } from "../src/codex/subagent-model-fallback"; import type { CodexAuthContext } from "../src/codex/auth-context"; import { handleResponses } from "../src/server/responses"; import type { OcxConfig } from "../src/types"; import type { RequestLogContext } from "../src/server/request-log"; +import type { ResponsesTerminalStatus } from "../src/bridge"; setDefaultTimeout(30_000); @@ -132,12 +137,12 @@ function poolNativePlusRoutedConfig(overrides: Partial = {}): OcxConf } as OcxConfig; } -function installPoolCredential(now: number): void { - saveCodexAccountCredential("pool-a", { - accessToken: "pool_token", - refreshToken: "pool_refresh", +function installPoolCredential(accountId: string, chatgptAccountId: string, now: number): void { + saveCodexAccountCredential(accountId, { + accessToken: `${accountId}_token`, + refreshToken: `${accountId}_refresh`, expiresAt: now + 24 * 60 * 60_000, - chatgptAccountId: "pool_acc", + chatgptAccountId, }); } @@ -162,16 +167,27 @@ function mockUpstream(capture: { }) as typeof fetch; } +function mockSseUpstream(sseBody: string, capture?: { urls: string[] }): void { + globalThis.fetch = (async (input) => { + capture?.urls.push(String(input)); + return new Response(sseBody, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }) as typeof fetch; +} + async function postSpawn( config: OcxConfig, body: Record, options: Parameters[3] = {}, logCtx: RequestLogContext = { model: "", provider: "" }, + headers: HeadersInit = {}, ): Promise { return handleResponses( new Request("http://localhost/v1/responses", { method: "POST", - headers: spawnHeaders(), + headers: spawnHeaders(headers), body: JSON.stringify(body), }), config, @@ -180,11 +196,11 @@ async function postSpawn( ); } -describe("subagent fallback probe lease release", () => { - test("releases abandoned probe lease when fallback leaves the cooled pool account", async () => { +describe("subagent fallback without primary auth cooldown failure", () => { + test("cooled primary with no probe lease selects healthy routed fallback", async () => { const now = 1_800_000_000_000; Date.now = () => now; - installPoolCredential(now); + installPoolCredential("pool-a", "pool_acc", now); const cfg = poolNativePlusRoutedConfig({ subagentModelFallback: ["xai/grok-4.5"], }); @@ -192,9 +208,6 @@ describe("subagent fallback probe lease release", () => { const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); - const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; - Date.now = () => probeAt; - const authPublications: Array = []; const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; mockUpstream(capture); @@ -207,51 +220,86 @@ describe("subagent fallback probe lease release", () => { expect(response.status).toBe(200); expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + expect(response.status).not.toBe(429); expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); - // Early pool auth published a probe-bearing context; final routed auth is undefined. - expect(authPublications.some((ctx) => ctx && "probeLeaseId" in ctx && ctx.probeLeaseId)).toBe(true); - expect(authPublications.at(-1)).toBeUndefined(); + // No auth is resolved before final route; routed final publishes undefined. + expect(authPublications).toEqual([undefined]); }); - test("same-provider model fallback reuses early probe auth without false cooldown", async () => { + test("cooled primary with no usable fallback still returns cooldown 429", async () => { const now = 1_800_000_000_000; Date.now = () => now; - installPoolCredential(now); + installPoolCredential("pool-a", "pool_acc", now); const cfg = poolNativePlusRoutedConfig({ - // Stay on the openai forward provider; only the model changes. subagentModelFallback: ["gpt-5.5"], }); updateAccountQuota("pool-a", 95, undefined, 20); const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + throw new Error("must not dispatch"); + }) as typeof fetch; + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(429); + expect(fetchCalls).toBe(0); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + }); + + test("same-provider native fallback at probe window authenticates only for final route", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + subagentModelFallback: ["gpt-5.5"], + }); + // Health-block the primary so fallback selects another native model; keep + // account below auto-switch threshold so the probe path is exercised. + updateAccountQuota("pool-a", 20, undefined, 20); + const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); + recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg, "pool-a"); + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; Date.now = () => probeAt; let finalAuth: CodexAuthContext | undefined; + const authPublications: Array = []; const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; mockUpstream(capture); const response = await postSpawn( cfg, { model: "gpt-5.6-sol", input: readableAgentInput(), stream: false }, - { onCodexAuthContextResolved: (ctx) => { finalAuth = ctx; } }, + { + onCodexAuthContextResolved: (ctx) => { + authPublications.push(ctx); + finalAuth = ctx; + }, + }, ); expect(response.status).toBe(200); expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); expect(finalAuth).toMatchObject({ kind: "pool", accountId: "pool-a" }); - // Early probe auth was reused (no false cooldown on same-account re-resolve). expect((finalAuth as { probeLeaseId?: string }).probeLeaseId).toBeTruthy(); - // Terminal handling may clear the live health lease after the successful probe; - // the important contract is that the request completed with the probe-bearing context. + expect(authPublications).toHaveLength(1); expect(response.status).not.toBe(429); }); - test("reroute auth failure releases the abandoned original probe lease", async () => { + test("final-route auth failure does not leave a primary probe lease", async () => { const now = 1_800_000_000_000; Date.now = () => now; - installPoolCredential(now); + installPoolCredential("pool-a", "pool_acc", now); const cfg: OcxConfig = { port: 0, defaultProvider: "openai", @@ -280,9 +328,8 @@ describe("subagent fallback probe lease release", () => { updateAccountQuota("pool-a", 95, undefined, 20); const resetAt = Math.floor((now + 4 * 24 * 60 * 60_000) / 1000); recordCodexUpstreamOutcome(cfg, "pool-a", 429, { resetAt, now }); - Date.now = () => now + CODEX_QUOTA_PROBE_INTERVAL_MS; - // Omit authorization so direct-mode final auth fails after releasing pool probe. + // Omit authorization so direct-mode final auth fails — primary never leased. const response = await handleResponses( new Request("http://localhost/v1/responses", { method: "POST", @@ -326,7 +373,6 @@ describe("subagent fallback final-route normalization", () => { subagentModelFallback: ["openai-apikey/gpt-5.6-sol-pro"], fastMode: true, }); - // Exhaust native primary via health block so fallback is chosen without pool quotas. const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); noteSubagentModelFailure("gpt-5.6-sol", "429", cfg); @@ -363,11 +409,137 @@ describe("subagent fallback final-route normalization", () => { expect(logCtx.providerAdapter).toBe("openai-responses"); }); + test("routed primary falling back to native gpt-5.5 clamps max effort to xhigh", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.5"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + const logCtx: RequestLogContext = { model: "", provider: "", requestedModel: "xai/grok-4.5" }; + + const response = await postSpawn( + cfg, + { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }, + {}, + logCtx, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { + model?: string; + reasoning?: { effort?: string }; + }; + expect(body.model).toBe("gpt-5.5"); + expect(body.reasoning?.effort).toBe("xhigh"); + }); + + test("routed primary falling back to native gpt-5.6 keeps real max effort", async () => { + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + subagentModelFallback: ["gpt-5.6-terra"], + activeCodexAccountId: undefined, + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + }, + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }); + + expect(response.status).toBe(200); + const body = JSON.parse(capture.bodies[0]!) as { reasoning?: { effort?: string } }; + expect(body.reasoning?.effort).toBe("max"); + }); + + test("native primary falling back to routed does not receive a native clamp", async () => { + const cfg = poolNativePlusRoutedConfig({ + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("gpt-5.6-sol", "rate limit exceeded", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "gpt-5.6-sol", + input: readableAgentInput(), + stream: false, + reasoning: { effort: "max" }, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + const body = JSON.parse(capture.bodies[0]!) as { reasoning?: { effort?: string } }; + // Routed adapters own effort mapping; the native clamp must not rewrite to xhigh. + expect(body.reasoning?.effort).not.toBe("xhigh"); + }); + test("routed primary falls back to native and preserves encrypted task passthrough", async () => { resetSubagentModelFallbackStateForTests(); const cfg = poolNativePlusRoutedConfig({ defaultProvider: "xai", - // Prefer a documented native slug that other tests in this file do not health-block. subagentModelFallback: ["gpt-5.6-terra"], activeCodexAccountId: undefined, providers: { @@ -438,6 +610,142 @@ describe("subagent fallback final-route normalization", () => { }); }); +describe("native fallback account preview", () => { + test("uses healthier pool account B when active A is above threshold", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + activeCodexAccountId: "pool-a", + subagentModelFallback: ["gpt-5.6-terra"], + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 10, undefined, 20); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const activeBefore = cfg.activeCodexAccountId; + expect(previewCodexAccountForRequest(null, cfg, now)).toBe("pool-b"); + expect(cfg.activeCodexAccountId).toBe(activeBefore); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")?.probeLeaseId).toBeUndefined(); + + let finalAuth: CodexAuthContext | undefined; + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn( + cfg, + { model: "xai/grok-4.5", input: readableAgentInput(), stream: false }, + { onCodexAuthContextResolved: (ctx) => { finalAuth = ctx; } }, + ); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("chatgpt.com/backend-api/codex"))).toBe(true); + expect(finalAuth).toMatchObject({ kind: "pool", accountId: "pool-b" }); + expect(capture.auths.some((auth) => auth?.includes("pool-b_token"))).toBe(true); + }); + + test("skips native fallback when every pool account is exhausted", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + defaultProvider: "xai", + activeCodexAccountId: "pool-a", + subagentModelFallback: ["gpt-5.6-terra", "xai/grok-3"], + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + providers: { + xai: { + adapter: "openai-chat", + baseUrl: "https://api.x.ai/v1", + authMode: "key", + apiKey: "xai-test", + }, + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 90, undefined, 20); + const { noteSubagentModelFailure } = await import("../src/codex/subagent-model-fallback"); + noteSubagentModelFailure("xai/grok-4.5", "429", cfg); + noteSubagentModelFailure("grok-4.5", "429", cfg); + + const capture = { urls: [] as string[], bodies: [] as string[], auths: [] as Array }; + mockUpstream(capture); + + const response = await postSpawn(cfg, { + model: "xai/grok-4.5", + input: readableAgentInput(), + stream: false, + }); + + expect(response.status).toBe(200); + expect(capture.urls.some((url) => url.includes("api.x.ai"))).toBe(true); + expect(capture.urls.some((url) => url.includes("chatgpt.com"))).toBe(false); + }); + + test("preview selection does not mutate affinity or acquire probe leases", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + installPoolCredential("pool-b", "pool_acc_b", now); + const cfg = poolNativePlusRoutedConfig({ + activeCodexAccountId: "pool-a", + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_acc_a" }, + { id: "pool-b", email: "b@example.test", isMain: false, chatgptAccountId: "pool_acc_b" }, + ], + }); + updateAccountQuota("pool-a", 95, undefined, 20); + updateAccountQuota("pool-b", 10, undefined, 20); + + // Bind affinity to pool-a via normal resolution once. + const bound = resolveCodexAccountForThreadDetailed("thread-1", cfg, now); + expect(bound).toMatchObject({ status: "selected", accountId: "pool-b" }); + const activeAfterBind = cfg.activeCodexAccountId; + + const previewed = previewCodexAccountForRequest("thread-1", cfg, now); + expect(previewed).toBe("pool-b"); + expect(cfg.activeCodexAccountId).toBe(activeAfterBind); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")?.probeLeaseId).toBeUndefined(); + }); +}); + describe("encrypted child native-only fallback", () => { test("rejects encrypted routed primary when only routed fallbacks exist", async () => { const cfg = poolNativePlusRoutedConfig({ @@ -539,3 +847,97 @@ describe("encrypted child native-only fallback", () => { expect(response.status).toBe(400); }); }); + +describe("native passthrough terminal finalization", () => { + function failedSse(message: string, type = "rate_limit_error"): string { + return `event: response.failed\ndata: ${JSON.stringify({ + type: "response.failed", + response: { + status: "failed", + error: { type, message }, + }, + })}\n\n`; + } + + async function runStreamingSpawn( + streamMode: "legacy-tee" | "eager-relay", + sseBody: string, + ): Promise<{ + terminals: ResponsesTerminalStatus[]; + healthBlocked: boolean; + responseText: string; + }> { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc", now); + const cfg = poolNativePlusRoutedConfig({ + streamMode, + activeCodexAccountId: "pool-a", + subagentModelFallback: ["xai/grok-4.5"], + }); + updateAccountQuota("pool-a", 20, undefined, 20); + + const terminals: ResponsesTerminalStatus[] = []; + mockSseUpstream(sseBody); + + const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); + // Force win32 so eager-relay decision path is reachable via streamMode override. + Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + try { + const response = await postSpawn( + cfg, + { model: "gpt-5.6-sol", input: readableAgentInput(), stream: true }, + { + onNativePassthroughTerminal: (status) => terminals.push(status), + }, + ); + const responseText = await response.text(); + // Allow inspection consumer microtasks to settle. + await Bun.sleep(20); + return { + terminals, + healthBlocked: isModelHealthBlocked("gpt-5.6-sol", cfg, "pool-a"), + responseText, + }; + } finally { + if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor); + } + } + + for (const streamMode of ["legacy-tee", "eager-relay"] as const) { + test(`${streamMode}: 429 failed records health and invokes terminal callback`, async () => { + const result = await runStreamingSpawn(streamMode, failedSse("rate limited", "rate_limit_error")); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(true); + expect(result.responseText).toContain("response.failed"); + }); + + test(`${streamMode}: 402-style insufficient_quota records health and invokes callback`, async () => { + const result = await runStreamingSpawn( + streamMode, + failedSse("insufficient quota", "insufficient_quota"), + ); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(true); + }); + + test(`${streamMode}: generic 500 failure invokes terminal callback without health block`, async () => { + const result = await runStreamingSpawn( + streamMode, + failedSse("internal server error", "server_error"), + ); + expect(result.terminals).toEqual(["failed"]); + expect(result.healthBlocked).toBe(false); + }); + + test(`${streamMode}: completed terminal fires exactly once`, async () => { + const sse = `event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { id: "r1", status: "completed", output: [] }, + })}\n\n`; + const result = await runStreamingSpawn(streamMode, sse); + expect(result.terminals).toEqual(["completed"]); + expect(result.healthBlocked).toBe(false); + }); + } +}); From dc3e500af76ece79343c51fc3861fff78277e68d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:34:17 +0200 Subject: [PATCH 11/12] fix(subagents): classify fallback nativeness from resolved routes Use routeModel and canonical/pool route metadata for encrypted eligibility and Codex quota checks instead of slug-only nativeSlugSet matching. --- src/codex/subagent-model-fallback.ts | 90 +++++++------ tests/subagent-model-fallback.test.ts | 186 +++++++++++++++++++++++++- 2 files changed, 235 insertions(+), 41 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index cd02462b68..4e1e09788c 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -19,15 +19,16 @@ import { getPoolAccountPlan, isCodexAccountInCooldown, } from "./routing"; -import { nativeOpenAiSlugs } from "./catalog"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; +import { routeModel, type RouteResult } from "../router"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; export type SubagentAvailabilityOptions = { /** - * When true, native candidates require an explicit usable account id. + * When true, pool-mode Codex candidates require an explicit usable account id. * Used for Codex pool configs where a null preview means no non-cooled account. */ requireNativeAccount?: boolean; @@ -44,20 +45,22 @@ type ModelHealth = { const modelHealth = new Map(); const quotaPrimedAt = new Map(); -const NATIVE_SLUG_CACHE_MS = 60_000; -let nativeSlugCache: { cachedAt: number; slugs: Set } | null = null; -function nativeSlugSet(now = Date.now()): Set { - if (nativeSlugCache && (now - nativeSlugCache.cachedAt) < NATIVE_SLUG_CACHE_MS) { - return nativeSlugCache.slugs; +const knownProviderIdSet = new Set(PROVIDER_REGISTRY.map(entry => entry.id.toLowerCase())); + +function tryRouteFallbackModel(config: OcxConfig, model: string): RouteResult | null { + try { + return routeModel(config, model); + } catch { + return null; } - const slugs = new Set(nativeOpenAiSlugs().map(slug => slug.toLowerCase())); - nativeSlugCache = { cachedAt: now, slugs }; - return slugs; } -const knownProviderIdSet = new Set(PROVIDER_REGISTRY.map(entry => entry.id.toLowerCase())); -function healthKey(model: string, accountId: string | null): string { - const scopedAccountId = nativeSlugSet().has(model.toLowerCase()) ? accountId : null; +function isPoolCodexRoute(route: RouteResult): boolean { + return route.codexAccountMode === "pool"; +} + +function healthKey(model: string, accountId: string | null, poolScoped: boolean): string { + const scopedAccountId = poolScoped ? accountId : null; return `${scopedAccountId ?? "none"}::${model.toLowerCase()}`; } @@ -106,10 +109,6 @@ export function buildSubagentModelChain( return normalizedChain(primary, config, extraFallback); } -function isNativeOpenAiSlug(model: string): boolean { - return nativeSlugSet().has(model.toLowerCase()); -} - function quotaThreshold(config: OcxConfig): number { const threshold = config.autoSwitchThreshold ?? 80; return threshold > 0 ? threshold : Number.POSITIVE_INFINITY; @@ -152,7 +151,8 @@ export function isNativeModelQuotaExhausted( now = Date.now(), options: SubagentAvailabilityOptions = {}, ): boolean { - if (!isNativeOpenAiSlug(model)) return false; + const route = tryRouteFallbackModel(config, model); + if (!route || !isPoolCodexRoute(route)) return false; const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); if (!resolvedAccountId) return false; const quota = getAccountQuota(resolvedAccountId); @@ -168,7 +168,11 @@ export function isModelHealthBlocked( now = Date.now(), options: SubagentAvailabilityOptions = {}, ): boolean { - const health = modelHealth.get(healthKey(model, resolveFallbackAccountId(config, accountId, options))); + const route = tryRouteFallbackModel(config, model); + const poolScoped = !!route && isPoolCodexRoute(route); + const health = modelHealth.get( + healthKey(model, resolveFallbackAccountId(config, accountId, options), poolScoped), + ); return !!health && health.unavailableUntil > now; } @@ -181,20 +185,21 @@ export function isSubagentModelUnavailable( ): boolean { if (isDisabledFallbackModel(model, config)) return true; if (!isRoutableFallbackModel(model, config)) return true; + const route = tryRouteFallbackModel(config, model); + if (!route || route.provider.disabled === true) return true; if (isModelHealthBlocked(model, config, accountId, now, options)) return true; - if (isNativeOpenAiSlug(model)) { - const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); - if (options.requireNativeAccount && !resolvedAccountId) return true; - if ( - resolvedAccountId - && isCodexAccountInCooldown(resolvedAccountId, now) - && !canAcquireCodexQuotaProbeLease(resolvedAccountId, now) - ) { - return true; - } - return isNativeModelQuotaExhausted(model, config, accountId, now, options); + if (!isPoolCodexRoute(route)) return false; + + const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); + if (options.requireNativeAccount && !resolvedAccountId) return true; + if ( + resolvedAccountId + && isCodexAccountInCooldown(resolvedAccountId, now) + && !canAcquireCodexQuotaProbeLease(resolvedAccountId, now) + ) { + return true; } - return false; + return isNativeModelQuotaExhausted(model, config, accountId, now, options); } export function selectAvailableSubagentModel( @@ -209,9 +214,12 @@ export function selectAvailableSubagentModel( const chain = normalizedChain(primary, config, extraFallback); const skipped: string[] = []; for (const candidate of chain) { - if (nativeFallbackOnly && !isNativeOpenAiSlug(candidate)) { - skipped.push(candidate); - continue; + if (nativeFallbackOnly) { + const route = tryRouteFallbackModel(config, candidate); + if (!route || !isCanonicalOpenAiForwardProvider(route.provider)) { + skipped.push(candidate); + continue; + } } if (isSubagentModelUnavailable(candidate, config, accountId, now, options)) { skipped.push(candidate); @@ -232,10 +240,15 @@ export function noteSubagentModelFailure( ): void { const interval = ttlMs ?? DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS; if (!isRateLimitOrQuotaFailureMessage(message)) return; - modelHealth.set(healthKey(model, resolveFallbackAccountId(config, accountId)), { - unavailableUntil: now + interval, - reason: "quota_exhausted", - }); + const route = tryRouteFallbackModel(config, model); + const poolScoped = !!route && isPoolCodexRoute(route); + modelHealth.set( + healthKey(model, resolveFallbackAccountId(config, accountId), poolScoped), + { + unavailableUntil: now + interval, + reason: "quota_exhausted", + }, + ); } export function configUsesCodexAccountPool(config: OcxConfig): boolean { @@ -251,7 +264,6 @@ export function resetSubagentModelFallbackStateForTests(): void { quotaPrimedAt.clear(); quotaPrimeInFlight = null; subagentQuotaPrimeForTests = null; - nativeSlugCache = null; } /** Test-only: inject the quota prime implementation used by {@link maybePrimeSubagentQuota}. */ diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 10af64a856..73aad21740 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -6,6 +6,7 @@ import { applySubagentModelFallback, buildSubagentModelChain, getSubagentQuotaPrimeStateForTests, + isNativeModelQuotaExhausted, isSubagentModelUnavailable, maybePrimeSubagentQuota, noteSubagentModelFailure, @@ -216,7 +217,7 @@ describe("subagent model fallback chain", () => { expect(getSubagentQuotaPrimeStateForTests().primedAt).toBeGreaterThan(0); }); - test("reset clears timestamp, in-flight, health, and native slug cache", async () => { + test("reset clears timestamp, in-flight, and health state", async () => { resetSubagentModelFallbackStateForTests(); let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); @@ -279,6 +280,185 @@ describe("subagent model fallback chain", () => { }); }); + test("direct bare GPT route ignores exhausted retained pool account quota", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + autoSwitchThreshold: 80, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["kimi/k3"], + }); + expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a", Date.now(), { + requireNativeAccount: true, + })).toBe(false); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + false, + { requireNativeAccount: true }, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + + test("another pool provider in config does not affect a direct GPT candidate", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + "openai-pool": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["kimi/k3"], + }); + expect(isNativeModelQuotaExhausted("gpt-5.6-sol", config, "pool-a")).toBe(false); + expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a")).toBe(false); + }); + + test("openai-direct/gpt-5.5 is accepted as encrypted-task fallback when canonical", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-direct/gpt-5.5", "kimi/k3"], + }); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + true, + { requireNativeAccount: true }, + ); + expect(selected).toEqual({ + model: "openai-direct/gpt-5.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("namespaced noncanonical OpenAI-compatible provider is rejected for encrypted tasks", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-compat": { + adapter: "openai-responses", + baseUrl: "https://api.example.com/v1", + authMode: "key", + apiKey: "sk-test", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-compat/gpt-5.5", "kimi/k3"], + }); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + true, + { requireNativeAccount: true }, + ); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: ["gpt-5.6-sol", "openai-compat/gpt-5.5", "kimi/k3"], + }); + }); + + test("pool quota affects only candidates whose resolved route uses pool mode", () => { + resetSubagentModelFallbackStateForTests(); + updateAccountQuota("pool-a", 95, undefined, 20); + const config = cfg({ + activeCodexAccountId: "pool-a", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + "openai-direct": { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + }, + subagentModelFallback: ["openai-direct/gpt-5.5", "kimi/k3"], + }); + expect(isNativeModelQuotaExhausted("gpt-5.6-sol", config, "pool-a")).toBe(true); + expect(isNativeModelQuotaExhausted("openai-direct/gpt-5.5", config, "pool-a")).toBe(false); + expect(isNativeModelQuotaExhausted("kimi/k3", config, "pool-a")).toBe(false); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + config, + [], + "pool-a", + Date.now(), + false, + { requireNativeAccount: true }, + ); + expect(selected).toEqual({ + model: "openai-direct/gpt-5.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + test("noteSubagentModelFailure records failures under the configured fallback slug", () => { resetSubagentModelFallbackStateForTests(); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg(), "main"); @@ -337,7 +517,9 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel allows raw slash model ids without provider namespaces", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + // Health-block the primary model only. A raw vendor/model id still routes through the + // default provider; it must remain selectable (not rejected as an unknown namespace). + noteSubagentModelFailure("gpt-5.6-sol", "429", cfg()); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg({ From 2fd77d81307f9651983ad8e74476a4b1a151355b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:05:09 +0200 Subject: [PATCH 12/12] fix(subagents): require pool accounts from resolved routes Drop the global explicit-pool config probe. Pool candidates now require a usable preview account based on each candidate routeModel result, including default openai pool mode. --- src/codex/subagent-model-fallback.ts | 56 +++------ src/server/responses/core.ts | 2 - tests/subagent-model-fallback.test.ts | 168 +++++++++++++++++++++----- 3 files changed, 157 insertions(+), 69 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 4e1e09788c..ec07d51db3 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -19,6 +19,7 @@ import { getPoolAccountPlan, isCodexAccountInCooldown, } from "./routing"; +import { isCodexAccountUsable } from "./account-usability"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; @@ -26,14 +27,6 @@ import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; import { routeModel, type RouteResult } from "../router"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; -export type SubagentAvailabilityOptions = { - /** - * When true, pool-mode Codex candidates require an explicit usable account id. - * Used for Codex pool configs where a null preview means no non-cooled account. - */ - requireNativeAccount?: boolean; -}; - type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; let quotaPrimeInFlight: Promise | null = null; @@ -118,14 +111,17 @@ function activeCodexAccountId(config: OcxConfig): string | null { return config.activeCodexAccountId ?? null; } -function resolveFallbackAccountId( +/** + * Resolve the account id used for pool-scoped quota/health checks. + * Explicit `null` means the pre-fallback preview found no usable account — do not + * substitute `activeCodexAccountId` (that active id may itself be unusable). + */ +function resolvePoolFallbackAccountId( config: OcxConfig, accountId?: string | null, - options: SubagentAvailabilityOptions = {}, ): string | null { if (typeof accountId === "string") return accountId; - // With requireNativeAccount, null means the pool preview found no usable account. - if (accountId === null && options.requireNativeAccount) return null; + if (accountId === null) return null; return activeCodexAccountId(config); } @@ -149,11 +145,10 @@ export function isNativeModelQuotaExhausted( config: OcxConfig, accountId?: string | null, now = Date.now(), - options: SubagentAvailabilityOptions = {}, ): boolean { const route = tryRouteFallbackModel(config, model); if (!route || !isPoolCodexRoute(route)) return false; - const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); + const resolvedAccountId = resolvePoolFallbackAccountId(config, accountId); if (!resolvedAccountId) return false; const quota = getAccountQuota(resolvedAccountId); const usage = computeCodexUsageScore(quota, getPoolAccountPlan(config, resolvedAccountId)); @@ -166,12 +161,11 @@ export function isModelHealthBlocked( config: OcxConfig, accountId?: string | null, now = Date.now(), - options: SubagentAvailabilityOptions = {}, ): boolean { const route = tryRouteFallbackModel(config, model); const poolScoped = !!route && isPoolCodexRoute(route); const health = modelHealth.get( - healthKey(model, resolveFallbackAccountId(config, accountId, options), poolScoped), + healthKey(model, resolvePoolFallbackAccountId(config, accountId), poolScoped), ); return !!health && health.unavailableUntil > now; } @@ -181,25 +175,26 @@ export function isSubagentModelUnavailable( config: OcxConfig, accountId?: string | null, now = Date.now(), - options: SubagentAvailabilityOptions = {}, ): boolean { if (isDisabledFallbackModel(model, config)) return true; if (!isRoutableFallbackModel(model, config)) return true; const route = tryRouteFallbackModel(config, model); if (!route || route.provider.disabled === true) return true; - if (isModelHealthBlocked(model, config, accountId, now, options)) return true; + if (isModelHealthBlocked(model, config, accountId, now)) return true; if (!isPoolCodexRoute(route)) return false; - const resolvedAccountId = resolveFallbackAccountId(config, accountId, options); - if (options.requireNativeAccount && !resolvedAccountId) return true; + // Pool candidates need a usable account. Derive requirement from the resolved + // route (canonical openai defaults to pool even when codexAccountMode is omitted). + const resolvedAccountId = resolvePoolFallbackAccountId(config, accountId); + if (!resolvedAccountId) return true; + if (!isCodexAccountUsable(config, resolvedAccountId)) return true; if ( - resolvedAccountId - && isCodexAccountInCooldown(resolvedAccountId, now) + isCodexAccountInCooldown(resolvedAccountId, now) && !canAcquireCodexQuotaProbeLease(resolvedAccountId, now) ) { return true; } - return isNativeModelQuotaExhausted(model, config, accountId, now, options); + return isNativeModelQuotaExhausted(model, config, accountId, now); } export function selectAvailableSubagentModel( @@ -209,7 +204,6 @@ export function selectAvailableSubagentModel( accountId?: string | null, now = Date.now(), nativeFallbackOnly = false, - options: SubagentAvailabilityOptions = {}, ): { model: string; rewritten: boolean; skipped: string[] } { const chain = normalizedChain(primary, config, extraFallback); const skipped: string[] = []; @@ -221,7 +215,7 @@ export function selectAvailableSubagentModel( continue; } } - if (isSubagentModelUnavailable(candidate, config, accountId, now, options)) { + if (isSubagentModelUnavailable(candidate, config, accountId, now)) { skipped.push(candidate); continue; } @@ -243,7 +237,7 @@ export function noteSubagentModelFailure( const route = tryRouteFallbackModel(config, model); const poolScoped = !!route && isPoolCodexRoute(route); modelHealth.set( - healthKey(model, resolveFallbackAccountId(config, accountId), poolScoped), + healthKey(model, resolvePoolFallbackAccountId(config, accountId), poolScoped), { unavailableUntil: now + interval, reason: "quota_exhausted", @@ -251,14 +245,6 @@ export function noteSubagentModelFailure( ); } -export function configUsesCodexAccountPool(config: OcxConfig): boolean { - return Object.values(config.providers).some( - provider => provider != null - && typeof provider === "object" - && (provider as { codexAccountMode?: string }).codexAccountMode === "pool", - ); -} - export function resetSubagentModelFallbackStateForTests(): void { modelHealth.clear(); quotaPrimedAt.clear(); @@ -391,7 +377,6 @@ export function applySubagentModelFallback( accountId?: string | null, now = Date.now(), nativeFallbackOnly = false, - options: SubagentAvailabilityOptions = {}, ): { from?: string; to?: string; skipped?: string[] } | null { if (!isThreadSpawnRequest(headers)) return null; const roleFallback = resolveAgentModelFallbackForPrimary(parsed.modelId, getCodexHome()); @@ -404,7 +389,6 @@ export function applySubagentModelFallback( accountId, now, nativeFallbackOnly, - options, ); if (!selection.rewritten) return selection.skipped.length > 0 ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 26fb764fd9..824d5dda7c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -80,7 +80,6 @@ import { supportedLadderFor } from "../effort-policy"; import { isThreadSpawnRequest } from "../effort-policy"; import { applySubagentModelFallback, - configUsesCodexAccountPool, maybePrimeSubagentQuota, recordSubagentQuotaFailureForThreadSpawn, } from "../../codex/subagent-model-fallback"; @@ -911,7 +910,6 @@ export async function handleResponses( previewAccountId, Date.now(), unreadableEncryptedAgentTask, - { requireNativeAccount: configUsesCodexAccountPool(config) }, ); if (fallback) { (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 73aad21740..3e13782d0b 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -1,5 +1,5 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -17,22 +17,43 @@ import { setSubagentQuotaPrimeForTests, subagentFallbackGuidanceText, } from "../src/codex/subagent-model-fallback"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../src/codex/account-runtime-state"; import { clearAccountQuota, updateAccountQuota } from "../src/codex/quota"; import type { OcxConfig } from "../src/types"; const savedCodexHome = process.env.CODEX_HOME; +const savedOpencodexHome = process.env.OPENCODEX_HOME; +let testDir: string; + +function installPoolCredential(accountId: string, now = Date.now()): void { + saveCodexAccountCredential(accountId, { + accessToken: `${accountId}_token`, + refreshToken: `${accountId}_refresh`, + expiresAt: now + 24 * 60 * 60_000, + chatgptAccountId: `${accountId}_acc`, + }); +} function cfg(overrides: Partial = {}): OcxConfig { return { port: 10100, providers: { + // Omitted codexAccountMode — canonical openai defaults to pool via routeModel. openai: { adapter: "openai-responses" }, "alibaba-token-plan": { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, kimi: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://example.invalid" }, + xai: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://api.x.ai/v1" }, }, defaultProvider: "openai", - activeCodexAccountId: "main", + activeCodexAccountId: "pool-a", autoSwitchThreshold: 80, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "a@example.test", isMain: false, chatgptAccountId: "pool_a_acc" }, + { id: "account-a", email: "aa@example.test", isMain: false, chatgptAccountId: "aa_acc" }, + { id: "account-b", email: "bb@example.test", isMain: false, chatgptAccountId: "bb_acc" }, + ], subagentModelFallback: [ "gpt-5.6-sol", "alibaba-token-plan/qwen3.8-max-preview", @@ -49,11 +70,31 @@ function codexHomeFixture(): string { return dir; } +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-fb-")); + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + installPoolCredential("pool-a"); + installPoolCredential("account-a"); + installPoolCredential("account-b"); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("account-a"); + clearAccountNeedsReauth("account-b"); + clearAccountNeedsReauth("main"); +}); + afterEach(() => { if (savedCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = savedCodexHome; + if (savedOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = savedOpencodexHome; clearAccountQuota(); resetSubagentModelFallbackStateForTests(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("account-a"); + clearAccountNeedsReauth("account-b"); + clearAccountNeedsReauth("main"); + rmSync(testDir, { recursive: true, force: true }); }); describe("subagent model fallback chain", () => { @@ -72,7 +113,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel skips quota-exhausted native models", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); expect(selected).toEqual({ model: "alibaba-token-plan/qwen3.8-max-preview", @@ -95,7 +136,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel skips cached routed failures", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "quota exhausted", cfg()); const selected = selectAvailableSubagentModel("gpt-5.6-sol", cfg()); expect(selected.model).toBe("kimi/k3"); @@ -104,7 +145,7 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel skips stale fallback entries that cannot route", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg({ @@ -158,7 +199,7 @@ describe("subagent model fallback chain", () => { setSubagentQuotaPrimeForTests(async () => { midRefreshModel = selectAvailableSubagentModel("gpt-5.6-sol", cfg()).model; await gate; - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); }); const priming = maybePrimeSubagentQuota(cfg()); @@ -181,7 +222,7 @@ describe("subagent model fallback chain", () => { setSubagentQuotaPrimeForTests(async () => { calls += 1; await gate; - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); }); const a = maybePrimeSubagentQuota(cfg()); @@ -210,7 +251,7 @@ describe("subagent model fallback chain", () => { setSubagentQuotaPrimeForTests(async () => { calls += 1; - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); }); await maybePrimeSubagentQuota(cfg()); expect(calls).toBe(2); @@ -241,17 +282,17 @@ describe("subagent model fallback chain", () => { resetSubagentModelFallbackStateForTests(); noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg()); expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg())).toBe(true); - expect(isSubagentModelUnavailable("qwen3.8-max-preview", cfg())).toBe(false); + expect(isSubagentModelUnavailable("kimi/k3", cfg())).toBe(false); }); test("selectAvailableSubagentModel can require native-only fallback for encrypted tasks", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg(), [], - "main", + "pool-a", Date.now(), true, ); @@ -264,12 +305,12 @@ describe("subagent model fallback chain", () => { test("selectAvailableSubagentModel can stay native-only for encrypted spawns", () => { resetSubagentModelFallbackStateForTests(); - updateAccountQuota("main", 95, undefined, 20); + updateAccountQuota("pool-a", 95, undefined, 20); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", cfg(), [], - "main", + "pool-a", Date.now(), true, ); @@ -297,17 +338,12 @@ describe("subagent model fallback chain", () => { }, subagentModelFallback: ["kimi/k3"], }); - expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a", Date.now(), { - requireNativeAccount: true, - })).toBe(false); + expect(isSubagentModelUnavailable("gpt-5.6-sol", config, "pool-a")).toBe(false); const selected = selectAvailableSubagentModel( "gpt-5.6-sol", config, [], "pool-a", - Date.now(), - false, - { requireNativeAccount: true }, ); expect(selected).toEqual({ model: "gpt-5.6-sol", @@ -371,7 +407,6 @@ describe("subagent model fallback chain", () => { "pool-a", Date.now(), true, - { requireNativeAccount: true }, ); expect(selected).toEqual({ model: "openai-direct/gpt-5.5", @@ -409,7 +444,6 @@ describe("subagent model fallback chain", () => { "pool-a", Date.now(), true, - { requireNativeAccount: true }, ); expect(selected).toEqual({ model: "gpt-5.6-sol", @@ -448,9 +482,6 @@ describe("subagent model fallback chain", () => { config, [], "pool-a", - Date.now(), - false, - { requireNativeAccount: true }, ); expect(selected).toEqual({ model: "openai-direct/gpt-5.5", @@ -459,11 +490,86 @@ describe("subagent model fallback chain", () => { }); }); + test("omitted openai codexAccountMode still requires a usable pool account", () => { + resetSubagentModelFallbackStateForTests(); + // Default cfg omits codexAccountMode on openai (defaults to pool via routeModel). + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ subagentModelFallback: ["xai/grok-4.5"] }), + [], + null, + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("no usable pool account falls back to XAI", () => { + resetSubagentModelFallbackStateForTests(); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ + activeCodexAccountId: undefined, + codexAccounts: [{ id: "main", email: "main@example.test", isMain: true }], + subagentModelFallback: ["xai/grok-4.5"], + }), + [], + null, + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("reauthentication-required pool account falls back to XAI", () => { + resetSubagentModelFallbackStateForTests(); + markAccountNeedsReauth("pool-a"); + const selected = selectAvailableSubagentModel( + "gpt-5.6-sol", + cfg({ subagentModelFallback: ["xai/grok-4.5"] }), + [], + "pool-a", + ); + expect(selected).toEqual({ + model: "xai/grok-4.5", + rewritten: true, + skipped: ["gpt-5.6-sol"], + }); + }); + + test("explicit direct mode remains unaffected by missing pool accounts", () => { + resetSubagentModelFallbackStateForTests(); + const config = cfg({ + activeCodexAccountId: undefined, + codexAccounts: [{ id: "main", email: "main@example.test", isMain: true }], + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + }, + xai: { adapter: "openai-chat", apiKey: "test", baseUrl: "https://api.x.ai/v1" }, + }, + subagentModelFallback: ["xai/grok-4.5"], + }); + const selected = selectAvailableSubagentModel("gpt-5.6-sol", config, [], null); + expect(selected).toEqual({ + model: "gpt-5.6-sol", + rewritten: false, + skipped: [], + }); + }); + test("noteSubagentModelFailure records failures under the configured fallback slug", () => { resetSubagentModelFallbackStateForTests(); - noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg(), "main"); - expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "main")).toBe(true); - expect(isSubagentModelUnavailable("qwen3.8-max-preview", cfg(), "main")).toBe(false); + noteSubagentModelFailure("alibaba-token-plan/qwen3.8-max-preview", "429", cfg(), "pool-a"); + expect(isSubagentModelUnavailable("alibaba-token-plan/qwen3.8-max-preview", cfg(), "pool-a")).toBe(true); + expect(isSubagentModelUnavailable("kimi/k3", cfg(), "pool-a")).toBe(false); }); test("readCodexAgentModelFallback parses multiline TOML arrays", () => { @@ -543,7 +649,7 @@ describe("subagent model fallback chain", () => { }); test("applySubagentModelFallback rewrites parsed request model", () => { - updateAccountQuota("main", 95); + updateAccountQuota("pool-a", 95); const parsed = { modelId: "gpt-5.6-sol", options: {}, @@ -565,7 +671,7 @@ describe("subagent model fallback chain", () => { }); test("applySubagentModelFallback is a no-op for main turns", () => { - updateAccountQuota("main", 95); + updateAccountQuota("pool-a", 95); const parsed = { modelId: "gpt-5.6-sol", options: {}, @@ -584,7 +690,7 @@ describe("subagent model fallback chain", () => { "model_fallback = [\"alibaba-token-plan/qwen3.8-max-preview\"]", "", ].join("\n"), "utf8"); - updateAccountQuota("main", 95); + updateAccountQuota("pool-a", 95); const parsed = { modelId: "gpt-5.6-sol", options: {},