diff --git a/src/codex/pool-refresh-backoff.ts b/src/codex/pool-refresh-backoff.ts index d9474ceb21..acbe213389 100644 --- a/src/codex/pool-refresh-backoff.ts +++ b/src/codex/pool-refresh-backoff.ts @@ -43,6 +43,12 @@ const backoffByAccount = new Map(); * must not re-quarantine the credential that replaced it. */ const fenceByAccount = new Map(); +/** + * Invalidates every account fence without having to know which refresh flights are currently in + * progress. A bulk routing-state reset can race a first failure for an account that has no map + * entry yet, so iterating either map cannot close this boundary. + */ +let globalFence = 0; let nowOverride: number | undefined; export function setCodexPoolRefreshFailureNowForTests(now?: number): void { @@ -52,12 +58,13 @@ export function setCodexPoolRefreshFailureNowForTests(now?: number): void { export function resetCodexPoolRefreshFailureBackoffForTests(): void { backoffByAccount.clear(); fenceByAccount.clear(); + globalFence = 0; nowOverride = undefined; } /** The value a refresh flight captures before it starts, to be handed back on failure. */ -export function codexPoolRefreshFence(accountId: string): number { - return fenceByAccount.get(accountId) ?? 0; +export function codexPoolRefreshFence(accountId: string): string { + return `${globalFence}:${fenceByAccount.get(accountId) ?? 0}`; } export function clearCodexPoolRefreshFailure(accountId: string): void { @@ -72,6 +79,8 @@ export function clearCodexPoolRefreshFailure(accountId: string): void { */ export function clearAllCodexPoolRefreshFailures(): void { backoffByAccount.clear(); + fenceByAccount.clear(); + globalFence += 1; } function currentNow(now?: number): number { @@ -115,7 +124,7 @@ export function noteCodexPoolRefreshFailure( accountId: string, reason: string, now = currentNow(), - fence?: number, + fence?: string, ): { consecutiveFailures: number; cooldownUntil: number; openedWindow: boolean } { const existing = backoffByAccount.get(accountId); // A flight that started before the account's failures were cleared is speaking for a grant diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 4a6a664cad..575372f173 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -234,6 +234,10 @@ Native Responses participates in the same pre-stream OAuth HTTP-429 account rota bridge. It uses the existing account quorum, cooldown and three-rotation request cap, refreshes the complete credential/transport/replay identity, and attributes usage to the serving account. Single-account installs do not retry; a missing alternate credential preserves the original error. +Credential-refresh failures are fenced by both the account generation and a global routing-state +generation. Reauthentication advances the account fence; replacing the whole routing roster +advances the global fence. A late failure from either obsolete state is ignored, while failures +captured after the reset still contribute to the bounded cooldown. Startup removes legacy Grok 4.5/4.6 Chat overrides once and persists the provider-owned `xaiResponsesDefaultVersion` marker. Later explicit Chat choices survive restarts. The migration diff --git a/tests/codex-integration/codex-pool-refresh-backoff.test.ts b/tests/codex-integration/codex-pool-refresh-backoff.test.ts index 1b13db292b..4f71f50535 100644 --- a/tests/codex-integration/codex-pool-refresh-backoff.test.ts +++ b/tests/codex-integration/codex-pool-refresh-backoff.test.ts @@ -4,6 +4,7 @@ import { CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES, CODEX_POOL_REFRESH_FAILURE_BACKOFF_MS, CodexPoolRefreshCooldownError, + clearAllCodexPoolRefreshFailures, clearCodexPoolRefreshFailure, codexPoolRefreshFence, getCodexPoolRefreshCooldownUntil, @@ -220,3 +221,31 @@ describe("a late failure from the replaced credential cannot re-cool the new one expect(source.slice(reported, reported + 200)).toContain("refreshFence"); }); }); + +/** + * The routing layer bulk-clears account state when its roster is replaced. A refresh that began + * before that reset may not have recorded any failure yet, so it is absent from both state maps. + * The global fence generation is what makes that unknown in-flight attempt stale. + */ +describe("a bulk routing reset fences every in-flight refresh", () => { + test("a pre-reset failure is ignored while a post-reset failure still counts", () => { + const now = 4_000_000; + setCodexPoolRefreshFailureNowForTests(now); + const staleFence = codexPoolRefreshFence("acct-bulk-fenced"); + + clearAllCodexPoolRefreshFailures(); + expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(false); + + for (let attempt = 0; attempt < CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES; attempt += 1) { + noteCodexPoolRefreshFailure("acct-bulk-fenced", "unknown", undefined, staleFence); + } + expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(false); + + const freshFence = codexPoolRefreshFence("acct-bulk-fenced"); + expect(freshFence).not.toBe(staleFence); + for (let attempt = 0; attempt < CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES; attempt += 1) { + noteCodexPoolRefreshFailure("acct-bulk-fenced", "unknown", undefined, freshFence); + } + expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(true); + }); +});