Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions src/codex/pool-refresh-backoff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ const backoffByAccount = new Map<string, RefreshFailureBackoff>();
* must not re-quarantine the credential that replaced it.
*/
const fenceByAccount = new Map<string, number>();
/**
* Invalidates every account fence without having to know which refresh flights are currently in
* progress. A bulk routing-state reset can race a first failure for an account that has no map
* entry yet, so iterating either map cannot close this boundary.
*/
let globalFence = 0;
let nowOverride: number | undefined;

export function setCodexPoolRefreshFailureNowForTests(now?: number): void {
Expand All @@ -52,12 +58,13 @@ export function setCodexPoolRefreshFailureNowForTests(now?: number): void {
export function resetCodexPoolRefreshFailureBackoffForTests(): void {
backoffByAccount.clear();
fenceByAccount.clear();
globalFence = 0;
nowOverride = undefined;
}

/** The value a refresh flight captures before it starts, to be handed back on failure. */
export function codexPoolRefreshFence(accountId: string): number {
return fenceByAccount.get(accountId) ?? 0;
export function codexPoolRefreshFence(accountId: string): string {
return `${globalFence}:${fenceByAccount.get(accountId) ?? 0}`;
}

export function clearCodexPoolRefreshFailure(accountId: string): void {
Expand All @@ -72,6 +79,8 @@ export function clearCodexPoolRefreshFailure(accountId: string): void {
*/
export function clearAllCodexPoolRefreshFailures(): void {
backoffByAccount.clear();
fenceByAccount.clear();
globalFence += 1;
}

function currentNow(now?: number): number {
Expand Down Expand Up @@ -115,7 +124,7 @@ export function noteCodexPoolRefreshFailure(
accountId: string,
reason: string,
now = currentNow(),
fence?: number,
fence?: string,
): { consecutiveFailures: number; cooldownUntil: number; openedWindow: boolean } {
const existing = backoffByAccount.get(accountId);
// A flight that started before the account's failures were cleared is speaking for a grant
Expand Down
4 changes: 4 additions & 0 deletions structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,10 @@ Native Responses participates in the same pre-stream OAuth HTTP-429 account rota
bridge. It uses the existing account quorum, cooldown and three-rotation request cap, refreshes
the complete credential/transport/replay identity, and attributes usage to the serving account.
Single-account installs do not retry; a missing alternate credential preserves the original error.
Credential-refresh failures are fenced by both the account generation and a global routing-state
generation. Reauthentication advances the account fence; replacing the whole routing roster
advances the global fence. A late failure from either obsolete state is ignored, while failures
captured after the reset still contribute to the bounded cooldown.

Startup removes legacy Grok 4.5/4.6 Chat overrides once and persists the provider-owned
`xaiResponsesDefaultVersion` marker. Later explicit Chat choices survive restarts. The migration
Expand Down
29 changes: 29 additions & 0 deletions tests/codex-integration/codex-pool-refresh-backoff.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES,
CODEX_POOL_REFRESH_FAILURE_BACKOFF_MS,
CodexPoolRefreshCooldownError,
clearAllCodexPoolRefreshFailures,
clearCodexPoolRefreshFailure,
codexPoolRefreshFence,
getCodexPoolRefreshCooldownUntil,
Expand Down Expand Up @@ -220,3 +221,31 @@ describe("a late failure from the replaced credential cannot re-cool the new one
expect(source.slice(reported, reported + 200)).toContain("refreshFence");
});
});

/**
* The routing layer bulk-clears account state when its roster is replaced. A refresh that began
* before that reset may not have recorded any failure yet, so it is absent from both state maps.
* The global fence generation is what makes that unknown in-flight attempt stale.
*/
describe("a bulk routing reset fences every in-flight refresh", () => {
test("a pre-reset failure is ignored while a post-reset failure still counts", () => {
const now = 4_000_000;
setCodexPoolRefreshFailureNowForTests(now);
const staleFence = codexPoolRefreshFence("acct-bulk-fenced");

clearAllCodexPoolRefreshFailures();
expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(false);

for (let attempt = 0; attempt < CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES; attempt += 1) {
noteCodexPoolRefreshFailure("acct-bulk-fenced", "unknown", undefined, staleFence);
}
expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(false);

const freshFence = codexPoolRefreshFence("acct-bulk-fenced");
expect(freshFence).not.toBe(staleFence);
for (let attempt = 0; attempt < CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES; attempt += 1) {
noteCodexPoolRefreshFailure("acct-bulk-fenced", "unknown", undefined, freshFence);
}
expect(isCodexPoolRefreshCooling("acct-bulk-fenced")).toBe(true);
});
});
Loading