From 8367dea365f66ecc721f131fbd9a5af220cdaa59 Mon Sep 17 00:00:00 2001 From: agentHits <140916359+agentHits@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:46:50 -0400 Subject: [PATCH 1/4] feat(oauth): rank Antigravity failover by Gemini vs Claude quota family Keep Claude exhaustion from starving Gemini selection, and cool 429s per model family so a Claude limit still leaves the account eligible for Gemini. Co-authored-by: agentHits <140916359+agentHits@users.noreply.github.com> --- scripts/test-layout/layout.json | 1 + src/oauth/account-quota-rank.ts | 84 ++++++++++--- src/oauth/generic-account-failover.ts | 57 +++++---- src/server/responses/adapter-continuation.ts | 2 + src/server/responses/adapter-dispatch.ts | 2 + src/server/responses/passthrough-dispatch.ts | 2 + src/server/responses/request-transport.ts | 2 +- src/server/responses/run-turn-execution.ts | 2 + src/server/responses/sidecar-execution.ts | 2 + tests/fixtures/test-layout-expected.json | 1 + tests/oauth/oauth-account-quota-rank.test.ts | 117 +++++++++++++++++++ 11 files changed, 231 insertions(+), 41 deletions(-) create mode 100644 tests/oauth/oauth-account-quota-rank.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 885d2abcd4..cd21f3c691 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -952,6 +952,7 @@ "nvidia-nim-hardening.test.ts": "providers", "oauth-account-attribution.test.ts": "oauth", "oauth-account-id-collision.test.ts": "oauth", + "oauth-account-quota-rank.test.ts": "oauth", "oauth-accounts-api.test.ts": "oauth", "oauth-callback-binds.test.ts": "oauth", "oauth-callback-server.test.ts": "oauth", diff --git a/src/oauth/account-quota-rank.ts b/src/oauth/account-quota-rank.ts index 0cae484fa5..76fe3c05a4 100644 --- a/src/oauth/account-quota-rank.ts +++ b/src/oauth/account-quota-rank.ts @@ -13,6 +13,35 @@ import { getCachedProviderAccountQuota, hasPassiveAccountQuota } from "../providers/quota"; import { getKiroAccountExhaustion } from "../providers/kiro-usage"; +/** Antigravity hosts Gemini and Claude windows on one account; ranking must not mix them. */ +export type QuotaModelFamily = "gem" | "cla"; + +export function classifyModelFamilyForQuota( + provider: string, + modelId?: string | null, +): QuotaModelFamily | undefined { + if (provider !== "google-antigravity" || typeof modelId !== "string" || !modelId.trim()) { + return undefined; + } + const id = modelId.toLowerCase(); + // Gemma is not Gemini: a substring/prefix match would poison Gemini ranking. + if (/(?:^|[^a-z])gemma(?:[^a-z]|$)/.test(id)) return undefined; + if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])gem(?:[^a-z]|$)/.test(id)) return "gem"; + if ( + /(?:^|[^a-z])claude(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])opus(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])sonnet(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])haiku(?:[^a-z]|$)/.test(id) + ) return "cla"; + return undefined; +} + +function windowMatchesFamily(label: string, family: QuotaModelFamily): boolean { + const token = label.trim().split(/[\s(/]+/)[0] ?? ""; + if (family === "gem") return /^gem(?:ini)?$/i.test(token); + return /^cla(?:ude)?$/i.test(token); +} + /** Lower sorts earlier. Unknown sits between measured-healthy and measured-empty. */ const RANK_HEALTHY = 0; const RANK_UNKNOWN = 1; @@ -48,15 +77,24 @@ const PASSIVE_HEADROOM_MAX_AGE_MS = 60 * 60_000; /** * Remaining headroom across every window the provider reports. * - * The minimum wins: an account at 5% of its five-hour window is unusable right now even if - * its monthly allowance is barely touched. - */ -function headroomOf(provider: string, accountId: string): number | null { +* The minimum wins: an account at 5% of its five-hour window is unusable right now even if +* its monthly allowance is barely touched. +*/ +function headroomOf(provider: string, accountId: string, requestedModelId?: string | null): number | null { const quota = getCachedProviderAccountQuota(provider, accountId); if (!quota) return null; // Null, not a low rank: this must reproduce "no evidence" so a stale roster degrades to // the unranked ring rather than to a differently wrong answer. if (hasPassiveAccountQuota(provider) && Date.now() - quota.updatedAt > PASSIVE_HEADROOM_MAX_AGE_MS) return null; + const family = classifyModelFamilyForQuota(provider, requestedModelId); + if (family) { + const percents = (quota.customWindows ?? []) + .filter(window => windowMatchesFamily(window.label, family)) + .map(window => window.percent) + .filter((value): value is number => typeof value === "number"); + if (percents.length === 0) return null; + return 100 - Math.max(...percents); + } const percents = [ quota.fiveHourPercent, quota.weeklyPercent, @@ -74,15 +112,23 @@ function headroomOf(provider: string, accountId: string): number | null { * than an ordering. Null stays null all the way out: a caller must decide what "unmeasured" * means for its own rule instead of being handed a fabricated 0 or 100. */ -export function accountHeadroomPercent(provider: string, accountId: string): number | null { - return headroomOf(provider, accountId); +export function accountHeadroomPercent( + provider: string, + accountId: string, + requestedModelId?: string | null, +): number | null { + return headroomOf(provider, accountId, requestedModelId); } /** Unknown usage is not exhaustion; Kiro's explicit overage verdict is authoritative. */ -export function isAccountQuotaExhausted(provider: string, accountId: string): boolean { +export function isAccountQuotaExhausted( + provider: string, + accountId: string, + requestedModelId?: string | null, +): boolean { const exhaustion = provider === "kiro" ? getKiroAccountExhaustion(`${provider}\u0000${accountId}`) : null; if (exhaustion !== null) return exhaustion.exhausted; - const headroom = headroomOf(provider, accountId); + const headroom = headroomOf(provider, accountId, requestedModelId); return headroom !== null && headroom <= 0; } @@ -92,24 +138,28 @@ export function isAccountQuotaExhausted(provider: string, accountId: string): bo * Returns the input untouched when no candidate has quota evidence, which keeps every * provider without per-account quota on exactly the behaviour it has today. */ -export function rankAccountsByHeadroom(provider: string, ring: readonly string[]): string[] { +export function rankAccountsByHeadroom( + provider: string, + ring: readonly string[], + requestedModelId?: string | null, +): string[] { if (ring.length < 2) return [...ring]; let sawEvidence = false; // Same rule as hasHeadroomEvidence: a passive provider's partial roster must not rank // at all. The failover path calls this directly (selectFailoverAccount), so the guard // cannot live only in the pre-dispatch predicate. - if (hasPassiveAccountQuota(provider) && !ring.every(id => headroomOf(provider, id) !== null)) { + if (hasPassiveAccountQuota(provider) && !ring.every(id => headroomOf(provider, id, requestedModelId) !== null)) { return [...ring]; } const ranked: Ranked[] = ring.map((id, index) => { // A provider-declared exhaustion verdict outranks the percentage: an account may sit at // 100% and still be servable when overage is enabled, and the verdict knows that. const exhaustion = provider === "kiro" ? getKiroAccountExhaustion(`${provider}\u0000${id}`) : null; - const headroom = headroomOf(provider, id); + const headroom = headroomOf(provider, id, requestedModelId); if (exhaustion !== null || headroom !== null) sawEvidence = true; - if (isAccountQuotaExhausted(provider, id)) return { id, bucket: RANK_EXHAUSTED, headroom: 0, index }; + if (isAccountQuotaExhausted(provider, id, requestedModelId)) return { id, bucket: RANK_EXHAUSTED, headroom: 0, index }; if (headroom === null) return { id, bucket: RANK_UNKNOWN, headroom: 0, index }; return { id, bucket: RANK_HEALTHY, headroom, index }; }); @@ -129,7 +179,11 @@ export function rankAccountsByHeadroom(provider: string, ring: readonly string[] * told "ranked" when nothing was measured. Pre-dispatch selection asks this first so it * can decline to act on a roster it knows nothing about. */ -export function hasHeadroomEvidence(provider: string, ids: readonly string[]): boolean { +export function hasHeadroomEvidence( + provider: string, + ids: readonly string[], + requestedModelId?: string | null, +): boolean { // A PASSIVE provider needs evidence for EVERY candidate, not any one of them. // // A probe fills the whole roster in one pass (fetchProviderAccountQuotas), so "any" @@ -140,10 +194,10 @@ export function hasHeadroomEvidence(provider: string, ids: readonly string[]): b // AWAY from an unmeasured account and TOWARD the one account known to be spent, which // is the exact inversion of what ranking is for. if (hasPassiveAccountQuota(provider)) { - return ids.length > 0 && ids.every(id => headroomOf(provider, id) !== null); + return ids.length > 0 && ids.every(id => headroomOf(provider, id, requestedModelId) !== null); } return ids.some(id => - headroomOf(provider, id) !== null + headroomOf(provider, id, requestedModelId) !== null || (provider === "kiro" && getKiroAccountExhaustion(`${provider}\u0000${id}`) !== null)); } /** diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index 6b444ea435..b944a6aaf1 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -22,6 +22,8 @@ import { hasHeadroomEvidence, isAccountQuotaExhausted, rankAccountsByHeadroom, + classifyModelFamilyForQuota, + type QuotaModelFamily, } from "./account-quota-rank"; import { genericPoolKey, @@ -81,13 +83,14 @@ const health = new Map(); /** Provider -> recent eligible-account count. TTL-bounded; never holds credential material. */ const presence = new Map(); -const healthKey = (provider: string, accountId: string) => `${provider}\u0000${accountId}`; +const healthKey = (provider: string, accountId: string, family?: QuotaModelFamily) => + family ? `${provider}\u0000${accountId}\u0000${family}` : `${provider}\u0000${accountId}`; -function isCooled(provider: string, accountId: string, now: number): boolean { - const entry = health.get(healthKey(provider, accountId)); +function isCooled(provider: string, accountId: string, now: number, family?: QuotaModelFamily): boolean { + const entry = health.get(healthKey(provider, accountId, family)); if (!entry) return false; if (entry.cooldownUntil <= now) { - health.delete(healthKey(provider, accountId)); + health.delete(healthKey(provider, accountId, family)); return false; } return true; @@ -175,11 +178,11 @@ function isProactivePreferenceEnabled(config: OcxConfig, providerName: string, n } /** Accounts that may serve traffic right now: not cooled, not flagged for reauth. */ -export function eligibleFailoverAccounts(providerName: string, now = Date.now()): string[] { +export function eligibleFailoverAccounts(providerName: string, now = Date.now(), family?: QuotaModelFamily): string[] { const set = getAccountSet(providerName); if (!set) return []; return set.accounts - .filter(account => account.needsReauth !== true && !isCooled(providerName, account.id, now)) + .filter(account => account.needsReauth !== true && !isCooled(providerName, account.id, now, family)) .map(account => account.id); } @@ -230,8 +233,8 @@ function stableGenericRoster(providerName: string): string[] { * statement about observed usage, and treating "no observation" as "spent" would evacuate every * quota-less provider off its active account on the very first request. */ -function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number): boolean { - const headroom = accountHeadroomPercent(providerName, accountId); +function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number, requestedModelId?: string | null): boolean { + const headroom = accountHeadroomPercent(providerName, accountId, requestedModelId); if (headroom === null) return false; return 100 - headroom >= threshold; } @@ -245,15 +248,17 @@ function pickFillFirstGenericAccount( providerName: string, activeId: string | undefined, now: number, + requestedModelId?: string | null, ): string | null { const stableAll = stableGenericRoster(providerName); if (stableAll.length < 2) return null; - const eligible = new Set(eligibleFailoverAccounts(providerName, now)); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const eligible = new Set(eligibleFailoverAccounts(providerName, now, family)); const stored = config.providers?.[providerName]?.oauthAccountFailover?.autoSwitchThreshold; const threshold = typeof stored === "number" && Number.isInteger(stored) && stored >= 0 && stored <= 100 ? stored : DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD; - if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold)) { + if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold, requestedModelId)) { return null; } const start = activeId ? stableAll.indexOf(activeId) : -1; @@ -302,6 +307,7 @@ export function rotateGenericOAuthAccountOn429( failedAccountId: string, retryAfterHeader: string | null | undefined, now = Date.now(), + requestedModelId?: string | null, ): string | null { if (!isGenericOAuthFailoverEnabled(config, providerName)) return null; const set = getAccountSet(providerName); @@ -314,13 +320,14 @@ export function rotateGenericOAuthAccountOn429( // A Retry-After from upstream still wins — it is the server's own instruction. const exhausted = parsed === undefined ? exhaustedCooldownMs(providerName, failedAccountId, now) : null; const cooldownMs = exhausted ?? Math.min(parsed ?? DEFAULT_COOLDOWN_MS, MAX_COOLDOWN_MS); - health.set(healthKey(providerName, failedAccountId), { + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + health.set(healthKey(providerName, failedAccountId, family), { cooldownUntil: now + cooldownMs, cooldownSource: parsed ? "retry-after" : "default", }); sweepExpiredOnWrite(now); - const eligible = eligibleFailoverAccounts(providerName, now).filter(id => id !== failedAccountId); + const eligible = eligibleFailoverAccounts(providerName, now, family).filter(id => id !== failedAccountId); if (eligible.length === 0) return null; // A rotation means the roster in use just changed; do not answer the next activation question // from a count read before the failure. @@ -359,7 +366,7 @@ export function rotateGenericOAuthAccountOn429( } // With no quota evidence this returns the ring untouched, so providers without // per-account quota keep exactly the traversal they have today. - return rankAccountsByHeadroom(providerName, candidates)[0] ?? null; + return rankAccountsByHeadroom(providerName, candidates, requestedModelId)[0] ?? null; } /** @@ -392,6 +399,7 @@ export function preferredInitialAccount( config: OcxConfig, providerName: string, now = Date.now(), + requestedModelId?: string | null, ): string | null { // The PROACTIVE predicate, not the reactive one: this steers a request upstream has not // refused, so `oauthAccountFailover.enabled: false` must still be able to refuse it. @@ -411,7 +419,8 @@ export function preferredInitialAccount( // would never reach its own test. Cooldowns and reauth are still honoured inside each pick. const strategy = activeGenericStrategy(config, providerName); if (strategy === "round-robin") { - const eligibleNow = eligibleFailoverAccounts(providerName, now); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const eligibleNow = eligibleFailoverAccounts(providerName, now, family); if (eligibleNow.length === 0) return null; // PEEK, not pick: this proposal is discardable, and advancing the ring for an account the // resolver then rejects would skip a turn for nothing. noteGenericPoolSelection commits. @@ -423,26 +432,26 @@ export function preferredInitialAccount( return picked && picked !== active ? picked : null; } if (strategy === "fill-first") { - const picked = pickFillFirstGenericAccount(config, providerName, active, now); + const picked = pickFillFirstGenericAccount(config, providerName, active, now, requestedModelId); return picked && picked !== active ? picked : null; } const activeRow = selected.accounts.find(account => account.id === active); if (activeRow && activeRow.needsReauth !== true - && !isCooled(providerName, activeRow.id, now) - && !isAccountQuotaExhausted(providerName, activeRow.id)) return null; + && !isCooled(providerName, activeRow.id, now, classifyModelFamilyForQuota(providerName, requestedModelId)) + && !isAccountQuotaExhausted(providerName, activeRow.id, requestedModelId)) return null; // Evidence is required BEFORE eligibility narrows the field. Without this, a provider // with no quota data at all could still be redirected: cool the active account with a // 429 and the eligible list collapses to one candidate, which any ranking returns // unchanged — an answer that looks ranked but was never measured. The no-op guarantee // for quota-less providers has to be checked on the full roster. - if (!hasHeadroomEvidence(providerName, order)) return null; + if (!hasHeadroomEvidence(providerName, order, requestedModelId)) return null; // Cooldowns are respected here, unlike in the presence count: this picks the account to // send to right now, and one inside its 429 window is the single candidate we hold // positive evidence against. - const eligible = order.filter(id => !isCooled(providerName, id, now)); + const eligible = order.filter(id => !isCooled(providerName, id, now, classifyModelFamilyForQuota(providerName, requestedModelId))); if (eligible.length === 0) return null; // Start the ring at the active account so an unranked outcome reproduces today's choice. @@ -451,7 +460,7 @@ export function preferredInitialAccount( const candidates = ring.filter(id => eligible.includes(id)); if (candidates.length === 0) return null; - const best = rankAccountsByHeadroom(providerName, candidates)[0] ?? null; + const best = rankAccountsByHeadroom(providerName, candidates, requestedModelId)[0] ?? null; // Nothing to do when the ranking agrees with the account we would have used anyway. // // A proposal still needs guarded selection commit after credential resolution: a @@ -461,12 +470,10 @@ export function preferredInitialAccount( /** Earliest remaining cooldown, for a client-facing Retry-After when every account is cooled. */ export function genericFailoverRetryAfterSeconds(providerName: string, now = Date.now()): number | null { - const set = getAccountSet(providerName); - if (!set) return null; + const prefix = `${providerName}\u0000`; let earliest: number | null = null; - for (const account of set.accounts) { - const entry = health.get(healthKey(providerName, account.id)); - if (!entry || entry.cooldownUntil <= now) continue; + for (const [key, entry] of health) { + if (!key.startsWith(prefix) || entry.cooldownUntil <= now) continue; if (earliest === null || entry.cooldownUntil < earliest) earliest = entry.cooldownUntil; } return earliest === null ? null : Math.max(1, Math.ceil((earliest - now) / 1000)); diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index a1db9398d5..ba1533ae75 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -395,6 +395,8 @@ export function createAdapterContinuations( route.providerName, transportState.genericFailoverAccountId, response.headers.get("retry-after"), + Date.now(), + route.modelId, ) : null; if (!nextAccountId) hop.permit?.release(); diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 57eadf6a28..28a7c9f8c6 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -732,6 +732,8 @@ export async function prepareAdapterExchange( route.providerName, transportState.genericFailoverAccountId, upstreamResponse.headers.get("retry-after"), + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index f5c5b94694..1d5557d4d1 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -1121,6 +1121,8 @@ export async function preparePassthroughExchange( const nextAccountId = rotateGenericOAuthAccountOn429( config, route.providerName, transportState.genericFailoverAccountId, upstreamResponse.headers.get("retry-after"), + Date.now(), + route.modelId, ); let snapshot: OAuthAccessSnapshot | undefined; if (nextAccountId) { diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index c80f242121..ba913bb2c7 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -431,7 +431,7 @@ export async function prepareResponsesTransport( // measured as spent. A null answer means "use the active account", so every provider // without quota evidence keeps the resolution it has today. const preferredAccountId = isGenericFailoverProvider(route.providerName, route.provider) - ? preferredInitialAccount(config, route.providerName) + ? preferredInitialAccount(config, route.providerName, Date.now(), route.modelId) : null; // Resolved account-scoped, NOT through failoverAccountSnapshot: that helper marks a // rotation site, and rotation sites must apply their credential through diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index 24e96802fb..9e2de70158 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -214,6 +214,8 @@ export async function executeResponsesRunTurn( route.providerName, transportState.genericFailoverAccountId, null, + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 7987d5ca93..9075d85a4f 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -184,6 +184,8 @@ export async function executeResponsesSidecars( route.providerName, transportState.genericFailoverAccountId, retryAfter, + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d2eb5d244b..e647184832 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -780,6 +780,7 @@ "nvidia-nim-hardening.test.ts": "providers", "oauth-account-attribution.test.ts": "oauth", "oauth-account-id-collision.test.ts": "oauth", + "oauth-account-quota-rank.test.ts": "oauth", "oauth-accounts-api.test.ts": "oauth", "oauth-callback-binds.test.ts": "oauth", "oauth-callback-server.test.ts": "oauth", diff --git a/tests/oauth/oauth-account-quota-rank.test.ts b/tests/oauth/oauth-account-quota-rank.test.ts new file mode 100644 index 0000000000..f96050d3ca --- /dev/null +++ b/tests/oauth/oauth-account-quota-rank.test.ts @@ -0,0 +1,117 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + classifyModelFamilyForQuota, + hasHeadroomEvidence, + isAccountQuotaExhausted, + rankAccountsByHeadroom, +} from "../../src/oauth/account-quota-rank"; +import { + clearGenericFailoverHealth, + preferredInitialAccount, + rotateGenericOAuthAccountOn429, +} from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../src/providers/quota"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalHome = process.env.OPENCODEX_HOME; +let home: string; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-ag-family-rank-")); + process.env.OPENCODEX_HOME = home; + clearGenericFailoverHealth(); +}); + +afterEach(() => { + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +const PROVIDER = { + adapter: "google", + authMode: "oauth", +} as unknown as OcxProviderConfig; + +function config(): OcxConfig { + return { + providers: { + "google-antigravity": { ...PROVIDER, oauthAccountFailover: { enabled: true } }, + }, + oauthAccountFailover: { enabled: true }, + } as unknown as OcxConfig; +} + +function seedWindows(accountId: string, gem: number, cla: number): void { + setCachedProviderAccountQuotaForTests("google-antigravity", accountId, { + updatedAt: Date.now(), + customWindows: [ + { label: "Gem", percent: gem }, + { label: "Gem (Weekly)", percent: gem }, + { label: "Cla", percent: cla }, + { label: "Cla (Weekly)", percent: cla }, + ], + }); +} + +describe("classifyModelFamilyForQuota", () => { + test("maps Gemini and Claude ids, and ignores Gemma", () => { + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-3.8-flash")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-sonnet-4-5")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "gemma-3-27b")).toBeUndefined(); + expect(classifyModelFamilyForQuota("xai", "gemini-3.8-flash")).toBeUndefined(); + expect(classifyModelFamilyForQuota("google-antigravity", undefined)).toBeUndefined(); + }); +}); + +describe("Antigravity family ranking", () => { + test("does not treat a spent Claude window as Gemini exhaustion", () => { + seedWindows("a", 10, 100); + seedWindows("b", 80, 5); + expect(isAccountQuotaExhausted("google-antigravity", "a", "gemini-3.8-flash")).toBe(false); + expect(isAccountQuotaExhausted("google-antigravity", "a", "claude-sonnet-4-5")).toBe(true); + expect(rankAccountsByHeadroom("google-antigravity", ["a", "b"], "gemini-3.8-flash")[0]).toBe("a"); + expect(rankAccountsByHeadroom("google-antigravity", ["a", "b"], "claude-sonnet-4-5")[0]).toBe("b"); + }); + + test("falls back to the unranked ring when family labels are missing", () => { + setCachedProviderAccountQuotaForTests("google-antigravity", "a", { + updatedAt: Date.now(), + customWindows: [{ label: "Other", percent: 1 }], + }); + setCachedProviderAccountQuotaForTests("google-antigravity", "b", { + updatedAt: Date.now(), + customWindows: [{ label: "Other", percent: 99 }], + }); + expect(hasHeadroomEvidence("google-antigravity", ["a", "b"], "gemini-3.8-flash")).toBe(false); + expect(rankAccountsByHeadroom("google-antigravity", ["b", "a"], "gemini-3.8-flash")).toEqual(["b", "a"]); + }); +}); + +describe("Antigravity family-scoped cooldown", () => { + test("a Claude 429 still keeps the account for Gemini", async () => { + for (const accountId of ["acct-a", "acct-b"]) { + await saveCredential("google-antigravity", { + access: "access-" + accountId, + refresh: "refresh-" + accountId, + expires: Date.now() + 3_600_000, + accountId, + } as never, { addAccount: true }); + } + const ids = getAccountSet("google-antigravity")?.accounts.map((account) => account.id) ?? []; + expect(ids.length).toBe(2); + await setActiveAccount("google-antigravity", ids[0]!); + seedWindows(ids[0]!, 10, 100); + seedWindows(ids[1]!, 80, 5); + const cfg = config(); + expect(rotateGenericOAuthAccountOn429(cfg, "google-antigravity", ids[0]!, null, Date.now(), "claude-sonnet-4-5")).toBe(ids[1]); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); + }); +}); From 3defbd0530e8042eaecb3230f6b13dd7aa43cb3f Mon Sep 17 00:00:00 2001 From: agentHits <140916359+agentHits@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:22:13 -0400 Subject: [PATCH 2/4] test(oauth): pin Antigravity family strategies and catalog ids Classify only gemini-* model ids, keep family cooldown off the other family's ring, and pass the request model into round-robin admission. --- src/oauth/account-quota-rank.ts | 4 +- src/oauth/generic-account-failover.ts | 10 +++- src/server/responses/request-transport.ts | 2 +- tests/oauth/oauth-account-quota-rank.test.ts | 57 ++++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/oauth/account-quota-rank.ts b/src/oauth/account-quota-rank.ts index 76fe3c05a4..a86859874f 100644 --- a/src/oauth/account-quota-rank.ts +++ b/src/oauth/account-quota-rank.ts @@ -26,7 +26,9 @@ export function classifyModelFamilyForQuota( const id = modelId.toLowerCase(); // Gemma is not Gemini: a substring/prefix match would poison Gemini ranking. if (/(?:^|[^a-z])gemma(?:[^a-z]|$)/.test(id)) return undefined; - if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])gem(?:[^a-z]|$)/.test(id)) return "gem"; + // Catalog ids are gemini-*, never a bare gem- token. Window labels still match Gem via + // windowMatchesFamily; this classifier is only for request model ids. + if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id)) return "gem"; if ( /(?:^|[^a-z])claude(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])opus(?:[^a-z]|$)/.test(id) diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index b944a6aaf1..b4c912c70d 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -282,11 +282,17 @@ function pickFillFirstGenericAccount( * advance and round-robin would propose the same account forever. This is the same shape * `commitAnthropicSelectionRouting` already commits with. */ -export function noteGenericPoolSelection(config: OcxConfig, providerName: string, accountId: string): void { +export function noteGenericPoolSelection( + config: OcxConfig, + providerName: string, + accountId: string, + requestedModelId?: string | null, +): void { if (activeGenericStrategy(config, providerName) !== "round-robin") return; const poolKey = genericPoolKey(providerName); const limit = genericStickyLimit(config, providerName); - const picked = pickRoundRobinAccount(poolKey, eligibleFailoverAccounts(providerName), limit); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const picked = pickRoundRobinAccount(poolKey, eligibleFailoverAccounts(providerName, Date.now(), family), limit); // The resolver may have admitted a different account than the ring proposed: a removal, a // reauth verdict or a manual selection can land during credential resolution. Realign the // cursor onto what actually served rather than leaving it on a road not taken. diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index ba913bb2c7..cf7aa88516 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -494,7 +494,7 @@ export async function prepareResponsesTransport( // Advance the pool cursor only now that this account is actually admitted. The // helper returns immediately unless the kernel is on AND the strategy is // round-robin, so quota and fill-first pools reach it without being touched. - noteGenericPoolSelection(config, route.providerName, resolved.accountId); + noteGenericPoolSelection(config, route.providerName, resolved.accountId, route.modelId); } // Anthropic is excluded from isGenericFailoverProvider -- its own pool owns affinity and // a fail-closed local-cli credential rule -- so without this stamp its identity is diff --git a/tests/oauth/oauth-account-quota-rank.test.ts b/tests/oauth/oauth-account-quota-rank.test.ts index f96050d3ca..63f91c4d73 100644 --- a/tests/oauth/oauth-account-quota-rank.test.ts +++ b/tests/oauth/oauth-account-quota-rank.test.ts @@ -10,6 +10,8 @@ import { } from "../../src/oauth/account-quota-rank"; import { clearGenericFailoverHealth, + eligibleFailoverAccounts, + genericFailoverRetryAfterSeconds, preferredInitialAccount, rotateGenericOAuthAccountOn429, } from "../../src/oauth/generic-account-failover"; @@ -68,6 +70,11 @@ describe("classifyModelFamilyForQuota", () => { expect(classifyModelFamilyForQuota("google-antigravity", "gemma-3-27b")).toBeUndefined(); expect(classifyModelFamilyForQuota("xai", "gemini-3.8-flash")).toBeUndefined(); expect(classifyModelFamilyForQuota("google-antigravity", undefined)).toBeUndefined(); + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-pro-agent")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-3.1-flash-image")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-sonnet-4-6")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-opus-4-6-thinking")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "gem-experimental")).toBeUndefined(); }); }); @@ -115,3 +122,53 @@ describe("Antigravity family-scoped cooldown", () => { expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); }); }); + +function kernelConfig(strategy: "fill-first" | "round-robin"): OcxConfig { + return { + pool: { kernel: true }, + providers: { + "google-antigravity": { + ...PROVIDER, + oauthAccountFailover: { enabled: true, strategy, autoSwitchThreshold: 80 }, + }, + }, + oauthAccountFailover: { enabled: true }, + } as unknown as OcxConfig; +} + +async function seedPair(): Promise { + for (const accountId of ["acct-a", "acct-b"]) { + await saveCredential("google-antigravity", { + access: "access-" + accountId, + refresh: "refresh-" + accountId, + expires: Date.now() + 3_600_000, + accountId, + } as never, { addAccount: true }); + } + const ids = getAccountSet("google-antigravity")?.accounts.map((account) => account.id) ?? []; + expect(ids.length).toBe(2); + await setActiveAccount("google-antigravity", ids[0]!); + return ids; +} + +describe("Antigravity family strategies behind pool.kernel", () => { + test("fill-first stays on Gemini headroom when only Claude is over the threshold", async () => { + const ids = await seedPair(); + seedWindows(ids[0]!, 40, 90); + seedWindows(ids[1]!, 10, 10); + const cfg = kernelConfig("fill-first"); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "claude-sonnet-4-6")).toBe(ids[1]); + }); + + test("a Claude 429 does not hide the account from Gemini round-robin", async () => { + const ids = await seedPair(); + seedWindows(ids[0]!, 20, 20); + seedWindows(ids[1]!, 20, 20); + const cfg = kernelConfig("round-robin"); + expect(rotateGenericOAuthAccountOn429(cfg, "google-antigravity", ids[0]!, null, Date.now(), "claude-sonnet-4-6")).toBe(ids[1]); + expect(eligibleFailoverAccounts("google-antigravity", Date.now(), "gem")).toContain(ids[0]); + expect(eligibleFailoverAccounts("google-antigravity", Date.now(), "cla")).not.toContain(ids[0]); + expect(genericFailoverRetryAfterSeconds("google-antigravity")).toBeGreaterThan(0); + }); +}); From 48aaf72635975fc073ddf3ab9103d6187515ecb5 Mon Sep 17 00:00:00 2001 From: agentHits <140916359+agentHits@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:46:50 -0400 Subject: [PATCH 3/4] feat(oauth): rank Antigravity failover by Gemini vs Claude quota family Keep Claude exhaustion from starving Gemini selection, and cool 429s per model family so a Claude limit still leaves the account eligible for Gemini. Co-authored-by: agentHits <140916359+agentHits@users.noreply.github.com> --- scripts/test-layout/layout.json | 1 + src/oauth/account-quota-rank.ts | 84 ++++++++++--- src/oauth/generic-account-failover.ts | 57 +++++---- src/server/responses/adapter-continuation.ts | 2 + src/server/responses/adapter-dispatch.ts | 2 + src/server/responses/passthrough-dispatch.ts | 2 + src/server/responses/request-transport.ts | 2 +- src/server/responses/run-turn-execution.ts | 2 + src/server/responses/sidecar-execution.ts | 2 + tests/fixtures/test-layout-expected.json | 1 + tests/oauth/oauth-account-quota-rank.test.ts | 117 +++++++++++++++++++ 11 files changed, 231 insertions(+), 41 deletions(-) create mode 100644 tests/oauth/oauth-account-quota-rank.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 30bcc28b47..0524810856 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -958,6 +958,7 @@ "nvidia-nim-hardening.test.ts": "providers", "oauth-account-attribution.test.ts": "oauth", "oauth-account-id-collision.test.ts": "oauth", + "oauth-account-quota-rank.test.ts": "oauth", "oauth-accounts-api.test.ts": "oauth", "oauth-callback-binds.test.ts": "oauth", "oauth-callback-server.test.ts": "oauth", diff --git a/src/oauth/account-quota-rank.ts b/src/oauth/account-quota-rank.ts index 0cae484fa5..76fe3c05a4 100644 --- a/src/oauth/account-quota-rank.ts +++ b/src/oauth/account-quota-rank.ts @@ -13,6 +13,35 @@ import { getCachedProviderAccountQuota, hasPassiveAccountQuota } from "../providers/quota"; import { getKiroAccountExhaustion } from "../providers/kiro-usage"; +/** Antigravity hosts Gemini and Claude windows on one account; ranking must not mix them. */ +export type QuotaModelFamily = "gem" | "cla"; + +export function classifyModelFamilyForQuota( + provider: string, + modelId?: string | null, +): QuotaModelFamily | undefined { + if (provider !== "google-antigravity" || typeof modelId !== "string" || !modelId.trim()) { + return undefined; + } + const id = modelId.toLowerCase(); + // Gemma is not Gemini: a substring/prefix match would poison Gemini ranking. + if (/(?:^|[^a-z])gemma(?:[^a-z]|$)/.test(id)) return undefined; + if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])gem(?:[^a-z]|$)/.test(id)) return "gem"; + if ( + /(?:^|[^a-z])claude(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])opus(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])sonnet(?:[^a-z]|$)/.test(id) + || /(?:^|[^a-z])haiku(?:[^a-z]|$)/.test(id) + ) return "cla"; + return undefined; +} + +function windowMatchesFamily(label: string, family: QuotaModelFamily): boolean { + const token = label.trim().split(/[\s(/]+/)[0] ?? ""; + if (family === "gem") return /^gem(?:ini)?$/i.test(token); + return /^cla(?:ude)?$/i.test(token); +} + /** Lower sorts earlier. Unknown sits between measured-healthy and measured-empty. */ const RANK_HEALTHY = 0; const RANK_UNKNOWN = 1; @@ -48,15 +77,24 @@ const PASSIVE_HEADROOM_MAX_AGE_MS = 60 * 60_000; /** * Remaining headroom across every window the provider reports. * - * The minimum wins: an account at 5% of its five-hour window is unusable right now even if - * its monthly allowance is barely touched. - */ -function headroomOf(provider: string, accountId: string): number | null { +* The minimum wins: an account at 5% of its five-hour window is unusable right now even if +* its monthly allowance is barely touched. +*/ +function headroomOf(provider: string, accountId: string, requestedModelId?: string | null): number | null { const quota = getCachedProviderAccountQuota(provider, accountId); if (!quota) return null; // Null, not a low rank: this must reproduce "no evidence" so a stale roster degrades to // the unranked ring rather than to a differently wrong answer. if (hasPassiveAccountQuota(provider) && Date.now() - quota.updatedAt > PASSIVE_HEADROOM_MAX_AGE_MS) return null; + const family = classifyModelFamilyForQuota(provider, requestedModelId); + if (family) { + const percents = (quota.customWindows ?? []) + .filter(window => windowMatchesFamily(window.label, family)) + .map(window => window.percent) + .filter((value): value is number => typeof value === "number"); + if (percents.length === 0) return null; + return 100 - Math.max(...percents); + } const percents = [ quota.fiveHourPercent, quota.weeklyPercent, @@ -74,15 +112,23 @@ function headroomOf(provider: string, accountId: string): number | null { * than an ordering. Null stays null all the way out: a caller must decide what "unmeasured" * means for its own rule instead of being handed a fabricated 0 or 100. */ -export function accountHeadroomPercent(provider: string, accountId: string): number | null { - return headroomOf(provider, accountId); +export function accountHeadroomPercent( + provider: string, + accountId: string, + requestedModelId?: string | null, +): number | null { + return headroomOf(provider, accountId, requestedModelId); } /** Unknown usage is not exhaustion; Kiro's explicit overage verdict is authoritative. */ -export function isAccountQuotaExhausted(provider: string, accountId: string): boolean { +export function isAccountQuotaExhausted( + provider: string, + accountId: string, + requestedModelId?: string | null, +): boolean { const exhaustion = provider === "kiro" ? getKiroAccountExhaustion(`${provider}\u0000${accountId}`) : null; if (exhaustion !== null) return exhaustion.exhausted; - const headroom = headroomOf(provider, accountId); + const headroom = headroomOf(provider, accountId, requestedModelId); return headroom !== null && headroom <= 0; } @@ -92,24 +138,28 @@ export function isAccountQuotaExhausted(provider: string, accountId: string): bo * Returns the input untouched when no candidate has quota evidence, which keeps every * provider without per-account quota on exactly the behaviour it has today. */ -export function rankAccountsByHeadroom(provider: string, ring: readonly string[]): string[] { +export function rankAccountsByHeadroom( + provider: string, + ring: readonly string[], + requestedModelId?: string | null, +): string[] { if (ring.length < 2) return [...ring]; let sawEvidence = false; // Same rule as hasHeadroomEvidence: a passive provider's partial roster must not rank // at all. The failover path calls this directly (selectFailoverAccount), so the guard // cannot live only in the pre-dispatch predicate. - if (hasPassiveAccountQuota(provider) && !ring.every(id => headroomOf(provider, id) !== null)) { + if (hasPassiveAccountQuota(provider) && !ring.every(id => headroomOf(provider, id, requestedModelId) !== null)) { return [...ring]; } const ranked: Ranked[] = ring.map((id, index) => { // A provider-declared exhaustion verdict outranks the percentage: an account may sit at // 100% and still be servable when overage is enabled, and the verdict knows that. const exhaustion = provider === "kiro" ? getKiroAccountExhaustion(`${provider}\u0000${id}`) : null; - const headroom = headroomOf(provider, id); + const headroom = headroomOf(provider, id, requestedModelId); if (exhaustion !== null || headroom !== null) sawEvidence = true; - if (isAccountQuotaExhausted(provider, id)) return { id, bucket: RANK_EXHAUSTED, headroom: 0, index }; + if (isAccountQuotaExhausted(provider, id, requestedModelId)) return { id, bucket: RANK_EXHAUSTED, headroom: 0, index }; if (headroom === null) return { id, bucket: RANK_UNKNOWN, headroom: 0, index }; return { id, bucket: RANK_HEALTHY, headroom, index }; }); @@ -129,7 +179,11 @@ export function rankAccountsByHeadroom(provider: string, ring: readonly string[] * told "ranked" when nothing was measured. Pre-dispatch selection asks this first so it * can decline to act on a roster it knows nothing about. */ -export function hasHeadroomEvidence(provider: string, ids: readonly string[]): boolean { +export function hasHeadroomEvidence( + provider: string, + ids: readonly string[], + requestedModelId?: string | null, +): boolean { // A PASSIVE provider needs evidence for EVERY candidate, not any one of them. // // A probe fills the whole roster in one pass (fetchProviderAccountQuotas), so "any" @@ -140,10 +194,10 @@ export function hasHeadroomEvidence(provider: string, ids: readonly string[]): b // AWAY from an unmeasured account and TOWARD the one account known to be spent, which // is the exact inversion of what ranking is for. if (hasPassiveAccountQuota(provider)) { - return ids.length > 0 && ids.every(id => headroomOf(provider, id) !== null); + return ids.length > 0 && ids.every(id => headroomOf(provider, id, requestedModelId) !== null); } return ids.some(id => - headroomOf(provider, id) !== null + headroomOf(provider, id, requestedModelId) !== null || (provider === "kiro" && getKiroAccountExhaustion(`${provider}\u0000${id}`) !== null)); } /** diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index 6b444ea435..b944a6aaf1 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -22,6 +22,8 @@ import { hasHeadroomEvidence, isAccountQuotaExhausted, rankAccountsByHeadroom, + classifyModelFamilyForQuota, + type QuotaModelFamily, } from "./account-quota-rank"; import { genericPoolKey, @@ -81,13 +83,14 @@ const health = new Map(); /** Provider -> recent eligible-account count. TTL-bounded; never holds credential material. */ const presence = new Map(); -const healthKey = (provider: string, accountId: string) => `${provider}\u0000${accountId}`; +const healthKey = (provider: string, accountId: string, family?: QuotaModelFamily) => + family ? `${provider}\u0000${accountId}\u0000${family}` : `${provider}\u0000${accountId}`; -function isCooled(provider: string, accountId: string, now: number): boolean { - const entry = health.get(healthKey(provider, accountId)); +function isCooled(provider: string, accountId: string, now: number, family?: QuotaModelFamily): boolean { + const entry = health.get(healthKey(provider, accountId, family)); if (!entry) return false; if (entry.cooldownUntil <= now) { - health.delete(healthKey(provider, accountId)); + health.delete(healthKey(provider, accountId, family)); return false; } return true; @@ -175,11 +178,11 @@ function isProactivePreferenceEnabled(config: OcxConfig, providerName: string, n } /** Accounts that may serve traffic right now: not cooled, not flagged for reauth. */ -export function eligibleFailoverAccounts(providerName: string, now = Date.now()): string[] { +export function eligibleFailoverAccounts(providerName: string, now = Date.now(), family?: QuotaModelFamily): string[] { const set = getAccountSet(providerName); if (!set) return []; return set.accounts - .filter(account => account.needsReauth !== true && !isCooled(providerName, account.id, now)) + .filter(account => account.needsReauth !== true && !isCooled(providerName, account.id, now, family)) .map(account => account.id); } @@ -230,8 +233,8 @@ function stableGenericRoster(providerName: string): string[] { * statement about observed usage, and treating "no observation" as "spent" would evacuate every * quota-less provider off its active account on the very first request. */ -function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number): boolean { - const headroom = accountHeadroomPercent(providerName, accountId); +function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number, requestedModelId?: string | null): boolean { + const headroom = accountHeadroomPercent(providerName, accountId, requestedModelId); if (headroom === null) return false; return 100 - headroom >= threshold; } @@ -245,15 +248,17 @@ function pickFillFirstGenericAccount( providerName: string, activeId: string | undefined, now: number, + requestedModelId?: string | null, ): string | null { const stableAll = stableGenericRoster(providerName); if (stableAll.length < 2) return null; - const eligible = new Set(eligibleFailoverAccounts(providerName, now)); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const eligible = new Set(eligibleFailoverAccounts(providerName, now, family)); const stored = config.providers?.[providerName]?.oauthAccountFailover?.autoSwitchThreshold; const threshold = typeof stored === "number" && Number.isInteger(stored) && stored >= 0 && stored <= 100 ? stored : DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD; - if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold)) { + if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold, requestedModelId)) { return null; } const start = activeId ? stableAll.indexOf(activeId) : -1; @@ -302,6 +307,7 @@ export function rotateGenericOAuthAccountOn429( failedAccountId: string, retryAfterHeader: string | null | undefined, now = Date.now(), + requestedModelId?: string | null, ): string | null { if (!isGenericOAuthFailoverEnabled(config, providerName)) return null; const set = getAccountSet(providerName); @@ -314,13 +320,14 @@ export function rotateGenericOAuthAccountOn429( // A Retry-After from upstream still wins — it is the server's own instruction. const exhausted = parsed === undefined ? exhaustedCooldownMs(providerName, failedAccountId, now) : null; const cooldownMs = exhausted ?? Math.min(parsed ?? DEFAULT_COOLDOWN_MS, MAX_COOLDOWN_MS); - health.set(healthKey(providerName, failedAccountId), { + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + health.set(healthKey(providerName, failedAccountId, family), { cooldownUntil: now + cooldownMs, cooldownSource: parsed ? "retry-after" : "default", }); sweepExpiredOnWrite(now); - const eligible = eligibleFailoverAccounts(providerName, now).filter(id => id !== failedAccountId); + const eligible = eligibleFailoverAccounts(providerName, now, family).filter(id => id !== failedAccountId); if (eligible.length === 0) return null; // A rotation means the roster in use just changed; do not answer the next activation question // from a count read before the failure. @@ -359,7 +366,7 @@ export function rotateGenericOAuthAccountOn429( } // With no quota evidence this returns the ring untouched, so providers without // per-account quota keep exactly the traversal they have today. - return rankAccountsByHeadroom(providerName, candidates)[0] ?? null; + return rankAccountsByHeadroom(providerName, candidates, requestedModelId)[0] ?? null; } /** @@ -392,6 +399,7 @@ export function preferredInitialAccount( config: OcxConfig, providerName: string, now = Date.now(), + requestedModelId?: string | null, ): string | null { // The PROACTIVE predicate, not the reactive one: this steers a request upstream has not // refused, so `oauthAccountFailover.enabled: false` must still be able to refuse it. @@ -411,7 +419,8 @@ export function preferredInitialAccount( // would never reach its own test. Cooldowns and reauth are still honoured inside each pick. const strategy = activeGenericStrategy(config, providerName); if (strategy === "round-robin") { - const eligibleNow = eligibleFailoverAccounts(providerName, now); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const eligibleNow = eligibleFailoverAccounts(providerName, now, family); if (eligibleNow.length === 0) return null; // PEEK, not pick: this proposal is discardable, and advancing the ring for an account the // resolver then rejects would skip a turn for nothing. noteGenericPoolSelection commits. @@ -423,26 +432,26 @@ export function preferredInitialAccount( return picked && picked !== active ? picked : null; } if (strategy === "fill-first") { - const picked = pickFillFirstGenericAccount(config, providerName, active, now); + const picked = pickFillFirstGenericAccount(config, providerName, active, now, requestedModelId); return picked && picked !== active ? picked : null; } const activeRow = selected.accounts.find(account => account.id === active); if (activeRow && activeRow.needsReauth !== true - && !isCooled(providerName, activeRow.id, now) - && !isAccountQuotaExhausted(providerName, activeRow.id)) return null; + && !isCooled(providerName, activeRow.id, now, classifyModelFamilyForQuota(providerName, requestedModelId)) + && !isAccountQuotaExhausted(providerName, activeRow.id, requestedModelId)) return null; // Evidence is required BEFORE eligibility narrows the field. Without this, a provider // with no quota data at all could still be redirected: cool the active account with a // 429 and the eligible list collapses to one candidate, which any ranking returns // unchanged — an answer that looks ranked but was never measured. The no-op guarantee // for quota-less providers has to be checked on the full roster. - if (!hasHeadroomEvidence(providerName, order)) return null; + if (!hasHeadroomEvidence(providerName, order, requestedModelId)) return null; // Cooldowns are respected here, unlike in the presence count: this picks the account to // send to right now, and one inside its 429 window is the single candidate we hold // positive evidence against. - const eligible = order.filter(id => !isCooled(providerName, id, now)); + const eligible = order.filter(id => !isCooled(providerName, id, now, classifyModelFamilyForQuota(providerName, requestedModelId))); if (eligible.length === 0) return null; // Start the ring at the active account so an unranked outcome reproduces today's choice. @@ -451,7 +460,7 @@ export function preferredInitialAccount( const candidates = ring.filter(id => eligible.includes(id)); if (candidates.length === 0) return null; - const best = rankAccountsByHeadroom(providerName, candidates)[0] ?? null; + const best = rankAccountsByHeadroom(providerName, candidates, requestedModelId)[0] ?? null; // Nothing to do when the ranking agrees with the account we would have used anyway. // // A proposal still needs guarded selection commit after credential resolution: a @@ -461,12 +470,10 @@ export function preferredInitialAccount( /** Earliest remaining cooldown, for a client-facing Retry-After when every account is cooled. */ export function genericFailoverRetryAfterSeconds(providerName: string, now = Date.now()): number | null { - const set = getAccountSet(providerName); - if (!set) return null; + const prefix = `${providerName}\u0000`; let earliest: number | null = null; - for (const account of set.accounts) { - const entry = health.get(healthKey(providerName, account.id)); - if (!entry || entry.cooldownUntil <= now) continue; + for (const [key, entry] of health) { + if (!key.startsWith(prefix) || entry.cooldownUntil <= now) continue; if (earliest === null || entry.cooldownUntil < earliest) earliest = entry.cooldownUntil; } return earliest === null ? null : Math.max(1, Math.ceil((earliest - now) / 1000)); diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 6d4ffcfb43..bc50107dc6 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -410,6 +410,8 @@ export function createAdapterContinuations( route.providerName, transportState.genericFailoverAccountId, response.headers.get("retry-after"), + Date.now(), + route.modelId, ) : null; if (!nextAccountId) hop.permit?.release(); diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 2863105d7a..42e87efe54 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -767,6 +767,8 @@ export async function prepareAdapterExchange( route.providerName, transportState.genericFailoverAccountId, upstreamResponse.headers.get("retry-after"), + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 65eb3512da..cfff2a045d 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -1133,6 +1133,8 @@ export async function preparePassthroughExchange( const nextAccountId = rotateGenericOAuthAccountOn429( config, route.providerName, transportState.genericFailoverAccountId, upstreamResponse.headers.get("retry-after"), + Date.now(), + route.modelId, ); let snapshot: OAuthAccessSnapshot | undefined; if (nextAccountId) { diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index c5b92a177d..faef746036 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -487,7 +487,7 @@ export async function prepareResponsesTransport( // measured as spent. A null answer means "use the active account", so every provider // without quota evidence keeps the resolution it has today. const preferredAccountId = isGenericFailoverProvider(route.providerName, route.provider) - ? preferredInitialAccount(config, route.providerName) + ? preferredInitialAccount(config, route.providerName, Date.now(), route.modelId) : null; // Resolved account-scoped, NOT through failoverAccountSnapshot: that helper marks a // rotation site, and rotation sites must apply their credential through diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index 20524edd3a..501c13f702 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -237,6 +237,8 @@ export async function executeResponsesRunTurn( route.providerName, transportState.genericFailoverAccountId, null, + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 7aeb9d452d..de1156b222 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -186,6 +186,8 @@ export async function executeResponsesSidecars( route.providerName, transportState.genericFailoverAccountId, retryAfter, + Date.now(), + route.modelId, ); if (!nextAccountId) { hop.permit?.release(); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 5553d5a7e1..b77e3cbf6e 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -786,6 +786,7 @@ "nvidia-nim-hardening.test.ts": "providers", "oauth-account-attribution.test.ts": "oauth", "oauth-account-id-collision.test.ts": "oauth", + "oauth-account-quota-rank.test.ts": "oauth", "oauth-accounts-api.test.ts": "oauth", "oauth-callback-binds.test.ts": "oauth", "oauth-callback-server.test.ts": "oauth", diff --git a/tests/oauth/oauth-account-quota-rank.test.ts b/tests/oauth/oauth-account-quota-rank.test.ts new file mode 100644 index 0000000000..f96050d3ca --- /dev/null +++ b/tests/oauth/oauth-account-quota-rank.test.ts @@ -0,0 +1,117 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + classifyModelFamilyForQuota, + hasHeadroomEvidence, + isAccountQuotaExhausted, + rankAccountsByHeadroom, +} from "../../src/oauth/account-quota-rank"; +import { + clearGenericFailoverHealth, + preferredInitialAccount, + rotateGenericOAuthAccountOn429, +} from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../src/providers/quota"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalHome = process.env.OPENCODEX_HOME; +let home: string; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-ag-family-rank-")); + process.env.OPENCODEX_HOME = home; + clearGenericFailoverHealth(); +}); + +afterEach(() => { + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +const PROVIDER = { + adapter: "google", + authMode: "oauth", +} as unknown as OcxProviderConfig; + +function config(): OcxConfig { + return { + providers: { + "google-antigravity": { ...PROVIDER, oauthAccountFailover: { enabled: true } }, + }, + oauthAccountFailover: { enabled: true }, + } as unknown as OcxConfig; +} + +function seedWindows(accountId: string, gem: number, cla: number): void { + setCachedProviderAccountQuotaForTests("google-antigravity", accountId, { + updatedAt: Date.now(), + customWindows: [ + { label: "Gem", percent: gem }, + { label: "Gem (Weekly)", percent: gem }, + { label: "Cla", percent: cla }, + { label: "Cla (Weekly)", percent: cla }, + ], + }); +} + +describe("classifyModelFamilyForQuota", () => { + test("maps Gemini and Claude ids, and ignores Gemma", () => { + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-3.8-flash")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-sonnet-4-5")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "gemma-3-27b")).toBeUndefined(); + expect(classifyModelFamilyForQuota("xai", "gemini-3.8-flash")).toBeUndefined(); + expect(classifyModelFamilyForQuota("google-antigravity", undefined)).toBeUndefined(); + }); +}); + +describe("Antigravity family ranking", () => { + test("does not treat a spent Claude window as Gemini exhaustion", () => { + seedWindows("a", 10, 100); + seedWindows("b", 80, 5); + expect(isAccountQuotaExhausted("google-antigravity", "a", "gemini-3.8-flash")).toBe(false); + expect(isAccountQuotaExhausted("google-antigravity", "a", "claude-sonnet-4-5")).toBe(true); + expect(rankAccountsByHeadroom("google-antigravity", ["a", "b"], "gemini-3.8-flash")[0]).toBe("a"); + expect(rankAccountsByHeadroom("google-antigravity", ["a", "b"], "claude-sonnet-4-5")[0]).toBe("b"); + }); + + test("falls back to the unranked ring when family labels are missing", () => { + setCachedProviderAccountQuotaForTests("google-antigravity", "a", { + updatedAt: Date.now(), + customWindows: [{ label: "Other", percent: 1 }], + }); + setCachedProviderAccountQuotaForTests("google-antigravity", "b", { + updatedAt: Date.now(), + customWindows: [{ label: "Other", percent: 99 }], + }); + expect(hasHeadroomEvidence("google-antigravity", ["a", "b"], "gemini-3.8-flash")).toBe(false); + expect(rankAccountsByHeadroom("google-antigravity", ["b", "a"], "gemini-3.8-flash")).toEqual(["b", "a"]); + }); +}); + +describe("Antigravity family-scoped cooldown", () => { + test("a Claude 429 still keeps the account for Gemini", async () => { + for (const accountId of ["acct-a", "acct-b"]) { + await saveCredential("google-antigravity", { + access: "access-" + accountId, + refresh: "refresh-" + accountId, + expires: Date.now() + 3_600_000, + accountId, + } as never, { addAccount: true }); + } + const ids = getAccountSet("google-antigravity")?.accounts.map((account) => account.id) ?? []; + expect(ids.length).toBe(2); + await setActiveAccount("google-antigravity", ids[0]!); + seedWindows(ids[0]!, 10, 100); + seedWindows(ids[1]!, 80, 5); + const cfg = config(); + expect(rotateGenericOAuthAccountOn429(cfg, "google-antigravity", ids[0]!, null, Date.now(), "claude-sonnet-4-5")).toBe(ids[1]); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); + }); +}); From 815624408f3b265e0ef458535fe914f578dfccf2 Mon Sep 17 00:00:00 2001 From: agentHits <140916359+agentHits@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:22:13 -0400 Subject: [PATCH 4/4] test(oauth): pin Antigravity family strategies and catalog ids Classify only gemini-* model ids, keep family cooldown off the other family's ring, and pass the request model into round-robin admission. Co-authored-by: agentHits <140916359+agentHits@users.noreply.github.com> --- src/oauth/account-quota-rank.ts | 4 +- src/oauth/generic-account-failover.ts | 10 +++- src/server/responses/request-transport.ts | 2 +- tests/oauth/oauth-account-quota-rank.test.ts | 57 ++++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/oauth/account-quota-rank.ts b/src/oauth/account-quota-rank.ts index 76fe3c05a4..a86859874f 100644 --- a/src/oauth/account-quota-rank.ts +++ b/src/oauth/account-quota-rank.ts @@ -26,7 +26,9 @@ export function classifyModelFamilyForQuota( const id = modelId.toLowerCase(); // Gemma is not Gemini: a substring/prefix match would poison Gemini ranking. if (/(?:^|[^a-z])gemma(?:[^a-z]|$)/.test(id)) return undefined; - if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])gem(?:[^a-z]|$)/.test(id)) return "gem"; + // Catalog ids are gemini-*, never a bare gem- token. Window labels still match Gem via + // windowMatchesFamily; this classifier is only for request model ids. + if (/(?:^|[^a-z])gemini(?:[^a-z]|$)/.test(id)) return "gem"; if ( /(?:^|[^a-z])claude(?:[^a-z]|$)/.test(id) || /(?:^|[^a-z])opus(?:[^a-z]|$)/.test(id) diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index b944a6aaf1..b4c912c70d 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -282,11 +282,17 @@ function pickFillFirstGenericAccount( * advance and round-robin would propose the same account forever. This is the same shape * `commitAnthropicSelectionRouting` already commits with. */ -export function noteGenericPoolSelection(config: OcxConfig, providerName: string, accountId: string): void { +export function noteGenericPoolSelection( + config: OcxConfig, + providerName: string, + accountId: string, + requestedModelId?: string | null, +): void { if (activeGenericStrategy(config, providerName) !== "round-robin") return; const poolKey = genericPoolKey(providerName); const limit = genericStickyLimit(config, providerName); - const picked = pickRoundRobinAccount(poolKey, eligibleFailoverAccounts(providerName), limit); + const family = classifyModelFamilyForQuota(providerName, requestedModelId); + const picked = pickRoundRobinAccount(poolKey, eligibleFailoverAccounts(providerName, Date.now(), family), limit); // The resolver may have admitted a different account than the ring proposed: a removal, a // reauth verdict or a manual selection can land during credential resolution. Realign the // cursor onto what actually served rather than leaving it on a road not taken. diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index faef746036..3e878959e2 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -550,7 +550,7 @@ export async function prepareResponsesTransport( // Advance the pool cursor only now that this account is actually admitted. The // helper returns immediately unless the kernel is on AND the strategy is // round-robin, so quota and fill-first pools reach it without being touched. - noteGenericPoolSelection(config, route.providerName, resolved.accountId); + noteGenericPoolSelection(config, route.providerName, resolved.accountId, route.modelId); } // Anthropic is excluded from isGenericFailoverProvider -- its own pool owns affinity and // a fail-closed local-cli credential rule -- so without this stamp its identity is diff --git a/tests/oauth/oauth-account-quota-rank.test.ts b/tests/oauth/oauth-account-quota-rank.test.ts index f96050d3ca..63f91c4d73 100644 --- a/tests/oauth/oauth-account-quota-rank.test.ts +++ b/tests/oauth/oauth-account-quota-rank.test.ts @@ -10,6 +10,8 @@ import { } from "../../src/oauth/account-quota-rank"; import { clearGenericFailoverHealth, + eligibleFailoverAccounts, + genericFailoverRetryAfterSeconds, preferredInitialAccount, rotateGenericOAuthAccountOn429, } from "../../src/oauth/generic-account-failover"; @@ -68,6 +70,11 @@ describe("classifyModelFamilyForQuota", () => { expect(classifyModelFamilyForQuota("google-antigravity", "gemma-3-27b")).toBeUndefined(); expect(classifyModelFamilyForQuota("xai", "gemini-3.8-flash")).toBeUndefined(); expect(classifyModelFamilyForQuota("google-antigravity", undefined)).toBeUndefined(); + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-pro-agent")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "gemini-3.1-flash-image")).toBe("gem"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-sonnet-4-6")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "claude-opus-4-6-thinking")).toBe("cla"); + expect(classifyModelFamilyForQuota("google-antigravity", "gem-experimental")).toBeUndefined(); }); }); @@ -115,3 +122,53 @@ describe("Antigravity family-scoped cooldown", () => { expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); }); }); + +function kernelConfig(strategy: "fill-first" | "round-robin"): OcxConfig { + return { + pool: { kernel: true }, + providers: { + "google-antigravity": { + ...PROVIDER, + oauthAccountFailover: { enabled: true, strategy, autoSwitchThreshold: 80 }, + }, + }, + oauthAccountFailover: { enabled: true }, + } as unknown as OcxConfig; +} + +async function seedPair(): Promise { + for (const accountId of ["acct-a", "acct-b"]) { + await saveCredential("google-antigravity", { + access: "access-" + accountId, + refresh: "refresh-" + accountId, + expires: Date.now() + 3_600_000, + accountId, + } as never, { addAccount: true }); + } + const ids = getAccountSet("google-antigravity")?.accounts.map((account) => account.id) ?? []; + expect(ids.length).toBe(2); + await setActiveAccount("google-antigravity", ids[0]!); + return ids; +} + +describe("Antigravity family strategies behind pool.kernel", () => { + test("fill-first stays on Gemini headroom when only Claude is over the threshold", async () => { + const ids = await seedPair(); + seedWindows(ids[0]!, 40, 90); + seedWindows(ids[1]!, 10, 10); + const cfg = kernelConfig("fill-first"); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "gemini-3.8-flash")).toBeNull(); + expect(preferredInitialAccount(cfg, "google-antigravity", Date.now(), "claude-sonnet-4-6")).toBe(ids[1]); + }); + + test("a Claude 429 does not hide the account from Gemini round-robin", async () => { + const ids = await seedPair(); + seedWindows(ids[0]!, 20, 20); + seedWindows(ids[1]!, 20, 20); + const cfg = kernelConfig("round-robin"); + expect(rotateGenericOAuthAccountOn429(cfg, "google-antigravity", ids[0]!, null, Date.now(), "claude-sonnet-4-6")).toBe(ids[1]); + expect(eligibleFailoverAccounts("google-antigravity", Date.now(), "gem")).toContain(ids[0]); + expect(eligibleFailoverAccounts("google-antigravity", Date.now(), "cla")).not.toContain(ids[0]); + expect(genericFailoverRetryAfterSeconds("google-antigravity")).toBeGreaterThan(0); + }); +});