From 5de7d9227bd680c265affaa794d728473361c5b5 Mon Sep 17 00:00:00 2001 From: Veritas-7 <234569343+Veritas-7@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:03:29 +0900 Subject: [PATCH] fix(combos): shorten request-rate cooldowns and send Retry-After on combo 503 Request-rate 429s without Retry-After no longer inherit the 60s quota cooldown. Combo unavailable responses advertise the soonest remaining target cooldown. --- src/combos/failover.ts | 71 +++++++++++++++++++++++++++++++++++- src/combos/index.ts | 4 ++ src/combos/resolve.ts | 3 ++ src/server/responses/core.ts | 32 ++++++++++++---- tests/combos.test.ts | 53 +++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 9 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 2dc8462ba2..84d30f7713 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -13,6 +13,21 @@ interface TargetCooldown { const DEFAULT_COOLDOWN_MS = 60_000; const MAX_COOLDOWN_MS = 10 * 60_000; +/** Short cooldown for request-rate 429s (for example provider code 1302) that omit Retry-After. */ +export const COMBO_REQUEST_RATE_COOLDOWN_MS = 5_000; + +const QUOTA_LIMIT_CODES = new Set([ + "1308", + "1310", + "1316", + "1317", + "1318", + "1319", + "1320", + "1321", + "insufficient_quota", +]); +const TRANSIENT_REQUEST_RATE_CODES = new Set(["1302", "1305"]); const IMF_FIXDATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/i; const RFC850_DATE_RE = /^(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/i; const ASCTIME_DATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ( \d|\d{2}) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/i; @@ -136,10 +151,58 @@ export function isComboTargetInCooldown( return true; } +export function isTransientRequestRateLimit(input: { + status?: number; + code?: string | null; + message?: string; +}): boolean { + const code = (input.code ?? "").trim().toLowerCase().replaceAll("-", "_"); + if (QUOTA_LIMIT_CODES.has(code)) return false; + if (TRANSIENT_REQUEST_RATE_CODES.has(code)) return true; + const text = (input.message ?? "").toLowerCase(); + if ( + text.includes("usage limit reached") + || text.includes("insufficient_quota") + || text.includes("quota exhausted") + ) { + return false; + } + return text.includes("rate limit reached for requests"); +} + +export function remainingComboCooldownMs(comboId: string, now = Date.now()): number | undefined { + const prefix = `${comboId}\0`; + let soonest: number | undefined; + for (const [key, cooldown] of targetCooldowns) { + if (!key.startsWith(prefix)) continue; + const remaining = cooldown.cooldownUntil - now; + if (remaining <= 0) { + targetCooldowns.delete(key); + continue; + } + if (soonest === undefined || remaining < soonest) soonest = remaining; + } + return soonest; +} + +export function comboCooldownRetryAfterSeconds(comboId: string, now = Date.now()): string | undefined { + const remainingMs = remainingComboCooldownMs(comboId, now); + if (remainingMs === undefined) return undefined; + return String(Math.max(1, Math.ceil(remainingMs / 1000))); +} + export function coolComboTarget( comboId: string, target: Pick, - options?: { retryAfter?: string | null; now?: number; cooldownMs?: number; writerGeneration?: number }, + options?: { + retryAfter?: string | null; + now?: number; + cooldownMs?: number; + writerGeneration?: number; + status?: number; + code?: string | null; + message?: string; + }, ): void { const now = options?.now ?? Date.now(); const writerGeneration = options?.writerGeneration ?? captureConfigGeneration(); @@ -147,7 +210,11 @@ export function coolComboTarget( if (writerGeneration < lastReconciledGeneration && !liveComboTargets.has(ownerKey)) return; const cooldownMs = options?.cooldownMs ?? parseRetryAfterMs(options?.retryAfter, now) - ?? DEFAULT_COOLDOWN_MS; + ?? (isTransientRequestRateLimit({ + status: options?.status, + code: options?.code, + message: options?.message, + }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS); targetCooldowns.set(cooldownMapKey(comboId, target), { cooldownUntil: now + Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS), }); diff --git a/src/combos/index.ts b/src/combos/index.ts index 502e210dc6..74efee0f79 100644 --- a/src/combos/index.ts +++ b/src/combos/index.ts @@ -31,9 +31,13 @@ export { } from "./resolve"; export { clearComboTargetCooldowns, + comboCooldownRetryAfterSeconds, + COMBO_REQUEST_RATE_COOLDOWN_MS, coolComboTarget, isComboTargetInCooldown, + isTransientRequestRateLimit, parseRetryAfterMs, + remainingComboCooldownMs, comboFailureDecision, type ComboFailureDecision, } from "./failover"; diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 4dc5cc0298..7896873454 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -250,6 +250,9 @@ export function advanceComboAfterFailure( retryAfter?: string | null; now?: number; eligible?: (target: Required) => boolean; + status?: number; + code?: string | null; + message?: string; } = {}, ): ComboPick | null { noteComboFailure(pick.comboId, pick.target, pick.writerGeneration); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 24c6d4021d..d439d559a6 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -74,6 +74,7 @@ import { concreteComboRequestBody, getCombo, isComboTargetInCooldown, + comboCooldownRetryAfterSeconds, NoAvailableComboTargetsError, noteComboSuccess, parseRetryAfterMs, @@ -1435,15 +1436,29 @@ export function decodeRequestErrorResponse(err: unknown, label: string): Respons -export function comboUnavailableResponse(message: string): Response { +export function comboUnavailableResponse( + message: string, + options?: { retryAfter?: string | null }, +): Response { + const headers = new Headers({ "Content-Type": "application/json" }); + const retryAfter = options?.retryAfter?.trim(); + if (retryAfter && retryAfter.length > 0 && retryAfter.length <= 128) { + headers.set("Retry-After", retryAfter); + } return new Response( JSON.stringify({ error: { message, type: "server_error", code: "combo_unavailable" }, }), - { status: 503, headers: { "Content-Type": "application/json" } }, + { status: 503, headers }, ); } +function comboUnavailable(comboId: string, now = Date.now()): Response { + return comboUnavailableResponse(`No available targets for combo: ${comboId}`, { + retryAfter: comboCooldownRetryAfterSeconds(comboId, now), + }); +} + export interface ConsumedComboFailure { @@ -2298,7 +2313,7 @@ export async function handleComboResponses( config, { parentThreadId: inboundClientThreadId }, ); - return comboUnavailableResponse(`No available targets for combo: ${comboId}`); + return comboUnavailable(comboId); } let recovered = false; try { @@ -2335,7 +2350,7 @@ export async function handleComboResponses( } if (!pick) { - return comboUnavailableResponse(`No available targets for combo: ${comboId}`); + return comboUnavailable(comboId); } // One immutable combo selection trace, before any child dispatch; child // adoption below must never replace it with a concrete child route trace. @@ -2543,6 +2558,9 @@ export async function handleComboResponses( retryAfter: failure.retryAfter, now: Date.now(), eligible: payloadEligible, + status: failure.response.status, + code: failure.upstreamCode, + message: failure.classificationText, }); if (!nextPick) adoptFailedChildLog(childLog); pick = nextPick; @@ -2913,7 +2931,7 @@ async function handleResponsesInner( logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailableResponse(err.message); + return comboUnavailable(err.comboId); } if (err instanceof NoEligiblePolicyCandidateError) { // Persist the evaluation trace (per-candidate exclusions + the @@ -3023,7 +3041,7 @@ async function handleResponsesInner( logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailableResponse(err.message); + return comboUnavailable(err.comboId); } if (err instanceof NoEligiblePolicyCandidateError) { logCtx.routeDecision = err.trace; @@ -3147,7 +3165,7 @@ async function handleResponsesInner( logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailableResponse(err.message); + return comboUnavailable(err.comboId); } if (err instanceof NoEligiblePolicyCandidateError) { logCtx.routeDecision = err.trace; diff --git a/tests/combos.test.ts b/tests/combos.test.ts index 83a1f6d1df..a8ed3f1580 100644 --- a/tests/combos.test.ts +++ b/tests/combos.test.ts @@ -17,6 +17,8 @@ import { comboPublicModelId, comboRequestHasImageInput, concreteComboRequestBody, + comboCooldownRetryAfterSeconds, + COMBO_REQUEST_RATE_COOLDOWN_MS, coolComboTarget, earliestQuotaResetAt, getCombo, @@ -29,6 +31,7 @@ import { normalizeComboConfig, parseComboModelId, parseRetryAfterMs, + remainingComboCooldownMs, pickComboTarget, preservesPhysicalComboProvider, resetComboEffortWarningStateForTests, @@ -38,6 +41,7 @@ import { UnknownComboError, } from "../src/combos"; import { comboFailureDecision } from "../src/combos/failover"; +import { comboUnavailableResponse } from "../src/server/responses/core"; import { getConfigPath, readConfigDiagnostics, saveConfig } from "../src/config"; import { routeModel } from "../src/router"; import { handleManagementAPI } from "../src/server/management-api"; @@ -414,6 +418,55 @@ describe("combo target cooldowns", () => { clearComboTargetCooldowns("other"); expect(isComboTargetInCooldown("other", target, 1_050)).toBe(false); }); + + test("uses a short cooldown for request-rate 1302 without Retry-After", () => { + coolComboTarget("free", target, { + now: 1_000, + code: "1302", + message: "Rate limit reached for requests", + }); + expect(isComboTargetInCooldown("free", target, 1_000 + COMBO_REQUEST_RATE_COOLDOWN_MS - 1)).toBe(true); + expect(isComboTargetInCooldown("free", target, 1_000 + COMBO_REQUEST_RATE_COOLDOWN_MS)).toBe(false); + }); + + test("keeps the default cooldown for usage-window 1308", () => { + coolComboTarget("free", target, { + now: 1_000, + code: "1308", + message: "Usage limit reached for 5 hour", + }); + expect(isComboTargetInCooldown("free", target, 1_000 + 59_999)).toBe(true); + expect(isComboTargetInCooldown("free", target, 1_000 + 60_000)).toBe(false); + }); + + test("honors explicit Retry-After over the request-rate default", () => { + coolComboTarget("free", target, { + now: 1_000, + retryAfter: "30", + code: "1302", + }); + expect(isComboTargetInCooldown("free", target, 1_000 + 29_999)).toBe(true); + expect(isComboTargetInCooldown("free", target, 1_000 + 30_000)).toBe(false); + }); + + test("reports the soonest remaining cooldown as Retry-After seconds", () => { + const later = { provider: "b", model: "m2" }; + coolComboTarget("free", target, { now: 1_000, cooldownMs: 5_000 }); + coolComboTarget("free", later, { now: 1_000, cooldownMs: 20_000 }); + expect(remainingComboCooldownMs("free", 1_000)).toBe(5_000); + expect(comboCooldownRetryAfterSeconds("free", 1_000)).toBe("5"); + expect(comboCooldownRetryAfterSeconds("free", 3_500)).toBe("3"); + expect(comboCooldownRetryAfterSeconds("missing", 1_000)).toBeUndefined(); + }); + + test("combo unavailable responses advertise remaining cooldown as Retry-After", () => { + coolComboTarget("free", target, { now: 1_000, cooldownMs: 5_000 }); + const response = comboUnavailableResponse("No available targets for combo: free", { + retryAfter: comboCooldownRetryAfterSeconds("free", 1_000), + }); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("5"); + }); }); describe("combo failure policy and advancement", () => {