From 324716774f217927fccd1d6f6700bac72e6fdc19 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 28 Sep 2026 11:41:02 -0400 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=94=81=20fix:=20Retry=20Code=20API=20?= =?UTF-8?q?Rate=20Limits=20That=20Reject=20Before=20a=20Workspace=20Call?= =?UTF-8?q?=20Starts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/api/src/code/workspace.spec.ts | 104 ++++++++++++++++++++++++ packages/api/src/code/workspace.ts | 50 +++++++++--- 2 files changed, 143 insertions(+), 11 deletions(-) diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index 149f5bd4daa..185bb7cb287 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -338,6 +338,106 @@ describe('workspace admission feedback', () => { expect(fetchImpl).toHaveBeenCalledTimes(1); }); + test('retries a Code API rate limit, which rejects before the operation starts', async () => { + const rateLimited = () => + new Response( + JSON.stringify({ + error: 'rate_limited', + message: 'Too many CodeAPI execution requests. Please retry in 1 second.', + retry_after_seconds: 1, + }), + { status: 429, headers: { 'Retry-After': '0' } }, + ); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce(rateLimited()) + .mockResolvedValueOnce(rateLimited()) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + replacements: 1, + bytesWritten: 24, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }), + ).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 }); + + expect(fetchImpl).toHaveBeenCalledTimes(3); + const bodies = fetchImpl.mock.calls.map((call) => call[1]?.body); + expect(new Set(bodies).size).toBe(1); + }); + + test('reports a rate limit as not started once the retry horizon is spent', async () => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), { + status: 429, + headers: { 'Retry-After': '1' }, + }), + ); + + const failure = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 0, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'echo test', + }, + }); + + await expect(failure).rejects.toThrow('The operation was not started'); + await expect(failure).rejects.toMatchObject({ upstreamStatus: 429 }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test('never retries a 429 that does not carry the typed rate-limit body', async () => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response('Too Many Requests', { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ); + + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'src/app.ts', + edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }], + }, + }), + ).rejects.toMatchObject({ upstreamStatus: 429 }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + test('never retries an ambiguous capacity-looking failure', async () => { const fetchImpl = jest.fn().mockResolvedValue( new Response('Gateway unavailable', { @@ -369,6 +469,10 @@ describe('workspace admission feedback', () => { [503, 'Gateway unavailable', false], [503, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', true], [503, 'null', false], + [503, '{"error":"rate_limited"}', false], + [429, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', false], + [429, '{"error":"rate_limited"}', true], + [429, 'Too Many Requests', false], ] as const)( 'does not infer non-execution from an ambiguous response', (status, body, truncated) => { diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index c39d4002f90..957d401a7ad 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -274,13 +274,18 @@ export class WorkspaceToolHttpError extends Error { public readonly upstreamBodyTruncated = false, ) { let message = `Workspace tool request ${reason}`; - if ( - reason === 'rejected' && - isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated) - ) { + const admissionRejection = + reason === 'rejected' + ? getWorkspaceAdmissionRejection(upstreamStatus, upstreamBody, upstreamBodyTruncated) + : undefined; + if (admissionRejection === 'queue_timeout') { message = 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.'; } + if (admissionRejection === 'rate_limited') { + message = + 'The Code API request rate limit was reached. The operation was not started. Wait a few seconds before retrying, or make fewer concurrent workspace calls.'; + } if (reason === 'insufficient_time') { message = 'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.'; @@ -295,15 +300,38 @@ export class WorkspaceToolHttpError extends Error { } } -function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated = false): boolean { - if (status !== 503 || !body || truncated || body.length > MAX_ERROR_BODY_BYTES) { - return false; +type WorkspaceAdmissionRejection = 'queue_timeout' | 'rate_limited'; + +/** + * Rejections the Code API issues before an operation is assigned, so retrying cannot repeat a + * mutation: a queue deadline that expired before admission, or its rate limiter, which runs as + * middleware ahead of the workspace router. Anything else, including an unparsable body under + * either status, keeps an unknown outcome and is never retried. + */ +function getWorkspaceAdmissionRejection( + status?: number, + body?: string, + truncated = false, +): WorkspaceAdmissionRejection | undefined { + if ( + (status !== 503 && status !== 429) || + !body || + truncated || + body.length > MAX_ERROR_BODY_BYTES + ) { + return undefined; } try { - const parsed: { code?: string } | null = JSON.parse(body); - return parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT'; + const parsed: { code?: string; error?: string } | null = JSON.parse(body); + if (status === 503 && parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT') { + return 'queue_timeout'; + } + if (status === 429 && parsed?.error === 'rate_limited') { + return 'rate_limited'; + } + return undefined; } catch { - return false; + return undefined; } } @@ -904,7 +932,7 @@ export async function executeWorkspaceTool({ signal?.throwIfAborted(); const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated); if ( - !isWorkspaceAdmissionTimeout(response.status, body, truncated) || + getWorkspaceAdmissionRejection(response.status, body, truncated) == null || Date.now() >= queueDeadlineAt ) { throw rejection; From b1565d86240f7b7c9fd49a45ec481e66f7f13a5e Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 16:06:14 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=81=20fix:=20Bound=20Workspace=20R?= =?UTF-8?q?ate=20Limit=20Retries=20by=20Code=20API=20Budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/server/services/Files/Code/process.js | 6 + .../services/Files/Code/process.spec.js | 26 +++ api/server/services/ToolService.js | 1 + .../services/__tests__/ToolService.spec.js | 2 + packages/api/src/code/command.spec.ts | 18 +++ packages/api/src/code/command.ts | 3 + packages/api/src/code/workspace.spec.ts | 148 ++++++++++++++++++ packages/api/src/code/workspace.ts | 48 ++++-- 8 files changed, 242 insertions(+), 10 deletions(-) diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 6371b1c40fe..a5c2bbdfe97 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -1094,6 +1094,7 @@ async function readWorkspaceFile({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ @@ -1147,6 +1148,7 @@ async function searchWorkspace({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ @@ -1200,6 +1202,7 @@ async function listWorkspaceFiles({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ @@ -1239,6 +1242,7 @@ async function writeWorkspaceFile({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ @@ -1278,6 +1282,7 @@ async function editWorkspaceFile({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ @@ -1316,6 +1321,7 @@ async function previewWorkspaceEdit({ return executeWorkspaceTool({ baseURL: codeApiBaseUrl, maxQueueWaitMs, + codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs, deadlineAtMs, /** Minted per admission attempt: a queued call outlives one token TTL. */ diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 875368196fd..65175294c02 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -2033,6 +2033,28 @@ describe('Code Process', () => { }); describe('readWorkspaceFile', () => { + it('forwards the configured rate-limit budget to the workspace transport', async () => { + const req = { + ...mockReq, + config: { ...mockReq.config, endpoints: { agents: { codeApiMaxRetryWaitMs: 0 } } }, + }; + mockExecuteWorkspaceTool.mockResolvedValueOnce({ operation: 'read_file' }); + + await readWorkspaceFile({ + file_path: 'src/app.ts', + workspace_id: 'primary', + start_line: 1, + max_lines: 10, + codeApiBaseUrl: 'https://attached-code.example.com/v1', + executionProfile: 'stateful', + req, + }); + + expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith( + expect.objectContaining({ codeApiMaxRetryWaitMs: 0 }), + ); + }); + it('forwards authenticated reads to the selected attached worker', async () => { const controller = new AbortController(); const result = { @@ -2083,6 +2105,7 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: undefined, maxRequestTimeoutMs: 125_000, deadlineAtMs: 160_000, request: { @@ -2144,6 +2167,7 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: undefined, maxRequestTimeoutMs: undefined, deadlineAtMs: undefined, request: { @@ -2204,6 +2228,7 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: undefined, maxRequestTimeoutMs: undefined, deadlineAtMs: undefined, request: { @@ -2266,6 +2291,7 @@ describe('Code Process', () => { baseURL: 'https://attached-code.example.com/v1', authHeaders: expect.any(Function), maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: undefined, maxRequestTimeoutMs: undefined, deadlineAtMs: undefined, request: { diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index f215d60cda3..41a79b7ba32 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -2315,6 +2315,7 @@ async function loadToolsForExecution({ maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs( codeExecutionContext.codeEnvironmentConfigSchema, ), + codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs( codeExecutionContext.codeEnvironmentConfigSchema, ), diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index dd388f232dc..d385a5d1bb2 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -2930,6 +2930,7 @@ describe('ToolService - Action Capability Gating', () => { AgentCapabilities.stateful_code_sessions, ]; const req = createMockReq(capabilities); + req.config.endpoints[EModelEndpoint.agents].codeApiMaxRetryWaitMs = 0; req.body = { codeWorkspaces: [{ environmentId: 'personal-machine', workspaceId: 'project-a' }], }; @@ -2970,6 +2971,7 @@ describe('ToolService - Action Capability Gating', () => { gitIdentity: { name: 'LibreChat Agent', email: 'agent@example.com' }, maxTimeoutMs: 120000, maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: 0, }); expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith( expect.objectContaining({ requestedSelections: req.body.codeWorkspaces }), diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts index 4ba032bbde9..b4cc19e0c95 100644 --- a/packages/api/src/code/command.spec.ts +++ b/packages/api/src/code/command.spec.ts @@ -164,6 +164,24 @@ function commandResponse(overrides: Record = {}): Response { } describe('createAttachedWorkspaceBashTool', () => { + test('honors a zero Code API rate-limit retry budget', async () => { + const fetchImpl: CodeBridgeFetch = jest.fn( + async () => new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }), + ); + const bashTool = createAttachedWorkspaceBashTool({ + baseUrl: 'https://code.example.com/v1', + authHeaders: () => ({}), + workspaceId: 'project-a', + codeApiMaxRetryWaitMs: 0, + fetchImpl, + }); + + await expect(bashTool.invoke({ command: 'pwd' })).rejects.toThrow( + 'The operation was not started', + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + test('dispatches commands to the resolved conversation workspace instance', async () => { const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse()); const workspaceInstanceId = 'e'.repeat(64); diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts index 2ec67f9885b..da191a5ec9b 100644 --- a/packages/api/src/code/command.ts +++ b/packages/api/src/code/command.ts @@ -302,6 +302,7 @@ export function createAttachedWorkspaceBashTool({ gitIdentity, maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, maxQueueWaitMs, + codeApiMaxRetryWaitMs, maxRequestTimeoutMs, fetchImpl, }: { @@ -315,6 +316,7 @@ export function createAttachedWorkspaceBashTool({ maxTimeoutMs?: number; /** Retry horizon across typed queue expirations, not an admission budget. */ maxQueueWaitMs?: number; + codeApiMaxRetryWaitMs?: number; /** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */ maxRequestTimeoutMs?: number; fetchImpl?: CodeBridgeFetch; @@ -395,6 +397,7 @@ export function createAttachedWorkspaceBashTool({ signal, fetchImpl, ...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }), + ...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }), ...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }), }); if (result.operation !== 'execute_command') { diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index 185bb7cb287..e5ef2f3f6fa 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -386,6 +386,154 @@ describe('workspace admission feedback', () => { expect(new Set(bodies).size).toBe(1); }); + test.each([undefined, 'invalid'])( + 'uses the typed 429 body delay when Retry-After is %s', + async (retryAfter) => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), { + status: 429, + ...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}), + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + codeApiMaxRetryWaitMs: 3_000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'x', + }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(1_999); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await jest.advanceTimersByTimeAsync(1); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }, + ); + + test.each([0, 150])('bounds typed 429 retries to the %i ms Code API budget', async (budget) => { + jest.useFakeTimers(); + try { + const fetchImpl = jest.fn().mockImplementation( + () => + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + codeApiMaxRetryWaitMs: budget, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(budget); + await expect(result).resolves.toMatchObject({ + upstreamStatus: 429, + message: expect.stringContaining('The operation was not started'), + }); + expect(fetchImpl).toHaveBeenCalledTimes(budget === 0 ? 1 : 2); + } finally { + jest.useRealTimers(); + } + }); + + test('does not wait when a 429 retry hint exceeds the Code API budget', async () => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 30 }), { + status: 429, + }), + ); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }), + ).rejects.toMatchObject({ upstreamStatus: 429 }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + test('keeps queue retries enabled when Code API rate-limit retries are disabled', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }), + ); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + codeApiMaxRetryWaitMs: 0, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toMatchObject({ upstreamStatus: 429 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('does not charge queue waits to the Code API rate-limit budget', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + codeApiMaxRetryWaitMs: 100, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(200); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(3); + } finally { + jest.useRealTimers(); + } + }); + test('reports a rate limit as not started once the retry horizon is spent', async () => { const fetchImpl = jest.fn().mockResolvedValue( new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), { diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 957d401a7ad..034172deb22 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -4,6 +4,7 @@ import { CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS, } from 'librechat-data-provider'; import type { CodeBridgeFetch } from './bridge'; +import { MAX_CODE_API_RATE_LIMIT_WAIT_MS } from '~/utils/code'; const WORKSPACE_TOOL_TIMEOUT_MS = 30_000; const MAX_PATH_LENGTH = 4096; @@ -354,9 +355,22 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi }); } -function workspaceAdmissionRetryDelay(value: string | null): number { - if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS; - return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); +function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number { + if (value != null && /^\d+$/.test(value)) { + return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); + } + if (rateLimitBody) { + try { + const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody); + const seconds = parsed?.retry_after_seconds; + if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) { + return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); + } + } catch { + // Fall back when the delay hint is unusable. + } + } + return WORKSPACE_QUEUE_RETRY_DELAY_MS; } /** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */ @@ -829,6 +843,7 @@ export async function executeWorkspaceTool({ signal, fetchImpl = fetch, maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS, + codeApiMaxRetryWaitMs = MAX_CODE_API_RATE_LIMIT_WAIT_MS, maxRequestTimeoutMs, deadlineAtMs, }: { @@ -838,6 +853,8 @@ export async function executeWorkspaceTool({ signal?: AbortSignal; fetchImpl?: CodeBridgeFetch; maxQueueWaitMs?: number; + /** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */ + codeApiMaxRetryWaitMs?: number; /** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */ maxRequestTimeoutMs?: number; /** Optional earlier caller deadline; a signal alone has no remaining-time value. */ @@ -848,6 +865,8 @@ export async function executeWorkspaceTool({ !Number.isSafeInteger(maxQueueWaitMs) || maxQueueWaitMs < 0 || maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS || + !Number.isSafeInteger(codeApiMaxRetryWaitMs) || + codeApiMaxRetryWaitMs < 0 || (maxRequestTimeoutMs !== undefined && (!Number.isSafeInteger(maxRequestTimeoutMs) || maxRequestTimeoutMs < 1 || @@ -867,6 +886,7 @@ export async function executeWorkspaceTool({ const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs); const body = JSON.stringify(request); let lastAdmissionRejection: WorkspaceToolHttpError | undefined; + let rateLimitWaitedMs = 0; while (true) { try { signal?.throwIfAborted(); @@ -931,20 +951,28 @@ export async function executeWorkspaceTool({ const { body, truncated } = await readErrorBody(response, requestSignal); signal?.throwIfAborted(); const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated); + const admission = getWorkspaceAdmissionRejection(response.status, body, truncated); + if (admission == null || Date.now() >= queueDeadlineAt) throw rejection; + const retryAfterMs = workspaceAdmissionRetryDelay( + response.headers.get('Retry-After'), + admission === 'rate_limited' ? body : undefined, + ); if ( - getWorkspaceAdmissionRejection(response.status, body, truncated) == null || - Date.now() >= queueDeadlineAt + admission === 'rate_limited' && + retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs ) { throw rejection; } - const delayMs = Math.min( - workspaceAdmissionRetryDelay(response.headers.get('Retry-After')), - retryDeadlineAt - Date.now(), - ); + const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now()); lastAdmissionRejection = rejection; if (delayMs <= 0) throw rejection; + const waitStartedAt = Date.now(); await waitForWorkspaceAdmission(delayMs, signal); - /** A clamped delay can land exactly on either deadline. Never open + if (admission === 'rate_limited') { + rateLimitWaitedMs += Date.now() - waitStartedAt; + if (rateLimitWaitedMs > codeApiMaxRetryWaitMs) throw rejection; + } + /** A clamped delay can land exactly on the queue or HTTP deadline. Never open * another admission window without the full execution reserve. */ if (Date.now() >= retryDeadlineAt) { throw rejection; From 9c406fdfb1c3434ffa515d1de7bf4a3da50ad8f3 Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 16:10:35 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=94=97=20fix:=20Isolate=20Code=20API?= =?UTF-8?q?=20Retry=20Default=20from=20Utility=20Graph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/api/src/code/limits.ts | 1 + packages/api/src/code/workspace.ts | 4 ++-- packages/api/src/utils/code.ts | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 packages/api/src/code/limits.ts diff --git a/packages/api/src/code/limits.ts b/packages/api/src/code/limits.ts new file mode 100644 index 00000000000..43646744827 --- /dev/null +++ b/packages/api/src/code/limits.ts @@ -0,0 +1 @@ +export const CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS: number = 20_000; diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 034172deb22..7cbe3476248 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -4,7 +4,7 @@ import { CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS, } from 'librechat-data-provider'; import type { CodeBridgeFetch } from './bridge'; -import { MAX_CODE_API_RATE_LIMIT_WAIT_MS } from '~/utils/code'; +import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from './limits'; const WORKSPACE_TOOL_TIMEOUT_MS = 30_000; const MAX_PATH_LENGTH = 4096; @@ -843,7 +843,7 @@ export async function executeWorkspaceTool({ signal, fetchImpl = fetch, maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS, - codeApiMaxRetryWaitMs = MAX_CODE_API_RATE_LIMIT_WAIT_MS, + codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS, maxRequestTimeoutMs, deadlineAtMs, }: { diff --git a/packages/api/src/utils/code.ts b/packages/api/src/utils/code.ts index 344c1d07ace..cc97220bc77 100644 --- a/packages/api/src/utils/code.ts +++ b/packages/api/src/utils/code.ts @@ -3,6 +3,7 @@ import https from 'https'; import { isAxiosError } from 'axios'; import { setTimeout as delay } from 'node:timers/promises'; import type { ServerRequest } from '~/types'; +import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from '~/code/limits'; /** * Dedicated agents for code-server requests, preventing socket pool contamination. @@ -44,7 +45,7 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null { /** Total time one operation may spend waiting out Code API rate limits. * Keeps recovery bounded inside a live chat turn. */ -export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = 20_000; +export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS; export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3; export interface CodeApiUploadRegistry { From de32233b99d9fcdf1ca9c03597c8967b52a6e17d Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 16:12:26 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=A7=A9=20fix:=20Type=20the=20Shared?= =?UTF-8?q?=20Code=20API=20Retry=20Default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/api/src/utils/code.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/utils/code.ts b/packages/api/src/utils/code.ts index cc97220bc77..974fa4b2af7 100644 --- a/packages/api/src/utils/code.ts +++ b/packages/api/src/utils/code.ts @@ -45,7 +45,7 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null { /** Total time one operation may spend waiting out Code API rate limits. * Keeps recovery bounded inside a live chat turn. */ -export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS; +export const MAX_CODE_API_RATE_LIMIT_WAIT_MS: number = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS; export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3; export interface CodeApiUploadRegistry { From c19a73114cdb21d9f24c6712cd56d24716275d4a Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 17:27:42 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=94=81=20fix:=20Keep=20Workspace=20Ad?= =?UTF-8?q?mission=20Retry=20Budgets=20Independent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/server/services/ToolService.js | 2 + .../services/__tests__/ToolService.spec.js | 37 ++ packages/api/src/agents/handlers.spec.ts | 12 +- packages/api/src/agents/handlers.ts | 13 +- packages/api/src/code/instructions.spec.ts | 66 ++++ packages/api/src/code/instructions.ts | 9 +- packages/api/src/code/workspace.spec.ts | 316 ++++++++++++++++-- packages/api/src/code/workspace.ts | 26 +- 8 files changed, 425 insertions(+), 56 deletions(-) diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 41a79b7ba32..b047447ec16 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1578,6 +1578,7 @@ async function loadToolDefinitionsWrapper({ enabled: codeExecutionEnabled, context: resolvedCodeExecutionContext, principalId: JSON.stringify([getTenantId(), req.user.id]), + codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, getAuthHeaders: (workerId) => getCodeApiAuthHeaders(req, workerId), }), }; @@ -1781,6 +1782,7 @@ async function loadAgentTools({ enabled: codeExecutionEnabled, context: codeExecutionContext, principalId: JSON.stringify([getTenantId(), req.user.id]), + codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs, getAuthHeaders: (workerId) => getCodeApiAuthHeaders(req, workerId), }); const { loadedTools, toolContextMap, dynamicToolContextMap, primedCodeFiles } = await loadTools({ diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index d385a5d1bb2..9c4d57eefda 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -713,6 +713,43 @@ describe('ToolService - Action Capability Gating', () => { }); }); + it.each([true, false])( + 'passes the Code API retry limit to repository instructions (definitionsOnly=%s)', + async (definitionsOnly) => { + const capabilities = [ + AgentCapabilities.tools, + AgentCapabilities.execute_code, + AgentCapabilities.stateful_code_sessions, + ]; + const req = createMockReq(capabilities); + req.config.endpoints[EModelEndpoint.agents].codeApiMaxRetryWaitMs = 0; + mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); + mockResolveCodeExecutionContext.mockReturnValueOnce({ + baseUrl: 'https://attached-code.example.com/v1', + codeSessionKey: 'attached-session', + executionProfile: 'stateful', + statefulSessions: true, + environmentType: 'attached', + environmentId: 'personal-machine', + }); + + const result = await loadAgentTools({ + req, + res: {}, + agent: { + id: 'attached-agent', + tools: [Tools.execute_code], + stateful_code_sessions: true, + }, + definitionsOnly, + }); + + expect(result.repositoryInstructionSource).toEqual( + expect.objectContaining({ codeApiMaxRetryWaitMs: 0 }), + ); + }, + ); + describe('isActionTool — cross-delimiter collision guard', () => { it('should identify real action tools', () => { expect(isActionTool(`get_weather${actionDelimiter}api_example_com`)).toBe(true); diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index afb806d0f82..367fb4b7a6e 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -5843,11 +5843,11 @@ describe('createToolExecuteHandler', () => { it.each([ { budget: 1200, elapsed: 0, remaining: 1200 }, { budget: 1200, elapsed: 400, remaining: 800 }, - { budget: 1200, elapsed: 1200, remaining: null }, - { budget: 1200, elapsed: 1500, remaining: null }, + { budget: 1200, elapsed: 1200, remaining: 0 }, + { budget: 1200, elapsed: 1500, remaining: 0 }, { budget: 0, elapsed: 400, remaining: 0 }, ])( - 'shares a protected edit retry horizon ($budget ms, preview $elapsed ms)', + 'shares a protected edit capacity retry horizon ($budget ms, preview $elapsed ms)', async ({ budget, elapsed, remaining }) => { const startedAt = Date.now(); let nowMs = startedAt; @@ -5931,12 +5931,6 @@ describe('createToolExecuteHandler', () => { deadlineAtMs: startedAt + 125_000, }), ); - if (remaining == null) { - expect(result.status).toBe('error'); - expect(result.errorMessage).toContain('The file was not modified'); - expect(editWorkspaceFile).not.toHaveBeenCalled(); - return; - } expect(result.status).toBe('success'); expect(editWorkspaceFile).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 7eb99e4e4dc..a239557eb96 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -4111,17 +4111,10 @@ async function handleAttachedWorkspaceEditFileCall({ const filteredContent = filteredFileResult(tc, req, path.filePath, preview.content); if (filteredContent != null) return filteredContent; expectedBaseSha256 = preview.baseSha256; - /** Zero disables retries, not the two operations required for a protected - * edit. Positive horizons must not restart after a successful preview. */ + /** A spent queue horizon disables capacity retries, not the required + * hash-guarded edit attempt or its independent rate-limit recovery. */ if (workspaceParams.maxQueueWaitMs > 0) { - const remainingMs = queueDeadlineAt - Date.now(); - if (remainingMs <= 0) { - return errorResult( - tc, - 'The workspace retry budget expired after preview. The file was not modified.', - ); - } - workspaceParams.maxQueueWaitMs = remainingMs; + workspaceParams.maxQueueWaitMs = Math.max(0, queueDeadlineAt - Date.now()); } } const result = await options.editWorkspaceFile({ diff --git a/packages/api/src/code/instructions.spec.ts b/packages/api/src/code/instructions.spec.ts index f5b92a0e5e9..e6ad0f7e5e9 100644 --- a/packages/api/src/code/instructions.spec.ts +++ b/packages/api/src/code/instructions.spec.ts @@ -37,6 +37,72 @@ const response = () => ); describe('repository instruction loading', () => { + it('does not retry a typed 429 when the configured Code API wait budget is zero', async () => { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(response()); + + const result = await createRepositoryInstructionLoader()({ + enabled: true, + context, + principalId: 'alice', + authHeaders: async () => ({}), + assertContent: jest.fn(), + codeApiMaxRetryWaitMs: 0, + fetchImpl, + }); + + expect(result).toBeUndefined(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('refreshes instruction-read authorization for a typed 429 retry', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(response()); + let minted = 0; + const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` })); + const load = createRepositoryInstructionLoader(); + const input = { + enabled: true, + context, + principalId: 'alice', + authHeaders, + fetchImpl, + codeApiMaxRetryWaitMs: 100, + timeoutMs: 500, + assertContent: jest.fn(), + }; + const result = load(input); + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toContain('Use the project test command.'); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(fetchImpl.mock.calls.map((call) => call[1]?.headers.Authorization)).toEqual([ + 'Bearer token-1', + 'Bearer token-2', + ]); + await expect(load(input)).resolves.toContain('Use the project test command.'); + expect(authHeaders).toHaveBeenCalledTimes(3); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + it('bounds optional authorization waits and preserves explicit cancellation', async () => { jest.useFakeTimers(); try { diff --git a/packages/api/src/code/instructions.ts b/packages/api/src/code/instructions.ts index 74db097616a..ce100b2f2d3 100644 --- a/packages/api/src/code/instructions.ts +++ b/packages/api/src/code/instructions.ts @@ -9,6 +9,7 @@ export interface RepositoryInstructionSource { context: CodeExecutionContext; principalId: string; authHeaders: () => Promise>; + codeApiMaxRetryWaitMs?: number; load: ReturnType; } @@ -54,6 +55,7 @@ export function createRepositoryInstructionLoader() { signal, fetchImpl, assertContent, + codeApiMaxRetryWaitMs, timeoutMs = 2000, }: { enabled: boolean; @@ -64,6 +66,7 @@ export function createRepositoryInstructionLoader() { signal?: AbortSignal; fetchImpl?: CodeBridgeFetch; assertContent: (content: string) => void; + codeApiMaxRetryWaitMs?: number; timeoutMs?: number; }): Promise => { const workspace = context.codeWorkspace; @@ -93,13 +96,13 @@ export function createRepositoryInstructionLoader() { const timer = setTimeout(() => deadline.abort(), budget); const readSignal = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal; try { - const headers = await abortable(authHeaders(), readSignal); if (content === undefined) { const result = await executeWorkspaceTool({ baseURL: context.baseUrl, - authHeaders: headers, + authHeaders: () => abortable(authHeaders(), readSignal), signal: readSignal, fetchImpl, + codeApiMaxRetryWaitMs, request: { protocolVersion: 1, operation: 'read_file', @@ -121,6 +124,8 @@ export function createRepositoryInstructionLoader() { content = result.content; if (cache.size >= 64) cache.delete(cache.keys().next().value!); cache.set(key, content); + } else { + await abortable(authHeaders(), readSignal); } } catch { if (signal?.aborted) throw signal.reason; diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index e5ef2f3f6fa..fb48d989333 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -338,6 +338,39 @@ describe('workspace admission feedback', () => { expect(fetchImpl).toHaveBeenCalledTimes(1); }); + test('cancels a typed rate-limit wait without replaying the request', async () => { + jest.useFakeTimers(); + try { + const controller = new AbortController(); + const reason = new DOMException('Stopped', 'AbortError'); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '1' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const request = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + signal: controller.signal, + fetchImpl, + maxQueueWaitMs: 0, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }); + + await jest.advanceTimersByTimeAsync(1); + controller.abort(reason); + await expect(request).rejects.toBe(reason); + await jest.advanceTimersByTimeAsync(1_000); + expect(fetchImpl).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + test('retries a Code API rate limit, which rejects before the operation starts', async () => { const rateLimited = () => new Response( @@ -386,41 +419,64 @@ describe('workspace admission feedback', () => { expect(new Set(bodies).size).toBe(1); }); - test.each([undefined, 'invalid'])( - 'uses the typed 429 body delay when Retry-After is %s', - async (retryAfter) => { - jest.useFakeTimers(); - try { - const fetchImpl = jest - .fn() - .mockResolvedValueOnce( - new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), { - status: 429, - ...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}), - }), - ) - .mockResolvedValueOnce(new Response('{}', { status: 504 })); - const result = executeWorkspaceTool({ + test.each([ + ['missing', undefined], + ['invalid', 'invalid'], + ['overflowing', '9'.repeat(400)], + ] as const)('uses the typed 429 body delay when Retry-After is %s', async (_case, retryAfter) => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), { + status: 429, + ...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}), + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + codeApiMaxRetryWaitMs: 3_000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'x', + }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(1_999); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await jest.advanceTimersByTimeAsync(1); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test.each([-1, 300_001, 1.5, Number.NaN])( + 'rejects an invalid Code API retry limit of %p before transport', + async (codeApiMaxRetryWaitMs) => { + const fetchImpl = jest.fn(); + await expect( + executeWorkspaceTool({ baseURL: 'https://code.example/v1', authHeaders: {}, fetchImpl, - codeApiMaxRetryWaitMs: 3_000, + codeApiMaxRetryWaitMs, request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x', }, - }).catch((error: WorkspaceToolHttpError) => error); - - await jest.advanceTimersByTimeAsync(1_999); - expect(fetchImpl).toHaveBeenCalledTimes(1); - await jest.advanceTimersByTimeAsync(1); - await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); - expect(fetchImpl).toHaveBeenCalledTimes(2); - } finally { - jest.useRealTimers(); - } + }), + ).rejects.toMatchObject({ reason: 'invalid' }); + expect(fetchImpl).not.toHaveBeenCalled(); }, ); @@ -534,7 +590,211 @@ describe('workspace admission feedback', () => { } }); - test('reports a rate limit as not started once the retry horizon is spent', async () => { + test.each([0, 50])( + 'retries typed 429s after a %i ms queue horizon without re-enabling capacity retries', + async (maxQueueWaitMs) => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs, + codeApiMaxRetryWaitMs: 100, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'x', + }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }, + ); + + test('continues 429 recovery after a queue retry but never extends its capacity horizon', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 150, + codeApiMaxRetryWaitMs: 100, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(200); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(3); + } finally { + jest.useRealTimers(); + } + }); + + test('never borrows the rate-limit horizon to retry a later capacity timeout', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), { + status: 503, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 50, + codeApiMaxRetryWaitMs: 200, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toMatchObject({ upstreamStatus: 503 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('retries after an authorized rate-limit wait despite timer jitter', async () => { + jest.useFakeTimers(); + const startedAt = Date.now(); + let currentTime = startedAt; + jest.spyOn(Date, 'now').mockImplementation(() => currentTime); + try { + const fetchImpl = jest + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: 100, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(0); + expect(fetchImpl).toHaveBeenCalledTimes(1); + currentTime += 101; + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toMatchObject({ upstreamStatus: 504 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('charges an overslept 429 wait before considering another retry', async () => { + jest.useFakeTimers(); + const startedAt = Date.now(); + let currentTime = startedAt; + jest.spyOn(Date, 'now').mockImplementation(() => currentTime); + try { + const rateLimited = () => + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }); + const fetchImpl = jest + .fn() + .mockResolvedValueOnce(rateLimited()) + .mockResolvedValueOnce(rateLimited()) + .mockResolvedValueOnce(new Response('{}', { status: 504 })); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: 200, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(0); + currentTime += 150; + await jest.advanceTimersByTimeAsync(100); + await expect(result).resolves.toMatchObject({ upstreamStatus: 429 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('reserves the caller HTTP execution budget when capacity retries are disabled', async () => { + jest.useFakeTimers(); + try { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(JSON.stringify({ error: 'rate_limited' }), { + status: 429, + headers: { 'Retry-After': '0' }, + }), + ); + const result = executeWorkspaceTool({ + baseURL: 'https://code.example/v1', + authHeaders: {}, + fetchImpl, + maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: 1_000, + maxRequestTimeoutMs: 35_050, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' }, + }).catch((error: WorkspaceToolHttpError) => error); + + await jest.advanceTimersByTimeAsync(50); + await expect(result).resolves.toMatchObject({ upstreamStatus: 429 }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + test('reports a rate limit as not started when its retry budget is disabled', async () => { const fetchImpl = jest.fn().mockResolvedValue( new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), { status: 429, @@ -546,7 +806,7 @@ describe('workspace admission feedback', () => { baseURL: 'https://code.example/v1', authHeaders: {}, fetchImpl, - maxQueueWaitMs: 0, + codeApiMaxRetryWaitMs: 0, request: { protocolVersion: 1, operation: 'execute_command', diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 7cbe3476248..7f43a2ea984 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -357,7 +357,10 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number { if (value != null && /^\d+$/.test(value)) { - return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); + const seconds = Number(value); + if (Number.isFinite(seconds)) { + return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS)); + } } if (rateLimitBody) { try { @@ -867,6 +870,7 @@ export async function executeWorkspaceTool({ maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS || !Number.isSafeInteger(codeApiMaxRetryWaitMs) || codeApiMaxRetryWaitMs < 0 || + codeApiMaxRetryWaitMs > 300_000 || (maxRequestTimeoutMs !== undefined && (!Number.isSafeInteger(maxRequestTimeoutMs) || maxRequestTimeoutMs < 1 || @@ -883,9 +887,10 @@ export async function executeWorkspaceTool({ maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs, ); const queueDeadlineAt = Date.now() + maxQueueWaitMs; - const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs); + const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs; const body = JSON.stringify(request); let lastAdmissionRejection: WorkspaceToolHttpError | undefined; + let lastRetryDeadlineAt = Infinity; let rateLimitWaitedMs = 0; while (true) { try { @@ -919,7 +924,7 @@ export async function executeWorkspaceTool({ if (timeoutSignal.aborted) { throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time'); } - if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) { + if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) { throw lastAdmissionRejection; } const remainingMs = Math.min( @@ -952,7 +957,12 @@ export async function executeWorkspaceTool({ signal?.throwIfAborted(); const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated); const admission = getWorkspaceAdmissionRejection(response.status, body, truncated); - if (admission == null || Date.now() >= queueDeadlineAt) throw rejection; + if (admission == null) throw rejection; + const retryDeadlineAt = Math.min( + admission === 'queue_timeout' ? queueDeadlineAt : Infinity, + callerRetryDeadlineAt, + ); + if (Date.now() >= retryDeadlineAt) throw rejection; const retryAfterMs = workspaceAdmissionRetryDelay( response.headers.get('Retry-After'), admission === 'rate_limited' ? body : undefined, @@ -965,14 +975,16 @@ export async function executeWorkspaceTool({ } const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now()); lastAdmissionRejection = rejection; + lastRetryDeadlineAt = retryDeadlineAt; if (delayMs <= 0) throw rejection; const waitStartedAt = Date.now(); await waitForWorkspaceAdmission(delayMs, signal); if (admission === 'rate_limited') { - rateLimitWaitedMs += Date.now() - waitStartedAt; - if (rateLimitWaitedMs > codeApiMaxRetryWaitMs) throw rejection; + /** A timer may run late. Charge the actual wait, but honor the retry that + * was authorized before waiting; another 429 cannot exceed the balance. */ + rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt); } - /** A clamped delay can land exactly on the queue or HTTP deadline. Never open + /** A clamped delay can land exactly on this admission's retry deadline. Never open * another admission window without the full execution reserve. */ if (Date.now() >= retryDeadlineAt) { throw rejection;