diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index a6ee34b5e..afee235ab 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -3182,7 +3182,7 @@ describe('OpenCodeServerHarness', () => { } }); - it('terminates an OpenCode retry loop from a structured 4xx status without a message', async () => { + it('terminates an OpenCode retry loop for an explicit OpenRouter credit limit', async () => { const { client, harness } = createHarness(); const taskEvents: TaskEvent[] = []; const persistedEnvelopes: AcpPersistedEnvelope[] = []; @@ -3227,6 +3227,20 @@ describe('OpenCodeServerHarness', () => { type: 'retry', attempt: 1, statusCode: 402, + message: JSON.stringify({ + name: 'APIError', + data: { + message: + 'This request requires more credits than your account balance.', + statusCode: 402, + isRetryable: false, + responseBody: JSON.stringify({ + error: { + metadata: { limit_source: 'openrouter_credits' }, + }, + }), + }, + }), next: Date.now() + 2_000, }, }, @@ -3248,7 +3262,7 @@ describe('OpenCodeServerHarness', () => { (envelope) => envelope.eventType === ACP_ENVELOPE_EVENT_TYPES.AssistantMessage && String(envelope.payload.text ?? '').includes( - 'Provider request failed with a non-retryable error.', + 'This request requires more credits than your account balance.', ), ), ).toBe(true); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts index 1784025b5..e484dac9b 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts @@ -92,16 +92,19 @@ describe('getOpenCodeProviderErrorRecovery', () => { }); }); - it('classifies HTTP client-error statuses inside JSON strings as terminal', () => { + it('retries an otherwise-unclassified 402 response', () => { const error = JSON.stringify({ error: { code: 402, - message: 'There is not enough credit to run this request.', + message: 'Provider-defined payment response.', }, }); - expect(isOpenCodeTerminalProviderError(error)).toBe(true); - expect(getOpenCodeProviderErrorRecovery(error)).toBeNull(); + expect(isOpenCodeTerminalProviderError(error)).toBe(false); + expect(getOpenCodeProviderErrorRecovery(error)).toMatchObject({ + kind: 'provider_error', + maxRetries: 6, + }); }); it.each([408, 429, 500, 503])( @@ -137,14 +140,20 @@ describe('getOpenCodeProviderErrorRecovery', () => { expect(getOpenCodeProviderErrorRecovery(error)).toBeNull(); }); - it('classifies payment-required responses as terminal', () => { + it('classifies an explicit OpenRouter credit limit as terminal', () => { expect( isOpenCodeTerminalProviderError({ name: 'APIError', data: { - message: 'Payment required', + message: + 'This request requires more credits than your account balance.', statusCode: 402, isRetryable: true, + responseBody: JSON.stringify({ + error: { + metadata: { limit_source: 'openrouter_credits' }, + }, + }), }, }), ).toBe(true); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts index a0fccb4b3..35a052647 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts @@ -33,8 +33,9 @@ const TERMINAL_ERROR_NAMES = new Set(['contextoverflowerror']); const POLICY_ERROR_NAMES = new Set(['contentfiltererror']); // Client errors are terminal because replaying the same request cannot -// succeed, except timeouts (408) and rate limits (429) which are transient. -const RETRYABLE_CLIENT_STATUS_CODES = new Set([408, 429]); +// succeed, except timeouts (408), rate limits (429), and provider-defined 402s +// without structured evidence of a permanent account limit. +const RETRYABLE_CLIENT_STATUS_CODES = new Set([402, 408, 429]); export function collectProviderErrorValues(error: unknown): unknown[] { const pending: Array<{ value: unknown; depth: number }> = [ @@ -46,7 +47,7 @@ export function collectProviderErrorValues(error: unknown): unknown[] { while (pending.length > 0) { const current = pending.shift(); - if (!current || current.depth > 4) { + if (!current || current.depth > 6) { continue; } @@ -130,6 +131,15 @@ function hasErrorName(values: unknown[], names: Set): boolean { }); } +function isOpenRouterCreditsLimitError(values: unknown[]): boolean { + return ( + extractProviderErrorHttpStatus(values) === 402 && + values.some( + (value) => asRecord(value)?.limit_source === 'openrouter_credits', + ) + ); +} + export function isOpenCodeContextOverflowError(error: unknown): boolean { return hasErrorName(collectProviderErrorValues(error), TERMINAL_ERROR_NAMES); } @@ -143,6 +153,10 @@ function isExplicitlyTerminal(values: unknown[]): boolean { return true; } + if (isOpenRouterCreditsLimitError(values)) { + return true; + } + const status = extractProviderErrorHttpStatus(values); return ( diff --git a/packages/cloud-agents/src/server/__tests__/non-task-provider-usage.test.ts b/packages/cloud-agents/src/server/__tests__/non-task-provider-usage.test.ts index af3d59b00..acb4baf0d 100644 --- a/packages/cloud-agents/src/server/__tests__/non-task-provider-usage.test.ts +++ b/packages/cloud-agents/src/server/__tests__/non-task-provider-usage.test.ts @@ -2164,6 +2164,52 @@ describe('resolveOpenCodeSmallModel', () => { ).toMatchObject({ retryable: false }); }); + it('classifies structured OpenRouter credit exhaustion as terminal', async () => { + const { classifyNonTaskInferenceError } = + await import('../non-task-provider-usage.js'); + + expect( + classifyNonTaskInferenceError({ + name: 'APIError', + data: { + message: + 'This request requires more credits than your account balance.', + statusCode: 402, + isRetryable: false, + responseBody: JSON.stringify({ + error: { + metadata: { limit_source: 'openrouter_credits' }, + }, + }), + }, + }), + ).toEqual({ + message: + 'The inference provider account does not have enough credits or quota.', + reason: 'insufficient_credits', + retryable: false, + }); + }); + + it('does not infer credit exhaustion from an arbitrary 402 response', async () => { + const { classifyNonTaskInferenceError } = + await import('../non-task-provider-usage.js'); + + expect( + classifyNonTaskInferenceError({ + name: 'APIError', + data: { + message: 'Payment required: insufficient credits.', + statusCode: 402, + }, + }), + ).toEqual({ + message: 'The inference provider rejected the validation request.', + reason: 'provider_error', + retryable: true, + }); + }); + it('continues observing provider errors when retry reporting fails', async () => { process.env = { ...originalEnv, @@ -2727,8 +2773,8 @@ describe('resolveOpenCodeSmallModel', () => { }, { providerError: '402 payment required: insufficient credits', - reason: 'insufficient_credits', - retryable: false, + reason: 'provider_error', + retryable: true, }, { providerError: '429 too many requests', @@ -2787,8 +2833,8 @@ describe('resolveOpenCodeSmallModel', () => { { providerMessage: 'upgrade your plan', statusCode: 402, - reason: 'insufficient_credits', - retryable: false, + reason: 'provider_error', + retryable: true, }, { providerMessage: 'slow down', diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index 3aefd2a5d..1c458d18d 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -7764,6 +7764,30 @@ describe('answerFastAgentQuestion native OpenCode tools', () => { } }); + it('does not retry non-retryable platform inference failures', async () => { + mocks.generateText.mockRejectedValue({ + name: 'APIError', + data: { + message: + 'This request requires more credits than your account balance.', + statusCode: 402, + isRetryable: false, + responseBody: JSON.stringify({ + error: { metadata: { limit_source: 'openrouter_credits' } }, + }), + }, + }); + + await expect( + answerFastAgentQuestion({ + ...baseParams, + turnSource: 'platform_event', + adapter: callbacks(), + }), + ).rejects.toMatchObject({ name: 'FastAgentInferenceError' }); + expect(mocks.generateText).toHaveBeenCalledOnce(); + }); + it('rethrows native prompt failures for platform event retry', async () => { mocks.generateText.mockRejectedValue(new Error('OpenCode unavailable')); const activity = { diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts index 4bfe84aee..476d1a244 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts @@ -709,6 +709,10 @@ class FastAgentInferenceError extends Error { } } +export function isNonRetryableFastAgentInferenceError(error: unknown): boolean { + return error instanceof FastAgentInferenceError && !error.failure.retryable; +} + function resolveFastAgentInferenceMaxRetries( failure: FastAgentInferenceFailure, ): number { diff --git a/packages/cloud-agents/src/server/non-task-provider-usage.ts b/packages/cloud-agents/src/server/non-task-provider-usage.ts index 14fcc8d30..8036eb54c 100644 --- a/packages/cloud-agents/src/server/non-task-provider-usage.ts +++ b/packages/cloud-agents/src/server/non-task-provider-usage.ts @@ -1919,6 +1919,43 @@ function isInferenceErrorExplicitlyNonRetryable(error: unknown): boolean { return false; } +function isOpenRouterCreditsLimitError( + error: unknown, + statusCode: number | undefined, +): boolean { + if (statusCode !== 402) return false; + + const pending: Array<{ value: unknown; depth: number }> = [ + { value: error, depth: 0 }, + ]; + const seen = new Set(); + + while (pending.length > 0) { + const current = pending.shift(); + if (!current || current.depth > 6) continue; + + const { value, depth } = current; + if (typeof value === 'string') { + try { + pending.push({ value: JSON.parse(value), depth: depth + 1 }); + } catch { + // Only structured OpenRouter metadata identifies this account limit. + } + continue; + } + if (!value || typeof value !== 'object' || seen.has(value)) continue; + + seen.add(value); + const record = value as Record; + if (record.limit_source === 'openrouter_credits') return true; + for (const nested of Object.values(record)) { + pending.push({ value: nested, depth: depth + 1 }); + } + } + + return false; +} + function isContentFilterInferenceError(error: unknown): boolean { const pending: Array<{ value: unknown; depth: number }> = [ { value: error, depth: 0 }, @@ -1985,6 +2022,10 @@ export function classifyNonTaskInferenceError( ? (record.data as Record) : undefined; const statusCode = findInferenceErrorStatusCode(inferenceError); + const openRouterCreditsLimited = isOpenRouterCreditsLimitError( + inferenceError, + statusCode, + ); const responseBody = typeof data?.responseBody === 'string' ? data.responseBody : ''; const detail = @@ -2004,6 +2045,15 @@ export function classifyNonTaskInferenceError( }; } + if (openRouterCreditsLimited) { + return { + message: + 'The inference provider account does not have enough credits or quota.', + reason: 'insufficient_credits', + retryable: false, + }; + } + if (isInferenceErrorExplicitlyNonRetryable(inferenceError)) { return { message: 'The inference provider rejected the request.', @@ -2084,15 +2134,6 @@ export function classifyNonTaskInferenceError( }; } - if (statusCode === 402) { - return { - message: - 'The inference provider account does not have enough credits or quota.', - reason: 'insufficient_credits', - retryable: false, - }; - } - if (statusCode === 404) { return { message: 'The selected model is unavailable with these credentials.', @@ -2123,12 +2164,13 @@ export function classifyNonTaskInferenceError( } if ( - detail.includes('insufficient_quota') || - detail.includes('insufficient quota') || - detail.includes('insufficient credit') || - detail.includes('payment required') || - detail.includes('billing') || - /\b402\b/u.test(detail) + statusCode !== 402 && + !/\b402\b/u.test(detail) && + (detail.includes('insufficient_quota') || + detail.includes('insufficient quota') || + detail.includes('insufficient credit') || + detail.includes('payment required') || + detail.includes('billing')) ) { return { message: @@ -2199,11 +2241,13 @@ export function classifyNonTaskInferenceError( // Remaining structured 4xx responses (400, 413, 422, …) are client errors: // resending the same request cannot recover them. 408 stays retryable as a - // timeout; 401/402/403/404/429 were classified above. + // timeout; provider-defined 402 responses stay retryable unless structured + // metadata or an explicit isRetryable=false signal classified them above. if ( statusCode !== undefined && statusCode >= 400 && statusCode < 500 && + statusCode !== 402 && statusCode !== 408 ) { return { diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event-queue.test.ts b/packages/sdk/src/server/lib/fast-agent-parent-event-queue.test.ts index 3aa002b3e..b46d2b48e 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event-queue.test.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event-queue.test.ts @@ -56,6 +56,11 @@ vi.mock('@roomote/cloud-agents/server', () => ({ error.cause.name === 'FastAgentDurableRetryScheduledError' ? error.cause : null, + isNonRetryableFastAgentInferenceError: (error: unknown) => + error instanceof Error && + error.name === 'FastAgentInferenceError' && + 'failure' in error && + (error.failure as { retryable?: boolean }).retryable === false, })); vi.mock('@roomote/db/server', () => ({ @@ -874,6 +879,39 @@ describe('Fast parent event durable queue', () => { expect(mocks.releaseLock).toHaveBeenCalledOnce(); }); + it('discards a non-retryable inference failure and continues', async () => { + const first = pendingRow('event-1'); + const second = pendingRow('event-2', { ...event, messageId: 'message-2' }); + mocks.findPending + .mockResolvedValueOnce(first) + .mockResolvedValueOnce(first) + .mockResolvedValueOnce(second) + .mockResolvedValueOnce(undefined); + const terminalError = Object.assign( + new Error('Fast mode inference failed (insufficient_credits).'), + { + name: 'FastAgentInferenceError', + failure: { reason: 'insufficient_credits', retryable: false }, + }, + ); + mocks.deliver + .mockRejectedValueOnce(terminalError) + .mockResolvedValueOnce('delivered'); + + await drainFastAgentParentEvents({ + conversationId: parent.sessionId, + eventKey: first.eventKey, + }); + + expect(mocks.deliver).toHaveBeenCalledTimes(2); + expect(mocks.updateSet).toHaveBeenCalledWith( + expect.objectContaining({ + discardedAt: expect.any(Date), + lastError: terminalError.message, + }), + ); + }); + it('discards a permanent head failure and continues to the next event', async () => { const first = pendingRow('event-1'); const second = pendingRow('event-2', { ...event, messageId: 'message-2' }); diff --git a/packages/sdk/src/server/lib/fast-agent-parent-event-queue.ts b/packages/sdk/src/server/lib/fast-agent-parent-event-queue.ts index 6b4d75693..b4dd506e9 100644 --- a/packages/sdk/src/server/lib/fast-agent-parent-event-queue.ts +++ b/packages/sdk/src/server/lib/fast-agent-parent-event-queue.ts @@ -5,6 +5,7 @@ import { Queue } from 'bullmq'; import { acquireFastAgentTurnLock, findFastAgentDurableRetryScheduledError, + isNonRetryableFastAgentInferenceError, } from '@roomote/cloud-agents/server'; import { and, @@ -445,6 +446,11 @@ export async function drainFastAgentParentEvents( await markDiscarded(row.id, deliveryError); continue; } + if (isNonRetryableFastAgentInferenceError(error)) { + await finalizeAutomationLaunch(row.event, 'failed', error); + await markDiscarded(row.id, error); + continue; + } await db .update(fastAgentParentEvents)