From 3ce9fc289dc2b6a40c0f60ee9d8b66722d3af2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Tue, 6 Oct 2026 14:50:19 +0200 Subject: [PATCH] fix(agent): name the status and the body of a provider error the SDK could not parse --- .changeset/provider-error-body.md | 5 + docs/reference/errors.mdx | 2 +- packages/e2e/src/agent/model/sdk.ts | 8 +- packages/e2e/src/agent/tool-loop.ts | 4 +- packages/e2e/src/internal/ai-trace.ts | 3 +- packages/e2e/src/internal/errors.ts | 30 +++++ packages/e2e/tests/unit/ai-trace.test.ts | 17 ++- .../unit/provider-failure-message.test.ts | 124 ++++++++++++++++++ 8 files changed, 181 insertions(+), 12 deletions(-) create mode 100644 .changeset/provider-error-body.md create mode 100644 packages/e2e/tests/unit/provider-failure-message.test.ts diff --git a/.changeset/provider-error-body.md b/.changeset/provider-error-body.md new file mode 100644 index 000000000..12bb740c8 --- /dev/null +++ b/.changeset/provider-error-body.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +When the provider package can't read a provider's error body, `MODEL_PROVIDER_FAILED` no longer shows an empty message. One example is the ChatGPT backend's HTTP 400 `{"detail": "Unsupported service_tier: fast"}`. The message and the `--ai-trace` entry now show the HTTP status and the body, with secrets redacted and the body cut at 1 KB. An `agent.act` step that used up its transport retries now names the last attempt's failure, as judgments already did. diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 99e598ec9..64feeaec1 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -60,7 +60,7 @@ try { | `POLICY_DENIED` | A denied destination (a navigation scheme other than `http:` or `https:`, `about:blank` aside, a malformed URL), a forbidden fill (an undeclared secret, a non-editable sink, a password into a field that is not a password field), or pixel evidence after a secret fill (`vision: 'only'` with pixels withheld, or an `act` or `vision: true` step whose semantic capture timed out before the step did). Also reading a value, text, or attribute from a secure field, the getters and the `toHaveValue`, `toHaveText`, `toContainText`, and `toHaveAttribute` matchers alike. An unknown tool or an undeclared field is refused to the model as the call's result and is not this code | configuration | 2 | | `APP_UNREACHABLE` | The app URL could not be reached, or an app command exited early or exceeded `startupTimeout`. The startup error includes recent log output with `command.env` values redacted, and suggests adding a `log` if none is configured. Under the agent, also any engine fault with no more specific code: on a device, an iOS automation runner that is busy or wedged reports here with a message that names the runner, the session and device, and the recovery; see [Mobile](/mobile#troubleshooting) | infrastructure | 3 | | `APP_ALREADY_RUNNING` | `readyUrl` already answered before the target's `app.command` was spawned and `reuseExisting` was not in effect; stop that process or set `reuseExisting: true` locally | infrastructure | 3 | -| `MODEL_PROVIDER_FAILED` | Provider transport failure (network error, 5xx, rate limit, or a request with no response within 120 s) after the adapter's transport retries (up to 5 per model call, honoring `retry-after`, bounded by the remaining step timeout). A request that stalls is aborted at 120 s and sent again, and an `agent.act` step's `--debug` transcript notes it. The bound covers the whole request, so a model that needs longer than 120 s for one answer fails here too. Evicts no [recording](/cache) | infrastructure | 3 | +| `MODEL_PROVIDER_FAILED` | Provider transport failure (network error, 5xx, rate limit, or a request with no response within 120 s) after the adapter's transport retries (up to 5 per model call, honoring `retry-after`, bounded by the remaining step timeout). A request that stalls is aborted at 120 s and sent again, and an `agent.act` step's `--debug` transcript notes it. The bound covers the whole request, so a model that needs longer than 120 s for one answer fails here too. An error body the provider package cannot read shows as the HTTP status and the body, secrets redacted, cut at 1 KB. Evicts no [recording](/cache) | infrastructure | 3 | | `CANCELLED` | The runner cancelled the operation, without a process signal: an `e2e mcp` `open_session` whose request the client cancelled or whose connection closed, or that the server's shutdown cut short, after stopping what it had started | infrastructure | 3 | | `MODEL_OUTPUT_INVALID` | Structured output still failed validation after the repair budget was spent | test | 1 | | `APP_NOT_OPEN` | An operation needed an open app; `app.open()` had not run, or the engine reported invalid state | test | 1 | diff --git a/packages/e2e/src/agent/model/sdk.ts b/packages/e2e/src/agent/model/sdk.ts index beb29279a..362306013 100644 --- a/packages/e2e/src/agent/model/sdk.ts +++ b/packages/e2e/src/agent/model/sdk.ts @@ -8,7 +8,7 @@ */ import type { ModelMessage } from 'ai'; -import { withHint } from '../../internal/errors.ts'; +import { providerFailureMessage, withHint } from '../../internal/errors.ts'; import { missingModelError, type ResolvedModel } from '../../config/agent.ts'; import { aiSdk, asSdkLanguageModel, loadAiSdk, type SdkLanguageModel } from '../ai-sdk.ts'; import { MODEL_REQUEST_HEADERS } from '../../internal/client-identity.ts'; @@ -292,20 +292,20 @@ function translateModelError( if (isContextOverflow(cause)) { return new AgentError( 'CONTEXT_OVERFLOW', - `the model request exceeded the context window: ${cause instanceof Error ? cause.message : String(cause)}`, + `the model request exceeded the context window: ${providerFailureMessage(cause)}`, { cause }, ); } if (APICallError.isInstance(cause)) { return new AgentError( 'MODEL_PROVIDER_FAILED', - withHint(`model provider failed: ${cause.message}`, failureHint(cause)), + withHint(`model provider failed: ${providerFailureMessage(cause)}`, failureHint(cause)), { cause }, ); } return new AgentError( 'MODEL_PROVIDER_FAILED', - `model provider failed: ${cause instanceof Error ? cause.message : String(cause)}`, + `model provider failed: ${providerFailureMessage(cause)}`, { cause }, ); } diff --git a/packages/e2e/src/agent/tool-loop.ts b/packages/e2e/src/agent/tool-loop.ts index cf3d15d7d..60c099ed0 100644 --- a/packages/e2e/src/agent/tool-loop.ts +++ b/packages/e2e/src/agent/tool-loop.ts @@ -15,7 +15,7 @@ import type { StepTurn } from '../run/steps.ts'; import type { AssistantModelMessage, ModelMessage, StepResult, ToolCallPart, ToolSet } from 'ai'; import { asSdkLanguageModel, loadAiSdk, type AiSdk, type SdkLanguageModel } from './ai-sdk.ts'; import { MODEL_REQUEST_HEADERS } from '../internal/client-identity.ts'; -import { ConfigurationError, withHint } from '../internal/errors.ts'; +import { ConfigurationError, providerFailureMessage, withHint } from '../internal/errors.ts'; import type { ProviderOptions } from '../types.ts'; import { failureHint, isAbort, TRANSPORT_RETRIES } from './model/sdk.ts'; import { isContextOverflow } from './model/overflow.ts'; @@ -330,7 +330,7 @@ class LoopRun { if (isAbort(cause)) { throw new AgentError('STEP_TIMEOUT', 'model call exceeded the remaining step timeout', { cause }); } - const message = cause instanceof Error ? cause.message : String(cause); + const message = providerFailureMessage(cause); if (isContextOverflow(cause)) { throw new AgentError( 'CONTEXT_OVERFLOW', diff --git a/packages/e2e/src/internal/ai-trace.ts b/packages/e2e/src/internal/ai-trace.ts index 0ca321870..3d7fc664a 100644 --- a/packages/e2e/src/internal/ai-trace.ts +++ b/packages/e2e/src/internal/ai-trace.ts @@ -19,6 +19,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { randomUUID } from 'node:crypto'; import type { Telemetry } from 'ai'; +import { providerFailureMessage } from './errors.ts'; /** Longest step label quoted in a run name before truncation. */ const MAX_LABEL_CHARS = 80; @@ -396,7 +397,7 @@ export class AiTraceRecorder { const state = this.calls.get(callId); if (state === undefined) return; const cause = event?.error; - const message = cause instanceof Error ? cause.message : String(cause ?? 'unknown error'); + const message = cause === undefined || cause === null ? 'unknown error' : providerFailureMessage(cause); for (const open of state.openSteps.values()) { this.steps.push(this.closeStep(open, { error: message })); } diff --git a/packages/e2e/src/internal/errors.ts b/packages/e2e/src/internal/errors.ts index 17449ad73..638579dcc 100644 --- a/packages/e2e/src/internal/errors.ts +++ b/packages/e2e/src/internal/errors.ts @@ -24,6 +24,36 @@ export function setErrorRedactor(redact: ((text: string) => string) | undefined) else redactorSlot.set(globalThis, redact); } +/** Response body bytes `providerFailureMessage` quotes. */ +const MAX_PROVIDER_BODY_BYTES = 1024; + +/** + * The message of a failed model call; for a spent retry chain, that of its + * last attempt. When a provider's error body does not match the AI SDK's + * error schema, the SDK leaves it unparsed and keeps only the HTTP status + * text, empty over HTTP/2, as the message. The status and the body, + * secret-redacted and cut short, then say what went wrong. + */ +export function providerFailureMessage(cause: unknown): string { + if (!(cause instanceof Error)) return errorMessage(cause); + const failure = cause as Error & { reason?: unknown; lastError?: unknown }; + if (failure.name === 'AI_RetryError' && failure.reason !== 'abort' && failure.lastError !== undefined) { + return providerFailureMessage(failure.lastError); + } + if (failure.name !== 'AI_APICallError') return failure.message; + const { statusCode, responseBody, data } = failure as { statusCode?: unknown; responseBody?: unknown; data?: unknown }; + if (typeof statusCode !== 'number' || statusCode < 400 || typeof responseBody !== 'string' || data !== undefined) { + return failure.message; + } + const status = `HTTP ${String(statusCode)}${failure.message === '' ? '' : ` ${failure.message}`}`; + const body = responseBody.trim(); + if (body === '') return status; + const redact = redactorSlot.get(globalThis) ?? ((text: string): string => text); + const redacted = redact(body); + const quoted = truncateUtf8(redacted, MAX_PROVIDER_BODY_BYTES); + return `${status}: ${quoted}${quoted.length < redacted.length ? '…' : ''}`; +} + /** * The message of a thrown value led by its code when it is an e2e error, as * the CLI prints one: `POLICY_DENIED: forbidden URL scheme: file:`. Any diff --git a/packages/e2e/tests/unit/ai-trace.test.ts b/packages/e2e/tests/unit/ai-trace.test.ts index f354df340..6cdc69bf2 100644 --- a/packages/e2e/tests/unit/ai-trace.test.ts +++ b/packages/e2e/tests/unit/ai-trace.test.ts @@ -1,3 +1,4 @@ +import { APICallError } from '@ai-sdk/provider'; import { describe, expect, it } from 'vitest'; import { AiTraceCollector, @@ -12,7 +13,7 @@ import { gate } from '../helpers/gate.ts'; const SCOPE = { test: 'todos › adds one', testId: 't1', target: 'web', agent: 'default', attempt: 0 }; /** Drives one two-step generation through the recorder the way the SDK does. */ -async function generation(recorder: AiTraceRecorder, callId: string, options: { fail?: boolean } = {}) { +async function generation(recorder: AiTraceRecorder, callId: string, options: { fail?: Error } = {}) { const t = recorder.telemetry; const fire = (callback: ((event: E) => unknown) | undefined, event: E) => callback?.(event); await fire(t.onStart, { callId, operationId: 'ai.generateText' } as never); @@ -39,8 +40,8 @@ async function generation(recorder: AiTraceRecorder, callId: string, options: { toolChoice: { type: 'required' }, } as never); await fire(t.onLanguageModelCallEnd, { callId, performance: { responseTimeMs: 321 } } as never); - if (options.fail === true) { - await fire(t.onError, { callId, error: new Error('provider exploded') }); + if (options.fail !== undefined) { + await fire(t.onError, { callId, error: options.fail }); return; } await fire(t.onStepEnd, { @@ -110,7 +111,7 @@ describe('AiTraceRecorder', () => { it('closes an open step with the error when the generation fails', async () => { const recorder = new AiTraceRecorder(); - await generation(recorder, 'call-1', { fail: true }); + await generation(recorder, 'call-1', { fail: new Error('provider exploded') }); const { steps } = recorder.drain(); expect(steps).toHaveLength(1); expect(steps[0]!.error).toBe('provider exploded'); @@ -118,6 +119,14 @@ describe('AiTraceRecorder', () => { expect(recorder.pending).toBe(false); }); + it('records the status and the body of a provider error the SDK could not parse', async () => { + const recorder = new AiTraceRecorder(); + const body = '{"detail":"Unsupported service_tier: fast"}'; + const fail = new APICallError({ message: '', url: 'https://provider.test', requestBodyValues: {}, statusCode: 400, responseBody: body }); + await generation(recorder, 'call-1', { fail }); + expect(recorder.drain().steps[0]!.error).toBe(`HTTP 400: ${body}`); + }); + it('names a run after the scope; outside a scope it keeps the SDK function id', async () => { const recorder = new AiTraceRecorder(); await recorder.telemetry.onStart?.({ callId: 'x', operationId: 'ai.generateText', functionId: 'my-agent' } as never); diff --git a/packages/e2e/tests/unit/provider-failure-message.test.ts b/packages/e2e/tests/unit/provider-failure-message.test.ts new file mode 100644 index 000000000..62d756114 --- /dev/null +++ b/packages/e2e/tests/unit/provider-failure-message.test.ts @@ -0,0 +1,124 @@ +import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; +import { afterEach, describe, expect, it } from 'vitest'; +import { loadAiSdk } from '../../src/agent/ai-sdk.ts'; +import { resolveConfig } from '../../src/config/resolve.ts'; +import { defineEngine } from '../../src/engine/index.ts'; +import { createEngineSession } from '../../src/engine/session.ts'; +import { providerFailureMessage, setErrorRedactor } from '../../src/internal/errors.ts'; +import { Deadline } from '../../src/internal/time.ts'; +import { AttemptBudget } from '../../src/run/budget.ts'; +import { createFixtures } from '../../src/run/fixtures.ts'; +import { StepRecorder } from '../../src/run/steps.ts'; +import { WorkerModels } from '../../src/run/worker-models.ts'; +import type { E2EConfig } from '../../src/types.ts'; +import { runAgentStepsOnFakeTime } from '../helpers/agent-fake-time.ts'; +import { snapshot } from '../helpers/snapshot.ts'; + +runAgentStepsOnFakeTime(); + +const DETAIL = '{"detail":"Unsupported service_tier: fast"}'; + +/** An OpenAI-compatible model whose server answers every request with `status`, `body`, and an empty status text, as HTTP/2 has none. */ +function rejectingModel(status: number, body: string) { + const fetch = async () => new Response(body, { status, statusText: '', headers: { 'content-type': 'application/json' } }); + return createOpenAICompatible({ name: 'fake', baseURL: 'https://provider.test/v1', apiKey: 'key', fetch }).chatModel('fake-model'); +} + +/** The error the AI SDK throws for one call to `model`. */ +async function thrownBy(model: ReturnType): Promise { + try { + await model.doGenerate({ prompt: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }] }); + } catch (error) { + return error; + } + throw new Error('the call did not fail'); +} + +/** A real fixture graph with an in-memory engine and no runner process. */ +function runtime(overrides: Partial) { + const engine = defineEngine({ name: 'fake', version: '1', spiVersion: 1, observe: async () => snapshot([]) }); + const config = resolveConfig( + { targets: [{ name: 'fake', platform: 'custom', engine }], cache: 'off', ...overrides }, + { projectRoot: process.cwd(), env: {} }, + ); + const signal = new AbortController().signal; + const steps = new StepRecorder('attempt'); + const { fixtures } = createFixtures({ + config, + target: config.targets[0]!, + session: createEngineSession({ engine, targetName: 'fake' }), + steps, + budget: new AttemptBudget(signal, new Deadline(600_000)), + runId: 'run', + attemptId: 'attempt', + attempt: { testId: 'test', attemptId: 'attempt', index: 0, signal, memory: new Map() }, + artifacts: { dir: '/tmp', register: () => 'artifact', link: () => 'artifact' }, + priorSteps: () => steps.completed(), + agentContext: undefined, + saveSession: undefined, + models: new WorkerModels(() => {}), + }); + return { fixtures }; +} + +describe('providerFailureMessage', () => { + afterEach(() => { + setErrorRedactor(undefined); + }); + + it('quotes the status and the body the SDK could not parse', async () => { + expect(providerFailureMessage(await thrownBy(rejectingModel(400, DETAIL)))).toBe(`HTTP 400: ${DETAIL}`); + expect(providerFailureMessage(await thrownBy(rejectingModel(502, '')))).toBe('HTTP 502'); + }); + + it("keeps the provider's message when the SDK parsed the body", async () => { + const error = await thrownBy(rejectingModel(400, '{"error":{"message":"model not found"}}')); + expect(providerFailureMessage(error)).toBe('model not found'); + expect(providerFailureMessage(new Error('socket hang up'))).toBe('socket hang up'); + }); + + it('redacts secrets in the body before cutting it short', async () => { + setErrorRedactor((text) => text.replaceAll('hunter2', '')); + const echoed = await thrownBy(rejectingModel(400, '{"detail":"bad field: hunter2"}')); + expect(providerFailureMessage(echoed)).toBe('HTTP 400: {"detail":"bad field: "}'); + // The secret straddles the cut: redacted whole first, no prefix of it survives. + const straddling = await thrownBy(rejectingModel(500, `${'x'.repeat(1020)}hunter2${'y'.repeat(100)}`)); + const message = providerFailureMessage(straddling); + expect(message).toBe(`HTTP 500: ${'x'.repeat(1020)} { + const { RetryError } = await loadAiSdk(); + const lastError = await thrownBy(rejectingModel(503, 'upstream unavailable')); + const chain = new RetryError({ message: 'Failed after 6 attempts. Last error: ', reason: 'maxRetriesExceeded', errors: [lastError] }); + expect(providerFailureMessage(chain)).toBe('HTTP 503: upstream unavailable'); + }); + + it('names the status and the body in the failed agent step', async () => { + const { fixtures } = runtime({ agents: { default: { model: rejectingModel(400, DETAIL) } } }); + + await expect(fixtures.agent.act('open the page')).rejects.toMatchObject({ + code: 'MODEL_PROVIDER_FAILED', + message: `the model provider failed: HTTP 400: ${DETAIL}`, + }); + }); + + it('names the last attempt when a retryable failure spent its retries', async () => { + const { fixtures } = runtime({ agents: { default: { model: rejectingModel(502, 'bad gateway') } } }); + + await expect(fixtures.agent.act('open the page')).rejects.toMatchObject({ + code: 'MODEL_PROVIDER_FAILED', + message: 'the model provider failed: HTTP 502: bad gateway', + }); + }); + + it('names the status and the body in a failed judgment', async () => { + const { fixtures } = runtime({ agents: { default: { model: rejectingModel(400, DETAIL) } } }); + + await expect(fixtures.agent.assert('the page is open')).rejects.toMatchObject({ + code: 'MODEL_PROVIDER_FAILED', + message: `model provider failed: HTTP 400: ${DETAIL}`, + }); + }); +});