diff --git a/.changeset/recover-openai-compatible-web-search.md b/.changeset/recover-openai-compatible-web-search.md new file mode 100644 index 000000000..b82d37f22 --- /dev/null +++ b/.changeset/recover-openai-compatible-web-search.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Recover when an OpenAI-compatible endpoint rejects the native web-search include. eve now removes `web_search` and retries the model call once. diff --git a/packages/eve/src/harness/model-call-error.test.ts b/packages/eve/src/harness/model-call-error.test.ts index 7044135ca..a14e4a381 100644 --- a/packages/eve/src/harness/model-call-error.test.ts +++ b/packages/eve/src/harness/model-call-error.test.ts @@ -4,7 +4,7 @@ import { classifyModelCallError, EmptyModelResponseError, extractModelCallErrorDetails, - extractUnsupportedProviderToolTypes, + extractUnsupportedProviderToolIdentifiers, isNoOutputGeneratedError, normalizeModelStreamError, extractUpstreamRejectionMessage, @@ -469,11 +469,11 @@ function gatewayProviderToolFailure(input: { }); } -describe("extractUnsupportedProviderToolTypes", () => { +describe("extractUnsupportedProviderToolIdentifiers", () => { it("returns the upstream tool type from a Bedrock fallback rejection", () => { const error = gatewayProviderToolFailure({ unsupportedTypes: ["web_search_20250305"] }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]); }); it("deduplicates when multiple providerAttempts reference the same tool type", () => { @@ -481,7 +481,7 @@ describe("extractUnsupportedProviderToolTypes", () => { unsupportedTypes: ["web_search_20250305", "web_search_20250305"], }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]); }); it("returns multiple types when distinct tools were rejected", () => { @@ -489,7 +489,7 @@ describe("extractUnsupportedProviderToolTypes", () => { unsupportedTypes: ["web_search_20250305", "computer_20251022"], }); - expect([...extractUnsupportedProviderToolTypes(error)].sort()).toEqual([ + expect([...extractUnsupportedProviderToolIdentifiers(error)].sort()).toEqual([ "computer_20251022", "web_search_20250305", ]); @@ -506,7 +506,86 @@ describe("extractUnsupportedProviderToolTypes", () => { truncateResponseBody: true, }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]); + }); + + it("returns the rejected OpenAI web-search include", () => { + const data = { + error: { + code: "invalid_value", + message: + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.", + param: "include", + type: "invalid_request_error", + }, + }; + const error = directApiCallError({ + data, + responseBody: JSON.stringify(data), + statusCode: 400, + }); + + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([ + "web_search_call.action.sources", + ]); + }); + + it.each([ + '{"error":{"message":"Invalid value: \'web_search_call.action.sources\'. Supported...', + '{"error":{"message":"Invalid value: \\"web_search_call.action.sources\\". Supported...', + ])("returns the rejected web-search include from a truncated response body", (responseBody) => { + const error = directApiCallError({ responseBody, statusCode: 400 }); + + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([ + "web_search_call.action.sources", + ]); + }); + + it("does not treat a supported value or request echo as a rejection", () => { + const enumeration = directApiCallError({ + data: { + error: { + message: + "Invalid value: 'code_interpreter_call.outputs'. Supported values are: 'web_search_call.action.sources'.", + param: "include", + }, + }, + statusCode: 400, + }); + const fullRejectionPhrase = + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'."; + const parsedRequestEcho = directApiCallError({ + data: { + error: { message: "Internal error" }, + request: { prompt: fullRejectionPhrase }, + }, + statusCode: 400, + }); + const truncatedRequestEcho = directApiCallError({ + responseBody: JSON.stringify({ + error: { message: "Internal error" }, + request: { prompt: fullRejectionPhrase }, + }).slice(0, -1), + statusCode: 400, + }); + const nestedRequestError = directApiCallError({ + responseBody: `{"request":{"error":{"message":"${fullRejectionPhrase}...`, + statusCode: 400, + }); + + expect(extractUnsupportedProviderToolIdentifiers(enumeration)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(parsedRequestEcho)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(truncatedRequestEcho)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(nestedRequestError)).toEqual([]); + }); + + it("scans malformed response bodies without ambiguous regex backtracking", () => { + const error = directApiCallError({ + responseBody: `{"error":{"message":"${"\\a".repeat(10_000)}...`, + statusCode: 400, + }); + + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]); }); it("returns empty for the ambiguous internal server error case (no tool rejection)", () => { @@ -528,7 +607,7 @@ describe("extractUnsupportedProviderToolTypes", () => { type: "internal_server_error", }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]); }); it("returns empty for generic structural 4xx errors", () => { @@ -537,7 +616,7 @@ describe("extractUnsupportedProviderToolTypes", () => { statusCode: 401, }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]); }); it("returns empty for non-tool 'not supported' phrasing", () => { @@ -552,13 +631,13 @@ describe("extractUnsupportedProviderToolTypes", () => { statusCode: 400, }); - expect(extractUnsupportedProviderToolTypes(error)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]); }); it("returns empty for plain Error and null/undefined inputs", () => { - expect(extractUnsupportedProviderToolTypes(new Error("mystery"))).toEqual([]); - expect(extractUnsupportedProviderToolTypes(null)).toEqual([]); - expect(extractUnsupportedProviderToolTypes(undefined)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(new Error("mystery"))).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(null)).toEqual([]); + expect(extractUnsupportedProviderToolIdentifiers(undefined)).toEqual([]); }); }); diff --git a/packages/eve/src/harness/model-call-error.ts b/packages/eve/src/harness/model-call-error.ts index 91e32f73e..e368c1c9e 100644 --- a/packages/eve/src/harness/model-call-error.ts +++ b/packages/eve/src/harness/model-call-error.ts @@ -20,6 +20,18 @@ const GATEWAY_MODEL_REQUEST_REJECTED_MESSAGE = * in unrelated "not supported" errors. */ const UNSUPPORTED_TOOL_TYPE_REGEX = /tool type ['"]([\w.-]+)['"] is not supported/i; +// OpenAI-compatible endpoints can reject the include added by the native +// web-search tool without naming the tool itself. Structured responses are +// checked only at `error.message` when `error.param` identifies `include`. +const UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX = + /invalid value:\s*['"](web_search_call\.action\.sources)['"]/i; +// A response body may be truncated before it can be parsed. Match only a +// top-level `error.message`; the disjoint string alternatives keep scanning +// linear on malformed provider payloads. +const TRUNCATED_API_ERROR_MESSAGE_REGEX = + /^\s*\{\s*"error"\s*:\s*\{\s*"message"\s*:\s*"((?:\\.|[^"\\])*)/i; +const ESCAPED_UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX = + /invalid value:\s*\\?['"](web_search_call\.action\.sources)\\?['"]/i; /** * The most informative human-readable rejection a model-call error @@ -93,9 +105,9 @@ function isTransientHttpStatus(status: number | undefined): boolean { } /** - * Returns the distinct upstream tool types referenced by any - * "tool type 'X' is not supported" rejection in an AI Gateway error's - * provider attempt list. + * Returns the distinct upstream identifiers from provider-tool rejections: + * AI Gateway's "tool type 'X' is not supported" provider attempts and + * OpenAI-compatible endpoints rejecting the web-search sources include. * * Walks the cause chain to find the gateway error and inspects both the * structured `data` field and the raw `responseBody` JSON. Returns an @@ -103,29 +115,27 @@ function isTransientHttpStatus(status: number | undefined): boolean { * * Used by the harness recovery path to identify which framework tools * to drop before retrying the failing step. Detection is by string - * match on the upstream tool type — see - * {@link resolveFrameworkToolFromUpstreamType} for the mapping back to + * match on the upstream identifier — see + * {@link resolveFrameworkToolFromUpstreamIdentifier} for the mapping back to * framework tool names. */ -export function extractUnsupportedProviderToolTypes(error: unknown): readonly string[] { +export function extractUnsupportedProviderToolIdentifiers(error: unknown): readonly string[] { const found = new Set(); for (const candidate of walkCauseChain(error)) { - collectUnsupportedToolTypesFromValue(readObjectField(candidate, "data"), found); + collectUnsupportedProviderToolIdentifiers(readObjectField(candidate, "data"), found); const responseBody = readStringField(candidate, "responseBody"); if (responseBody !== undefined) { try { - collectUnsupportedToolTypesFromValue(JSON.parse(responseBody), found); + collectUnsupportedProviderToolIdentifiers(JSON.parse(responseBody), found); } catch { // The response body may be truncated mid-JSON when the upstream // includes a large request snapshot. Fall back to a raw string // scan so we still surface the tool name when the regex match // lies before the truncation boundary. - const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(responseBody); - if (match?.[1] !== undefined) { - found.add(match[1]); - } + collectUnsupportedToolTypes(responseBody, found); + collectTruncatedWebSearchIncludeError(responseBody, found); } } } @@ -133,14 +143,42 @@ export function extractUnsupportedProviderToolTypes(error: unknown): readonly st return [...found]; } +function collectUnsupportedProviderToolIdentifiers(value: unknown, out: Set): void { + collectUnsupportedToolTypesFromValue(value, out); + + if (!isObject(value)) return; + const apiError = readObjectField(value, "error"); + if (apiError === undefined || readStringField(apiError, "param") !== "include") return; + + const message = readStringField(apiError, "message"); + if (message !== undefined) { + collectRegexMatch(message, UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX, out); + } +} + +function collectUnsupportedToolTypes(value: string, out: Set): void { + collectRegexMatch(value, UNSUPPORTED_TOOL_TYPE_REGEX, out); +} + +function collectTruncatedWebSearchIncludeError(value: string, out: Set): void { + const message = TRUNCATED_API_ERROR_MESSAGE_REGEX.exec(value)?.[1]; + if (message !== undefined) { + collectRegexMatch(message, ESCAPED_UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX, out); + } +} + +function collectRegexMatch(value: string, regex: RegExp, out: Set): void { + const match = regex.exec(value); + if (match?.[1] !== undefined) { + out.add(match[1]); + } +} + function collectUnsupportedToolTypesFromValue(value: unknown, out: Set): void { if (value === null || value === undefined) return; if (typeof value === "string") { - const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(value); - if (match?.[1] !== undefined) { - out.add(match[1]); - } + collectUnsupportedToolTypes(value, out); return; } diff --git a/packages/eve/src/harness/provider-tools.test.ts b/packages/eve/src/harness/provider-tools.test.ts index 52e658987..fcdfe809c 100644 --- a/packages/eve/src/harness/provider-tools.test.ts +++ b/packages/eve/src/harness/provider-tools.test.ts @@ -8,7 +8,7 @@ import { WEB_SEARCH_PARALLEL_OUTPUT_SCHEMA, } from "#runtime/framework-tools/web-search.js"; import { - resolveFrameworkToolFromUpstreamType, + resolveFrameworkToolFromUpstreamIdentifier, resolveWebSearchBackend, resolveWebSearchOutputSchema, resolveWebSearchProviderTool, @@ -195,13 +195,19 @@ describe("resolveWebSearchBackend", () => { }); }); -describe("resolveFrameworkToolFromUpstreamType", () => { +describe("resolveFrameworkToolFromUpstreamIdentifier", () => { it("maps the Anthropic web_search_20250305 type back to web_search", () => { - expect(resolveFrameworkToolFromUpstreamType("web_search_20250305")).toBe("web_search"); + expect(resolveFrameworkToolFromUpstreamIdentifier("web_search_20250305")).toBe("web_search"); + }); + + it("maps the OpenAI web-search sources include back to web_search", () => { + expect(resolveFrameworkToolFromUpstreamIdentifier("web_search_call.action.sources")).toBe( + "web_search", + ); }); it("returns null for unknown upstream tool types", () => { - expect(resolveFrameworkToolFromUpstreamType("computer_20251022")).toBeNull(); - expect(resolveFrameworkToolFromUpstreamType("some.future.tool")).toBeNull(); + expect(resolveFrameworkToolFromUpstreamIdentifier("computer_20251022")).toBeNull(); + expect(resolveFrameworkToolFromUpstreamIdentifier("some.future.tool")).toBeNull(); }); }); diff --git a/packages/eve/src/harness/provider-tools.ts b/packages/eve/src/harness/provider-tools.ts index fb516131e..e347fbc2f 100644 --- a/packages/eve/src/harness/provider-tools.ts +++ b/packages/eve/src/harness/provider-tools.ts @@ -16,35 +16,37 @@ import type { JsonObject } from "#shared/json.js"; export type WebSearchBackend = "anthropic" | "google" | "openai" | "parallel"; /** - * Maps an upstream provider tool type (the literal `type` string the AI SDK - * sends to the provider) back to the framework tool name that injected it. + * Maps an upstream provider rejection identifier back to the framework tool + * that injected it. Identifiers include literal provider tool `type` strings + * and request fields that an OpenAI-compatible endpoint attributes to a tool. * - * Used when the AI Gateway routes a request to a fallback provider that - * does not support a provider-specific tool — the upstream error references - * the provider-specific type (e.g. `web_search_20250305`), but the harness - * needs to drop the framework tool by its public name (`web_search`). + * The harness needs this mapping because upstream errors do not use eve's + * public framework tool names. For example, both `web_search_20250305` and + * `web_search_call.action.sources` originate from `web_search`. * * Adding a new provider tool requires adding the corresponding mapping * entry here alongside its {@link resolveWebSearchProviderTool} switch * arm so detection stays in lockstep with injection. */ -const UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME: Readonly> = { +const UPSTREAM_REJECTION_IDENTIFIER_TO_FRAMEWORK_NAME: Readonly> = { // Anthropic's stable web search tool. The Bedrock and Vertex // Anthropic backends reject this type because they only host the // older Claude Messages surface. web_search_20250305: WEB_SEARCH_TOOL_DEFINITION.name, + // OpenAI-compatible endpoints quote this AI SDK include value instead of a tool type. + "web_search_call.action.sources": WEB_SEARCH_TOOL_DEFINITION.name, }; /** - * Returns the framework tool name that produced an upstream provider tool - * `type`, or `null` when the type is not one we know how to remove. + * Returns the framework tool name that produced an upstream rejection + * identifier, or `null` when it is not one we know how to remove. * * Used by the harness recovery path to decide which tools to drop when a - * gateway fallback provider rejects a tool. Unknown types fall through to + * provider rejects a tool. Unknown identifiers fall through to * the existing terminal/recoverable handling. */ -export function resolveFrameworkToolFromUpstreamType(type: string): string | null { - return UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME[type] ?? null; +export function resolveFrameworkToolFromUpstreamIdentifier(identifier: string): string | null { + return UPSTREAM_REJECTION_IDENTIFIER_TO_FRAMEWORK_NAME[identifier] ?? null; } /** diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts index c451d5c3d..b4f435980 100644 --- a/packages/eve/src/harness/tool-loop.test.ts +++ b/packages/eve/src/harness/tool-loop.test.ts @@ -4029,7 +4029,7 @@ describe("createToolLoopHarness", () => { } }); - it("retries with the offending tool dropped and a one-shot system note", async () => { + it("retries an OpenAI-compatible include rejection without web_search", async () => { const resolveRuntimeContext = vi.fn((input: InstrumentationStepStartedEventInput) => ({ runtimeContext: { "test.attempt": typeof input.modelInput.instructions === "string" ? "original" : "retry", @@ -4040,8 +4040,24 @@ describe("createToolLoopHarness", () => { "step.started": resolveRuntimeContext, }, }); + const responseBodyValue = { + error: { + code: "invalid_value", + message: + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.", + param: "include", + type: "invalid_request_error", + }, + }; + const failure = Object.assign(new Error(responseBodyValue.error.message), { + data: responseBodyValue, + isRetryable: false, + name: "AI_APICallError", + responseBody: JSON.stringify(responseBodyValue), + statusCode: 400, + }); const { constructedCalls } = setupRecoveryAgent({ - failure: createGatewayUnsupportedToolError({ unsupportedTypes: ["web_search_20250305"] }), + failure, successResult: { finishReason: "stop", response: { messages: [{ content: "ok", role: "assistant" }] }, diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts index 8c0487925..e55c126e3 100644 --- a/packages/eve/src/harness/tool-loop.ts +++ b/packages/eve/src/harness/tool-loop.ts @@ -107,8 +107,6 @@ import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-c import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js"; import { createInstrumentationHandleEvent } from "#harness/instrumentation-native-events.js"; import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js"; -import { resolveParentLineage } from "#harness/parent-lineage.js"; -import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js"; import { consumeDeferredStepInput, getApprovedTools, @@ -136,7 +134,7 @@ import { classifyModelCallError, EmptyModelResponseError, extractModelCallErrorDetails, - extractUnsupportedProviderToolTypes, + extractUnsupportedProviderToolIdentifiers, isNoOutputGeneratedError, type UpstreamRejectionSummary, extractUpstreamRejectionMessage, @@ -158,7 +156,7 @@ import { detectPromptCachePath, getAnthropicCacheMarker, } from "#harness/prompt-cache.js"; -import { resolveFrameworkToolFromUpstreamType } from "#harness/provider-tools.js"; +import { resolveFrameworkToolFromUpstreamIdentifier } from "#harness/provider-tools.js"; import { createRuntimeActionRequestFromToolCall, resolvePendingRuntimeActions, @@ -544,7 +542,6 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { agentName: config.runtimeIdentity?.agentName, handleEvent: baseEmit, hooks: config.instrumentation?.hooks, - parentLineage: resolveParentLineage(parent, store?.get(ChannelKey)), parentTraceContext: store?.get(ParentTraceContextKey), rootSessionId: parent?.rootSessionId, sessionId: session.sessionId, @@ -1677,10 +1674,9 @@ function extractToolResultCallIds(messages: readonly StepResponseMessage[]): Rea } /** - * Inspects a model-call failure for the "tool type 'X' is not supported" - * provider-attempt rejection that AI Gateway returns when a fallback - * provider cannot serve a provider-specific tool. On a match, retries the - * step once with the offending tool dropped and a one-shot system note + * Inspects a model-call failure for a known upstream rejection caused by a + * provider-specific tool. On a match, retries the step once with the offending + * tool dropped and a one-shot system note * telling the model which capability has been removed. * * Returns `recovered` when the retry succeeded so the caller can hand @@ -1689,8 +1685,8 @@ function extractToolResultCallIds(messages: readonly StepResponseMessage[]): Rea * threw) otherwise so the caller's existing terminal/recoverable * cascade still runs. * - * Recovery is intentionally scoped to known provider tools — entries in - * {@link UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME} — so an unrelated + * Recovery is intentionally scoped to known provider-tool rejection + * identifiers so an unrelated * upstream rejection cannot accidentally drop a user-authored tool. */ async function attemptUnsupportedProviderToolRecovery(input: { @@ -1699,14 +1695,14 @@ async function attemptUnsupportedProviderToolRecovery(input: { readonly sessionId: string; readonly turnId: string; }): Promise { - const unsupportedTypes = extractUnsupportedProviderToolTypes(input.error); - if (unsupportedTypes.length === 0) { + const unsupportedIdentifiers = extractUnsupportedProviderToolIdentifiers(input.error); + if (unsupportedIdentifiers.length === 0) { return { outcome: "skipped" }; } const toolsToDisable: string[] = []; - for (const type of unsupportedTypes) { - const frameworkName = resolveFrameworkToolFromUpstreamType(type); + for (const identifier of unsupportedIdentifiers) { + const frameworkName = resolveFrameworkToolFromUpstreamIdentifier(identifier); if (frameworkName !== null && !toolsToDisable.includes(frameworkName)) { toolsToDisable.push(frameworkName); } @@ -1720,7 +1716,7 @@ async function attemptUnsupportedProviderToolRecovery(input: { disabled: toolsToDisable, sessionId: input.sessionId, turnId: input.turnId, - upstreamTypes: unsupportedTypes, + upstreamIdentifiers: unsupportedIdentifiers, }); const retryCallOptions: RecoveryRetryCallOptions = {