diff --git a/.changeset/web-search-compat-recovery.md b/.changeset/web-search-compat-recovery.md new file mode 100644 index 000000000..d45605d11 --- /dev/null +++ b/.changeset/web-search-compat-recovery.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Recover automatically when an OpenAI-compatible endpoint (e.g. AWS Bedrock Mantle) rejects the injected provider-managed `web_search` tool. The harness now recognizes the `web_search_call.action.sources` include rejection, drops `web_search`, and retries the step, so agents on such endpoints complete turns instead of failing every model call. diff --git a/packages/eve/src/harness/model-call-error.test.ts b/packages/eve/src/harness/model-call-error.test.ts index 7044135ca..d0a18f093 100644 --- a/packages/eve/src/harness/model-call-error.test.ts +++ b/packages/eve/src/harness/model-call-error.test.ts @@ -509,6 +509,81 @@ describe("extractUnsupportedProviderToolTypes", () => { expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]); }); + it("returns the OpenAI web-search include value from an OpenAI-compatible endpoint rejection", () => { + // Bedrock Mantle (and other OpenAI-compatible endpoints without native + // web search) reject the `web_search_call.action.sources` include the + // AI SDK adds for the injected OpenAI web-search tool. The rejection + // names the include value, not a tool type. + const data = { + error: { + message: + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.", + type: "invalid_request_error", + param: "include", + code: "invalid_value", + }, + }; + const error = directApiCallError({ + data, + responseBody: JSON.stringify(data), + statusCode: 400, + }); + + expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_call.action.sources"]); + }); + + it("returns the include value when only the responseBody carries it", () => { + const responseBody = JSON.stringify({ + error: { + message: + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.", + type: "invalid_request_error", + }, + }); + const error = directApiCallError({ responseBody, statusCode: 400 }); + + expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_call.action.sources"]); + }); + + it("recovers the include value from a truncated responseBody via raw string scan", () => { + const fullBody = JSON.stringify({ + error: { + message: + "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.", + type: "invalid_request_error", + }, + request: { input: "large snapshot ".repeat(50) }, + }); + const responseBody = `${fullBody.slice(0, fullBody.indexOf("Supported values") + 20)}...`; + const error = directApiCallError({ responseBody, statusCode: 400 }); + + expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_call.action.sources"]); + }); + + it("ignores the include value when it appears only as a supported value or request echo", () => { + // An endpoint that supports web-search sources but rejects a different + // include enumerates it after "Supported values are:"; a truncated body + // can also echo the request's include array. Neither is a web_search + // rejection, so recovery must not drop the tool. + const enumeration = directApiCallError({ + data: { + error: { + message: + "Invalid value: 'code_interpreter_call.outputs'. Supported values are: 'web_search_call.action.sources', 'reasoning.encrypted_content'.", + type: "invalid_request_error", + }, + }, + statusCode: 400, + }); + expect(extractUnsupportedProviderToolTypes(enumeration)).toEqual([]); + + const requestEcho = directApiCallError({ + responseBody: `{"error":{"message":"Internal error"},"request":{"include":["web_search_call.action.sources"],"input":"...`, + statusCode: 400, + }); + expect(extractUnsupportedProviderToolTypes(requestEcho)).toEqual([]); + }); + it("returns empty for the ambiguous internal server error case (no tool rejection)", () => { const innerBody = { error: { message: "Bad Request", type: "internal_server_error" }, diff --git a/packages/eve/src/harness/model-call-error.ts b/packages/eve/src/harness/model-call-error.ts index 91e32f73e..378ad0fad 100644 --- a/packages/eve/src/harness/model-call-error.ts +++ b/packages/eve/src/harness/model-call-error.ts @@ -21,6 +21,34 @@ const GATEWAY_MODEL_REQUEST_REJECTED_MESSAGE = */ const UNSUPPORTED_TOOL_TYPE_REGEX = /tool type ['"]([\w.-]+)['"] is not supported/i; +/** + * Anchored regex for OpenAI-compatible endpoints that reject the + * `web_search_call.action.sources` include instead of naming a tool + * type (e.g. Bedrock Mantle's "Invalid value: + * 'web_search_call.action.sources'. Supported values are: + * 'reasoning.encrypted_content'."). The AI SDK adds that include only + * when the OpenAI web-search provider tool is in the request, so this + * rejection can only be caused by the injected web-search tool. The + * `invalid value` prefix keeps the match anchored to the rejected + * value — the same token also shows up in supported-value enumerations + * for other rejected includes and in request snapshots echoed into + * truncated error bodies, and neither of those means web search is + * unsupported. + */ +const UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX = + /invalid value:\s*['"](web_search_call\.action\.sources)['"]/i; + +/** + * Every rejection shape the recovery path can map back to a framework + * tool: the gateway's "tool type 'X' is not supported" phrasing and the + * OpenAI-compatible include rejection. Group 1 of each regex is the + * upstream token handed to `resolveFrameworkToolFromUpstreamType`. + */ +const UPSTREAM_TOOL_REJECTION_REGEXES: readonly RegExp[] = [ + UNSUPPORTED_TOOL_TYPE_REGEX, + UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX, +]; + /** * The most informative human-readable rejection a model-call error * carries, extracted from the upstream response. Not a semantic-error @@ -93,17 +121,19 @@ function isTransientHttpStatus(status: number | undefined): boolean { } /** - * Returns the distinct upstream tool types referenced by any - * "tool type 'X' is not supported" rejection in an AI Gateway error's - * provider attempt list. + * Returns the distinct upstream tokens referenced by any provider-tool + * rejection the recovery path knows how to act on: the AI Gateway's + * "tool type 'X' is not supported" provider-attempt phrasing, and the + * OpenAI-compatible "Invalid value: 'web_search_call.action.sources'" + * include rejection. * * Walks the cause chain to find the gateway error and inspects both the * structured `data` field and the raw `responseBody` JSON. Returns an - * empty array for errors that are not of this shape. + * empty array for errors that are not of these shapes. * * Used by the harness recovery path to identify which framework tools * to drop before retrying the failing step. Detection is by string - * match on the upstream tool type — see + * match on the upstream token — see * {@link resolveFrameworkToolFromUpstreamType} for the mapping back to * framework tool names. */ @@ -122,10 +152,7 @@ export function extractUnsupportedProviderToolTypes(error: unknown): readonly st // includes a large request snapshot. Fall back to a raw string // scan so we still surface the tool name when the regex match // lies before the truncation boundary. - const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(responseBody); - if (match?.[1] !== undefined) { - found.add(match[1]); - } + collectUpstreamToolRejections(responseBody, found); } } } @@ -133,14 +160,20 @@ export function extractUnsupportedProviderToolTypes(error: unknown): readonly st return [...found]; } +function collectUpstreamToolRejections(value: string, out: Set): void { + for (const regex of UPSTREAM_TOOL_REJECTION_REGEXES) { + const match = regex.exec(value); + if (match?.[1] !== undefined) { + out.add(match[1]); + } + } +} + function collectUnsupportedToolTypesFromValue(value: unknown, out: Set): void { if (value === null || value === undefined) return; if (typeof value === "string") { - const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(value); - if (match?.[1] !== undefined) { - out.add(match[1]); - } + collectUpstreamToolRejections(value, out); return; } diff --git a/packages/eve/src/harness/provider-tools.test.ts b/packages/eve/src/harness/provider-tools.test.ts index 52e658987..4168b5795 100644 --- a/packages/eve/src/harness/provider-tools.test.ts +++ b/packages/eve/src/harness/provider-tools.test.ts @@ -200,6 +200,12 @@ describe("resolveFrameworkToolFromUpstreamType", () => { expect(resolveFrameworkToolFromUpstreamType("web_search_20250305")).toBe("web_search"); }); + it("maps the OpenAI web-search sources include back to web_search", () => { + expect(resolveFrameworkToolFromUpstreamType("web_search_call.action.sources")).toBe( + "web_search", + ); + }); + it("returns null for unknown upstream tool types", () => { expect(resolveFrameworkToolFromUpstreamType("computer_20251022")).toBeNull(); expect(resolveFrameworkToolFromUpstreamType("some.future.tool")).toBeNull(); diff --git a/packages/eve/src/harness/provider-tools.ts b/packages/eve/src/harness/provider-tools.ts index fb516131e..9ed3511ec 100644 --- a/packages/eve/src/harness/provider-tools.ts +++ b/packages/eve/src/harness/provider-tools.ts @@ -33,6 +33,11 @@ const UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME: Readonly> = { // Anthropic backends reject this type because they only host the // older Claude Messages surface. web_search_20250305: WEB_SEARCH_TOOL_DEFINITION.name, + // The Responses API include the AI SDK adds whenever the OpenAI + // web-search tool is present. OpenAI-compatible endpoints without + // native web search (e.g. Bedrock Mantle) reject the request by + // quoting this include value instead of naming a tool type. + "web_search_call.action.sources": WEB_SEARCH_TOOL_DEFINITION.name, }; /**