fix(eve): recover from rejected web search includes

.changeset/recover-openai-compatible-web-search.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Recover when an OpenAI-compatible endpoint rejects the native web-search include. eve now removes `web_search` and retries the model call once.

packages/eve/src/harness/model-call-error.test.ts

@@ -4,7 +4,7 @@ import {
   classifyModelCallError,
   EmptyModelResponseError,
   extractModelCallErrorDetails,
-  extractUnsupportedProviderToolTypes,
+  extractUnsupportedProviderToolIdentifiers,
   isNoOutputGeneratedError,
   normalizeModelStreamError,
   extractUpstreamRejectionMessage,
@@ -469,27 +469,27 @@ function gatewayProviderToolFailure(input: {
   });
 }
 
-describe("extractUnsupportedProviderToolTypes", () => {
+describe("extractUnsupportedProviderToolIdentifiers", () => {
   it("returns the upstream tool type from a Bedrock fallback rejection", () => {
     const error = gatewayProviderToolFailure({ unsupportedTypes: ["web_search_20250305"] });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]);
   });
 
   it("deduplicates when multiple providerAttempts reference the same tool type", () => {
     const error = gatewayProviderToolFailure({
       unsupportedTypes: ["web_search_20250305", "web_search_20250305"],
     });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]);
   });
 
   it("returns multiple types when distinct tools were rejected", () => {
     const error = gatewayProviderToolFailure({
       unsupportedTypes: ["web_search_20250305", "computer_20251022"],
     });
 
-    expect([...extractUnsupportedProviderToolTypes(error)].sort()).toEqual([
+    expect([...extractUnsupportedProviderToolIdentifiers(error)].sort()).toEqual([
       "computer_20251022",
       "web_search_20250305",
     ]);
@@ -506,7 +506,86 @@ describe("extractUnsupportedProviderToolTypes", () => {
       truncateResponseBody: true,
     });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual(["web_search_20250305"]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual(["web_search_20250305"]);
+  });
+
+  it("returns the rejected OpenAI web-search include", () => {
+    const data = {
+      error: {
+        code: "invalid_value",
+        message:
+          "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.",
+        param: "include",
+        type: "invalid_request_error",
+      },
+    };
+    const error = directApiCallError({
+      data,
+      responseBody: JSON.stringify(data),
+      statusCode: 400,
+    });
+
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([
+      "web_search_call.action.sources",
+    ]);
+  });
+
+  it.each([
+    '{"error":{"message":"Invalid value: \'web_search_call.action.sources\'. Supported...<truncated>',
+    '{"error":{"message":"Invalid value: \\"web_search_call.action.sources\\". Supported...<truncated>',
+  ])("returns the rejected web-search include from a truncated response body", (responseBody) => {
+    const error = directApiCallError({ responseBody, statusCode: 400 });
+
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([
+      "web_search_call.action.sources",
+    ]);
+  });
+
+  it("does not treat a supported value or request echo as a rejection", () => {
+    const enumeration = directApiCallError({
+      data: {
+        error: {
+          message:
+            "Invalid value: 'code_interpreter_call.outputs'. Supported values are: 'web_search_call.action.sources'.",
+          param: "include",
+        },
+      },
+      statusCode: 400,
+    });
+    const fullRejectionPhrase =
+      "Invalid value: 'web_search_call.action.sources'. Supported values are: 'reasoning.encrypted_content'.";
+    const parsedRequestEcho = directApiCallError({
+      data: {
+        error: { message: "Internal error" },
+        request: { prompt: fullRejectionPhrase },
+      },
+      statusCode: 400,
+    });
+    const truncatedRequestEcho = directApiCallError({
+      responseBody: JSON.stringify({
+        error: { message: "Internal error" },
+        request: { prompt: fullRejectionPhrase },
+      }).slice(0, -1),
+      statusCode: 400,
+    });
+    const nestedRequestError = directApiCallError({
+      responseBody: `{"request":{"error":{"message":"${fullRejectionPhrase}...<truncated>`,
+      statusCode: 400,
+    });
+
+    expect(extractUnsupportedProviderToolIdentifiers(enumeration)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(parsedRequestEcho)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(truncatedRequestEcho)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(nestedRequestError)).toEqual([]);
+  });
+
+  it("scans malformed response bodies without ambiguous regex backtracking", () => {
+    const error = directApiCallError({
+      responseBody: `{"error":{"message":"${"\\a".repeat(10_000)}...<truncated>`,
+      statusCode: 400,
+    });
+
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]);
   });
 
   it("returns empty for the ambiguous internal server error case (no tool rejection)", () => {
@@ -528,7 +607,7 @@ describe("extractUnsupportedProviderToolTypes", () => {
       type: "internal_server_error",
     });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]);
   });
 
   it("returns empty for generic structural 4xx errors", () => {
@@ -537,7 +616,7 @@ describe("extractUnsupportedProviderToolTypes", () => {
       statusCode: 401,
     });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]);
   });
 
   it("returns empty for non-tool 'not supported' phrasing", () => {
@@ -552,13 +631,13 @@ describe("extractUnsupportedProviderToolTypes", () => {
       statusCode: 400,
     });
 
-    expect(extractUnsupportedProviderToolTypes(error)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(error)).toEqual([]);
   });
 
   it("returns empty for plain Error and null/undefined inputs", () => {
-    expect(extractUnsupportedProviderToolTypes(new Error("mystery"))).toEqual([]);
-    expect(extractUnsupportedProviderToolTypes(null)).toEqual([]);
-    expect(extractUnsupportedProviderToolTypes(undefined)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(new Error("mystery"))).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(null)).toEqual([]);
+    expect(extractUnsupportedProviderToolIdentifiers(undefined)).toEqual([]);
   });
 });
 

packages/eve/src/harness/model-call-error.ts

@@ -20,6 +20,18 @@ const GATEWAY_MODEL_REQUEST_REJECTED_MESSAGE =
  * in unrelated "not supported" errors.
  */
 const UNSUPPORTED_TOOL_TYPE_REGEX = /tool type ['"]([\w.-]+)['"] is not supported/i;
+// OpenAI-compatible endpoints can reject the include added by the native
+// web-search tool without naming the tool itself. Structured responses are
+// checked only at `error.message` when `error.param` identifies `include`.
+const UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX =
+  /invalid value:\s*['"](web_search_call\.action\.sources)['"]/i;
+// A response body may be truncated before it can be parsed. Match only a
+// top-level `error.message`; the disjoint string alternatives keep scanning
+// linear on malformed provider payloads.
+const TRUNCATED_API_ERROR_MESSAGE_REGEX =
+  /^\s*\{\s*"error"\s*:\s*\{\s*"message"\s*:\s*"((?:\\.|[^"\\])*)/i;
+const ESCAPED_UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX =
+  /invalid value:\s*\\?['"](web_search_call\.action\.sources)\\?['"]/i;
 
 /**
  * The most informative human-readable rejection a model-call error
@@ -93,54 +105,80 @@ function isTransientHttpStatus(status: number | undefined): boolean {
 }
 
 /**
- * Returns the distinct upstream tool types referenced by any
- * "tool type 'X' is not supported" rejection in an AI Gateway error's
- * provider attempt list.
+ * Returns the distinct upstream identifiers from provider-tool rejections:
+ * AI Gateway's "tool type 'X' is not supported" provider attempts and
+ * OpenAI-compatible endpoints rejecting the web-search sources include.
  *
  * Walks the cause chain to find the gateway error and inspects both the
  * structured `data` field and the raw `responseBody` JSON. Returns an
  * empty array for errors that are not of this shape.
  *
  * Used by the harness recovery path to identify which framework tools
  * to drop before retrying the failing step. Detection is by string
- * match on the upstream tool type — see
- * {@link resolveFrameworkToolFromUpstreamType} for the mapping back to
+ * match on the upstream identifier — see
+ * {@link resolveFrameworkToolFromUpstreamIdentifier} for the mapping back to
  * framework tool names.
  */
-export function extractUnsupportedProviderToolTypes(error: unknown): readonly string[] {
+export function extractUnsupportedProviderToolIdentifiers(error: unknown): readonly string[] {
   const found = new Set<string>();
 
   for (const candidate of walkCauseChain(error)) {
-    collectUnsupportedToolTypesFromValue(readObjectField(candidate, "data"), found);
+    collectUnsupportedProviderToolIdentifiers(readObjectField(candidate, "data"), found);
 
     const responseBody = readStringField(candidate, "responseBody");
     if (responseBody !== undefined) {
       try {
-        collectUnsupportedToolTypesFromValue(JSON.parse(responseBody), found);
+        collectUnsupportedProviderToolIdentifiers(JSON.parse(responseBody), found);
       } catch {
         // The response body may be truncated mid-JSON when the upstream
         // includes a large request snapshot. Fall back to a raw string
         // scan so we still surface the tool name when the regex match
         // lies before the truncation boundary.
-        const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(responseBody);
-        if (match?.[1] !== undefined) {
-          found.add(match[1]);
-        }
+        collectUnsupportedToolTypes(responseBody, found);
+        collectTruncatedWebSearchIncludeError(responseBody, found);
       }
     }
   }
 
   return [...found];
 }
 
+function collectUnsupportedProviderToolIdentifiers(value: unknown, out: Set<string>): void {
+  collectUnsupportedToolTypesFromValue(value, out);
+
+  if (!isObject(value)) return;
+  const apiError = readObjectField(value, "error");
+  if (apiError === undefined || readStringField(apiError, "param") !== "include") return;
+
+  const message = readStringField(apiError, "message");
+  if (message !== undefined) {
+    collectRegexMatch(message, UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX, out);
+  }
+}
+
+function collectUnsupportedToolTypes(value: string, out: Set<string>): void {
+  collectRegexMatch(value, UNSUPPORTED_TOOL_TYPE_REGEX, out);
+}
+
+function collectTruncatedWebSearchIncludeError(value: string, out: Set<string>): void {
+  const message = TRUNCATED_API_ERROR_MESSAGE_REGEX.exec(value)?.[1];
+  if (message !== undefined) {
+    collectRegexMatch(message, ESCAPED_UNSUPPORTED_WEB_SEARCH_INCLUDE_REGEX, out);
+  }
+}
+
+function collectRegexMatch(value: string, regex: RegExp, out: Set<string>): void {
+  const match = regex.exec(value);
+  if (match?.[1] !== undefined) {
+    out.add(match[1]);
+  }
+}
+
 function collectUnsupportedToolTypesFromValue(value: unknown, out: Set<string>): void {
   if (value === null || value === undefined) return;
 
   if (typeof value === "string") {
-    const match = UNSUPPORTED_TOOL_TYPE_REGEX.exec(value);
-    if (match?.[1] !== undefined) {
-      out.add(match[1]);
-    }
+    collectUnsupportedToolTypes(value, out);
     return;
   }
 

packages/eve/src/harness/provider-tools.test.ts

@@ -8,7 +8,7 @@ import {
   WEB_SEARCH_PARALLEL_OUTPUT_SCHEMA,
 } from "#runtime/framework-tools/web-search.js";
 import {
-  resolveFrameworkToolFromUpstreamType,
+  resolveFrameworkToolFromUpstreamIdentifier,
   resolveWebSearchBackend,
   resolveWebSearchOutputSchema,
   resolveWebSearchProviderTool,
@@ -195,13 +195,19 @@ describe("resolveWebSearchBackend", () => {
   });
 });
 
-describe("resolveFrameworkToolFromUpstreamType", () => {
+describe("resolveFrameworkToolFromUpstreamIdentifier", () => {
   it("maps the Anthropic web_search_20250305 type back to web_search", () => {
-    expect(resolveFrameworkToolFromUpstreamType("web_search_20250305")).toBe("web_search");
+    expect(resolveFrameworkToolFromUpstreamIdentifier("web_search_20250305")).toBe("web_search");
+  });
+
+  it("maps the OpenAI web-search sources include back to web_search", () => {
+    expect(resolveFrameworkToolFromUpstreamIdentifier("web_search_call.action.sources")).toBe(
+      "web_search",
+    );
   });
 
   it("returns null for unknown upstream tool types", () => {
-    expect(resolveFrameworkToolFromUpstreamType("computer_20251022")).toBeNull();
-    expect(resolveFrameworkToolFromUpstreamType("some.future.tool")).toBeNull();
+    expect(resolveFrameworkToolFromUpstreamIdentifier("computer_20251022")).toBeNull();
+    expect(resolveFrameworkToolFromUpstreamIdentifier("some.future.tool")).toBeNull();
   });
 });

packages/eve/src/harness/provider-tools.ts

@@ -16,35 +16,37 @@ import type { JsonObject } from "#shared/json.js";
 export type WebSearchBackend = "anthropic" | "google" | "openai" | "parallel";
 
 /**
- * Maps an upstream provider tool type (the literal `type` string the AI SDK
- * sends to the provider) back to the framework tool name that injected it.
+ * Maps an upstream provider rejection identifier back to the framework tool
+ * that injected it. Identifiers include literal provider tool `type` strings
+ * and request fields that an OpenAI-compatible endpoint attributes to a tool.
  *
- * Used when the AI Gateway routes a request to a fallback provider that
- * does not support a provider-specific tool — the upstream error references
- * the provider-specific type (e.g. `web_search_20250305`), but the harness
- * needs to drop the framework tool by its public name (`web_search`).
+ * The harness needs this mapping because upstream errors do not use eve's
+ * public framework tool names. For example, both `web_search_20250305` and
+ * `web_search_call.action.sources` originate from `web_search`.
  *
  * Adding a new provider tool requires adding the corresponding mapping
  * entry here alongside its {@link resolveWebSearchProviderTool} switch
  * arm so detection stays in lockstep with injection.
  */
-const UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME: Readonly<Record<string, string>> = {
+const UPSTREAM_REJECTION_IDENTIFIER_TO_FRAMEWORK_NAME: Readonly<Record<string, string>> = {
   // Anthropic's stable web search tool. The Bedrock and Vertex
   // Anthropic backends reject this type because they only host the
   // older Claude Messages surface.
   web_search_20250305: WEB_SEARCH_TOOL_DEFINITION.name,
+  // OpenAI-compatible endpoints quote this AI SDK include value instead of a tool type.
+  "web_search_call.action.sources": WEB_SEARCH_TOOL_DEFINITION.name,
 };
 
 /**
- * Returns the framework tool name that produced an upstream provider tool
- * `type`, or `null` when the type is not one we know how to remove.
+ * Returns the framework tool name that produced an upstream rejection
+ * identifier, or `null` when it is not one we know how to remove.
  *
  * Used by the harness recovery path to decide which tools to drop when a
- * gateway fallback provider rejects a tool. Unknown types fall through to
+ * provider rejects a tool. Unknown identifiers fall through to
  * the existing terminal/recoverable handling.
  */
-export function resolveFrameworkToolFromUpstreamType(type: string): string | null {
-  return UPSTREAM_TOOL_TYPE_TO_FRAMEWORK_NAME[type] ?? null;
+export function resolveFrameworkToolFromUpstreamIdentifier(identifier: string): string | null {
+  return UPSTREAM_REJECTION_IDENTIFIER_TO_FRAMEWORK_NAME[identifier] ?? null;
 }
 
 /**