fix(web-search): retry an empty forced answer once before failing the turn

src/web-search/loop.ts

@@ -2,6 +2,7 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../adapters/
 import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxProviderOpaqueToolCallMetadata, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types";
 import { namespacedToolName, toolChoiceToolPredicate } from "../types";
 import { cloneProviderOpaqueToolCallMetadata } from "../responses/provider-opaque-metadata";
+import { truncationReasonFor } from "../responses/truncated-stop-reason";
 import type { AttemptRecoveryKind } from "../usage/log";
 import { bridgeToResponsesSSE } from "../bridge";
 import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor";
@@ -230,6 +231,24 @@ function forcedAnswerNudge(): OcxMessage {
   };
 }
 
+/**
+ * Transient developer-role nudge for the ONE recovery pass after a forced answer came back empty.
+ * The recovery also removes every tool, so the model has nothing to call and can only return text;
+ * this turn says so explicitly rather than relying on the removal alone. Like {@link forcedAnswerNudge}
+ * it is iteration-local and never touches the persisted `messages`.
+ */
+function forcedAnswerRetryNudge(): OcxMessage {
+  return {
+    role: "developer",
+    content:
+      "Your previous response contained no usable answer. Web search has finished for this turn and " +
+      "no tools are available for this response. Answer the user's question now in assistant text, " +
+      "using the web search results already gathered above. If those results are insufficient, say " +
+      "what is missing instead of returning an empty response.",
+    timestamp: Date.now(),
+  };
+}
+
 function jsonError(status: number, message: string): Response {
   return new Response(JSON.stringify({ error: { message, type: "upstream_error", code: null } }), {
     status,
@@ -370,7 +389,9 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
   const signal = internalAbort.signal;
 
   // Hard iteration bound (termination safety net); forceAnswer normally ends the loop sooner.
+  // `maxSearches` search rounds, the forced answer, and at most one empty-answer recovery pass.
   const HARD_CAP = maxSearches + 2;
+  let emptyAnswerRetries = 0;
   const connectTimeoutMs = deps.connectTimeoutMs ?? 200_000;
   const routedModelStallTimeoutMs = deps.routedModelStallTimeoutMs ?? 200_000;
 
@@ -407,12 +428,25 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
     // ignores what the search found, which reads to the user as "the search did nothing". Nudge it
     // (iteration-locally — never mutate the shared `messages`) to actually use the gathered results.
     // Only when a REAL search ran (executedSearchCount, not empty-query/limit/repeat placeholders).
-    const iterMessages: OcxMessage[] = forceAnswer && executedSearchCount > 0
+    let iterMessages: OcxMessage[] = forceAnswer && executedSearchCount > 0
       ? [...messages, forcedAnswerNudge()]
       : messages;
+    // #1001 follow-up: the recovery pass for an empty forced answer. Removing every tool leaves the
+    // model nothing to call, and the extra developer turn asks it for the text it just failed to
+    // produce. Both an EMPTY tool list and `toolChoice: "none"` are applied: options alone are not
+    // enough, because adapters such as Devin put `context.tools` on the wire verbatim.
+    const recoveringEmptyAnswer = forceAnswer && emptyAnswerRetries > 0;
+    if (recoveringEmptyAnswer) iterMessages = [...iterMessages, forcedAnswerRetryNudge()];
     const iterParsed: OcxParsedRequest = {
       ...parsed, stream: true,
-      context: { ...parsed.context, messages: iterMessages, tools: forceAnswer ? toolsNoWebSearch : allTools },
+      ...(recoveringEmptyAnswer ? { options: { ...parsed.options, toolChoice: "none" as const } } : {}),
+      context: {
+        ...parsed.context,
+        messages: iterMessages,
+        // The recovery pass is answer-only, so it advertises NO tools at all. Dropping just the
+        // synthetic web_search would leave every remaining client tool exposed to the retry.
+        tools: recoveringEmptyAnswer ? [] : (forceAnswer ? toolsNoWebSearch : allTools),
+      },
     };
     // One cumulative header deadline spans every pool-key 429 rotation in this model iteration.
     // clear() stops only its timer after final headers; the direct turn signal remains attached to
@@ -850,7 +884,36 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
               if (terminalEvent?.type === "done"
                 && (split.hasMalformedToolCall
                   || (!split.hasRealToolCall && !hasVisibleAssistantText(split.passthrough)))) {
-                throw new LoopError(502, "forced-answer pass produced no usable assistant output");
+                // #1001 fixed the silent success by failing here. A malformed call still fails: it
+                // reports a protocol problem, and replaying it would only re-ask an unwell upstream.
+                // Silence is different — it is recoverable, so retry exactly once with the results
+                // already gathered before failing the turn.
+                //
+                // A `done` whose stop reason is in the bridge's truncation vocabulary is NOT
+                // silence. `content_filter`/`refusal`/`max_tokens` are terminal decisions the
+                // bridge already reports as `response.incomplete`, so retrying one would spend a
+                // second upstream call on a decision the provider already made — and could report
+                // success for a filtered turn. Only a clean completion reaches the retry; a
+                // truncated terminal is replayed unchanged below and still ends `response.incomplete`.
+                const truncated = truncationReasonFor(terminalEvent.stopReason) !== undefined;
+                console.warn("[web-search-loop] unusable forced answer", JSON.stringify({
+                  model: parsed.modelId,
+                  recoveryAttempt: emptyAnswerRetries,
+                  searchCalls: split.calls.length,
+                  malformed: split.hasMalformedToolCall,
+                  stopReason: terminalEvent.stopReason,
+                  truncated,
+                  eventTypes: [...new Set(split.passthrough.map(event => event.type))],
+                }));
+                if (!truncated && !split.hasMalformedToolCall && !split.hasRealToolCall && emptyAnswerRetries === 0) {
+                  emptyAnswerRetries++;
+                  console.warn("[web-search-loop] empty forced answer — retrying once without tools");
+                  yield { type: "heartbeat" };
+                  continue;
+                }
+                if (!truncated) {
+                  throw new LoopError(502, "forced-answer pass produced no usable assistant output");
+                }
               }
             }
             if (executedSearchCount > 0) {

structure/runtime.md

@@ -191,6 +191,27 @@ not an authentication or entitlement decision.
 
 Routed Responses continuations whose local replay state is missing resolve their recovery decision from the selected wire protocol, not the model name; the contract lives in [Responses transport](transports/responses.md).
 
+### Hosted web-search forced-answer contract
+
+`src/web-search/loop.ts` drives the routed model in rounds: while the model's only actionable output is
+the synthetic `web_search` call, each round runs the sidecar search and re-asks. Once the search budget
+is spent the loop takes a forced-answer pass with the synthetic tool removed, so the model has to answer
+from the tool results already in the conversation. The loop's hard iteration bound is therefore
+`maxSearches + 2` — the search rounds, the forced pass, and at most one recovery pass below.
+
+A forced pass that ends `done` with no visible text and no real tool call is silence, not an answer. It
+is retried exactly once, at the cost of one extra upstream model call: the retry empties
+`context.tools` as well as setting `toolChoice`, so no adapter can keep advertising a tool, appends a
+developer nudge asking for the missing text, and leaves the search/result history collected so far
+unchanged. A second empty pass fails the turn rather than reporting a silent success, and a malformed
+closed call fails immediately without a retry.
+
+Truncated terminals are a different outcome and are never retried: a forced pass that ends `done` with a
+`stopReason` from the truncation vocabulary in `src/responses/truncated-stop-reason.ts`
+(`content_filter`, `refusal`, `max_tokens`, ...) is replayed unchanged and spends no recovery call, so
+the bridge still reports it as `response.incomplete`. A provider's explicit filtered/truncated decision
+is preserved; only genuine silence is bought back with a second model call.
+
 ## Remote Hub hardening ownership
 
 `src/remote/protocol.ts` owns pure interval/feature negotiation. `src/remote/hub-state.ts` owns the `GET|HEAD /v1/hub-state` contract, its caps, and the parser both sides share. `src/client/hub-client.ts` owns bounded, schema-validated remote catalog consumption, hub-state reads, and key-id probes; `src/client/hub-state.ts` owns the resolution and the owner-stamped 0600 cache, and a failed read reports "unavailable" rather than degrading to the client's own local provider and login state. `src/client/hub-relay.ts` is a fixed-authority management relay with URL, header, body, redirect, and stream bounds. The public data listener remains the direct client→hub path; the loopback management ingress never serves data-plane routes.

tests/web-search/web-search.test.ts

@@ -133,6 +133,181 @@ describe("issue #1001 — forced-answer passes must produce usable output", () =
     expect(frames.some(frame => frame.event === "response.completed")).toBe(true);
     expect(frames.some(frame => frame.event === "response.failed")).toBe(false);
   });
+
+  // #1001 chose to fail rather than complete silently, which turned silence into a dead turn:
+  // the user sees "stream disconnected before completion: forced-answer pass produced no usable
+  // assistant output". Silence is recoverable, so the pass is retried once with no tools before
+  // the same error is reported. Malformed calls still fail immediately.
+  describe("empty forced answer recovery", () => {
+    const webSearchOnly = [{ type: "web_search" }];
+    // The client's ordinary tool must survive the forced pass yet disappear from the recovery pass:
+    // adapters such as Devin put context.tools on the wire verbatim, so toolChoice "none" alone
+    // still advertised it. Only a web_search + ordinary-tool fixture distinguishes the two.
+    const webSearchAndFileTool = [
+      { type: "web_search" },
+      { type: "function", name: "read_file", description: "Read file", parameters: { type: "object" } },
+    ];
+
+    function sequenceAdapter(
+      passes: AdapterEvent[][],
+      seen: OcxParsedRequest[],
+      onPass?: (pass: number) => void,
+    ): ProviderAdapter {
+      let pass = 0;
+      return {
+        name: "sequence",
+        buildRequest: (request) => {
+          seen.push(request);
+          return { url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" };
+        },
+        fetchResponse: async () => new Response("wire", { status: 200 }),
+        async *parseStream() {
+          const index = Math.min(pass++, passes.length - 1);
+          onPass?.(index);
+          for (const event of passes[index] ?? []) yield event;
+        },
+        async parseResponse() {
+          throw new Error("parseResponse must be unreachable");
+        },
+      };
+    }
+
+    async function drivePasses(
+      passes: AdapterEvent[][],
+      seen: OcxParsedRequest[] = [],
+      options: { tools?: unknown[]; abortSignal?: AbortSignal; onPass?: (pass: number) => void } = {},
+    ) {
+      const response = await runWithWebSearch({
+        parsed: parseRequest({
+          model: "routed/model",
+          input: "hi",
+          stream: true,
+          tools: options.tools ?? webSearchOnly,
+        }),
+        adapter: sequenceAdapter(passes, seen, options.onPass),
+        forwardProvider,
+        hostedTool: { type: "web_search" },
+        selectedForwardHeaders: new Headers({ authorization: "Bearer token" }),
+        settings: { model: "gpt-5.6-luna", reasoning: "low", timeoutMs: 30_000 },
+        maxSearches: 1,
+        ...(options.abortSignal ? { abortSignal: options.abortSignal } : {}),
+      });
+      return collectSse(response.body!);
+    }
+
+    /** Only the three frames that end a turn, in the order the bridge emitted them. */
+    function terminalFrames(frames: { event?: string }[]): string[] {
+      return frames
+        .map(frame => frame.event ?? "")
+        .filter(event => event === "response.completed" || event === "response.failed" || event === "response.incomplete");
+    }
+
+    test("an empty forced pass is retried once and completes", async () => {
+      const frames = await drivePasses([
+        webSearchFirstPass,
+        [{ type: "done" }],
+        [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+      ]);
+      expect(frames.some(frame => frame.event === "response.completed")).toBe(true);
+      expect(frames.some(frame => frame.event === "response.failed")).toBe(false);
+    });
+
+    test("the recovery pass asks for text with every tool removed", async () => {
+      const seen: OcxParsedRequest[] = [];
+      await drivePasses([
+        webSearchFirstPass,
+        [{ type: "done" }],
+        [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+      ], seen);
+      // The search pass plus the empty forced pass plus exactly one recovery — no extra upstream call.
+      expect(seen).toHaveLength(3);
+      const recovery = seen[2]!;
+      expect(recovery.options.toolChoice).toBe("none");
+      expect(recovery.context.tools).toEqual([]);
+      // The results gathered by the search reach the recovery turn as a tool result ...
+      expect(recovery.context.messages.filter(message => message.role === "toolResult")).toHaveLength(1);
+      // ... and the recovery turn carries the developer nudge that asks for the missing text.
+      expect(recovery.context.messages.some(message =>
+        message.role === "developer" && String(message.content).includes("no tools are available")))
+        .toBe(true);
+    });
+
+    test("an ordinary client tool is dropped from the recovery pass but kept for the forced pass", async () => {
+      const seen: OcxParsedRequest[] = [];
+      const frames = await drivePasses([
+        webSearchFirstPass,
+        [{ type: "done" }],
+        [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+      ], seen, { tools: webSearchAndFileTool });
+      expect(seen).toHaveLength(3);
+      // Forced pass: the synthetic web_search is gone, the client's own tool is still advertised.
+      expect(seen[1]!.context.tools?.map(tool => tool.name)).toEqual(["read_file"]);
+      expect(seen[1]!.options.toolChoice).toBeUndefined();
+      // Recovery pass: nothing to call at all — in the tool list AND in the tool choice.
+      expect(seen[2]!.context.tools).toEqual([]);
+      expect(seen[2]!.options.toolChoice).toBe("none");
+      expect(frames.some(frame => frame.event === "response.completed")).toBe(true);
+    });
+
+    test("a persistent empty forced pass still fails after the one recovery", async () => {
+      const seen: OcxParsedRequest[] = [];
+      const frames = await drivePasses([
+        webSearchFirstPass,
+        [{ type: "done" }],
+        [{ type: "done" }],
+      ], seen);
+      expect(seen).toHaveLength(3);
+      expect(frames.some(frame => frame.event === "response.failed")).toBe(true);
+      expect(frames.some(frame => frame.event === "response.completed")).toBe(false);
+    });
+
+    test("a malformed forced call is not retried", async () => {
+      const seen: OcxParsedRequest[] = [];
+      const frames = await drivePasses([
+        webSearchFirstPass,
+        [{ type: "tool_call_start", id: "", name: "" }, { type: "tool_call_end" }, { type: "done" }],
+        [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+      ], seen);
+      expect(seen).toHaveLength(2);
+      expect(frames.some(frame => frame.event === "response.failed")).toBe(true);
+    });
+
+    // A filtered/truncated forced pass is a provider DECISION, not silence. The bridge already
+    // reports it as response.incomplete, so the loop must spend no second upstream call on it and
+    // must not turn it into a success — the third pass below is deliberately a good answer that
+    // must never be requested.
+    for (const stopReason of ["content_filter", "max_tokens", "refusal"] as const) {
+      test("a " + stopReason + " forced terminal is replayed incompletely and never retried", async () => {
+        const seen: OcxParsedRequest[] = [];
+        const frames = await drivePasses([
+          webSearchFirstPass,
+          [{ type: "done", stopReason }],
+          [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+        ], seen);
+        expect(seen).toHaveLength(2);
+        expect(terminalFrames(frames)).toEqual(["response.incomplete"]);
+        const incomplete = frames.filter(frame => frame.event === "response.incomplete");
+        const snapshot = incomplete[0]!.data.response as { incomplete_details: { reason: string } };
+        expect(snapshot.incomplete_details.reason)
+          .toBe(stopReason === "max_tokens" ? "max_output_tokens" : "content_filter");
+      });
+    }
+
+    test("a cancelled turn does not spend the recovery attempt", async () => {
+      const seen: OcxParsedRequest[] = [];
+      const controller = new AbortController();
+      const frames = await drivePasses([
+        webSearchFirstPass,
+        [{ type: "done" }],
+        [{ type: "text_delta", text: "recovered answer" }, { type: "done" }],
+      ], seen, {
+        abortSignal: controller.signal,
+        onPass: pass => { if (pass === 1) controller.abort(new Error("client closed responses stream")); },
+      });
+      expect(seen).toHaveLength(2);
+      expect(frames.some(frame => frame.event === "response.completed")).toBe(false);
+    });
+  });
 });
 
 const routedProvider: OcxProviderConfig = {