From 05e7763253748c49531e7c9fd1415afb1ef738c7 Mon Sep 17 00:00:00 2001 From: Cortes Ventures Date: Fri, 11 Sep 2026 20:51:50 -0400 Subject: [PATCH] fix(web-search): retry an empty forced answer once before failing the turn An empty forced-answer pass (terminal `done`, no visible assistant text, no tool call) is retried exactly once instead of failing the whole turn with a 502. The recovery iteration appends one developer nudge and advertises NO tools at all -- an empty `context.tools` plus `toolChoice: "none"`, because adapters such as Devin serialize `context.tools` verbatim and would otherwise still expose a remaining client tool. A terminal whose stop reason is in the bridge's truncation vocabulary (`content_filter`, `refusal`, `max_tokens`, ...) is NOT silence: it is replayed unchanged so the bridge keeps reporting `response.incomplete`, and it never spends a second upstream call or turns a filtered turn into a success. Malformed calls still fail immediately, a persistent empty pass still fails, and cancellation does not consume the recovery attempt. `HARD_CAP` is `maxSearches + 2` (search rounds + the forced answer + at most one recovery pass). structure/runtime.md records the changed retry contract: a clean empty final pass can now cost one extra upstream call before the turn fails. --- src/web-search/loop.ts | 69 ++++++++++- structure/runtime.md | 21 ++++ tests/web-search/web-search.test.ts | 175 ++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 3 deletions(-) diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 99b275ed8d..828ad0effe 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -2,6 +2,7 @@ import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../adapters/ import type { AdapterEvent, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxProviderOpaqueToolCallMetadata, OcxThinkingContent, OcxUsage, RateLimitRetryPolicy } from "../types"; import { namespacedToolName, toolChoiceToolPredicate } from "../types"; import { cloneProviderOpaqueToolCallMetadata } from "../responses/provider-opaque-metadata"; +import { truncationReasonFor } from "../responses/truncated-stop-reason"; import type { AttemptRecoveryKind } from "../usage/log"; import { bridgeToResponsesSSE } from "../bridge"; import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor"; @@ -230,6 +231,24 @@ function forcedAnswerNudge(): OcxMessage { }; } +/** + * Transient developer-role nudge for the ONE recovery pass after a forced answer came back empty. + * The recovery also removes every tool, so the model has nothing to call and can only return text; + * this turn says so explicitly rather than relying on the removal alone. Like {@link forcedAnswerNudge} + * it is iteration-local and never touches the persisted `messages`. + */ +function forcedAnswerRetryNudge(): OcxMessage { + return { + role: "developer", + content: + "Your previous response contained no usable answer. Web search has finished for this turn and " + + "no tools are available for this response. Answer the user's question now in assistant text, " + + "using the web search results already gathered above. If those results are insufficient, say " + + "what is missing instead of returning an empty response.", + timestamp: Date.now(), + }; +} + function jsonError(status: number, message: string): Response { return new Response(JSON.stringify({ error: { message, type: "upstream_error", code: null } }), { status, @@ -370,7 +389,9 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise 0 + let iterMessages: OcxMessage[] = forceAnswer && executedSearchCount > 0 ? [...messages, forcedAnswerNudge()] : messages; + // #1001 follow-up: the recovery pass for an empty forced answer. Removing every tool leaves the + // model nothing to call, and the extra developer turn asks it for the text it just failed to + // produce. Both an EMPTY tool list and `toolChoice: "none"` are applied: options alone are not + // enough, because adapters such as Devin put `context.tools` on the wire verbatim. + const recoveringEmptyAnswer = forceAnswer && emptyAnswerRetries > 0; + if (recoveringEmptyAnswer) iterMessages = [...iterMessages, forcedAnswerRetryNudge()]; const iterParsed: OcxParsedRequest = { ...parsed, stream: true, - context: { ...parsed.context, messages: iterMessages, tools: forceAnswer ? toolsNoWebSearch : allTools }, + ...(recoveringEmptyAnswer ? { options: { ...parsed.options, toolChoice: "none" as const } } : {}), + context: { + ...parsed.context, + messages: iterMessages, + // The recovery pass is answer-only, so it advertises NO tools at all. Dropping just the + // synthetic web_search would leave every remaining client tool exposed to the retry. + tools: recoveringEmptyAnswer ? [] : (forceAnswer ? toolsNoWebSearch : allTools), + }, }; // One cumulative header deadline spans every pool-key 429 rotation in this model iteration. // clear() stops only its timer after final headers; the direct turn signal remains attached to @@ -850,7 +884,36 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise event.type))], + })); + if (!truncated && !split.hasMalformedToolCall && !split.hasRealToolCall && emptyAnswerRetries === 0) { + emptyAnswerRetries++; + console.warn("[web-search-loop] empty forced answer — retrying once without tools"); + yield { type: "heartbeat" }; + continue; + } + if (!truncated) { + throw new LoopError(502, "forced-answer pass produced no usable assistant output"); + } } } if (executedSearchCount > 0) { diff --git a/structure/runtime.md b/structure/runtime.md index ae3cc34bbb..95d2430025 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -191,6 +191,27 @@ not an authentication or entitlement decision. Routed Responses continuations whose local replay state is missing resolve their recovery decision from the selected wire protocol, not the model name; the contract lives in [Responses transport](transports/responses.md). +### Hosted web-search forced-answer contract + +`src/web-search/loop.ts` drives the routed model in rounds: while the model's only actionable output is +the synthetic `web_search` call, each round runs the sidecar search and re-asks. Once the search budget +is spent the loop takes a forced-answer pass with the synthetic tool removed, so the model has to answer +from the tool results already in the conversation. The loop's hard iteration bound is therefore +`maxSearches + 2` — the search rounds, the forced pass, and at most one recovery pass below. + +A forced pass that ends `done` with no visible text and no real tool call is silence, not an answer. It +is retried exactly once, at the cost of one extra upstream model call: the retry empties +`context.tools` as well as setting `toolChoice`, so no adapter can keep advertising a tool, appends a +developer nudge asking for the missing text, and leaves the search/result history collected so far +unchanged. A second empty pass fails the turn rather than reporting a silent success, and a malformed +closed call fails immediately without a retry. + +Truncated terminals are a different outcome and are never retried: a forced pass that ends `done` with a +`stopReason` from the truncation vocabulary in `src/responses/truncated-stop-reason.ts` +(`content_filter`, `refusal`, `max_tokens`, ...) is replayed unchanged and spends no recovery call, so +the bridge still reports it as `response.incomplete`. A provider's explicit filtered/truncated decision +is preserved; only genuine silence is bought back with a second model call. + ## Remote Hub hardening ownership `src/remote/protocol.ts` owns pure interval/feature negotiation. `src/remote/hub-state.ts` owns the `GET|HEAD /v1/hub-state` contract, its caps, and the parser both sides share. `src/client/hub-client.ts` owns bounded, schema-validated remote catalog consumption, hub-state reads, and key-id probes; `src/client/hub-state.ts` owns the resolution and the owner-stamped 0600 cache, and a failed read reports "unavailable" rather than degrading to the client's own local provider and login state. `src/client/hub-relay.ts` is a fixed-authority management relay with URL, header, body, redirect, and stream bounds. The public data listener remains the direct client→hub path; the loopback management ingress never serves data-plane routes. diff --git a/tests/web-search/web-search.test.ts b/tests/web-search/web-search.test.ts index e182308a0d..6d2747570e 100644 --- a/tests/web-search/web-search.test.ts +++ b/tests/web-search/web-search.test.ts @@ -133,6 +133,181 @@ describe("issue #1001 — forced-answer passes must produce usable output", () = expect(frames.some(frame => frame.event === "response.completed")).toBe(true); expect(frames.some(frame => frame.event === "response.failed")).toBe(false); }); + + // #1001 chose to fail rather than complete silently, which turned silence into a dead turn: + // the user sees "stream disconnected before completion: forced-answer pass produced no usable + // assistant output". Silence is recoverable, so the pass is retried once with no tools before + // the same error is reported. Malformed calls still fail immediately. + describe("empty forced answer recovery", () => { + const webSearchOnly = [{ type: "web_search" }]; + // The client's ordinary tool must survive the forced pass yet disappear from the recovery pass: + // adapters such as Devin put context.tools on the wire verbatim, so toolChoice "none" alone + // still advertised it. Only a web_search + ordinary-tool fixture distinguishes the two. + const webSearchAndFileTool = [ + { type: "web_search" }, + { type: "function", name: "read_file", description: "Read file", parameters: { type: "object" } }, + ]; + + function sequenceAdapter( + passes: AdapterEvent[][], + seen: OcxParsedRequest[], + onPass?: (pass: number) => void, + ): ProviderAdapter { + let pass = 0; + return { + name: "sequence", + buildRequest: (request) => { + seen.push(request); + return { url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }; + }, + fetchResponse: async () => new Response("wire", { status: 200 }), + async *parseStream() { + const index = Math.min(pass++, passes.length - 1); + onPass?.(index); + for (const event of passes[index] ?? []) yield event; + }, + async parseResponse() { + throw new Error("parseResponse must be unreachable"); + }, + }; + } + + async function drivePasses( + passes: AdapterEvent[][], + seen: OcxParsedRequest[] = [], + options: { tools?: unknown[]; abortSignal?: AbortSignal; onPass?: (pass: number) => void } = {}, + ) { + const response = await runWithWebSearch({ + parsed: parseRequest({ + model: "routed/model", + input: "hi", + stream: true, + tools: options.tools ?? webSearchOnly, + }), + adapter: sequenceAdapter(passes, seen, options.onPass), + forwardProvider, + hostedTool: { type: "web_search" }, + selectedForwardHeaders: new Headers({ authorization: "Bearer token" }), + settings: { model: "gpt-5.6-luna", reasoning: "low", timeoutMs: 30_000 }, + maxSearches: 1, + ...(options.abortSignal ? { abortSignal: options.abortSignal } : {}), + }); + return collectSse(response.body!); + } + + /** Only the three frames that end a turn, in the order the bridge emitted them. */ + function terminalFrames(frames: { event?: string }[]): string[] { + return frames + .map(frame => frame.event ?? "") + .filter(event => event === "response.completed" || event === "response.failed" || event === "response.incomplete"); + } + + test("an empty forced pass is retried once and completes", async () => { + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "done" }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ]); + expect(frames.some(frame => frame.event === "response.completed")).toBe(true); + expect(frames.some(frame => frame.event === "response.failed")).toBe(false); + }); + + test("the recovery pass asks for text with every tool removed", async () => { + const seen: OcxParsedRequest[] = []; + await drivePasses([ + webSearchFirstPass, + [{ type: "done" }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ], seen); + // The search pass plus the empty forced pass plus exactly one recovery — no extra upstream call. + expect(seen).toHaveLength(3); + const recovery = seen[2]!; + expect(recovery.options.toolChoice).toBe("none"); + expect(recovery.context.tools).toEqual([]); + // The results gathered by the search reach the recovery turn as a tool result ... + expect(recovery.context.messages.filter(message => message.role === "toolResult")).toHaveLength(1); + // ... and the recovery turn carries the developer nudge that asks for the missing text. + expect(recovery.context.messages.some(message => + message.role === "developer" && String(message.content).includes("no tools are available"))) + .toBe(true); + }); + + test("an ordinary client tool is dropped from the recovery pass but kept for the forced pass", async () => { + const seen: OcxParsedRequest[] = []; + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "done" }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ], seen, { tools: webSearchAndFileTool }); + expect(seen).toHaveLength(3); + // Forced pass: the synthetic web_search is gone, the client's own tool is still advertised. + expect(seen[1]!.context.tools?.map(tool => tool.name)).toEqual(["read_file"]); + expect(seen[1]!.options.toolChoice).toBeUndefined(); + // Recovery pass: nothing to call at all — in the tool list AND in the tool choice. + expect(seen[2]!.context.tools).toEqual([]); + expect(seen[2]!.options.toolChoice).toBe("none"); + expect(frames.some(frame => frame.event === "response.completed")).toBe(true); + }); + + test("a persistent empty forced pass still fails after the one recovery", async () => { + const seen: OcxParsedRequest[] = []; + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "done" }], + [{ type: "done" }], + ], seen); + expect(seen).toHaveLength(3); + expect(frames.some(frame => frame.event === "response.failed")).toBe(true); + expect(frames.some(frame => frame.event === "response.completed")).toBe(false); + }); + + test("a malformed forced call is not retried", async () => { + const seen: OcxParsedRequest[] = []; + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "tool_call_start", id: "", name: "" }, { type: "tool_call_end" }, { type: "done" }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ], seen); + expect(seen).toHaveLength(2); + expect(frames.some(frame => frame.event === "response.failed")).toBe(true); + }); + + // A filtered/truncated forced pass is a provider DECISION, not silence. The bridge already + // reports it as response.incomplete, so the loop must spend no second upstream call on it and + // must not turn it into a success — the third pass below is deliberately a good answer that + // must never be requested. + for (const stopReason of ["content_filter", "max_tokens", "refusal"] as const) { + test("a " + stopReason + " forced terminal is replayed incompletely and never retried", async () => { + const seen: OcxParsedRequest[] = []; + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "done", stopReason }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ], seen); + expect(seen).toHaveLength(2); + expect(terminalFrames(frames)).toEqual(["response.incomplete"]); + const incomplete = frames.filter(frame => frame.event === "response.incomplete"); + const snapshot = incomplete[0]!.data.response as { incomplete_details: { reason: string } }; + expect(snapshot.incomplete_details.reason) + .toBe(stopReason === "max_tokens" ? "max_output_tokens" : "content_filter"); + }); + } + + test("a cancelled turn does not spend the recovery attempt", async () => { + const seen: OcxParsedRequest[] = []; + const controller = new AbortController(); + const frames = await drivePasses([ + webSearchFirstPass, + [{ type: "done" }], + [{ type: "text_delta", text: "recovered answer" }, { type: "done" }], + ], seen, { + abortSignal: controller.signal, + onPass: pass => { if (pass === 1) controller.abort(new Error("client closed responses stream")); }, + }); + expect(seen).toHaveLength(2); + expect(frames.some(frame => frame.event === "response.completed")).toBe(false); + }); + }); }); const routedProvider: OcxProviderConfig = {