From 91e75e2938ad1af4d2b034b83d62e1fae4b76bcd Mon Sep 17 00:00:00 2001 From: DavidBabinec Date: Sat, 11 Jul 2026 17:13:29 +0200 Subject: [PATCH] fix(agent): recover interrupted browser tool turns --- docs/features/agent.md | 22 +- server/ai/conversations/history.ts | 8 +- server/ai/drivers/http/execTool.ts | 19 +- server/ai/drivers/http/toolLoop.ts | 50 +++- server/ai/handlers/chat.ts | 18 +- server/ai/handlers/toolResult.ts | 11 +- server/ai/mcp/server.test.ts | 26 ++ server/ai/mcp/server.ts | 34 ++- server/ai/runtime/runner.ts | 51 +++- src/__tests__/agent/agentApi.test.ts | 162 +++++++++++++ src/__tests__/agent/agentSlice.test.ts | 227 +++++++++++++++++- src/__tests__/agent/contentBridge.test.ts | 67 +++++- src/__tests__/agent/toolResultApi.test.ts | 98 ++++++++ src/__tests__/ai/chatImageHandler.test.ts | 85 +++++++ src/__tests__/ai/messageHistory.test.ts | 2 +- .../ai/runnerToolFinalization.test.ts | 175 ++++++++++++++ src/__tests__/ai/toolLoop.test.ts | 86 +++++++ src/admin/ai/toolResultApi.ts | 9 +- src/admin/ai/useMcpWorkspaceBridge.ts | 103 ++++---- src/admin/pages/site/agent/agentApi.ts | 58 ++++- src/admin/pages/site/agent/agentSlice.ts | 26 +- src/admin/pages/site/agent/index.ts | 1 + src/admin/pages/site/agent/streamEvents.ts | 19 +- .../pages/site/agent/toolCallLifecycle.ts | 23 ++ src/core/ai/index.ts | 1 + src/core/ai/toolOutput.ts | 9 + 26 files changed, 1252 insertions(+), 138 deletions(-) create mode 100644 src/__tests__/agent/agentApi.test.ts create mode 100644 src/__tests__/agent/toolResultApi.test.ts create mode 100644 src/__tests__/ai/runnerToolFinalization.test.ts create mode 100644 src/admin/pages/site/agent/toolCallLifecycle.ts diff --git a/docs/features/agent.md b/docs/features/agent.md index d3ee642fb..9be3a8fef 100644 --- a/docs/features/agent.md +++ b/docs/features/agent.md @@ -721,7 +721,7 @@ The server admits only one active writer per conversation. A concurrent tab rece **User attachments are not tool screenshots.** A pasted image is a persisted `kind:'image'` block on a user message. Conversation-detail responses, authenticated image responses, and chat streams use `Cache-Control: private, no-store`; the database remains the intentional durable copy. Images returned by `site_render_snapshot` or another browser tool instead travel transiently on the plural `AiToolOutput.images` channel, are subject to the heavy-evidence rules above, and remain session-only even though the panel exposes every returned image through the same gallery and draggable preview. The two paths share provider-native image mapping but have different storage and replay lifecycles. -**Tool outcomes are first-class.** A `role:'tool'` row records its result as a `{ kind: 'toolResult', ok, error? }` block — `ok` is an explicit boolean, never inferred from the emptiness of a text block. The persister writes it (`appendToolResult`), `buildMessageHistory` reads `ok`/`error` straight off the block to reconstruct the replay `AiToolOutput`, and the client folds it back into the matching tool-call badge (`rehydrateMessages`). The heavy successful `data` an `AiToolOutput` may carry is intentionally **not** persisted: the model already consumed it in the round that produced the result, so replay only needs `{ ok, error }` — re-feeding large tool payloads every turn would bloat the context for no benefit. +**Tool outcomes are first-class.** A `role:'tool'` row records its result as a `{ kind: 'toolResult', ok, error? }` block — `ok` is an explicit boolean, never inferred from the emptiness of a text block. The persister writes it (`appendToolResult`), `buildMessageHistory` reads `ok`/`error` straight off the block to reconstruct the replay `AiToolOutput`, and the client folds it back into the matching tool-call badge (`rehydrateMessages`). A loaded conversation never owns a live bridge from its previous process: any persisted call without a matching valid result is finalized as `INTERRUPTED_TOOL_RESULT_ERROR`, never restored as pending. The heavy successful `data` an `AiToolOutput` may carry is intentionally **not** persisted: the model already consumed it in the round that produced the result, so replay only needs `{ ok, error }` — re-feeding large tool payloads every turn would bloat the context for no benefit. --- @@ -767,16 +767,13 @@ unblocks deletion of the credential that had been protected by the default FK. ## Abort + crash recovery -- **Abort.** "Stop" calls `agentSlice.abortAgent()` → `AbortController.abort()` → the fetch stream closes. When the abort signal fires on the server: - - `req.signal` is passed straight to every `fetch()` call in the driver loop (`fetch(endpoint, { signal })`). The in-flight HTTP request to the provider is cancelled immediately — no further tokens are generated or billed. On `AbortError` the loop returns cleanly with no `error` event. - - Any `callBrowser` promise still waiting for a browser tool-result rejects via the `onAbort` listener registered per pending call (in `server/ai/runtime/transport.ts`). The listener fires, clears the timeout, and removes the pending entry. - - The stream's `destroy()` hook fires, rejects any remaining pending entries, and removes the bridge from the registry. -- **Interrupted tool calls.** If a stream aborts mid-turn — between the assistant's `tool_use` row write and the matching `tool_result` row write (e.g. `ERR_INCOMPLETE_CHUNKED_ENCODING`, server restart) — the persisted history has an unanswered `tool_use` block. `buildMessageHistory` in `server/ai/conversations/history.ts` heals the gap: every tool-call id that has no persisted `tool` result row gets a synthetic error result (`INTERRUPTED_TOOL_RESULT_ERROR`) injected before the next user turn. The model reads the error and can retry; the conversation is never permanently un-sendable. Adjacent synthetic results plus the following real user prompt are merged into one user turn by `pushUserContent` in `server/ai/drivers/anthropic.ts`, satisfying Anthropic's strict user/assistant alternation requirement. -- **Browser tool timeout.** If the browser never POSTs a tool-result, `callBrowser` rejects after 90 seconds (`BROWSER_TOOL_TIMEOUT_MS` in `server/ai/runtime/transport.ts`). The driver sees a rejection, emits an error, and the stream closes. This prevents a closed or unresponsive tab from hanging the tool loop indefinitely. +- **Abort owns the whole response.** "Stop" calls `agentSlice.abortAgent()` and aborts the chat fetch. The server also owns a response-lifecycle controller: `ReadableStream.cancel()` or a failed `controller.enqueue()` aborts the same turn even when the original request signal does not observe a disappearing response consumer. `AbortSignal.any()` threads that combined signal into the provider request and browser bridge, and the handler's `finally` destroys the bridge and releases the per-conversation writer lock. +- **Pending calls become terminal.** `runChat` persists `INTERRUPTED_TOOL_RESULT_ERROR` for every declared tool call still unresolved on a graceful abort or terminal driver event. A hard process stop can still land between those two writes, so both recovery projections enforce the same invariant: `buildMessageHistory` injects a synthetic error for provider replay, while `rehydrateMessages` renders an unmatched or malformed call as a failed historical badge. `pending` therefore means only work owned by the current live stream; a reload never shows an old spinner. Adjacent synthetic results plus the following real user prompt are merged into one user turn by `pushUserContent` in `server/ai/drivers/anthropic.ts`, satisfying Anthropic's strict user/assistant alternation requirement. +- **Browser bridge failures are terminal once.** A browser executor resolving `{ ok: false }` remains an ordinary model-correctable tool outcome. A rejected `callBrowser` is transport failure instead: the loop emits exactly one failed `toolResult`, then one terminal `error`, and does not spend another provider round retrying against the same dead bridge. A missing result still has a 90-second upper bound (`BROWSER_TOOL_TIMEOUT_MS`), but it now ends the turn rather than starting a chain of 90-second retries. - **Crash on server.** If `runChat` throws, the stream emits `{ type: 'error', message }`. The browser surfaces the message verbatim in the Agent Panel (admin-only surface, so info-disclosure is not a concern). - **Tool failure.** Browser executors wrap every call in try/catch. Failures return `{ ok: false, error }`. The model reads the error message in the next turn and retries with corrected input. -- **Bridge-result POST after abort.** If the browser POSTs a tool-result after the stream has closed, the server returns 404 and drops the result silently. -- **Page reload mid-stream.** The stream dies. The conversation row and its persisted messages survive. The user can reload the past thread via `loadAgentConversation` and re-send. +- **Tool-result delivery failure.** A 404 means the browser completed work for a bridge the active runtime no longer owns (commonly a server restart). While the chat signal is active, `postToolResult` propagates that failure, the client aborts the stale response, finalizes its pending badge, and asks the user to send again. Only a POST already being torn down by an aborted signal is ignored quietly. A clean NDJSON EOF without `done` or `error` is handled the same way instead of being mistaken for success. +- **Page reload mid-stream.** The response cancel hook aborts the provider and releases the writer lock. Conversation rows survive; loading the thread shows any unmatched call once as interrupted, with no reconstructed session-only screenshot and no live timeout/spinner. --- @@ -828,7 +825,7 @@ unblocks deletion of the credential that had been protected by the default FK. - `server/ai/handlers/chat.ts` — `POST /admin/api/ai/chat/:scope` endpoint - `server/ai/handlers/conversations.ts` — conversation CRUD plus the ownership-guarded lazy image endpoint - `server/ai/handlers/toolResult.ts` — `POST /admin/api/ai/tool-result` endpoint - - `server/ai/conversations/history.ts` — `buildMessageHistory()` + `INTERRUPTED_TOOL_RESULT_ERROR` (heals interrupted tool calls) + - `src/core/ai/toolOutput.ts` — canonical `AiToolOutput` envelope + shared `INTERRUPTED_TOOL_RESULT_ERROR` - `server/ai/conversations/store.ts` — `appendMessage`, `listMessagesForConversation`, `readConversationForUser` - `server/ai/runtime/runner.ts` — `runChat()` driver loop - `server/ai/contextTokens.ts` — `normalizeContextTokens()` — provider-normalised "context used" for the meter @@ -838,6 +835,9 @@ unblocks deletion of the credential that had been protected by the default FK. - `server/ai/runtime/persister.ts` — `ConversationsPersister` interface + `createConversationsPersister()` - `server/ai/runtime/types.ts` — canonical `AiStreamEvent`, `AiMessage`, `AiTool`, `ToolContext` types - `server/ai/runtime/transport.ts` — `createBridge()` / `resolveBridgeToolResult()` + - `src/admin/ai/toolResultApi.ts` — browser tool-result delivery; active failures terminate the stale chat turn + - `src/admin/pages/site/agent/agentApi.ts` — conversation bootstrap + terminal historical tool-call rehydration + - `src/admin/pages/site/agent/toolCallLifecycle.ts` — live-stream pending-call finalization - `server/ai/audit/store.ts` — `getUsageTotals`, `getUsageByUser`, `getUsageByScope`, `getUsageByModel`, `getUsageByDay` (usage rollup queries) - `server/ai/handlers/audit.ts` — `GET /admin/api/ai/audit` handler - `server/time.ts` — `resolveTimeZone` + `localDayKeyFactory` (shared timezone day-bucketing utilities) @@ -848,7 +848,7 @@ unblocks deletion of the credential that had been protected by the default FK. - `src/admin/pages/site/agent/agentSlice.ts` — scope-agnostic slice factory (`createAgentSlice`) - `src/admin/pages/site/agent/agentProviderUpdate.ts` — timed provider/model update and ambiguous-commit reconciliation - `src/admin/pages/site/agent/agentSliceConfig.site.ts` — site-editor scope config - - `src/admin/pages/site/agent/agentApi.ts` — tool-result POST, conversation bootstrap, message rehydration + - `src/admin/pages/site/agent/agentApi.ts` — conversation bootstrap and message rehydration - `src/admin/pages/site/agent/streamEvents.ts` — `ServerStreamEventSchema` + `processStreamEvent` - `src/admin/pages/site/panels/AgentPanel/AgentImageGallery.tsx` — shared compact gallery for persisted and session-only images - `src/admin/pages/site/panels/AgentPanel/AgentImagePreview.tsx` — draggable modeless image preview diff --git a/server/ai/conversations/history.ts b/server/ai/conversations/history.ts index 08f744b83..a12b0f7bb 100644 --- a/server/ai/conversations/history.ts +++ b/server/ai/conversations/history.ts @@ -23,16 +23,10 @@ * preceding tool call in the replayed history. */ +import { INTERRUPTED_TOOL_RESULT_ERROR } from '@core/ai' import type { AiContentBlock, AiMessage, AiToolOutput } from '../runtime/types' import type { MessageRecord } from './types' -/** - * Error surfaced to the model for a tool call whose result was never persisted - * because the turn was interrupted before it completed. - */ -export const INTERRUPTED_TOOL_RESULT_ERROR = - 'Tool call did not complete — the previous turn was interrupted before a result was produced.' - export const NON_VISION_USER_IMAGE_OMITTED = '[Attached image omitted because the selected model does not support image input.]' diff --git a/server/ai/drivers/http/execTool.ts b/server/ai/drivers/http/execTool.ts index cda59608e..ba7f8f64c 100644 --- a/server/ai/drivers/http/execTool.ts +++ b/server/ai/drivers/http/execTool.ts @@ -23,9 +23,13 @@ import type { import type { ToolContextBase } from '../types' /** - * Execute one tool call and return the canonical `AiToolOutput`. Never throws — - * validation, handler, and bridge failures all collapse to `{ ok: false, error }` - * so the loop can feed the failure back to the model and let it recover. + * Execute one tool call and return the canonical `AiToolOutput`. + * + * Input, permission, and server-handler failures are ordinary tool outcomes: + * the loop feeds `{ ok: false, error }` back to the model so it can recover. + * A rejected browser bridge is different — no result can reach the active + * turn, so that infrastructure failure propagates to the loop and terminates + * the turn instead of inviting the model to retry against the same dead bridge. */ export async function executeAiTool( aiTool: AiTool, @@ -64,12 +68,9 @@ export async function executeAiTool( } // Browser execution: forward to the bridge and wait for the POST-back. - try { - return await bridge.callBrowser(aiTool.name, validated) - } catch (err) { - const message = err instanceof Error ? err.message : `Tool ${aiTool.name} failed.` - return { ok: false, error: message } - } + // A resolved `{ ok: false }` remains a recoverable domain failure. Rejection + // means the transport itself is unavailable and deliberately propagates. + return await bridge.callBrowser(aiTool.name, validated) } /** diff --git a/server/ai/drivers/http/toolLoop.ts b/server/ai/drivers/http/toolLoop.ts index a09c8fd2e..09d14a6be 100644 --- a/server/ai/drivers/http/toolLoop.ts +++ b/server/ai/drivers/http/toolLoop.ts @@ -132,6 +132,15 @@ export async function* runToolLoop( let cacheCreationTokens = 0 let costUsd: number | undefined + const aggregateUsageEvent = (): Extract => ({ + type: 'usage', + promptTokens, + completionTokens, + costUsd, + cacheReadTokens: cacheReadTokens || undefined, + cacheCreationTokens: cacheCreationTokens || undefined, + }) + for (;;) { if (req.signal.aborted) return @@ -219,9 +228,35 @@ export async function* runToolLoop( for (const call of turn.toolCalls) { const tool = toolsByName.get(call.name) const input = prepareToolInput(call, req) - const output: AiToolOutput = tool - ? await executeAiTool(tool, input, req.bridge, req.signal, req.toolContextBase) - : { ok: false, error: `Unknown tool: ${call.name}` } + let output: AiToolOutput + try { + output = tool + ? await executeAiTool(tool, input, req.bridge, req.signal, req.toolContextBase) + : { ok: false, error: `Unknown tool: ${call.name}` } + } catch (err) { + // Browser tools communicate domain failures by resolving an + // `AiToolOutput`. Rejection means the bridge transport disappeared + // (timeout, server reload, closed stream). Retrying within this turn + // would hit the same dead bridge and can burn repeated provider rounds. + if (req.signal.aborted) return + const detail = err instanceof Error ? err.message : String(err) + yield { + type: 'toolResult', + toolCallId: call.id, + toolName: call.name, + ok: false, + error: detail, + } + // The provider already completed and billed this round before the + // bridge failed. Persist its accumulated usage before the terminal + // error so conversation totals and the failed-turn audit stay honest. + yield aggregateUsageEvent() + yield { + type: 'error', + message: `Browser tool transport failed: ${detail}`, + } + return + } yield { type: 'toolResult', toolCallId: call.id, @@ -240,14 +275,7 @@ export async function* runToolLoop( } } - yield { - type: 'usage', - promptTokens, - completionTokens, - costUsd, - cacheReadTokens: cacheReadTokens || undefined, - cacheCreationTokens: cacheCreationTokens || undefined, - } + yield aggregateUsageEvent() } /** diff --git a/server/ai/handlers/chat.ts b/server/ai/handlers/chat.ts index 5485ba2d8..2a0473dcd 100644 --- a/server/ai/handlers/chat.ts +++ b/server/ai/handlers/chat.ts @@ -320,6 +320,13 @@ async function handleAiChat( })() const { messages, systemPrompt, tokensAtStart } = prepared + // `req.signal` covers request-side aborts, but a streaming response consumer + // can disappear independently (tab reload, dev-server hot restart, proxy + // disconnect). Own a second lifecycle signal and abort it from the response + // stream's `cancel()` hook or when enqueue proves the consumer is gone. + const streamAbort = new AbortController() + const turnSignal = AbortSignal.any([req.signal, streamAbort.signal]) + const stream = new ReadableStream({ async start(controller) { let streamClosed = false @@ -348,6 +355,7 @@ async function handleAiChat( controller.enqueue(encodeStreamEvent(wireEvent)) } catch { streamClosed = true + streamAbort.abort() } } @@ -366,7 +374,7 @@ async function handleAiChat( } const { bridgeId, bridge, destroy } = createBridge( emit, - req.signal, + turnSignal, undefined, (next) => { toolContextBase.snapshot = next }, ) @@ -382,7 +390,7 @@ async function handleAiChat( modelId: conversation.modelId, modelCapabilities, credentials: resolvedCredential, - signal: req.signal, + signal: turnSignal, bridge, toolContextBase, } @@ -435,6 +443,12 @@ async function handleAiChat( } } }, + cancel() { + // Abort provider fetches and pending browser waiters immediately; the + // handler's finally block then destroys the bridge and releases the + // per-conversation writer lock. + streamAbort.abort() + }, }) return new Response(stream, { diff --git a/server/ai/handlers/toolResult.ts b/server/ai/handlers/toolResult.ts index f718b885b..539691211 100644 --- a/server/ai/handlers/toolResult.ts +++ b/server/ai/handlers/toolResult.ts @@ -52,9 +52,14 @@ async function handleAiToolResult(req: Request, db: DbClient): Promise const matched = resolveBridgeToolResult(bridgeId, requestId, result, snapshot) if (!matched) { - // Bridge gone or unknown requestId — likely the stream was aborted - // before the browser's POST arrived. Not a fatal client error. - return jsonResponse({ ok: false }, { status: 404 }) + // The browser completed work for a turn this runtime no longer owns — most + // commonly after a server restart. The active chat client must abort its + // old response instead of silently waiting 90 seconds and letting the + // model retry the same tool repeatedly. + return jsonResponse( + { error: 'The AI tool bridge is no longer active. The server may have restarted; send the message again.' }, + { status: 404 }, + ) } return jsonResponse({ ok: true }) } diff --git a/server/ai/mcp/server.test.ts b/server/ai/mcp/server.test.ts index be965ef4b..f58d8708b 100644 --- a/server/ai/mcp/server.test.ts +++ b/server/ai/mcp/server.test.ts @@ -150,6 +150,32 @@ describe('mcp server', () => { ]) }) + it('returns an MCP tool error when the live editor bridge disconnects mid-call', async () => { + const userId = 'u1-disconnected-workspace' + const controller = new AbortController() + const reader = createEditorBridgeStream(userId, 'site', controller.signal).getReader() + await readUntil(reader, (event) => event.type === 'bridgeReady') + const client = await connectClient( + db, + ['ai.chat', 'ai.tools.write', 'site.structure.edit'], + userId, + ) + + const call = client.callTool({ + name: 'site_insert_html', + arguments: { parentId: 'root', html: '

site

' }, + }) + await readUntil(reader, (event) => event.type === 'toolRequest') + controller.abort() + + const result = await call + expect(result.isError).toBe(true) + expect(JSON.stringify(result.content)).toContain('before tool result arrived') + + await client.close() + await reader.read().catch(() => {}) + }) + it('enforces an own-only connector grant before relaying through a full-power owner browser', async () => { await db` insert into users (id, email, email_normalized, display_name, password_hash, role_id) diff --git a/server/ai/mcp/server.ts b/server/ai/mcp/server.ts index 5a6334822..5da41bbaf 100644 --- a/server/ai/mcp/server.ts +++ b/server/ai/mcp/server.ts @@ -16,7 +16,8 @@ import { } from '@modelcontextprotocol/sdk/types.js' import type { DbClient } from '../../db/client' import type { CoreCapability } from '@core/capabilities' -import type { AiBrowserBridge, AiTool } from '../runtime/types' +import { getErrorMessage } from '@core/utils/errorMessage' +import type { AiBrowserBridge, AiTool, AiToolOutput } from '../runtime/types' import { executeAiTool } from '../drivers/http/execTool' import { mcpToolsForCapabilities } from './registry' import { authorizeMcpContentTool } from './contentAuthorization' @@ -111,14 +112,29 @@ export function buildMcpServer(ctx: McpServerContext): Server { } const controller = new AbortController() - const output = await executeAiTool(tool, args ?? {}, bridge, controller.signal, { - db: ctx.db, - userId: ctx.userId, - capabilities: ctx.capabilities, - scope: tool.scope === 'shared' ? 'content' : tool.scope, - conversationId: `mcp:${ctx.connectorId}`, - snapshot: null, - }) + let output: AiToolOutput + try { + output = await executeAiTool(tool, args ?? {}, bridge, controller.signal, { + db: ctx.db, + userId: ctx.userId, + capabilities: ctx.capabilities, + scope: tool.scope === 'shared' ? 'content' : tool.scope, + conversationId: `mcp:${ctx.connectorId}`, + snapshot: null, + }) + } catch (err) { + // Browser bridge rejection is terminal for a chat turn, but MCP has no + // surrounding provider loop to terminate. Translate the same transport + // failure into the protocol's normal tool-error result instead of + // letting the request handler reject with an internal MCP error. + return { + isError: true, + content: [{ + type: 'text', + text: getErrorMessage(err, `Browser tool "${tool.name}" could not return a result.`), + }], + } + } if (!output.ok) { return { isError: true, content: [{ type: 'text', text: output.error ?? 'Tool failed.' }] } diff --git a/server/ai/runtime/runner.ts b/server/ai/runtime/runner.ts index bb0e351e8..5f98516d1 100644 --- a/server/ai/runtime/runner.ts +++ b/server/ai/runtime/runner.ts @@ -20,6 +20,7 @@ import type { ConversationsPersister } from './persister' import type { AiProvider, AiStreamRequest } from '../drivers/types' import type { AiStreamEvent } from './types' +import { INTERRUPTED_TOOL_RESULT_ERROR } from '@core/ai' interface RunChatArgs { driver: AiProvider @@ -52,12 +53,40 @@ export async function runChat(args: RunChatArgs): Promise { await persister.appendAssistantText(text) } + /** + * A graceful abort/terminal driver event can arrive after the assistant's + * tool-call row was persisted but before its result. Close every such pair + * explicitly so normal cancellation never leaves history dangling. A hard + * process crash can still interrupt this write; replay/rehydration heals that + * unavoidable case separately. + */ + async function finalizePendingToolCalls(): Promise { + const pendingCalls = [...pendingToolCallsByCallId] + for (const [toolCallId, pending] of pendingCalls) { + const event: AiStreamEvent = { + type: 'toolResult', + toolCallId, + toolName: pending.name, + ok: false, + error: INTERRUPTED_TOOL_RESULT_ERROR, + } + await persister.appendToolResult({ + toolCallId, + toolName: pending.name, + ok: false, + error: INTERRUPTED_TOOL_RESULT_ERROR, + }) + pendingToolCallsByCallId.delete(toolCallId) + emit(event) + } + } + try { for await (const event of driver.stream(request)) { // Forward live events immediately. Usage is the one exception: its USD // value may need cache-aware server pricing, so that terminal event is // emitted after persistence resolves the authoritative cost below. - if (event.type !== 'usage') emit(event) + if (event.type !== 'usage' && event.type !== 'error') emit(event) switch (event.type) { case 'text': { @@ -66,15 +95,18 @@ export async function runChat(args: RunChatArgs): Promise { } case 'toolCall': { await flushPendingAssistantText() - pendingToolCallsByCallId.set(event.toolCallId, { - name: event.toolName, - input: event.input, - }) await persister.appendToolCall({ toolCallId: event.toolCallId, toolName: event.toolName, input: event.input, }) + // Track only after the declaration row exists. If that write fails, + // finalization must not append an orphan result for a call the + // persisted history never declared. + pendingToolCallsByCallId.set(event.toolCallId, { + name: event.toolName, + input: event.input, + }) break } case 'toolResult': { @@ -125,8 +157,11 @@ export async function runChat(args: RunChatArgs): Promise { case 'error': { // Driver reported a terminal error. Flush whatever text accumulated // before bailing — the user still sees the partial assistant - // message in their history. + // message in their history. Pending tool outcomes must precede the + // terminal wire error so the client can finalize its status rows. await flushPendingAssistantText() + await finalizePendingToolCalls() + emit(event) return } // `bridgeReady`, `toolRequest`, `done`: nothing to persist. @@ -137,6 +172,7 @@ export async function runChat(args: RunChatArgs): Promise { // Stream ended without explicit error or done — flush trailing text. await flushPendingAssistantText() + await finalizePendingToolCalls() emit({ type: 'done' }) } catch (err) { const detail = err instanceof Error ? err.message : String(err) @@ -146,6 +182,9 @@ export async function runChat(args: RunChatArgs): Promise { // admins, not end users. console.error('[ai/runner] driver.stream() threw:', err) await flushPendingAssistantText().catch(() => { /* noop */ }) + await finalizePendingToolCalls().catch((finalizeErr) => { + console.error('[ai/runner] pending tool finalization failed:', finalizeErr) + }) emit({ type: 'error', message: `AI runtime error: ${detail}` }) } } diff --git a/src/__tests__/agent/agentApi.test.ts b/src/__tests__/agent/agentApi.test.ts new file mode 100644 index 000000000..4fde8892b --- /dev/null +++ b/src/__tests__/agent/agentApi.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'bun:test' +import { INTERRUPTED_TOOL_RESULT_ERROR } from '@core/ai' +import type { ConversationDetail } from '@admin/ai/api' +import { rehydrateMessages, type AgentMessage, type AgentToolCall } from '@site/agent' + +type PersistedMessage = ConversationDetail['messages'][number] + +function message( + id: string, + role: PersistedMessage['role'], + content: PersistedMessage['content'], + toolCallId: string | null = null, + toolName: string | null = null, +): PersistedMessage { + return { + id, + position: 0, + role, + content, + toolCallId, + toolName, + createdAt: '2026-07-11T10:00:00.000Z', + } +} + +function toolCalls(messages: AgentMessage[]): AgentToolCall[] { + return messages.flatMap((entry) => entry.blocks.flatMap((block) => + block.kind === 'toolCall' ? [block.toolCall] : [], + )) +} + +describe('rehydrateMessages — persisted tool recovery', () => { + it('keeps completed results and finalizes an unanswered screenshot as interrupted', () => { + const messages = rehydrateMessages([ + message('user-1', 'user', [{ kind: 'text', text: 'Check both breakpoints.' }]), + message( + 'assistant-complete', + 'assistant', + [{ + kind: 'toolCall', + toolCallId: 'snapshot-complete', + toolName: 'site_render_snapshot', + input: { breakpointId: 'mobile' }, + }], + 'snapshot-complete', + 'site_render_snapshot', + ), + message( + 'result-complete', + 'tool', + [{ kind: 'toolResult', ok: true }], + 'snapshot-complete', + 'site_render_snapshot', + ), + message( + 'assistant-interrupted', + 'assistant', + [{ + kind: 'toolCall', + toolCallId: 'snapshot-interrupted', + toolName: 'site_render_snapshot', + input: { breakpointId: 'desktop' }, + }], + 'snapshot-interrupted', + 'site_render_snapshot', + ), + ]) + + const [completed, interrupted] = toolCalls(messages) + expect(completed).toMatchObject({ + externalId: 'snapshot-complete', + actionType: 'site_render_snapshot', + params: { breakpointId: 'mobile' }, + status: 'success', + result: { ok: true }, + }) + expect(completed?.previewImages).toBeUndefined() + expect(interrupted).toMatchObject({ + externalId: 'snapshot-interrupted', + actionType: 'site_render_snapshot', + params: { breakpointId: 'desktop' }, + status: 'error', + result: { ok: false, error: INTERRUPTED_TOOL_RESULT_ERROR }, + }) + expect(interrupted?.previewImages).toBeUndefined() + expect(toolCalls(messages).some((toolCall) => toolCall.status === 'pending')).toBe(false) + }) + + it('finalizes malformed matching results and ignores orphan tool rows', () => { + const messages = rehydrateMessages([ + message( + 'assistant-1', + 'assistant', + [{ + kind: 'toolCall', + toolCallId: 'snapshot-malformed', + toolName: 'site_render_snapshot', + input: ['invalid', 'params'], + }], + 'snapshot-malformed', + 'site_render_snapshot', + ), + // A role:tool row without a first-class result block is malformed. It + // must terminate the historical call rather than leave it running. + message( + 'malformed-result', + 'tool', + [{ kind: 'text', text: 'legacy result' }], + 'snapshot-malformed', + 'site_render_snapshot', + ), + // Missing ids and unmatched ids are ignored, never rendered as empty + // assistant messages. + message('missing-id-result', 'tool', [{ kind: 'toolResult', ok: true }]), + message( + 'orphan-result', + 'tool', + [{ kind: 'toolResult', ok: true }], + 'unknown-call', + 'site_render_snapshot', + ), + ]) + + expect(messages).toHaveLength(1) + expect(toolCalls(messages)[0]).toMatchObject({ + params: {}, + status: 'error', + result: { ok: false, error: INTERRUPTED_TOOL_RESULT_ERROR }, + }) + }) + + it('does not let a late result after the next user turn resurrect an interrupted call', () => { + const messages = rehydrateMessages([ + message( + 'assistant-1', + 'assistant', + [{ + kind: 'toolCall', + toolCallId: 'snapshot-late', + toolName: 'site_render_snapshot', + input: {}, + }], + 'snapshot-late', + 'site_render_snapshot', + ), + message('user-next', 'user', [{ kind: 'text', text: 'Try again.' }]), + message( + 'late-result', + 'tool', + [{ kind: 'toolResult', ok: true }], + 'snapshot-late', + 'site_render_snapshot', + ), + ]) + + expect(toolCalls(messages)[0]).toMatchObject({ + status: 'error', + result: { ok: false, error: INTERRUPTED_TOOL_RESULT_ERROR }, + }) + expect(messages.map((entry) => entry.role)).toEqual(['assistant', 'user']) + }) +}) diff --git a/src/__tests__/agent/agentSlice.test.ts b/src/__tests__/agent/agentSlice.test.ts index f389edadb..6e6d6c6d6 100644 --- a/src/__tests__/agent/agentSlice.test.ts +++ b/src/__tests__/agent/agentSlice.test.ts @@ -9,7 +9,7 @@ import { type AgentToolCall, } from '@site/agent' import type { ConversationView } from '@admin/ai/api' -import type { AiUserContentBlock } from '@core/ai' +import { INTERRUPTED_TOOL_RESULT_ERROR, type AiUserContentBlock } from '@core/ai' import '@modules/base' // --------------------------------------------------------------------------- @@ -142,9 +142,9 @@ const conversationCreateResponse = (id: string) => { status: 201, headers: { 'Content-Type': 'application/json' } }, ) -const conversationDetailResponse = ( +const conversationDetailMessagesResponse = ( id: string, - content: unknown[], + messages: unknown[], selection: { credentialId: string; modelId: string } = { credentialId: 'cred-1', modelId: 'claude-sonnet-4-6', @@ -165,21 +165,27 @@ const conversationDetailResponse = ( contextTokens: 0, createdAt: '2026-07-11T10:00:00.000Z', updatedAt: '2026-07-11T10:00:00.000Z', - messages: [{ - id: 'message-image', - position: 0, - role: 'user', - content, - toolCallId: null, - toolName: null, - createdAt: '2026-07-11T10:00:00.000Z', - }], + messages, }, }), { status: 200, headers: { 'Content-Type': 'application/json' }, }) +const conversationDetailResponse = ( + id: string, + content: unknown[], + selection?: { credentialId: string; modelId: string }, +) => conversationDetailMessagesResponse(id, [{ + id: 'message-image', + position: 0, + role: 'user', + content, + toolCallId: null, + toolName: null, + createdAt: '2026-07-11T10:00:00.000Z', +}], selection) + const toolResultAckResponse = () => new Response(JSON.stringify({ ok: true }), { status: 200, @@ -745,6 +751,106 @@ describe('sendAgentMessage — request lifecycle', () => { void rootId }) + it('aborts the active send and surfaces an active tool-result POST failure', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + let toolResultSignal: AbortSignal | null = null + + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': () => conversationCreateResponse('conv-tool-result-failure'), + '/admin/api/ai/chat/site': () => ndjsonResponse([ + { type: 'bridgeReady', bridgeId: 'bridge-tool-result-failure' }, + { + type: 'toolCall', + toolCallId: 'call-tool-result-failure', + toolName: 'site_apply_css', + input: { css: '.failure-test { display: block; }' }, + status: 'pending', + }, + { + type: 'toolRequest', + requestId: 'request-tool-result-failure', + toolName: 'site_apply_css', + input: { css: '.failure-test { display: block; }' }, + }, + { type: 'done' }, + ]), + '/admin/api/ai/tool-result': (_call, init) => { + toolResultSignal = init?.signal ?? null + return new Response( + JSON.stringify({ error: 'The active tool bridge no longer exists.' }), + { status: 404, headers: { 'Content-Type': 'application/json' } }, + ) + }, + }) + + let result: { accepted: boolean } + try { + result = await useEditorStore.getState().sendAgentMessage( + textContent('Apply a class, then continue.'), + ) + } finally { + intercept.restore() + } + + expect(result).toEqual({ accepted: true }) + expect(toolResultSignal).not.toBeNull() + expect(toolResultSignal!.aborted).toBe(true) + expect(useEditorStore.getState().isAgentStreaming).toBe(false) + expect(useEditorStore.getState().agentError).toContain( + 'The active tool bridge no longer exists.', + ) + const calls = useEditorStore.getState().agentMessages.flatMap(getToolCallBlocks) + expect(calls).toHaveLength(1) + expect(calls[0]).toMatchObject({ + status: 'error', + result: { ok: false, error: expect.stringContaining('The active tool bridge no longer exists.') }, + }) + }) + + it('does not mistake an active tool-result AbortError for an intentional Stop', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + let toolResultSignal: AbortSignal | null = null + + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': () => conversationCreateResponse('conv-active-abort'), + '/admin/api/ai/chat/site': () => ndjsonResponse([ + { type: 'bridgeReady', bridgeId: 'bridge-active-abort' }, + { + type: 'toolRequest', + requestId: 'request-active-abort', + toolName: 'site_apply_css', + input: { css: '.active-abort { display: block; }' }, + }, + { type: 'done' }, + ]), + '/admin/api/ai/tool-result': (_call, init) => { + toolResultSignal = init?.signal ?? null + const error = new Error('Tool-result delivery was aborted upstream.') + error.name = 'AbortError' + return Promise.reject(error) + }, + }) + + try { + expect(await useEditorStore.getState().sendAgentMessage( + textContent('Apply a class, then continue.'), + )).toEqual({ accepted: true }) + } finally { + intercept.restore() + } + + expect(toolResultSignal).not.toBeNull() + expect(toolResultSignal!.aborted).toBe(true) + expect(useEditorStore.getState().isAgentStreaming).toBe(false) + expect(useEditorStore.getState().agentError).toContain( + 'Tool-result delivery was aborted upstream.', + ) + }) + it('surfaces a clear error when no site default credential is configured', async () => { freshAgentState() useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) @@ -769,7 +875,7 @@ describe('sendAgentMessage — request lifecycle', () => { }) }) -describe('loadAgentConversation — image rehydration', () => { +describe('loadAgentConversation — rehydration', () => { it('restores persisted user image blocks and advances the composer epoch', async () => { freshAgentState() useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) @@ -837,6 +943,65 @@ describe('loadAgentConversation — image rehydration', () => { intercept.restore() } }) + + it('restores an interrupted screenshot as a failed historical call, never live work', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentMessages: [], + agentError: 'stale error', + }) + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-restart': () => + conversationDetailMessagesResponse('conv-restart', [ + { + id: 'prompt-1', + position: 0, + role: 'user', + content: [{ kind: 'text', text: 'Capture desktop.' }], + toolCallId: null, + toolName: null, + createdAt: '2026-07-11T10:00:00.000Z', + }, + { + id: 'snapshot-call', + position: 1, + role: 'assistant', + content: [{ + kind: 'toolCall', + toolCallId: 'snapshot-interrupted', + toolName: 'site_render_snapshot', + input: { breakpointId: 'desktop' }, + }], + toolCallId: 'snapshot-interrupted', + toolName: 'site_render_snapshot', + createdAt: '2026-07-11T10:00:01.000Z', + }, + ]), + }) + + try { + await useEditorStore.getState().loadAgentConversation('conv-restart') + } finally { + intercept.restore() + } + + const state = useEditorStore.getState() + const restoredCalls = state.agentMessages.flatMap(getToolCallBlocks) + expect(restoredCalls).toHaveLength(1) + expect(restoredCalls[0]).toMatchObject({ + externalId: 'snapshot-interrupted', + actionType: 'site_render_snapshot', + params: { breakpointId: 'desktop' }, + status: 'error', + result: { ok: false, error: INTERRUPTED_TOOL_RESULT_ERROR }, + }) + expect(restoredCalls[0]?.previewImages).toBeUndefined() + expect(state.agentError).toBeNull() + expect(state.isAgentStreaming).toBe(false) + expect(state.isAgentConversationPending).toBe(false) + expect(state.agentConversationId).toBe('conv-restart') + }) }) describe('conversation reset key-set', () => { @@ -939,6 +1104,42 @@ describe('conversation reset key-set', () => { }) describe('sendAgentMessage — streaming + error surfacing', () => { + it('treats a clean EOF without done/error as an interrupted turn', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': () => conversationCreateResponse('conv-truncated'), + '/admin/api/ai/chat/site': () => ndjsonResponse([ + { type: 'bridgeReady', bridgeId: 'b-truncated' }, + { + type: 'toolCall', + toolCallId: 'snapshot-truncated', + toolName: 'site_render_snapshot', + input: { breakpointId: 'desktop' }, + status: 'pending', + }, + ]), + }) + + try { + expect(await useEditorStore.getState().sendAgentMessage(textContent('Inspect it.'))).toEqual({ + accepted: true, + }) + } finally { + intercept.restore() + } + + const state = useEditorStore.getState() + expect(state.agentError).toContain('server may have restarted') + expect(state.isAgentStreaming).toBe(false) + expect(state.agentMessages.flatMap(getToolCallBlocks)[0]).toMatchObject({ + status: 'error', + result: { ok: false, error: expect.stringContaining('server may have restarted') }, + }) + }) + it('dispatches streamed events and surfaces a mid-stream error event once', async () => { freshAgentState() useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) diff --git a/src/__tests__/agent/contentBridge.test.ts b/src/__tests__/agent/contentBridge.test.ts index 4ae7a6c48..ab2652253 100644 --- a/src/__tests__/agent/contentBridge.test.ts +++ b/src/__tests__/agent/contentBridge.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it } from 'bun:test' import { executeContentTool } from '@content/agent/contentBridge' -import { executeMcpBridgeRequest } from '@admin/ai/useMcpWorkspaceBridge' +import { + executeMcpBridgeRequest, + runMcpWorkspaceBridgeConnection, +} from '@admin/ai/useMcpWorkspaceBridge' import { setContentBridgeHandle, type ContentBridgeHandle, @@ -178,3 +181,65 @@ describe('executeMcpBridgeRequest', () => { expect(persisted).toBe(false) }) }) + +describe('runMcpWorkspaceBridgeConnection', () => { + it('aborts a connection whose result POST fails and starts the next attempt fresh', async () => { + const realFetch = globalThis.fetch + const connectionSignals: AbortSignal[] = [] + let requestCount = 0 + globalThis.fetch = (async (_input, init) => { + requestCount += 1 + if (requestCount === 1) { + const signal = init?.signal as AbortSignal + expect(signal.aborted).toBe(false) + connectionSignals.push(signal) + return new Response([ + JSON.stringify({ type: 'bridgeReady', bridgeId: 'bridge-stale' }), + JSON.stringify({ + type: 'toolRequest', + requestId: 'request-stale', + toolName: 'site_apply_css', + input: {}, + }), + '', + ].join('\n'), { headers: { 'Content-Type': 'application/x-ndjson' } }) + } + if (requestCount === 2) { + return new Response( + JSON.stringify({ error: 'The editor bridge no longer owns this request.' }), + { status: 404, headers: { 'Content-Type': 'application/json' } }, + ) + } + + const signal = init?.signal as AbortSignal + expect(signal.aborted).toBe(false) + connectionSignals.push(signal) + return new Response(null, { status: 401 }) + }) as typeof fetch + + const lifecycleController = new AbortController() + try { + await expect(runMcpWorkspaceBridgeConnection( + 'site', + async () => ({ ok: true }), + undefined, + lifecycleController.signal, + )).rejects.toThrow('The editor bridge no longer owns this request.') + + expect(lifecycleController.signal.aborted).toBe(false) + expect(connectionSignals[0]?.aborted).toBe(true) + + await expect(runMcpWorkspaceBridgeConnection( + 'site', + async () => ({ ok: true }), + undefined, + lifecycleController.signal, + )).resolves.toBe('auth') + expect(connectionSignals[1]?.aborted).toBe(true) + expect(requestCount).toBe(3) + } finally { + lifecycleController.abort() + globalThis.fetch = realFetch + } + }) +}) diff --git a/src/__tests__/agent/toolResultApi.test.ts b/src/__tests__/agent/toolResultApi.test.ts new file mode 100644 index 000000000..f5d914d4c --- /dev/null +++ b/src/__tests__/agent/toolResultApi.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'bun:test' +import { postToolResult } from '@admin/ai/toolResultApi' + +function abortError(): Error { + const error = new Error('The operation was aborted.') + error.name = 'AbortError' + return error +} + +async function withFetch( + implementation: typeof fetch, + run: () => Promise, +): Promise { + const originalFetch = globalThis.fetch + globalThis.fetch = implementation + try { + await run() + } finally { + globalThis.fetch = originalFetch + } +} + +describe('postToolResult', () => { + it('rejects an active 404 bridge failure with the server error', async () => { + const controller = new AbortController() + + await withFetch( + (async () => new Response( + JSON.stringify({ error: 'The active tool bridge no longer exists.' }), + { status: 404, headers: { 'Content-Type': 'application/json' } }, + )) as typeof fetch, + async () => { + await expect(postToolResult( + 'bridge-1', + 'request-1', + { ok: true }, + controller.signal, + )).rejects.toThrow('The active tool bridge no longer exists.') + }, + ) + }) + + it('rejects other failures while the bridge is active', async () => { + const controller = new AbortController() + + await withFetch( + (async () => new Response( + JSON.stringify({ error: 'Tool-result storage failed.' }), + { status: 503, headers: { 'Content-Type': 'application/json' } }, + )) as typeof fetch, + async () => { + await expect(postToolResult( + 'bridge-1', + 'request-1', + { ok: true }, + controller.signal, + )).rejects.toThrow('Tool-result storage failed.') + }, + ) + }) + + it('keeps an already-aborted teardown quiet', async () => { + const controller = new AbortController() + controller.abort() + + await withFetch( + (async () => { + throw abortError() + }) as typeof fetch, + async () => { + await expect(postToolResult( + 'bridge-1', + 'request-1', + { ok: true }, + controller.signal, + )).resolves.toBeUndefined() + }, + ) + }) + + it('does not hide an abort failure while the bridge remains active', async () => { + const controller = new AbortController() + + await withFetch( + (async () => { + throw abortError() + }) as typeof fetch, + async () => { + await expect(postToolResult( + 'bridge-1', + 'request-1', + { ok: true }, + controller.signal, + )).rejects.toThrow('The operation was aborted.') + }, + ) + }) +}) diff --git a/src/__tests__/ai/chatImageHandler.test.ts b/src/__tests__/ai/chatImageHandler.test.ts index bd1c7aca3..a8505bfab 100644 --- a/src/__tests__/ai/chatImageHandler.test.ts +++ b/src/__tests__/ai/chatImageHandler.test.ts @@ -214,6 +214,74 @@ describe('AI chat user-image boundary', () => { expect(persistedUserImages).toHaveLength(1) }) + it('aborts the provider and releases the writer lock when the response is cancelled', async () => { + const providerStarted = deferred() + const providerAborted = deferred() + let providerCalls = 0 + globalThis.fetch = async (input, init) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['tools'] }) + } + if (url !== 'http://ollama.test/v1/chat/completions') { + throw new Error(`Unexpected fetch: ${url}`) + } + providerCalls += 1 + if (providerCalls === 1) { + providerStarted.resolve() + const signal = init?.signal + if (!signal) throw new Error('Provider request did not receive a turn signal.') + return await new Promise((_resolve, reject) => { + const onAbort = () => { + providerAborted.resolve() + reject(new DOMException('Provider request aborted.', 'AbortError')) + } + if (signal.aborted) onAbort() + else signal.addEventListener('abort', onAbort, { once: true }) + }) + } + return new Response([ + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } }) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'Wait while inspecting.' }], + }, + }) + expect(response.status).toBe(200) + await providerStarted.promise + + await response.body?.cancel() + await withTimeout(providerAborted.promise, 1_000) + + // Cancellation tears down the bridge/provider in the stream's finally + // path. Once that path releases admission, this conversation accepts a + // new turn instead of remaining stuck behind the abandoned request. + let retry: Response | null = null + for (let attempt = 0; attempt < 20; attempt += 1) { + retry = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'Continue.' }], + }, + }) + if (retry.status !== 409) break + await new Promise((resolve) => setTimeout(resolve, 0)) + } + + expect(retry?.status).toBe(200) + await retry?.text() + expect(providerCalls).toBe(2) + }) + it('does not persist a turn aborted during capability discovery', async () => { const image = await jpegBlock() const capabilityStarted = deferred() @@ -313,3 +381,20 @@ function deferred() { const promise = new Promise((done) => { resolve = done }) return { promise, resolve } } + +async function withTimeout(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | null = null + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`Timed out after ${timeoutMs}ms.`)), + timeoutMs, + ) + }), + ]) + } finally { + if (timer) clearTimeout(timer) + } +} diff --git a/src/__tests__/ai/messageHistory.test.ts b/src/__tests__/ai/messageHistory.test.ts index 0fc9a8e2e..70f79518c 100644 --- a/src/__tests__/ai/messageHistory.test.ts +++ b/src/__tests__/ai/messageHistory.test.ts @@ -1,7 +1,7 @@ import { describe, test, expect } from 'bun:test' +import { INTERRUPTED_TOOL_RESULT_ERROR } from '@core/ai' import { buildMessageHistory, - INTERRUPTED_TOOL_RESULT_ERROR, NON_VISION_USER_IMAGE_OMITTED, projectUserImagesForModel, } from '../../../server/ai/conversations/history' diff --git a/src/__tests__/ai/runnerToolFinalization.test.ts b/src/__tests__/ai/runnerToolFinalization.test.ts new file mode 100644 index 000000000..18d1bcd37 --- /dev/null +++ b/src/__tests__/ai/runnerToolFinalization.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, test } from 'bun:test' +import { INTERRUPTED_TOOL_RESULT_ERROR } from '@core/ai' +import type { AiProvider, AiStreamRequest } from '../../../server/ai/drivers/types' +import type { ConversationsPersister } from '../../../server/ai/runtime/persister' +import { runChat } from '../../../server/ai/runtime/runner' +import type { AiStreamEvent } from '../../../server/ai/runtime/types' + +function request(signal: AbortSignal): AiStreamRequest { + return { + systemPrompt: [], + messages: [], + tools: [], + modelId: 'test-model', + modelCapabilities: { + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, + }, + credentials: { + id: 'credential-1', + providerId: 'ollama', + authMode: 'baseUrl', + apiKey: null, + baseUrl: 'http://localhost:11434', + }, + signal, + bridge: { + async callBrowser() { + return { ok: true } + }, + }, + toolContextBase: { + db: {} as never, + userId: 'user-1', + capabilities: [], + scope: 'site', + conversationId: 'conversation-1', + snapshot: null, + }, + } +} + +function provider( + stream: AiProvider['stream'], +): AiProvider { + return { + id: 'ollama', + label: 'Test provider', + supportedAuthModes: ['baseUrl'], + capabilities: () => ({ + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, + }), + async listModels() { + return [] + }, + stream, + } +} + +function persister(onToolCall?: () => void): { + value: ConversationsPersister + results: Array[0]> +} { + const results: Array[0]> = [] + return { + results, + value: { + async appendAssistantText() {}, + async appendToolCall() { + onToolCall?.() + }, + async appendToolResult(result) { + results.push(result) + }, + async recordUsage() { + return 0 + }, + recordContext() {}, + }, + } +} + +describe('runChat pending tool finalization', () => { + test('persists an interrupted result when a graceful abort ends the turn', async () => { + const controller = new AbortController() + const toolCallPersisted = deferred() + const stored = persister(() => toolCallPersisted.resolve()) + const emitted: AiStreamEvent[] = [] + const driver = provider(async function* (req) { + yield { + type: 'toolCall', + toolCallId: 'tool-1', + toolName: 'site_render_snapshot', + input: {}, + status: 'pending', + } + await untilAborted(req.signal) + }) + + const running = runChat({ + driver, + request: request(controller.signal), + persister: stored.value, + emit(event) { + emitted.push(event) + }, + }) + await toolCallPersisted.promise + controller.abort() + await running + + expect(stored.results).toEqual([{ + toolCallId: 'tool-1', + toolName: 'site_render_snapshot', + ok: false, + error: INTERRUPTED_TOOL_RESULT_ERROR, + }]) + expect(emitted.map((event) => event.type)).toEqual([ + 'toolCall', + 'toolResult', + 'done', + ]) + }) + + test('emits the interrupted result before a terminal driver error', async () => { + const stored = persister() + const emitted: AiStreamEvent[] = [] + const driver = provider(async function* () { + yield { + type: 'toolCall', + toolCallId: 'tool-2', + toolName: 'site_apply_css', + input: {}, + status: 'pending', + } + yield { type: 'error', message: 'Provider stream failed.' } + }) + + await runChat({ + driver, + request: request(new AbortController().signal), + persister: stored.value, + emit(event) { + emitted.push(event) + }, + }) + + expect(stored.results).toHaveLength(1) + expect(emitted.map((event) => event.type)).toEqual([ + 'toolCall', + 'toolResult', + 'error', + ]) + expect(emitted.at(-1)).toEqual({ type: 'error', message: 'Provider stream failed.' }) + }) +}) + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} + +function untilAborted(signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve() + return new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(), { once: true }) + }) +} diff --git a/src/__tests__/ai/toolLoop.test.ts b/src/__tests__/ai/toolLoop.test.ts index bbc4a5c6a..08b91af35 100644 --- a/src/__tests__/ai/toolLoop.test.ts +++ b/src/__tests__/ai/toolLoop.test.ts @@ -153,6 +153,92 @@ describe('runToolLoop via anthropicDriver', () => { expect(contextEvents.map((e) => e.promptTokens)).toEqual([20, 30]) }) + test('keeps a resolved browser-tool domain failure recoverable', async () => { + const requestBodies: Array> = [] + globalThis.fetch = (async (_url: string, init: RequestInit) => { + requestBodies.push(JSON.parse(init.body as string)) + return sseResponse(requestBodies.length === 1 ? TURN1 : TURN2) + }) as typeof fetch + const req = makeRequest({ + async callBrowser() { + return { ok: false, error: 'Canvas node no longer exists.' } + }, + }, []) + + const events: AiStreamEvent[] = [] + for await (const event of anthropicDriver.stream(req)) events.push(event) + + expect(requestBodies).toHaveLength(2) + expect(events.filter((event) => event.type === 'toolResult')).toEqual([ + { + type: 'toolResult', + toolCallId: 't_echo', + toolName: 'echo', + ok: true, + error: undefined, + }, + { + type: 'toolResult', + toolCallId: 't_paint', + toolName: 'paint', + ok: false, + error: 'Canvas node no longer exists.', + }, + ]) + expect(events.some((event) => event.type === 'error')).toBe(false) + expect(JSON.stringify(requestBodies[1])).toContain('Canvas node no longer exists.') + }) + + test('terminates after one failed result when an active browser bridge rejects with AbortError', async () => { + const requestBodies: Array> = [] + globalThis.fetch = (async (_url: string, init: RequestInit) => { + requestBodies.push(JSON.parse(init.body as string)) + return sseResponse(TURN1) + }) as typeof fetch + const req = makeRequest({ + async callBrowser() { + const error = new Error('Browser tool "paint" result timed out.') + error.name = 'AbortError' + throw error + }, + }, []) + + const events: AiStreamEvent[] = [] + for await (const event of anthropicDriver.stream(req)) events.push(event) + + // The provider is never called for a second round against the same dead + // bridge. The successful server tool still records its own result first. + expect(requestBodies).toHaveLength(1) + expect(events.filter((event) => event.type === 'toolResult')).toEqual([ + { + type: 'toolResult', + toolCallId: 't_echo', + toolName: 'echo', + ok: true, + error: undefined, + }, + { + type: 'toolResult', + toolCallId: 't_paint', + toolName: 'paint', + ok: false, + error: 'Browser tool "paint" result timed out.', + }, + ]) + expect(events.at(-1)).toEqual({ + type: 'error', + message: 'Browser tool transport failed: Browser tool "paint" result timed out.', + }) + expect(events.filter((event) => event.type === 'usage')).toEqual([{ + type: 'usage', + promptTokens: 20, + completionTokens: 15, + costUsd: undefined, + cacheReadTokens: undefined, + cacheCreationTokens: undefined, + }]) + }) + test('returns an error event (not a throw) on a non-OK HTTP status', async () => { globalThis.fetch = (async () => new Response('{"error":{"message":"bad key"}}', { status: 401 })) as typeof fetch const req = makeRequest({ async callBrowser() { return { ok: true } } }, []) diff --git a/src/admin/ai/toolResultApi.ts b/src/admin/ai/toolResultApi.ts index d9d55453d..ccc5b6f47 100644 --- a/src/admin/ai/toolResultApi.ts +++ b/src/admin/ai/toolResultApi.ts @@ -1,6 +1,6 @@ /** Post a browser-executed AI/MCP tool result back to its server bridge. */ import { Type } from '@core/utils/typeboxHelpers' -import { ApiError, apiRequest, isAbortError } from '@core/http' +import { apiRequest } from '@core/http' import type { AiToolOutput } from '@core/ai' const TOOL_RESULT_PATH = '/admin/api/ai/tool-result' @@ -27,9 +27,8 @@ export async function postToolResult( fallbackMessage: 'Tool-result POST failed.', }) } catch (err) { - // The bridge may disappear while a result is in flight during teardown. - if (isAbortError(err)) return - if (err instanceof ApiError && err.status === 404) return - console.error('[tool-result] Failed to post tool result:', err) + // Once the caller has torn down the bridge, the POST outcome is irrelevant. + if (signal?.aborted) return + throw err } } diff --git a/src/admin/ai/useMcpWorkspaceBridge.ts b/src/admin/ai/useMcpWorkspaceBridge.ts index 02f0cfd58..ebed2c81e 100644 --- a/src/admin/ai/useMcpWorkspaceBridge.ts +++ b/src/admin/ai/useMcpWorkspaceBridge.ts @@ -6,7 +6,6 @@ */ import { useEffect } from 'react' import { Type } from '@core/utils/typeboxHelpers' -import { isAbortError } from '@core/http' import type { AiToolOutput } from '@core/ai' import { getErrorMessage } from '@core/utils/errorMessage' import { readNdjsonStream } from './ndjsonStream' @@ -56,57 +55,77 @@ export async function executeMcpBridgeRequest( } } +type McpBridgeConnectionOutcome = 'auth' | 'transient' + +/** + * Run one editor-bridge connection. Every attempt owns a fresh controller; + * its signal is also linked to the hook lifetime so unmount still tears down + * the current request. Leaving this function aborts the connection, which is + * essential when tool-result delivery fails: the server waiter must reject + * now instead of hanging until its 90-second timeout. + */ +export async function runMcpWorkspaceBridgeConnection( + scope: McpWorkspaceScope, + dispatchTool: McpToolDispatcher, + afterSuccessfulTool: McpAfterSuccessfulTool | undefined, + lifecycleSignal: AbortSignal, +): Promise { + const connectionController = new AbortController() + const signal = AbortSignal.any([lifecycleSignal, connectionController.signal]) + let bridgeId = '' + + try { + const res = await fetch(`${MCP_BRIDGE_PATH}?scope=${scope}`, { + method: 'GET', + credentials: 'same-origin', + headers: { Accept: 'application/x-ndjson' }, + signal, + }) + if (res.status === 401 || res.status === 403) return 'auth' + if (!res.ok || !res.body) return 'transient' + + for await (const event of readNdjsonStream(res.body.getReader(), BridgeEventSchema)) { + signal.throwIfAborted() + if (event.type === 'bridgeReady') { + bridgeId = event.bridgeId + console.info(`[mcp-workspace-bridge:${scope}] connected`) + continue + } + + const result = await executeMcpBridgeRequest( + dispatchTool, + event.toolName, + event.input, + afterSuccessfulTool, + ) + await postToolResult(bridgeId, event.requestId, result, signal) + } + return 'transient' + } finally { + connectionController.abort() + } +} + export function useMcpWorkspaceBridge( scope: McpWorkspaceScope, dispatchTool: McpToolDispatcher, afterSuccessfulTool?: McpAfterSuccessfulTool, ): void { useEffect(() => { - const controller = new AbortController() + const lifecycleController = new AbortController() let stopped = false let reconnectTimer: ReturnType | null = null // Returns 'auth' when the server rejected on auth (back off longer, but // keep retrying). Returns 'transient' when the stream ended or was not // ready. Unmount is the only permanent stop condition. - async function connectOnce(): Promise<'auth' | 'transient'> { - let bridgeId = '' - const res = await fetch(`${MCP_BRIDGE_PATH}?scope=${scope}`, { - method: 'GET', - credentials: 'same-origin', - headers: { Accept: 'application/x-ndjson' }, - signal: controller.signal, - }) - if (res.status === 401 || res.status === 403) return 'auth' - if (!res.ok || !res.body) return 'transient' - - for await (const event of readNdjsonStream(res.body.getReader(), BridgeEventSchema)) { - if (stopped) break - if (event.type === 'bridgeReady') { - bridgeId = event.bridgeId - console.info(`[mcp-workspace-bridge:${scope}] connected`) - continue - } - - const result = await executeMcpBridgeRequest( - dispatchTool, - event.toolName, - event.input, - afterSuccessfulTool, - ) - try { - await postToolResult( - bridgeId, - event.requestId, - result, - controller.signal, - ) - } catch (err) { - if (isAbortError(err) || stopped) break - console.error(`[mcp-workspace-bridge:${scope}] result post failed:`, err) - } - } - return 'transient' + async function connectOnce(): Promise { + return runMcpWorkspaceBridgeConnection( + scope, + dispatchTool, + afterSuccessfulTool, + lifecycleController.signal, + ) } async function loop(): Promise { @@ -116,7 +135,7 @@ export function useMcpWorkspaceBridge( const outcome = await connectOnce() if (outcome === 'auth') delay = AUTH_RETRY_DELAY_MS } catch (err) { - if (isAbortError(err) || stopped) break + if (stopped || lifecycleController.signal.aborted) break console.error(`[mcp-workspace-bridge:${scope}] stream error (will retry):`, err) } if (stopped) break @@ -131,7 +150,7 @@ export function useMcpWorkspaceBridge( return () => { stopped = true if (reconnectTimer) clearTimeout(reconnectTimer) - controller.abort() + lifecycleController.abort() } }, [scope, dispatchTool, afterSuccessfulTool]) } diff --git a/src/admin/pages/site/agent/agentApi.ts b/src/admin/pages/site/agent/agentApi.ts index da70286f4..56a3975a1 100644 --- a/src/admin/pages/site/agent/agentApi.ts +++ b/src/admin/pages/site/agent/agentApi.ts @@ -11,6 +11,7 @@ */ import { nanoid } from 'nanoid' +import { INTERRUPTED_TOOL_RESULT_ERROR, aiToolError } from '@core/ai' import { Type, type Static } from '@core/utils/typeboxHelpers' import { apiRequest, isAbortError } from '@core/http' import { @@ -46,27 +47,54 @@ export function rehydrateMessages( records: ConversationDetail['messages'], ): AgentMessage[] { const out: AgentMessage[] = [] - const toolCallIndex = new Map() // toolCallId → block + // Only calls still awaiting a persisted role:tool row remain here. Loading a + // conversation never resumes its old browser bridge, so anything left after + // the complete scan is historical interruption, not live work. + const unanswered = new Map() + + const markInterrupted = (toolCall: AgentToolCall): void => { + toolCall.status = 'error' + toolCall.result = aiToolError(INTERRUPTED_TOOL_RESULT_ERROR) + // Tool-result images are session-only and cannot be reconstructed after a + // reload. Be explicit so malformed future wire data cannot revive one. + delete toolCall.previewImages + } + const finalizeUnanswered = (): void => { + for (const toolCall of unanswered.values()) markInterrupted(toolCall) + unanswered.clear() + } for (const rec of records) { - if (rec.role === 'tool' && rec.toolCallId) { + if (rec.role === 'tool') { // Fold the first-class `toolResult` block into the matching tool-call // block. `ok` is read directly off the block — never inferred from the - // emptiness of a text block. - const existing = toolCallIndex.get(rec.toolCallId) - if (existing) { - const resultBlock = rec.content.find((b) => b.kind === 'toolResult') - if (resultBlock?.kind === 'toolResult') { - existing.status = resultBlock.ok ? 'success' : 'error' - existing.result = { - ok: resultBlock.ok, - error: resultBlock.ok ? undefined : resultBlock.error, + // emptiness of a text block. Orphan rows are ignored; malformed matching + // rows terminate the call as interrupted instead of leaving a spinner. + const toolCallId = rec.toolCallId + if (toolCallId) { + const existing = unanswered.get(toolCallId) + if (existing) { + const resultBlock = rec.content.find((b) => b.kind === 'toolResult') + if (resultBlock?.kind === 'toolResult') { + existing.status = resultBlock.ok ? 'success' : 'error' + existing.result = { + ok: resultBlock.ok, + error: resultBlock.ok ? undefined : resultBlock.error, + } + } else { + markInterrupted(existing) } + unanswered.delete(toolCallId) } } continue } + // A real user turn closes the preceding assistant run. Any result that + // appears later is stale/orphaned and must not resurrect the historical + // call as successful; this mirrors provider-history healing on the server. + if (rec.role === 'user') finalizeUnanswered() + const msg: AgentMessage = { id: rec.id, role: rec.role === 'user' ? 'user' : 'assistant', @@ -78,18 +106,20 @@ export function rehydrateMessages( if (block.kind === 'text') { msg.blocks.push({ kind: 'text', text: block.text }) } else if (block.kind === 'toolCall') { + const duplicate = unanswered.get(block.toolCallId) + if (duplicate) markInterrupted(duplicate) const toolCall: AgentToolCall = { id: nanoid(), externalId: block.toolCallId, actionType: block.toolName, - params: (block.input && typeof block.input === 'object' + params: (block.input && typeof block.input === 'object' && !Array.isArray(block.input) ? (block.input as Record) : {}), result: null, status: 'pending', } msg.blocks.push({ kind: 'toolCall', toolCall }) - toolCallIndex.set(block.toolCallId, toolCall) + unanswered.set(block.toolCallId, toolCall) } else if (rec.role === 'user' && block.kind === 'image') { msg.blocks.push({ kind: 'image', @@ -101,6 +131,8 @@ export function rehydrateMessages( out.push(msg) } + finalizeUnanswered() + return out } diff --git a/src/admin/pages/site/agent/agentSlice.ts b/src/admin/pages/site/agent/agentSlice.ts index 7d3f0f489..1c3260456 100644 --- a/src/admin/pages/site/agent/agentSlice.ts +++ b/src/admin/pages/site/agent/agentSlice.ts @@ -18,7 +18,7 @@ import { nanoid } from 'nanoid' import type { EditorStoreSliceCreator } from '@site/store/types' -import { ApiError, responseErrorMessage } from '@core/http' +import { ApiError, isAbortError, responseErrorMessage } from '@core/http' import type { AiChatRequestBody } from '@core/ai' import { pushToast } from '@ui/components/Toast' import { @@ -56,6 +56,7 @@ import { waitForProviderUpdate, type ConfirmedProviderSelection, } from './agentProviderUpdate' +import { failPendingToolCalls } from './toolCallLifecycle' // Session-id is in-memory only. While the editor stays open, follow-up // messages reuse the SDK session id (Claude has continuity across the @@ -188,6 +189,7 @@ function surfaceAssistantError( set((state) => { state.agentError = error const msg = state.agentMessages.find((m) => m.id === assistantId) + failPendingToolCalls(msg, error) if (msg && msg.blocks.length === 0) { msg.blocks.push({ kind: 'text', text: placeholder }) } @@ -625,7 +627,9 @@ export function createAgentSlice( }) if (!res.body) throw new Error('Agent response has no body') + let terminalEventSeen = false for await (const event of readNdjsonStream(res.body.getReader(), ServerStreamEventSchema)) { + if (event.type === 'done' || event.type === 'error') terminalEventSeen = true await processStreamEvent( event, assistantId, @@ -637,6 +641,11 @@ export function createAgentSlice( config.buildSnapshot, ) } + if (!terminalEventSeen) { + throw new Error( + 'AI response ended before the turn completed. The server may have restarted; send the message again.', + ) + } flushPendingText() return { accepted: true } @@ -644,10 +653,21 @@ export function createAgentSlice( // Abort the fetch so any in-flight MCP tool handler on the server // rejects cleanly (via destroyBridge in the stream's finally block) // instead of waiting forever for a tool-result that won't arrive. + const requestWasAlreadyAborted = controller.signal.aborted controller.abort() - if (err instanceof Error && err.name === 'AbortError') { - if (accepted) flushPendingText() + // Only an AbortError caused by our already-aborted controller is an + // intentional Stop/teardown. An AbortError raised while the request + // was active (for example a failed tool-result delivery) is a real + // operation failure and must remain visible with retry guidance. + if (requestWasAlreadyAborted && isAbortError(err)) { + if (accepted) { + flushPendingText() + set((state) => { + const message = state.agentMessages.find((item) => item.id === assistantId) + failPendingToolCalls(message) + }) + } } else { // Admin-only surface (capability gated) — show the actual // failure cause so the operator can act. Network / unexpected diff --git a/src/admin/pages/site/agent/index.ts b/src/admin/pages/site/agent/index.ts index 807bb1f71..d12e2ceb5 100644 --- a/src/admin/pages/site/agent/index.ts +++ b/src/admin/pages/site/agent/index.ts @@ -20,6 +20,7 @@ export { siteAgentSliceConfig } from './agentSliceConfig.site' // Stream protocol — schema + per-event reducer + NDJSON reader. export { processStreamEvent } from './streamEvents' +export { rehydrateMessages } from './agentApi' // Site-specific snapshot builder — emits the raw authoritative tree the server // renders into the agent's HTML read surface. diff --git a/src/admin/pages/site/agent/streamEvents.ts b/src/admin/pages/site/agent/streamEvents.ts index 9a8024588..4cd0c8526 100644 --- a/src/admin/pages/site/agent/streamEvents.ts +++ b/src/admin/pages/site/agent/streamEvents.ts @@ -34,6 +34,7 @@ import type { ServerStreamEvent, } from './types' import { getErrorMessage } from '@core/utils/errorMessage' +import { failPendingToolCalls } from './toolCallLifecycle' // --------------------------------------------------------------------------- // Stream-event schema @@ -261,11 +262,25 @@ export async function processStreamEvent( // went wrong" placeholder; this surface is admin-only (capability // gated) so info-disclosure concerns don't apply. console.error('[AgentSlice] Server error event:', event.message) - set({ agentError: event.message }) + set((state) => { + state.agentError = event.message + const message = state.agentMessages.find((item) => item.id === assistantId) + failPendingToolCalls(message, event.message) + }) + break + } + + case 'done': { + // `done` with a pending call is an inconsistent/truncated server turn. + // Keep the completed response usable, but never leave historical work + // rendered as though it were still running. + set((state) => { + const message = state.agentMessages.find((item) => item.id === assistantId) + failPendingToolCalls(message) + }) break } - case 'done': default: break } diff --git a/src/admin/pages/site/agent/toolCallLifecycle.ts b/src/admin/pages/site/agent/toolCallLifecycle.ts new file mode 100644 index 000000000..69d4f9495 --- /dev/null +++ b/src/admin/pages/site/agent/toolCallLifecycle.ts @@ -0,0 +1,23 @@ +import { INTERRUPTED_TOOL_RESULT_ERROR, aiToolError } from '@core/ai' +import type { AgentMessage } from './types' + +/** + * A pending tool call belongs only to the currently live response stream. + * Once that stream ends or fails, every unresolved call is historical and + * must become terminal so the panel never leaves a permanent spinner behind. + */ +export function failPendingToolCalls( + message: AgentMessage | undefined, + error: string = INTERRUPTED_TOOL_RESULT_ERROR, +): boolean { + if (!message) return false + + let changed = false + for (const block of message.blocks) { + if (block.kind !== 'toolCall' || block.toolCall.status !== 'pending') continue + block.toolCall.status = 'error' + block.toolCall.result = aiToolError(error) + changed = true + } + return changed +} diff --git a/src/core/ai/index.ts b/src/core/ai/index.ts index 790392fed..636fe323e 100644 --- a/src/core/ai/index.ts +++ b/src/core/ai/index.ts @@ -1,6 +1,7 @@ export * from './mcpConnectorSchemas' export { AiToolOutputSchema, + INTERRUPTED_TOOL_RESULT_ERROR, aiToolError, aiToolOk, } from './toolOutput' diff --git a/src/core/ai/toolOutput.ts b/src/core/ai/toolOutput.ts index 3fcd2b121..d42b59dc3 100644 --- a/src/core/ai/toolOutput.ts +++ b/src/core/ai/toolOutput.ts @@ -24,6 +24,15 @@ export const AiToolOutputSchema = Type.Object({ export type AiToolOutput = Static +/** + * Canonical failure for a persisted tool call whose matching result never + * landed because its turn was interrupted (reload, disconnect, or restart). + * Shared by provider-history healing and browser conversation rehydration so + * the model and the user see the same terminal outcome. + */ +export const INTERRUPTED_TOOL_RESULT_ERROR = + 'Tool call did not complete — the previous turn was interrupted before a result was produced.' + export function aiToolOk(data?: unknown, images?: AiToolImage[]): AiToolOutput { const out: AiToolOutput = { ok: true } if (data !== undefined) out.data = data