From f4f17eb842eae211d405a1ff6177983da2a8851f Mon Sep 17 00:00:00 2001 From: DavidBabinec Date: Sat, 11 Jul 2026 02:48:14 +0200 Subject: [PATCH 1/4] feat(agent): paste images into conversations --- .gitattributes | 1 + docs/features/agent.md | 87 +++- server/ai/conversations/history.ts | 57 ++- server/ai/conversations/store.ts | 25 + server/ai/drivers/anthropic.ts | 12 +- server/ai/drivers/http/toolLoop.ts | 6 +- server/ai/drivers/modelCapabilities.ts | 120 +++++ server/ai/drivers/modelList.ts | 42 ++ server/ai/drivers/ollama.ts | 115 ++++- server/ai/drivers/openai.ts | 18 +- server/ai/drivers/openaiCompatible.ts | 10 +- server/ai/drivers/openrouter.ts | 19 +- server/ai/drivers/types.ts | 29 +- server/ai/handlers/chat.ts | 300 +++++++++--- server/ai/handlers/conversations.ts | 5 +- server/ai/handlers/credentials.ts | 8 +- server/ai/handlers/models.ts | 3 +- server/ai/inputImages.ts | 201 ++++++++ server/ai/tools/site/writeTools.ts | 2 +- .../agent/agentImageAttachment.test.ts | 230 +++++++++ src/__tests__/agent/agentSlice.test.ts | 442 +++++++++++++++++- .../ai/auditUsagePersistence.test.ts | 1 + src/__tests__/ai/chatImageHandler.test.ts | 355 ++++++++++++++ src/__tests__/ai/chatRequest.test.ts | 77 +++ .../ai/conversationRoundtrip.test.ts | 60 +++ src/__tests__/ai/inputImages.test.ts | 181 +++++++ src/__tests__/ai/messageHistory.test.ts | 67 +++ src/__tests__/ai/modelCapabilities.test.ts | 163 +++++++ src/__tests__/ai/ollamaMapping.test.ts | 110 ++++- src/__tests__/ai/responsesMapping.test.ts | 8 +- src/__tests__/ai/toolEvidence.test.ts | 23 +- src/__tests__/ai/toolLoop.test.ts | 2 +- src/__tests__/panels/agentPanel.test.tsx | 361 +++++++++++++- src/__tests__/ui/modelPicker.test.tsx | 43 +- src/admin/ai/ModelPicker/ModelPicker.tsx | Bin 14783 -> 15045 bytes src/admin/ai/api.ts | 52 ++- src/admin/pages/site/agent/agentApi.ts | 25 +- src/admin/pages/site/agent/agentSlice.ts | 376 ++++++++++++--- src/admin/pages/site/agent/agentSliceTypes.ts | 10 +- src/admin/pages/site/agent/types.ts | 23 +- .../site/panels/AgentPanel/AgentComposer.tsx | 296 ++++++++++++ .../panels/AgentPanel/AgentPanel.module.css | 80 ++++ .../site/panels/AgentPanel/AgentPanel.tsx | 165 ++----- .../panels/AgentPanel/ConversationHistory.tsx | 12 +- .../site/panels/AgentPanel/ModelPicker.tsx | 3 + .../panels/AgentPanel/agentImageAttachment.ts | 313 +++++++++++++ .../AgentPanel/usePendingImageAttachment.ts | 123 +++++ .../site/sidebars/LeftSidebar/LeftSidebar.tsx | 15 +- src/admin/spotlight/commands/aiAssistant.ts | 2 +- src/core/ai/chatRequest.ts | 33 ++ src/core/ai/index.ts | 30 +- src/core/ai/userImage.ts | 44 ++ 52 files changed, 4403 insertions(+), 382 deletions(-) create mode 100644 .gitattributes create mode 100644 server/ai/drivers/modelCapabilities.ts create mode 100644 server/ai/drivers/modelList.ts create mode 100644 server/ai/inputImages.ts create mode 100644 src/__tests__/agent/agentImageAttachment.test.ts create mode 100644 src/__tests__/ai/chatImageHandler.test.ts create mode 100644 src/__tests__/ai/chatRequest.test.ts create mode 100644 src/__tests__/ai/inputImages.test.ts create mode 100644 src/__tests__/ai/modelCapabilities.test.ts create mode 100644 src/admin/pages/site/panels/AgentPanel/AgentComposer.tsx create mode 100644 src/admin/pages/site/panels/AgentPanel/agentImageAttachment.ts create mode 100644 src/admin/pages/site/panels/AgentPanel/usePendingImageAttachment.ts create mode 100644 src/core/ai/chatRequest.ts create mode 100644 src/core/ai/userImage.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..490e25e6f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/admin/ai/ModelPicker/ModelPicker.tsx diff diff --git a/docs/features/agent.md b/docs/features/agent.md index 855aef6fe..dda893cfd 100644 --- a/docs/features/agent.md +++ b/docs/features/agent.md @@ -108,7 +108,10 @@ src/admin/pages/content/agent/ └── useContentToolBridge.ts — always-mounted handle + content-scope MCP relay src/admin/pages/site/panels/AgentPanel/ -├── AgentPanel.tsx — main panel; resolves active model's contextWindow from the models endpoint +├── AgentPanel.tsx — panel shell, persisted message thread, and user-image rendering +├── AgentComposer.tsx — controlled draft, paste/send lifecycle, active-model capability check +├── agentImageAttachment.ts — browser decode, resize, JPEG normalisation, and base64 encoding +├── usePendingImageAttachment.ts — ref-backed one-image queue, cancellation, and bounded preview lifecycle ├── ModelPicker.tsx — credential + model selector used in the input bar ├── ConversationHistory.tsx — history popover (browse, restore, delete past threads) ├── ContextMeter.tsx — "context used / window" progress indicator (display only) @@ -139,7 +142,23 @@ While credentials are still loading, `lockReason` stays `null` so the panel does When the panel opens, `AgentPanel` calls `loadScopeDefault()` so the model picker immediately shows the configured scope default — no "Default" placeholder, no send-time no-provider surprise. `composerLocked` is gated by `hasActiveProvider` (`Boolean(activeCredentialId && activeModelId)`), meaning a stale "No AI provider configured" error string never locks out the UI once a credential + model is staged; picking a model via `setAgentProvider` clears `agentError` immediately, re-enabling the composer. -The composer area includes a `` that shows "context used / window" as a progress bar. `AgentPanel` resolves the active model's `contextWindow` from `GET /admin/api/ai/providers/:id/models?credentialId=…` (the same catalogue-enriched response the picker uses), so the meter appears as soon as a model is selected — before the first turn. The "used" half comes from `agentContextTokens` in the store (see slice state below). The meter is hidden when no context window is known (Ollama, uncatalogued models). +The composer area includes a `` that shows "context used / window" as a progress bar. `AgentComposer` resolves the full active-model descriptor from `GET /admin/api/ai/providers/:id/models?credentialId=…` (the same catalogue-enriched response the picker uses), then uses its `contextWindow`, `capabilities.visionInput`, and `capabilities.toolCalling`. A model known not to support tools is blocked with an inline "choose an agent-capable model" message; the server repeats that gate authoritatively. The meter appears as soon as a model is selected — before the first turn. The "used" half comes from `agentContextTokens` in the store (see slice state below). The meter is hidden when no context window is known (Ollama, uncatalogued models). + +### Pasting a user image + +The composer accepts one clipboard image alongside optional text. Pasting reserves the attachment slot synchronously, then normalises the file asynchronously; send stays disabled while the image is processing, while model support is being checked, after a processing failure, when the selected model is not vision-capable, or when it is known not to support agent tools. The pending card uses a placeholder during decoding, then shows the bounded normalised JPEG itself; it never points an `` at the potentially huge source file. A primitive `Button` removes it. Removing or replacing the composer aborts the preparation operation and stops all downstream resize/encode work after the browser's current decode returns (`createImageBitmap` itself has no cancellation API). A message may contain only an image: the server persists the turn normally and titles a new conversation `Image`. + +The provider-neutral v1 policy is defined once in `src/core/ai/userImage.ts` and enforced on both sides of the boundary: + +- accepted clipboard sources: PNG, JPEG, or WebP; +- maximum source file: 12 MiB; +- source-header guard: at most 16,384 px on either edge and 40,000,000 encoded pixels; PNG/JPEG/WebP dimensions (including JPEG EXIF orientation) are read before allocating a decoder, and `createImageBitmap` is asked for the bounded output size; +- at most one image per message and at most eight persisted user images per conversation (the ninth returns 413 and the user starts a new chat); +- browser output: metadata-stripped `image/jpeg`, transparent pixels composited over white; +- maximum output: 1,500,000 bytes, 1568 px on either edge, and 1,500,000 total pixels; +- complete chat-request envelope: 16 MiB, leaving room for the scope snapshot around the roughly 2 MB base64 image. + +`agentImageAttachment.ts` fits both the edge and pixel budgets, tries progressively lower JPEG qualities, then reduces dimensions when necessary. The server never trusts that browser work: `server/ai/inputImages.ts` checks canonical base64, decoded byte length, JPEG magic bytes and dimensions, then fully decodes and re-encodes the JPEG through Sharp before appending the message. That second canonicalisation rejects truncated pixel data and strips EXIF/XMP/ICC metadata even when a direct authenticated client bypasses the browser. Unsupported, malformed, or oversized content is rejected before it can enter conversation history. --- @@ -152,24 +171,26 @@ Each entry in **Settings → AI → Providers** stores one credential. The provi | `anthropic` | Anthropic (Claude) | `apiKey` | API key (`sk-ant-…`) | — | Static `claude-*` catalogue enriched with OpenRouter prices + context windows | | `openai` | OpenAI | `apiKey` | API key (`sk-…`) | — | Static `gpt-*` / `o*` catalogue enriched with OpenRouter prices + context windows | | `openrouter` | OpenRouter | `apiKey` | API key (`sk-or-…`) | — | Live `GET /api/v1/models` (cross-provider; native cost reporting) | -| `ollama` | Ollama (local) | `baseUrl` | Base URL (e.g. `http://localhost:11434`) | API key (bearer, for proxied deployments) | Live `GET {baseUrl}/api/tags`; static fallback list when unreachable | +| `ollama` | Ollama (local) | `baseUrl` | Base URL (e.g. `http://localhost:11434`) | API key (bearer, for proxied deployments) | Live `GET {baseUrl}/api/tags`, with `POST {baseUrl}/api/show` capability lookup per model; static fallback list when unreachable | | `openai-compatible` | Custom Provider | `baseUrl` | Base URL — any host serving the OpenAI `/v1/chat/completions` wire protocol | API key (bearer; cloud services need one, local servers often don't) | Live `GET {baseUrl}/v1/models` (standard OpenAI list shape); model `id` used as label | -**Custom Provider** (id `openai-compatible`) is the generic adapter for any endpoint that speaks the OpenAI chat/completions wire protocol — Groq (`https://api.groq.com/openai`), Together, DeepSeek, Mistral, Fireworks, self-hosted vLLM, LM Studio, and others. Capabilities default to `{ toolCalling: true, visionInput: false, promptCache: false, streaming: true }`; the operator is responsible for selecting a model that actually supports tool calling. Because arbitrary endpoints are not in the OpenRouter catalogue, no context-window enrichment is available and the context meter stays hidden for these models. +**Custom Provider** (id `openai-compatible`) is the generic adapter for any endpoint that speaks the OpenAI chat/completions wire protocol — Groq (`https://api.groq.com/openai`), Together, DeepSeek, Mistral, Fireworks, self-hosted vLLM, LM Studio, and others. Capabilities default to `{ toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }`; the operator is responsible for selecting a model that actually supports tool calling. Because arbitrary endpoints are not in the OpenRouter catalogue, no context-window enrichment is available and the context meter stays hidden for these models. + +**The server is the model-capability authority.** The composer catalogue flags are early UX gates, but they are not trusted for persistence, provider calls, or tool screenshots. `chat.ts` resolves the selected model on every turn through `resolveModelCapabilities`. Providers with stable capabilities (Anthropic, OpenAI, Custom Provider) use their driver default. Model-specific providers own a selected-model lookup: OpenRouter resolves the exact entry's `architecture.input_modalities`, while Ollama sends an authenticated `POST /api/show` for only the selected model. The shared resolver de-duplicates concurrent lookups, applies a ten-second provider timeout, includes credential/backend revisions in its cache key, and caches successful results for five minutes. Missing or unavailable model-specific metadata fails closed for vision input; Custom Provider also remains fail-closed in v1. An image targeting a non-vision model, or an editor-agent turn targeting a model known not to support tools, receives 422 before the user message is stored. --- ## Flow ```text -User types prompt → Agent Panel +User types text and/or pastes an image → Agent Panel │ ▼ -agentSlice.sendAgentMessage(content) +agentSlice.sendAgentMessage(contentBlocks) │ ├─→ buildSnapshot() → SiteAgentSnapshot or ContentSnapshot ├─→ ensure conversation row (lazily created from AI defaults on first call) - ├─→ POST /admin/api/ai/chat/ { conversationId, prompt, snapshot } + ├─→ POST /admin/api/ai/chat/ { conversationId, content, snapshot } │ ▼ Server: chat.ts @@ -177,13 +198,18 @@ Server: chat.ts ├─→ CSRF + requireCapability('ai.chat') ├─→ load conversation row (credentialId, modelId) + full message history ├─→ decrypt credential; resolveDriver(credential.providerId) + ├─→ validate text/image blocks + image bytes; enforce conversation image budget + ├─→ resolve/cache the selected model's capabilities (also gates tool screenshots) + ├─→ acquire the conversation's single-writer stream lease + ├─→ project persisted images for bounded provider replay ├─→ selectToolsForScope(scope, capabilities) │ — write tools excluded unless caller has ai.tools.write ├─→ build scope system prompt → [staticPrefix, BOUNDARY, dynamicSuffix] ├─→ createBridge(emit) → { bridgeId, bridge, destroy } ├─→ emit { type: 'bridgeReady', bridgeId } └─→ runChat({ driver, request, persister, emit }) — streaming begins - │ request carries the FULL conversation history as req.messages. + │ request carries the conversation history as req.messages; user + │ images are projected to the bounded replay form described below. │ Direct HTTP drivers have no server-side session — every turn │ replays the whole log, mapped into the provider's message array. │ @@ -459,7 +485,7 @@ The agent works **design-system-first**: it establishes or reuses tokens, then r | Tool | Input | Success `data` | What it does | |-------------------|-----------------------|----------------|------------------------------------------------------------------| -| `site_render_snapshot` | `{ breakpointId?, nodeId? }` | `{ breakpointId, nodeId?, label, width, capturedAt, layout, screenshot }` + optional `images[]` | Inspect the rendered canvas: always returns a layout report (viewport, per-node bounding boxes, overflow / broken-image / invisible warnings); on a vision-capable model a PNG is attached via the tool-output **image channel**. `breakpointId` picks the frame (defaults to active); `nodeId` scopes the capture to that node's subtree — image and report cover only that section, with coordinates relative to its box, and the report carries the same `nodeId`. Omit `nodeId` for the whole page; an unknown `nodeId` returns an `aiToolError` | +| `site_render_snapshot` | `{ breakpointId?, nodeId? }` | `{ breakpointId, nodeId?, label, width, capturedAt, layout, screenshot }` + optional `images[]` | Inspect the rendered canvas: always returns a layout report (viewport, per-node bounding boxes, overflow / broken-image / invisible warnings); when the active provider supports native image-bearing tool results, a PNG is attached via the tool-output **image channel**. `breakpointId` picks the frame (defaults to active); `nodeId` scopes the capture to that node's subtree — image and report cover only that section, with coordinates relative to its box, and the report carries the same `nodeId`. Omit `nodeId` for the whole page; an unknown `nodeId` returns an `aiToolError` | ### Auto-navigation @@ -507,7 +533,7 @@ The content system prompt is markdown-native: it tells the model to exchange bod `site_render_snapshot` (and `site_read_document` / `site_get_node_html`) return large payloads. Five rules keep them from exploding context (a screenshot inlined as base64 JSON text once pushed a single turn past 1M tokens): 1. **Image channel, not text.** `AiToolOutput` carries an optional `images: { mimeType, data }[]` (`src/core/ai/toolOutput.ts`). `site_render_snapshot` puts the PNG there — never in `data`. The Anthropic driver forwards it as a **native `image` block** inside the `tool_result` (billed at the rendered image's token cost). Text-only tool channels (Ollama / OpenAI-compatible `function_call_output`) **drop** the image and append a one-line `[N screenshot(s) omitted…]` note. The capture caps the screenshot's long edge at `MAX_IMAGE_EDGE` (1568px in `renderEvidence.ts`) — a tall landing page would otherwise exceed Anthropic's hard 8000px-per-dimension limit (400 error), and the model downsizes the long edge to ~1568px anyway. -2. **Capture is vision-gated.** The chat handler resolves `driver.capabilities(modelId)` into `AiStreamRequest.modelCapabilities`. The shared tool loop injects `captureScreenshot: visionInput` into every `site_render_snapshot` call, so a non-vision model never pays the html-to-image cost — it gets the layout report only. (The model never sets `captureScreenshot` itself.) +2. **Capture is provider-channel-gated.** The chat handler resolves the selected model on every turn and places the result in `AiStreamRequest.modelCapabilities`. `visionInput` means pasted user images are accepted; the separate `toolResultImages` flag means that provider's tool-result wire shape can actually carry a native image. The shared tool loop injects `captureScreenshot: visionInput && toolResultImages` into `site_render_snapshot`. Today Anthropic supports both; Responses and chat/completions providers accept user images but have text-only function/tool results, so they get the layout report without paying for a screenshot that would be discarded. (The model never sets `captureScreenshot` itself.) 3. **`site_read_document` CSS is document-relevant, not the public full-site CSS bundle.** Public pages can share page-invariant CSS files, but `site_read_document` inlines CSS into model context. It keeps framework variables/utilities, font token variables, target-document module CSS, used class rules, ambient selectors whose class tokens all exist on the target document, classless/global ambient selectors, and document-targeted user stylesheets. It omits browser-only `@font-face` file declarations and ambient selectors from unrelated imported pages. 4. **`site_read_document` is cleaned and paged before it reaches the model.** `renderAgentDocument` strips pathological strings from the broad read surface: long base64/data URLs become `data:;base64,[omitted N chars]`, and very long URLs are middle-truncated. The returned object always includes `pageInfo` with `part`, `totalParts`, `nextPart`, `ranges`, `serializedChars`, and cleanup counts. The hard budget is measured against `JSON.stringify({ html, css, pageInfo }).length`, because that is the text providers receive as the tool result. If `nextPart` is not `null`, the agent calls `site_read_document({ document, part: nextPart })` to continue. For exact node-level markup, use the `uid` with `site_get_node_html`. 5. **Stale evidence is elided.** Within one tool loop, only the **most recent** heavy result per tool name (`site_render_snapshot`, `site_read_document`, `site_get_node_html`, or anything with an image) is replayed at full fidelity; earlier ones are rewritten to a one-line breadcrumb (`"Earlier output removed… Call again…"`). Older snapshots describe page state the model has since mutated, so they carry no value. See `applyHeavyElision` in `server/ai/drivers/http/toolLoop.ts`. @@ -606,12 +632,18 @@ interface AgentSlice { * on loadAgentConversation; updated live from each turn's `usage` event. */ agentContextTokens: number | null + /** Blocks Send/navigation while a history load or delete may replace the active chat. */ + isAgentConversationPending: boolean + /** Blocks Send/navigation while an existing chat's model PUT is pending. */ + isAgentProviderPending: boolean + /** Incremented when a conversation is replaced so local text/image drafts remount cleanly. */ + agentComposerEpoch: number // ── Actions ─────────────────────────────────────────────────────────── openAgent(): void closeAgent(): void toggleAgent(): void - sendAgentMessage(content: string): Promise + sendAgentMessage(content: AiUserContentBlock[]): Promise<{ accepted: boolean }> abortAgent(): void clearAgentMessages(): void startNewAgentConversation(): void @@ -629,6 +661,31 @@ Conversations and their message history are persisted server-side in `ai_convers **Content blocks are one schema.** Every message body is an `AiContentBlock[]` — a discriminated union of `text` / `image` / `toolCall` / `toolResult` kinds defined once as a TypeBox schema in `@core/ai` (`src/core/ai/contentBlock.ts`). The server runtime type (`AiContentBlock`), the read boundary (`ContentBlocksSchema` in `conversations/store.ts`, which validates every block out of `content_json`), and the client wire schema (`MessageViewSchema` in `src/admin/ai/api.ts`) all derive from it. Add a kind there and every reader/writer sees it. +**User turns use the same canonical blocks at the HTTP boundary.** `AiChatRequestBodySchema` in `src/core/ai/chatRequest.ts` accepts `{ conversationId, content, snapshot? }`, where `content` is one or two `AiUserContentBlock`s and may contain at most one trimmed text block plus at most one canonical JPEG block. The server canonicalises a mixed turn as text then image and removes whitespace-only text. It does not accept `toolCall` or `toolResult` blocks from the browser. + +```ts +{ + conversationId, + content: [ + { kind: 'text', text: 'Use this mockup as the reference.' }, + { kind: 'image', mimeType: 'image/jpeg', data: '' }, + ], + snapshot, +} +``` + +**Persisted images and provider replay are deliberately different views.** Every accepted user JPEG is stored inline in `ai_messages.content_json` and rehydrates into the conversation UI, so reloads retain the thumbnail. Up to eight images may be stored in one conversation. Before a provider call, `projectUserImagesForModel` creates a non-mutating outbound projection: + +- a vision model receives the image from only the newest image-bearing user turn; images in older turns become a short "earlier image omitted" breadcrumb; +- a non-vision model receives no image bytes at all; every persisted image becomes a text breadcrumb, so switching models cannot poison the conversation; +- the database rows are never rewritten by projection, so the UI history remains intact and switching back to a vision model can use the newest persisted image again. + +User attachments are private chat data, not media-library assets: the normalised base64 bytes live in the database until the conversation is deleted and purged, are returned to authorised conversation reads for thumbnail rendering, and are sent to the configured AI provider whenever they survive the outbound replay projection. They are never written to public uploads or published with the site. + +The server admits only one active writer per conversation. A concurrent tab receives a retryable 409 before appending, which keeps message positions ordered and makes the eight-image budget atomic relative to persistence. In the browser, model changes for an existing conversation are serialized; Send waits for the provider/model update to reach the server, and conversation/model controls stay disabled while a turn streams. If a model PUT times out after an ambiguous commit, the browser re-reads the conversation before re-enabling Send, so the picker cannot disagree with server routing. Stop owns the whole first-send lifecycle, including default lookup and lazy conversation creation, so an aborted bootstrap cannot leave the composer locked. + +**User attachments are not tool screenshots.** A pasted image is a persisted `kind:'image'` block on a user message. Conversation-detail responses and chat streams use `Cache-Control: private, no-store`; the database remains the intentional durable copy. A `site_render_snapshot` PNG instead travels transiently on `AiToolOutput.images`, is subject to the heavy-evidence rules above, and its browser preview remains session-only. The two paths share provider-native image mapping but have different storage, replay, and elision lifecycles. + **Tool outcomes are first-class.** A `role:'tool'` row records its result as a `{ kind: 'toolResult', ok, error? }` block — `ok` is an explicit boolean, never inferred from the emptiness of a text block. The persister writes it (`appendToolResult`), `buildMessageHistory` reads `ok`/`error` straight off the block to reconstruct the replay `AiToolOutput`, and the client folds it back into the matching tool-call badge (`rehydrateMessages`). The heavy successful `data` an `AiToolOutput` may carry is intentionally **not** persisted: the model already consumed it in the round that produced the result, so replay only needs `{ ok, error }` — re-feeding large tool payloads every turn would bloat the context for no benefit. --- @@ -648,6 +705,8 @@ The `` shows how much of the active model's context window the cur ### Live model catalogue +The browser de-duplicates concurrent requests for the same credential, applies a ten-second timeout, and retains successful catalogues for five minutes so the composer capability check and model picker share one result across conversation switches. Credential deletion invalidates its cached catalogue. The server independently applies the same ten-second deadline and forwards request cancellation into provider fetches; Ollama resolves `/api/show` metadata in batches of six rather than launching an unbounded fan-out. + `server/ai/pricing/` is the single source for per-model prices **and context windows**. It sources from OpenRouter's public `/api/v1/models` endpoint (no key required), which publishes list prices and `context_length` for Anthropic and OpenAI models. The module lifecycle: - **Cold start**: loads the DB cache from `ai_model_pricing` (durable fallback) and kicks a background refresh. The first turn prices immediately off the last-known data. @@ -707,6 +766,8 @@ unblocks deletion of the credential that had been protected by the default FK. - `docs/features/content-workspace.md` — content workspace UI and content-scope Agent Panel mount - Source-of-truth files: - `src/core/ai/toolOutput.ts` — `AiToolOutput` type, `AiToolOutputSchema`, `aiToolOk`, `aiToolError` (canonical bridge result) + - `src/core/ai/chatRequest.ts` — canonical browser-to-server chat envelope and 16 MiB request ceiling + - `src/core/ai/userImage.ts` — accepted source formats, normalised JPEG schema, byte/dimension limits, and eight-image conversation budget - `src/core/ai/toolSchemas.ts` — all site browser-tool input schemas (single source of truth; imported by both the server registry and the browser executor) - `src/core/ai/documentRefs.ts` — document refs/descriptors for pages, templates, and visual components - `src/core/ai/readSurface.ts` — runtime-agnostic `renderAgentDocument` annotated HTML + compact CSS renderer @@ -718,6 +779,10 @@ unblocks deletion of the credential that had been protected by the default FK. - `server/ai/tools/site/snapshot.ts` — `SiteAgentSnapshotSchema` + `SiteAgentSnapshot` re-export + catalog output types (`ModuleInfo`, `SnapshotTokens`, …) - `server/ai/tools/content/readTools.ts` — 7 server-side content read tool definitions - `server/ai/tools/content/writeTools.ts` — 8 browser-bridged content write/navigation tool definitions + - `server/ai/inputImages.ts` — server-side base64, JPEG, byte, and dimension validation before persistence + - `server/ai/drivers/modelCapabilities.ts` — cached, timed, authoritative/fail-closed selected-model capability resolution on every turn + - `server/ai/drivers/modelList.ts` — bounded provider catalogue lookup with caller cancellation + - `server/ai/conversations/history.ts` — interrupted-tool healing plus outbound user-image replay projection - `server/ai/tools/content/systemPrompt.ts` — markdown-native content system prompt - `server/ai/tools/content/snapshot.ts` — `ContentSnapshot` shape consumed by the content prompt and tool context - `src/admin/pages/site/agent/siteAgentSnapshot.ts` — `SiteAgentSnapshotSchema` (TypeBox source of truth) + `SiteAgentSnapshot` (derived type) + `buildSiteAgentSnapshot` diff --git a/server/ai/conversations/history.ts b/server/ai/conversations/history.ts index a15bbdc99..444015de9 100644 --- a/server/ai/conversations/history.ts +++ b/server/ai/conversations/history.ts @@ -23,7 +23,7 @@ * preceding tool call in the replayed history. */ -import type { AiMessage, AiToolOutput } from '../runtime/types' +import type { AiContentBlock, AiMessage, AiToolOutput } from '../runtime/types' import type { MessageRecord } from './types' /** @@ -33,6 +33,11 @@ import type { MessageRecord } from './types' export const INTERRUPTED_TOOL_RESULT_ERROR = 'Tool call did not complete — the previous turn was interrupted before a result was produced.' +export const EARLIER_USER_IMAGE_OMITTED = + '[Earlier attached image omitted from model context. Ask the user to attach it again if needed.]' +export const NON_VISION_USER_IMAGE_OMITTED = + '[Attached image omitted because the selected model does not support image input.]' + /** * Reconstruct `AiMessage` history from persisted `MessageRecord` rows. * @@ -100,3 +105,53 @@ export function buildMessageHistory(records: MessageRecord[]): AiMessage[] { return out } + +/** + * Bound provider replay without mutating persisted history. + * + * Vision models receive only the newest image-bearing user turn; older images + * become breadcrumbs. Text-only models receive breadcrumbs for every image, + * which keeps a conversation usable after switching away from a vision model. + */ +export function projectUserImagesForModel( + messages: readonly AiMessage[], + visionInput: boolean, +): AiMessage[] { + let latestImageMessage = -1 + if (visionInput) { + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index] + if ( + message?.role === 'user' + && message.content.some((block) => block.kind === 'image') + ) { + latestImageMessage = index + break + } + } + } + + return messages.map((message, messageIndex) => { + if (message.role !== 'user' || !message.content.some((block) => block.kind === 'image')) { + return message + } + if (visionInput && messageIndex === latestImageMessage) return message + + const replacement = visionInput + ? EARLIER_USER_IMAGE_OMITTED + : NON_VISION_USER_IMAGE_OMITTED + let breadcrumbAdded = false + const content: AiContentBlock[] = [] + for (const block of message.content) { + if (block.kind !== 'image') { + content.push(block) + continue + } + if (!breadcrumbAdded) { + content.push({ kind: 'text', text: replacement }) + breadcrumbAdded = true + } + } + return { role: 'user', content } + }) +} diff --git a/server/ai/conversations/store.ts b/server/ai/conversations/store.ts index 1609f4a8b..d8cb4c268 100644 --- a/server/ai/conversations/store.ts +++ b/server/ai/conversations/store.ts @@ -320,6 +320,31 @@ export async function updateConversationForUser( return rows[0] ? conversationRowToRecord(rows[0]) : null } +/** + * Give a first turn its derived title without overwriting a rename that won + * the race in another tab. The placeholder predicate belongs in the UPDATE, + * not in a preceding read. + */ +export async function replaceDefaultConversationTitle( + db: DbClient, + userId: string, + conversationId: string, + title: string, +): Promise { + const nextTitle = title.trim() + if (!nextTitle) return false + const result = await db` + update ai_conversations + set title = ${nextTitle}, + updated_at = current_timestamp + where id = ${conversationId} + and user_id = ${userId} + and deleted_at is null + and title = ${DEFAULT_CONVERSATION_TITLE} + ` + return result.rowCount > 0 +} + /** * Soft-delete by setting `deleted_at`. Idempotent — calling on an * already-deleted row sets deleted_at to the current time again. diff --git a/server/ai/drivers/anthropic.ts b/server/ai/drivers/anthropic.ts index fbad77562..57176251e 100644 --- a/server/ai/drivers/anthropic.ts +++ b/server/ai/drivers/anthropic.ts @@ -60,13 +60,14 @@ export const anthropicDriver: AiProvider = { return { toolCalling: true, visionInput: true, + toolResultImages: true, promptCache: true, streaming: true, } }, - async listModels(creds) { - return fetchAnthropicModels(creds) + async listModels(creds, signal) { + return fetchAnthropicModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -122,7 +123,10 @@ const AnthropicModelsResponseSchema = Type.Object( * - a failed request or unparseable body throws, so the caller surfaces the * error rather than masking it with a stale hardcoded list. */ -async function fetchAnthropicModels(creds: AiResolvedCredential): Promise { +async function fetchAnthropicModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { if (creds.authMode !== 'apiKey' || !creds.apiKey) return [] const res = await fetch(`${ANTHROPIC_MODELS_ENDPOINT}?limit=1000`, { @@ -130,6 +134,7 @@ async function fetchAnthropicModels(creds: AiResolvedCredential): Promise( function prepareToolInput(call: TurnToolCall, req: AiStreamRequest): unknown { if (call.name === 'site_render_snapshot') { const base = call.input && typeof call.input === 'object' ? call.input : {} - return { ...base, captureScreenshot: req.modelCapabilities.visionInput } + return { + ...base, + captureScreenshot: + req.modelCapabilities.visionInput && req.modelCapabilities.toolResultImages, + } } return call.input } diff --git a/server/ai/drivers/modelCapabilities.ts b/server/ai/drivers/modelCapabilities.ts new file mode 100644 index 000000000..e6825594c --- /dev/null +++ b/server/ai/drivers/modelCapabilities.ts @@ -0,0 +1,120 @@ +import type { + AiProvider, + AiProviderCapabilities, + AiResolvedCredential, +} from './types' + +const CAPABILITY_CACHE_TTL_MS = 5 * 60 * 1000 +const CAPABILITY_LOOKUP_TIMEOUT_MS = 10_000 +const MAX_CACHE_ENTRIES_PER_DRIVER = 256 + +interface CapabilityCacheEntry { + expiresAt: number + value: AiProviderCapabilities +} + +const capabilityCache = new WeakMap>() +const capabilityLookups = new WeakMap>>() + +/** + * Resolve the selected model's runtime capabilities. Providers with + * model-specific capability metadata own the authoritative lookup; the shared + * layer bounds it with a short cache and de-duplicates concurrent requests. + */ +export async function resolveModelCapabilities( + driver: AiProvider, + credentials: AiResolvedCredential, + modelId: string, +): Promise { + const fallback = driver.capabilities(modelId) + if (!driver.resolveCapabilities) return fallback + + const key = `${credentialRevisionKey(credentials)}\0${modelId}` + const cache = mapFor(capabilityCache, driver) + const cached = cache.get(key) + if (cached && cached.expiresAt > Date.now()) return cached.value + if (cached) cache.delete(key) + + const lookups = mapFor(capabilityLookups, driver) + const existingLookup = lookups.get(key) + if (existingLookup) return existingLookup + + // Store the handled promise, not the raw provider request. Every concurrent + // waiter must receive the same fail-closed result when discovery rejects. + const lookup = (async () => { + try { + const value = await resolveWithTimeout(driver, credentials, modelId) + cache.set(key, { expiresAt: Date.now() + CAPABILITY_CACHE_TTL_MS, value }) + if (cache.size > MAX_CACHE_ENTRIES_PER_DRIVER) { + const oldestKey = cache.keys().next().value + if (oldestKey !== undefined) cache.delete(oldestKey) + } + return value + } catch (err) { + console.error(`[ai/${driver.id}] model capability lookup failed:`, err) + // A provider-specific lookup exists because its static vision flag is not + // authoritative. Network/schema failures therefore fail closed for image + // input while retaining the driver's safe defaults for other features. + return { ...fallback, visionInput: false } + } finally { + lookups.delete(key) + } + })() + lookups.set(key, lookup) + return lookup +} + +function mapFor( + root: WeakMap>, + driver: AiProvider, +): Map { + const existing = root.get(driver) + if (existing) return existing + const created = new Map() + root.set(driver, created) + return created +} + +function credentialRevisionKey(credentials: AiResolvedCredential): string { + // Credentials are edited in place. Include backend/auth material so a base + // URL or key rotation cannot reuse a positive result from the old backend; + // hash the secret rather than retaining it in a Map key. + return [ + credentials.id, + credentials.authMode, + credentials.baseUrl ?? '', + stableHash(credentials.apiKey ?? ''), + ].join('\0') +} + +function stableHash(value: string): string { + let hash = 2166136261 + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index) + hash = Math.imul(hash, 16777619) + } + return (hash >>> 0).toString(16) +} + +async function resolveWithTimeout( + driver: AiProvider, + credentials: AiResolvedCredential, + modelId: string, +): Promise { + const controller = new AbortController() + let timeoutId: ReturnType | null = null + const timeout = new Promise((_resolve, reject) => { + timeoutId = setTimeout(() => { + controller.abort() + reject(new Error(`Model capability lookup timed out after ${CAPABILITY_LOOKUP_TIMEOUT_MS}ms.`)) + }, CAPABILITY_LOOKUP_TIMEOUT_MS) + }) + try { + return await Promise.race([ + driver.resolveCapabilities!(credentials, modelId, controller.signal), + timeout, + ]) + } finally { + if (timeoutId) clearTimeout(timeoutId) + } +} diff --git a/server/ai/drivers/modelList.ts b/server/ai/drivers/modelList.ts new file mode 100644 index 000000000..25e15982e --- /dev/null +++ b/server/ai/drivers/modelList.ts @@ -0,0 +1,42 @@ +import type { AiProvider, AiProviderModel, AiResolvedCredential } from './types' + +const MODEL_LIST_TIMEOUT_MS = 10_000 + +/** + * Resolve a provider catalogue with both caller cancellation and a server-side + * deadline. The race releases the HTTP handler even if a future driver forgets + * to honour the signal; current fetch-based drivers also stop their upstream + * request when the composed controller aborts. + */ +export async function listProviderModels( + driver: AiProvider, + credentials: AiResolvedCredential, + parentSignal?: AbortSignal, +): Promise { + const controller = new AbortController() + const abortFromParent = () => controller.abort(parentSignal?.reason) + if (parentSignal?.aborted) abortFromParent() + else parentSignal?.addEventListener('abort', abortFromParent, { once: true }) + + const timeoutId = setTimeout(() => { + controller.abort(new Error('Model catalogue request timed out.')) + }, MODEL_LIST_TIMEOUT_MS) + let rejectAborted!: (reason?: unknown) => void + const aborted = new Promise((_resolve, reject) => { rejectAborted = reject }) + const rejectAbort = () => rejectAborted( + controller.signal.reason ?? new DOMException('The operation was aborted.', 'AbortError'), + ) + if (controller.signal.aborted) rejectAbort() + else controller.signal.addEventListener('abort', rejectAbort, { once: true }) + + try { + return await Promise.race([ + driver.listModels(credentials, controller.signal), + aborted, + ]) + } finally { + clearTimeout(timeoutId) + parentSignal?.removeEventListener('abort', abortFromParent) + controller.signal.removeEventListener('abort', rejectAbort) + } +} diff --git a/server/ai/drivers/ollama.ts b/server/ai/drivers/ollama.ts index 3e4767020..3e3ac017c 100644 --- a/server/ai/drivers/ollama.ts +++ b/server/ai/drivers/ollama.ts @@ -16,6 +16,7 @@ */ import { Type, parseValue } from '@core/utils/typeboxHelpers' +import { isAbortError } from '@core/http' import { type AiAuthMode, type AiProviderId, @@ -23,6 +24,7 @@ import { } from '../runtime/types' import type { AiProvider, + AiProviderCapabilities, AiProviderModel, AiResolvedCredential, AiStreamRequest, @@ -31,6 +33,7 @@ import { runToolLoop } from './http/toolLoop' import { makeChatCompletionsAdapter, trimSlash } from './http/chatCompletions' const SUPPORTED_AUTH_MODES: AiAuthMode[] = ['baseUrl'] +const OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY = 6 // Ollama models vary per-install. Defaults are common picks as of May 2026 and // only surface when the `/api/tags` catalogue fetch fails. @@ -40,21 +43,21 @@ const FALLBACK_MODELS: AiProviderModel[] = [ label: 'Llama 4', tier: 'smart', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, }, { id: 'llama3.3', label: 'Llama 3.3', tier: 'balanced', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, }, { id: 'qwen3', label: 'Qwen 3', tier: 'balanced', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, }, ] @@ -64,18 +67,23 @@ export const ollamaDriver: AiProvider = { supportedAuthModes: SUPPORTED_AUTH_MODES, capabilities(modelId: string) { - const model = FALLBACK_MODELS.find((m) => m.id === modelId) - return model?.capabilities ?? { - toolCalling: true, - visionInput: false, - promptCache: false, - streaming: true, + return fallbackCapabilities(modelId) + }, + + async resolveCapabilities(creds: AiResolvedCredential, modelId: string, signal: AbortSignal) { + const declared = await fetchOllamaDeclaredCapabilities(creds, modelId, signal) + const fallback = fallbackCapabilities(modelId) + if (!declared) return { ...fallback, visionInput: false } + return { + ...fallback, + toolCalling: declared.includes('tools'), + visionInput: declared.includes('vision'), } }, - async listModels(creds: AiResolvedCredential) { + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { if (!creds.baseUrl) return FALLBACK_MODELS - return fetchOllamaModels(creds.baseUrl) + return fetchOllamaModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -112,23 +120,92 @@ const OllamaTagsSchema = Type.Object({ ), }) -async function fetchOllamaModels(baseUrl: string): Promise { +const OllamaShowSchema = Type.Object( + { capabilities: Type.Optional(Type.Array(Type.String())) }, + { additionalProperties: true }, +) + +function fallbackCapabilities(modelId: string): AiProviderCapabilities { + const model = FALLBACK_MODELS.find((candidate) => candidate.id === modelId) + return model?.capabilities ?? { + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, + } +} + +async function fetchOllamaModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { + const baseUrl = creds.baseUrl + if (!baseUrl) return FALLBACK_MODELS try { - const res = await fetch(`${trimSlash(baseUrl)}/api/tags`) + const res = await fetch(`${trimSlash(baseUrl)}/api/tags`, { + headers: ollamaHeaders(creds.apiKey), + signal, + }) if (!res.ok) return FALLBACK_MODELS const parsed = parseValue(OllamaTagsSchema, await res.json()) - const models = (parsed.models ?? []) + const modelIds = (parsed.models ?? []) .map((m) => m.name ?? m.model) .filter((id): id is string => typeof id === 'string' && id.length > 0) - .map((id) => ({ - id, - label: id, - catalogueSource: 'live' as const, - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + const models: AiProviderModel[] = [] + for (let offset = 0; offset < modelIds.length; offset += OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY) { + signal?.throwIfAborted() + const batch = modelIds.slice(offset, offset + OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY) + const resolvedBatch = await Promise.all(batch.map(async (id) => { + const declared = await fetchOllamaDeclaredCapabilities(creds, id, signal).catch((err) => { + if (signal?.aborted || isAbortError(err)) throw err + return null + }) + return { + id, + label: id, + catalogueSource: 'live' as const, + capabilities: { + toolCalling: declared ? declared.includes('tools') : true, + visionInput: declared?.includes('vision') ?? false, + toolResultImages: false, + promptCache: false, + streaming: true, + }, + } satisfies AiProviderModel })) + models.push(...resolvedBatch) + } return models.length > 0 ? models : FALLBACK_MODELS } catch (err) { + if (signal?.aborted || isAbortError(err)) throw err console.error('[ai/ollama] models request failed:', err) return FALLBACK_MODELS } } + +async function fetchOllamaDeclaredCapabilities( + creds: AiResolvedCredential, + modelId: string, + signal?: AbortSignal, +): Promise { + if (!creds.baseUrl) return null + const res = await fetch(`${trimSlash(creds.baseUrl)}/api/show`, { + method: 'POST', + headers: ollamaHeaders(creds.apiKey, true), + body: JSON.stringify({ model: modelId }), + signal, + }) + if (!res.ok) { + throw new Error(`[ai/ollama] model capability request failed: ${res.status} ${res.statusText}`) + } + const parsed = parseValue(OllamaShowSchema, await res.json()) + return parsed.capabilities ?? null +} + +function ollamaHeaders(apiKey: string | null, json = false): Record { + const headers: Record = {} + if (json) headers['content-type'] = 'application/json' + if (apiKey) headers.Authorization = `Bearer ${apiKey}` + return headers +} diff --git a/server/ai/drivers/openai.ts b/server/ai/drivers/openai.ts index c3071c3ed..dda2049fa 100644 --- a/server/ai/drivers/openai.ts +++ b/server/ai/drivers/openai.ts @@ -65,20 +65,21 @@ export const openaiDriver: AiProvider = { capabilities(_modelId: string) { // OpenAI's catalogue reports no capability flags, so we can't key off the - // model id. Current GPT/o-series chat models tool-call and accept images; - // the tool loop only reads `visionInput` here, so a permissive default is - // correct. (Prompt caching is automatic on OpenAI, not the Anthropic-style + // model id. Current GPT/o-series chat models tool-call and accept user + // images, so a permissive input default is correct; Responses tool outputs + // remain text-only. (Prompt caching is automatic on OpenAI, not the Anthropic-style // cache_control prefix this flag controls, so it stays false.) return { toolCalling: true, visionInput: true, + toolResultImages: false, promptCache: false, streaming: true, } }, - async listModels(creds: AiResolvedCredential) { - return fetchOpenAiModels(creds) + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { + return fetchOpenAiModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -136,11 +137,15 @@ function stableHash(value: string): string { * moderation, …) with no metadata, so we filter to the chat/reasoning families * and derive the label + tier from the id — heuristic, not authoritative. */ -async function fetchOpenAiModels(creds: AiResolvedCredential): Promise { +async function fetchOpenAiModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { if (creds.authMode !== 'apiKey' || !creds.apiKey) return [] const res = await fetch(OPENAI_MODELS_ENDPOINT, { headers: { Authorization: `Bearer ${creds.apiKey}` }, + signal, }) if (!res.ok) { throw new Error(`[ai/openai] models request failed: ${res.status} ${res.statusText}`) @@ -157,6 +162,7 @@ async function fetchOpenAiModels(creds: AiResolvedCredential): Promise { @@ -86,11 +88,12 @@ const ModelsResponseSchema = Type.Object( async function fetchOpenAiCompatibleModels( baseUrl: string, apiKey: string | null, + signal?: AbortSignal, ): Promise { try { const headers: Record = {} if (apiKey) headers.Authorization = `Bearer ${apiKey}` - const res = await fetch(`${normalizeOpenAiBaseUrl(baseUrl)}/v1/models`, { headers }) + const res = await fetch(`${normalizeOpenAiBaseUrl(baseUrl)}/v1/models`, { headers, signal }) if (!res.ok) return [] const parsed = parseValue(ModelsResponseSchema, await res.json()) return parsed.data.map((m) => ({ @@ -100,6 +103,7 @@ async function fetchOpenAiCompatibleModels( capabilities: { ...GENERIC_CAPABILITIES }, })) } catch (err) { + if (signal?.aborted || isAbortError(err)) throw err console.error('[ai/openai-compatible] models request failed:', err) return [] } diff --git a/server/ai/drivers/openrouter.ts b/server/ai/drivers/openrouter.ts index 4ffcb5d3a..e656e1cee 100644 --- a/server/ai/drivers/openrouter.ts +++ b/server/ai/drivers/openrouter.ts @@ -43,6 +43,7 @@ const OPENROUTER_ENDPOINT = `${OPENROUTER_BASE_URL}/responses` const DEFAULT_CAPABILITIES: AiProviderCapabilities = { toolCalling: true, visionInput: false, + toolResultImages: false, promptCache: false, streaming: true, } @@ -67,8 +68,14 @@ export const openrouterDriver: AiProvider = { return DEFAULT_CAPABILITIES }, - async listModels(creds: AiResolvedCredential) { - return fetchOpenRouterModels(creds) + async resolveCapabilities(creds: AiResolvedCredential, modelId: string, signal: AbortSignal) { + const models = await fetchOpenRouterModels(creds, signal) + return models.find((model) => model.id === modelId)?.capabilities + ?? DEFAULT_CAPABILITIES + }, + + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { + return fetchOpenRouterModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -125,13 +132,16 @@ function perMTok(value: string | undefined): number | null { return perToken * 1_000_000 } -async function fetchOpenRouterModels(creds: AiResolvedCredential): Promise { +async function fetchOpenRouterModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { const headers: Record = {} // The catalogue endpoint is public, but sending the bearer lets per-key // availability (e.g. BYOK-only models) reflect in the list. if (creds.apiKey) headers.Authorization = `Bearer ${creds.apiKey}` - const res = await fetch(`${OPENROUTER_BASE_URL}/models`, { headers }) + const res = await fetch(`${OPENROUTER_BASE_URL}/models`, { headers, signal }) if (!res.ok) { throw new Error(`[ai/openrouter] models request failed: ${res.status} ${res.statusText}`) } @@ -156,6 +166,7 @@ async function fetchOpenRouterModels(creds: AiResolvedCredential): Promise + /** + * Optional selected-model lookup for providers whose vision/tool support is + * model-specific. The chat boundary caches and de-duplicates this result; + * drivers should query the narrowest authoritative provider endpoint they + * expose. A failed lookup is treated as non-vision by the caller. + */ + resolveCapabilities?( + credentials: AiResolvedCredential, + modelId: string, + signal: AbortSignal, + ): Promise + + listModels( + credentials: AiResolvedCredential, + signal?: AbortSignal, + ): Promise /** * Run one agent turn. Yields canonical AiStreamEvents as the model diff --git a/server/ai/handlers/chat.ts b/server/ai/handlers/chat.ts index 06b3f5dc8..7faa1b5fb 100644 --- a/server/ai/handlers/chat.ts +++ b/server/ai/handlers/chat.ts @@ -4,7 +4,7 @@ * Opens an NDJSON stream against a chat. Body: * { * conversationId: string, - * prompt: string, + * content: Array<{ kind: 'text' | 'image', ... }>, * snapshot?: unknown // scope-specific per-request context * } * @@ -19,8 +19,21 @@ * 6. Streams NDJSON events back as the driver produces them. */ -import { Type, safeParseValue } from '@core/utils/typeboxHelpers' -import { jsonResponse, readValidatedBody, badRequest } from '../../http' +import { safeParseValue } from '@core/utils/typeboxHelpers' +import { + AI_CHAT_MAX_REQUEST_BYTES, + AI_CONVERSATION_MAX_USER_IMAGES, + AiChatRequestBodySchema, + type AiChatRequestBody, + type AiContentBlock, +} from '@core/ai' +import { + RequestBodyTooLargeError, + badRequest, + jsonResponse, + payloadTooLarge, + readValidatedBody, +} from '../../http' import { requireCapability } from '../../auth/authz' import type { DbClient } from '../../db/client' import { createAuditEvent } from '../../repositories/audit' @@ -28,17 +41,26 @@ import { appendMessage, listMessagesForConversation, readConversationForUser, - updateConversationForUser, + replaceDefaultConversationTitle, deriveConversationTitle, DEFAULT_CONVERSATION_TITLE, } from '../conversations/store' -import { buildMessageHistory } from '../conversations/history' +import { + buildMessageHistory, + projectUserImagesForModel, +} from '../conversations/history' import { readCredentialForUser, resolveCredentialForDriver, touchCredentialLastUsed, } from '../credentials/store' import { resolveDriver } from '../drivers' +import { resolveModelCapabilities } from '../drivers/modelCapabilities' +import { + AiImageInputError, + canonicaliseAiUserContent, + preflightAiUserContent, +} from '../inputImages' import { selectToolsForScope } from '../tools' import { buildSiteSystemPrompt, @@ -62,16 +84,9 @@ import type { } from '../runtime/types' import type { AiStreamRequest } from '../drivers/types' -const ChatRequestBodySchema = Type.Object({ - conversationId: Type.String({ minLength: 1 }), - prompt: Type.String({ minLength: 1 }), - // snapshot stays loose here — scope-specific shape; tools cast it inside - // their handlers. The handler narrows below based on the conversation's - // scope before passing to the system-prompt builder. - snapshot: Type.Optional(Type.Unknown()), -}) - const VALID_SCOPES: ToolScope[] = ['site', 'content', 'data', 'plugin'] +const activeChatConversations = new Set() +const REQUEST_ABORTED = Symbol('request-aborted') /** * Match `/admin/api/ai/chat/:scope`. Returns `null` if path doesn't match. @@ -105,9 +120,19 @@ async function handleAiChat( if (userOrResponse instanceof Response) return userOrResponse const user = userOrResponse - const chatBody = await readValidatedBody(req, ChatRequestBodySchema) + let chatBody: AiChatRequestBody | null + try { + chatBody = await readValidatedBody(req, AiChatRequestBodySchema, { + maxBytes: AI_CHAT_MAX_REQUEST_BYTES, + }) + } catch (err) { + if (err instanceof RequestBodyTooLargeError) { + return payloadTooLarge('Chat request is too large.') + } + throw err + } if (!chatBody) return badRequest('Invalid request body.') - const { conversationId, prompt, snapshot } = chatBody + const { conversationId, content, snapshot } = chatBody const conversation = await readConversationForUser(db, user.id, conversationId) if (!conversation) { @@ -142,54 +167,172 @@ async function handleAiChat( } const driver = resolveDriver(credential.providerId) - // Capability-filtered toolset. Callers without `ai.tools.write` only see - // read tools registered with the driver — the model has no way to - // emit a write call. See B6 in the capabilities review. + let preflight: ReturnType + try { + preflight = preflightAiUserContent(content) + } catch (err) { + if (err instanceof AiImageInputError) { + return err.status === 413 ? payloadTooLarge(err.message) : badRequest(err.message) + } + throw err + } + const requestedImage = preflight.image !== null + if (requestedImage) { + // Avoid an expensive decode when the conversation is already at its image + // budget. The authoritative check is repeated under the writer lease. + const initialRecords = await listMessagesForConversation(db, conversation.id) + const existingImageCount = countUserImages(initialRecords) + if (existingImageCount >= AI_CONVERSATION_MAX_USER_IMAGES) { + return payloadTooLarge( + `This conversation already contains ${AI_CONVERSATION_MAX_USER_IMAGES} images. Start a new chat to attach another.`, + ) + } + } + + // Resolve every selected model, not only image-bearing turns: the same + // authoritative flag also gates browser-tool screenshots. Model-specific + // drivers are cached/de-duplicated by the shared resolver. + const modelCapabilities = await waitForRequest( + resolveModelCapabilities(driver, resolvedCredential, conversation.modelId), + req.signal, + ) + if (modelCapabilities === REQUEST_ABORTED) return clientClosedRequest() const tools = selectToolsForScope(scope, user.capabilities) + if (requestedImage && !modelCapabilities.visionInput) { + return jsonResponse( + { error: 'The selected model does not support image input. Choose a vision-capable model.' }, + { status: 422 }, + ) + } + if (tools.length > 0 && !modelCapabilities.toolCalling) { + return jsonResponse( + { error: 'The selected model does not support tool calling. Choose an agent-capable model.' }, + { status: 422 }, + ) + } + if (req.signal.aborted) return clientClosedRequest() - // Append the user's message BEFORE streaming so it's persisted even if - // the stream aborts mid-response. - await appendMessage(db, conversation.id, { - role: 'user', - content: [{ kind: 'text', text: prompt }], - }) + // Full decode/re-encode is deliberately after the cheap conversation budget + // and capability gates so rejected requests cannot force needless Sharp work. + let userContent: AiContentBlock[] + try { + userContent = await canonicaliseAiUserContent(preflight) + } catch (err) { + if (err instanceof AiImageInputError) { + return err.status === 413 ? payloadTooLarge(err.message) : badRequest(err.message) + } + throw err + } + if (req.signal.aborted) return clientClosedRequest() + + // One provider stream may write a conversation at a time. Besides avoiding + // interleaved assistant/tool rows, this lets the refreshed image-budget + // check below stay atomic with the append: concurrent tabs get a retryable + // 409 instead of overshooting the cap. + const releaseConversation = acquireConversationStream(conversation.id) + if (!releaseConversation) { + return jsonResponse( + { error: 'This conversation is already generating a response. Wait for it to finish.' }, + { status: 409 }, + ) + } + if (req.signal.aborted) { + releaseConversation() + return clientClosedRequest() + } - // The first prompt names the conversation: replace the placeholder title - // with an excerpt of what the user asked for. Only fires while the title is - // still the default, so a user-renamed chat is never overwritten. - if (conversation.title === DEFAULT_CONVERSATION_TITLE) { - const derivedTitle = deriveConversationTitle(prompt) - if (derivedTitle) { - await updateConversationForUser(db, user.id, conversation.id, { title: derivedTitle }) - .catch((err) => { console.error('[ai/chat] auto-title failed:', err) }) + let existingRecords: Awaited> + let latestConversation: NonNullable>> + try { + const refreshedConversation = await readConversationForUser(db, user.id, conversation.id) + if (!refreshedConversation) { + releaseConversation() + return jsonResponse({ error: 'Conversation not found' }, { status: 404 }) + } + latestConversation = refreshedConversation + if ( + latestConversation.credentialId !== conversation.credentialId + || latestConversation.modelId !== conversation.modelId + ) { + releaseConversation() + return jsonResponse( + { error: 'The conversation model changed while this message was being prepared. Send again.' }, + { status: 409 }, + ) } + existingRecords = await listMessagesForConversation(db, conversation.id) + } catch (err) { + releaseConversation() + throw err + } + if (req.signal.aborted) { + releaseConversation() + return clientClosedRequest() + } + const currentImageCount = countUserImages(existingRecords) + if (requestedImage && currentImageCount >= AI_CONVERSATION_MAX_USER_IMAGES) { + releaseConversation() + return payloadTooLarge( + `This conversation already contains ${AI_CONVERSATION_MAX_USER_IMAGES} images. Start a new chat to attach another.`, + ) } - const existingMessages = await listMessagesForConversation(db, conversation.id) - const messages = buildMessageHistory(existingMessages) + const prepared = await (async () => { + try { + // Append the user's message BEFORE streaming so it's persisted even if + // the stream aborts mid-response. + const appendedMessage = await appendMessage(db, conversation.id, { + role: 'user', + content: userContent, + }) - const systemPrompt = buildSystemPromptForScope(scope, snapshot) + // The first prompt names the conversation: replace the placeholder title + // with an excerpt of what the user asked for. Only fires while the title + // is still the default, so a user-renamed chat is never overwritten. + if (latestConversation.title === DEFAULT_CONVERSATION_TITLE) { + const text = userContent.find((block) => block.kind === 'text') + const derivedTitle = text?.kind === 'text' + ? deriveConversationTitle(text.text) + : 'Image' + if (derivedTitle) { + await replaceDefaultConversationTitle(db, user.id, conversation.id, derivedTitle) + .catch((err) => { console.error('[ai/chat] auto-title failed:', err) }) + } + } - // Capture totals reported by the persister so the audit row can hold - // them when the stream completes (we read them off the conversation row - // diff post-stream — see the post-loop block). - const tokensAtStart = { - prompt: conversation.promptTokensTotal, - completion: conversation.completionTokensTotal, - cost: conversation.costUsdTotal, - } + const messages = projectUserImagesForModel( + buildMessageHistory([...existingRecords, appendedMessage]), + modelCapabilities.visionInput, + ) + const systemPrompt = buildSystemPromptForScope(scope, snapshot) - await createAuditEvent(db, { - actorUserId: user.id, - action: 'ai.chat.started', - targetType: 'ai_conversation', - targetId: conversation.id, - metadata: { - scope, - providerId: credential.providerId, - modelId: conversation.modelId, - }, - }) + // Capture totals reported by the persister so the audit row can hold + // them when the stream completes (we read them off the conversation row + // diff post-stream — see the post-loop block). + const tokensAtStart = { + prompt: latestConversation.promptTokensTotal, + completion: latestConversation.completionTokensTotal, + cost: latestConversation.costUsdTotal, + } + + await createAuditEvent(db, { + actorUserId: user.id, + action: 'ai.chat.started', + targetType: 'ai_conversation', + targetId: conversation.id, + metadata: { + scope, + providerId: credential.providerId, + modelId: conversation.modelId, + }, + }) + return { messages, systemPrompt, tokensAtStart } + } catch (err) { + releaseConversation() + throw err + } + })() + const { messages, systemPrompt, tokensAtStart } = prepared const stream = new ReadableStream({ async start(controller) { @@ -251,7 +394,7 @@ async function handleAiChat( messages, tools, modelId: conversation.modelId, - modelCapabilities: driver.capabilities(conversation.modelId), + modelCapabilities, credentials: resolvedCredential, signal: req.signal, bridge, @@ -274,7 +417,6 @@ async function handleAiChat( emit({ type: 'error', message: `AI chat failed: ${detail}` }) } finally { if (destroyBridge) destroyBridge() - closeStream() // Emit the terminal audit event. Re-read the conversation row to // capture the deltas the persister just committed. try { @@ -301,6 +443,9 @@ async function handleAiChat( // Audit failures must never break the user-visible stream — the // request already finished by the time we hit this branch. console.error('[ai/chat] audit emit failed:', auditErr) + } finally { + releaseConversation() + closeStream() } } }, @@ -310,7 +455,7 @@ async function handleAiChat( status: 200, headers: { 'Content-Type': 'application/x-ndjson', - 'Cache-Control': 'no-cache', + 'Cache-Control': 'private, no-store', 'X-Accel-Buffering': 'no', }, }) @@ -320,6 +465,45 @@ async function handleAiChat( // Helpers // --------------------------------------------------------------------------- +function acquireConversationStream(conversationId: string): (() => void) | null { + if (activeChatConversations.has(conversationId)) return null + activeChatConversations.add(conversationId) + let released = false + return () => { + if (released) return + released = true + activeChatConversations.delete(conversationId) + } +} + +type StoredConversationMessage = Awaited< + ReturnType +>[number] + +function countUserImages(records: readonly StoredConversationMessage[]): number { + return records.reduce( + (count, record) => count + ( + record.role === 'user' + ? record.content.filter((block) => block.kind === 'image').length + : 0 + ), + 0, + ) +} + +function clientClosedRequest(): Response { + return new Response(null, { status: 499, statusText: 'Client Closed Request' }) +} + +function waitForRequest(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(REQUEST_ABORTED) + return new Promise((resolve, reject) => { + const onAbort = () => resolve(REQUEST_ABORTED) + signal.addEventListener('abort', onAbort, { once: true }) + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + export function buildSystemPromptForScope( scope: ToolScope, snapshot: unknown, diff --git a/server/ai/handlers/conversations.ts b/server/ai/handlers/conversations.ts index c5a21bebc..d5b44b957 100644 --- a/server/ai/handlers/conversations.ts +++ b/server/ai/handlers/conversations.ts @@ -117,7 +117,10 @@ async function handleRead(req: Request, db: DbClient, id: string): Promise { diff --git a/server/ai/handlers/credentials.ts b/server/ai/handlers/credentials.ts index a25d7e270..be264ca6d 100644 --- a/server/ai/handlers/credentials.ts +++ b/server/ai/handlers/credentials.ts @@ -24,6 +24,7 @@ import { updateCredentialForUser, } from '../credentials/store' import { resolveDriver } from '../drivers' +import { listProviderModels } from '../drivers/modelList' import type { AiProviderModel } from '../drivers/types' import type { CredentialRecord } from '../credentials/types' import { listDefaults, setDefaultForScope } from '../defaults/store' @@ -126,7 +127,7 @@ async function handleCreate(req: Request, db: DbClient): Promise { // credential. Never overwrites an existing choice; failures here must not // fail credential creation. try { - await seedEmptyDefaults(db, record, userOrResponse.id) + await seedEmptyDefaults(db, record, userOrResponse.id, req.signal) } catch (err) { console.warn( '[ai/credentials] auto-default skipped - default seeding failed:', @@ -160,6 +161,7 @@ async function seedEmptyDefaults( db: DbClient, record: CredentialRecord, userId: string, + signal?: AbortSignal, ): Promise { const existing = await listDefaults(db) const filled = new Set(existing.map((d) => d.scope)) @@ -172,7 +174,7 @@ async function seedEmptyDefaults( const resolved = await resolveCredentialForDriver(record) apiKeyForRedaction = resolved.apiKey const driver = resolveDriver(record.providerId) - const models = await driver.listModels(resolved) + const models = await listProviderModels(driver, resolved, signal) const liveModels = models.filter((model) => model.catalogueSource !== 'fallback') const top = liveModels.find((m) => m.tier === 'smartest') ?? liveModels[0] topModelId = top?.id ?? null @@ -300,7 +302,7 @@ async function dispatchTest(req: Request, db: DbClient, id: string): Promise { + return canonicaliseAiUserContent(preflightAiUserContent(content)) +} + +/** Cheap structural/byte checks that never invoke an image decoder. */ +export function preflightAiUserContent( + content: readonly AiUserContentBlock[], +): AiUserContentPreflight { + const textBlocks = content.filter((block) => block.kind === 'text') + const imageBlocks = content.filter((block) => block.kind === 'image') + + if (textBlocks.length > 1) { + throw new AiImageInputError('A message can contain at most one text block.') + } + if (imageBlocks.length > 1) { + throw new AiImageInputError('A message can contain at most one image.') + } + + const text = textBlocks[0]?.text.trim() ?? '' + const image = imageBlocks[0] + if (!text && !image) { + throw new AiImageInputError('Message must contain text or an image.') + } + + const imageBytes = image ? preflightAiUserImage(image) : null + return { text, image: image ?? null, imageBytes } +} + +/** Full decoder boundary + canonical block reconstruction after admission gates. */ +export async function canonicaliseAiUserContent( + preflight: AiUserContentPreflight, +): Promise { + const canonicalImage = preflight.image && preflight.imageBytes + ? await canonicaliseAiUserImage(preflight.imageBytes) + : null + + // Text precedes the image in the canonical provider-facing order. This also + // removes whitespace-only text so no driver emits an empty text part and + // reconstructs the image so request-only extra fields cannot be persisted. + const canonical: AiContentBlock[] = [] + if (preflight.text) canonical.push({ kind: 'text', text: preflight.text }) + if (canonicalImage) canonical.push(canonicalImage) + return canonical +} + +export async function validateAiUserImage( + image: AiUserImageBlock, +): Promise { + return canonicaliseAiUserImage(preflightAiUserImage(image)) +} + +function preflightAiUserImage(image: AiUserImageBlock): Buffer { + const bytes = decodeCanonicalBase64(image.data) + if (bytes.byteLength > AI_USER_IMAGE_MAX_BYTES) { + throw new AiImageInputError( + `Image exceeds the ${formatMegabytes(AI_USER_IMAGE_MAX_BYTES)} MB limit.`, + 413, + ) + } + + if (bytes[0] !== 0xff || bytes[1] !== 0xd8 || bytes[2] !== 0xff) { + throw new AiImageInputError('Image data is not a JPEG.') + } + return bytes +} + +async function canonicaliseAiUserImage(bytes: Buffer): Promise { + let metadata: sharp.Metadata + try { + metadata = await sharp(bytes).metadata() + } catch (err) { + throw new AiImageInputError('Image data could not be decoded.', 400, { cause: err }) + } + + if (metadata.format !== 'jpeg' || !metadata.width || !metadata.height) { + throw new AiImageInputError('Image data is not a valid JPEG.') + } + if ( + metadata.width > AI_USER_IMAGE_MAX_EDGE + || metadata.height > AI_USER_IMAGE_MAX_EDGE + || metadata.width * metadata.height > AI_USER_IMAGE_MAX_PIXELS + ) { + throw new AiImageInputError( + `Image dimensions exceed the ${AI_USER_IMAGE_MAX_EDGE}px / ${AI_USER_IMAGE_MAX_PIXELS.toLocaleString()}px limit.`, + 413, + ) + } + + const canonicalBytes = await canonicaliseJpeg(bytes, metadata.width, metadata.height) + return { + kind: 'image', + mimeType: 'image/jpeg', + data: canonicalBytes.toString('base64'), + } +} + +/** + * Fully decode and re-encode the image before persistence. Sharp strips source + * metadata by default; `failOn: 'warning'` also rejects truncated JPEGs whose + * headers are readable but whose pixel data is incomplete. + */ +async function canonicaliseJpeg( + bytes: Buffer, + sourceWidth: number, + sourceHeight: number, +): Promise { + let width = sourceWidth + let height = sourceHeight + let lastSize = bytes.byteLength + + try { + for (let resizeAttempt = 0; resizeAttempt < 3; resizeAttempt += 1) { + for (const quality of SERVER_JPEG_QUALITIES) { + let pipeline = sharp(bytes, { + failOn: 'warning', + limitInputPixels: AI_USER_IMAGE_MAX_PIXELS, + }) + .rotate() + .flatten({ background: { r: 255, g: 255, b: 255 } }) + + if (resizeAttempt > 0) { + pipeline = pipeline.resize({ + width, + height, + fit: 'inside', + withoutEnlargement: true, + }) + } + + const output = await pipeline.jpeg({ quality }).toBuffer() + lastSize = output.byteLength + if (output.byteLength <= AI_USER_IMAGE_MAX_BYTES) return output + } + + const scale = Math.max( + 0.5, + Math.sqrt(AI_USER_IMAGE_MAX_BYTES / lastSize) * 0.9, + ) + width = Math.max(1, Math.floor(width * scale)) + height = Math.max(1, Math.floor(height * scale)) + } + } catch (err) { + throw new AiImageInputError('Image data could not be fully decoded.', 400, { cause: err }) + } + + throw new AiImageInputError( + `Image could not be reduced below the ${formatMegabytes(AI_USER_IMAGE_MAX_BYTES)} MB limit.`, + 413, + ) +} + +function decodeCanonicalBase64(data: string): Buffer { + if (!data || data.length % 4 !== 0 || !CANONICAL_BASE64.test(data)) { + throw new AiImageInputError('Image data must be canonical base64.') + } + const bytes = Buffer.from(data, 'base64') + if (bytes.toString('base64') !== data) { + throw new AiImageInputError('Image data must be canonical base64.') + } + return bytes +} + +function formatMegabytes(bytes: number): string { + return (bytes / 1_000_000).toFixed(1) +} diff --git a/server/ai/tools/site/writeTools.ts b/server/ai/tools/site/writeTools.ts index 2f7b04199..212b67cbe 100644 --- a/server/ai/tools/site/writeTools.ts +++ b/server/ai/tools/site/writeTools.ts @@ -400,7 +400,7 @@ const renderSnapshotTool: AiTool = { scope: 'site', execution: 'browser', description: - "Inspect the rendered canvas. Returns a layout report: viewport size, per-node bounding boxes, image-load status, and warnings (overflow / broken-image / invisible-node) — enough to catch most layout bugs in text. On a vision-capable model a screenshot is also attached as an image. Pass `breakpointId` to choose which breakpoint frame (defaults to active). Pass `nodeId` to capture just that node's subtree — a sharper, cheaper image than the whole page, and a report scoped to that section with coordinates relative to the node; omit `nodeId` to capture the full page.", + "Inspect the rendered canvas. Returns a layout report: viewport size, per-node bounding boxes, image-load status, and warnings (overflow / broken-image / invisible-node) — enough to catch most layout bugs in text. When the active provider supports native image-bearing tool results, a screenshot is also attached as an image. Pass `breakpointId` to choose which breakpoint frame (defaults to active). Pass `nodeId` to capture just that node's subtree — a sharper, cheaper image than the whole page, and a report scoped to that section with coordinates relative to the node; omit `nodeId` to capture the full page.", inputSchema: RenderSnapshotInputSchema, } diff --git a/src/__tests__/agent/agentImageAttachment.test.ts b/src/__tests__/agent/agentImageAttachment.test.ts new file mode 100644 index 000000000..ee6d00189 --- /dev/null +++ b/src/__tests__/agent/agentImageAttachment.test.ts @@ -0,0 +1,230 @@ +import { afterEach, describe, expect, it, mock } from 'bun:test' +import sharp from 'sharp' +import { + AI_USER_IMAGE_MAX_EDGE, + AI_USER_IMAGE_MAX_PIXELS, +} from '@core/ai' +import { + fitAgentImageSize, + normaliseAgentImage, + readAgentImageSourceSize, +} from '@site/panels/AgentPanel/agentImageAttachment' + +interface BrowserImageMocks { + close: ReturnType + createImageBitmap: ReturnType + drawImage: ReturnType + fillRect: ReturnType + restore(): void +} + +let activeMocks: BrowserImageMocks | null = null + +function installBrowserImageMocks(blob: Blob | null = new Blob( + [new Uint8Array([1, 2, 3])], + { type: 'image/jpeg' }, +)): BrowserImageMocks { + activeMocks?.restore() + const canvasPrototype = Object.getPrototypeOf(document.createElement('canvas')) as object + const createBitmapDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + const getContextDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'getContext') + const toBlobDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'toBlob') + const close = mock(() => {}) + const drawImage = mock(() => {}) + const fillRect = mock(() => {}) + const createImageBitmap = mock(async () => ({ + width: 2000, + height: 1000, + close, + } as unknown as ImageBitmap)) + + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: createImageBitmap, + }) + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value: () => ({ fillStyle: '', fillRect, drawImage }), + }) + Object.defineProperty(canvasPrototype, 'toBlob', { + configurable: true, + value: (callback: BlobCallback) => callback(blob), + }) + + const installed: BrowserImageMocks = { + close, + createImageBitmap, + drawImage, + fillRect, + restore() { + restoreProperty(globalThis, 'createImageBitmap', createBitmapDescriptor) + restoreProperty(canvasPrototype, 'getContext', getContextDescriptor) + restoreProperty(canvasPrototype, 'toBlob', toBlobDescriptor) + }, + } + activeMocks = installed + return installed +} + +function restoreProperty( + target: object, + key: PropertyKey, + descriptor: PropertyDescriptor | undefined, +): void { + if (descriptor) Object.defineProperty(target, key, descriptor) + else Reflect.deleteProperty(target, key) +} + +describe('agent image attachment preparation', () => { + afterEach(() => { + activeMocks?.restore() + activeMocks = null + }) + + it('does not upscale images that already fit the policy', () => { + expect(fitAgentImageSize(640, 480)).toEqual({ width: 640, height: 480 }) + }) + + it('fits wide images inside both the long-edge and pixel limits', () => { + const size = fitAgentImageSize(4000, 2000) + expect(size).toEqual({ width: AI_USER_IMAGE_MAX_EDGE, height: AI_USER_IMAGE_MAX_EDGE / 2 }) + expect(size.width * size.height).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_PIXELS) + }) + + it('never rounds a pixel-limited image back above the pixel budget', () => { + const size = fitAgentImageSize(2000, 2000) + expect(size.width * size.height).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_PIXELS) + }) + + it('rejects invalid dimensions before allocating a canvas', () => { + expect(() => fitAgentImageSize(0, 100)).toThrow('Image has invalid dimensions.') + expect(() => fitAgentImageSize(Number.NaN, 100)).toThrow('Image has invalid dimensions.') + }) + + it('normalises a pasted image to a bounded JPEG and closes the decoded bitmap', async () => { + const browser = installBrowserImageMocks() + const block = await normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + ) + + expect(block).toEqual({ kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }) + expect(browser.fillRect).toHaveBeenCalledTimes(1) + expect(browser.drawImage).toHaveBeenCalledTimes(1) + expect(browser.close).toHaveBeenCalledTimes(1) + expect(browser.createImageBitmap.mock.calls[0]?.[1]).toMatchObject({ + imageOrientation: 'from-image', + resizeWidth: 1568, + resizeHeight: 784, + resizeQuality: 'high', + }) + }) + + it('closes the decoded bitmap when browser encoding fails', async () => { + const browser = installBrowserImageMocks(null) + await expect(normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + )).rejects.toThrow('This browser could not encode the pasted image.') + expect(browser.close).toHaveBeenCalledTimes(1) + }) + + it('closes a decoded bitmap and skips encoding when preparation is cancelled', async () => { + const browser = installBrowserImageMocks() + const decoded = deferred() + browser.createImageBitmap.mockImplementation(() => decoded.promise) + const controller = new AbortController() + const preparing = normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + controller.signal, + ) + + while (browser.createImageBitmap.mock.calls.length === 0) await Promise.resolve() + controller.abort() + decoded.resolve({ + width: 1568, + height: 784, + close: browser.close, + } as unknown as ImageBitmap) + + await expect(preparing).rejects.toHaveProperty('name', 'AbortError') + expect(browser.close).toHaveBeenCalledTimes(1) + expect(browser.drawImage).not.toHaveBeenCalled() + }) + + it('reads bounded PNG dimensions before decoding and rejects decompression bombs', async () => { + expect(readAgentImageSourceSize(pngHeader(640, 480), 'image/png')) + .toEqual({ width: 640, height: 480 }) + const browser = installBrowserImageMocks() + + await expect(normaliseAgentImage( + new File([pngHeader(20_000, 20_000)], 'bomb.png', { type: 'image/png' }), + )).rejects.toThrow('Source image dimensions exceed') + expect(browser.createImageBitmap).not.toHaveBeenCalled() + }) + + it('reads real JPEG and WebP dimensions without a full browser decode', async () => { + const source = sharp({ + create: { + width: 321, + height: 123, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }) + const [jpeg, webp] = await Promise.all([ + source.clone().jpeg().toBuffer(), + source.clone().webp().toBuffer(), + ]) + + expect(readAgentImageSourceSize(jpeg, 'image/jpeg')).toEqual({ width: 321, height: 123 }) + expect(readAgentImageSourceSize(webp, 'image/webp')).toEqual({ width: 321, height: 123 }) + }) + + it('uses EXIF display orientation when sizing a JPEG decode target', async () => { + const oriented = await sharp({ + create: { + width: 400, + height: 300, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer() + + expect(readAgentImageSourceSize(oriented, 'image/jpeg')) + .toEqual({ width: 300, height: 400 }) + }) + + it('rejects unsupported clipboard formats before decoding', async () => { + const createImageBitmap = mock(async () => { + throw new Error('should not decode') + }) + const descriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: createImageBitmap, + }) + try { + await expect(normaliseAgentImage( + new File([new Uint8Array([1])], 'animation.gif', { type: 'image/gif' }), + )).rejects.toThrow('Use a PNG, JPEG, or WebP image.') + expect(createImageBitmap).not.toHaveBeenCalled() + } finally { + restoreProperty(globalThis, 'createImageBitmap', descriptor) + } + }) +}) + +function pngHeader(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(24) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + bytes.set([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52], 8) + const view = new DataView(bytes.buffer) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} diff --git a/src/__tests__/agent/agentSlice.test.ts b/src/__tests__/agent/agentSlice.test.ts index cb91026e3..62e5b492a 100644 --- a/src/__tests__/agent/agentSlice.test.ts +++ b/src/__tests__/agent/agentSlice.test.ts @@ -9,6 +9,7 @@ import { type AgentToolCall, } from '@site/agent' import type { ConversationView } from '@admin/ai/api' +import type { AiUserContentBlock } from '@core/ai' import '@modules/base' // --------------------------------------------------------------------------- @@ -34,6 +35,9 @@ function freshAgentState() { agentActiveCredentialId: null, agentActiveModelId: null, agentContextTokens: null, + isAgentConversationPending: false, + isAgentProviderPending: false, + agentComposerEpoch: 0, agentConversations: [], hasUnsavedChanges: false, }) @@ -78,7 +82,7 @@ interface InterceptedFetch { * any unexpected call instead of hanging. */ function captureFetchByRoute( - routes: Record Response>, + routes: Record Response | Promise>, ): { restore: () => void; calls: InterceptedFetch[] } { const original = globalThis.fetch const calls: InterceptedFetch[] = [] @@ -129,12 +133,58 @@ const conversationCreateResponse = (id: string) => { status: 201, headers: { 'Content-Type': 'application/json' } }, ) +const conversationDetailResponse = ( + id: string, + content: unknown[], + selection: { credentialId: string; modelId: string } = { + credentialId: 'cred-1', + modelId: 'claude-sonnet-4-6', + }, +) => + new Response(JSON.stringify({ + conversation: { + id, + scope: 'site', + title: 'Image', + credentialId: selection.credentialId, + modelId: selection.modelId, + promptTokensTotal: 0, + completionTokensTotal: 0, + costUsdTotal: 0, + cacheReadTokensTotal: 0, + cacheCreationTokensTotal: 0, + contextTokens: 0, + createdAt: '2026-07-11T10:00:00.000Z', + updatedAt: '2026-07-11T10:00:00.000Z', + messages: [{ + id: 'message-image', + position: 0, + role: 'user', + content, + toolCallId: null, + toolName: null, + createdAt: '2026-07-11T10:00:00.000Z', + }], + }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + const toolResultAckResponse = () => new Response(JSON.stringify({ ok: true }), { status: 200, headers: { 'Content-Type': 'application/json' }, }) +const textContent = (text: string): AiUserContentBlock[] => [{ kind: 'text', text }] + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} + // --------------------------------------------------------------------------- // processStreamEvent — bridge handshake + tool requests // --------------------------------------------------------------------------- @@ -385,8 +435,9 @@ describe('sendAgentMessage — request lifecycle', () => { ]), }) + let result: { accepted: boolean } | undefined try { - await useEditorStore.getState().sendAgentMessage('Add a hero') + result = await useEditorStore.getState().sendAgentMessage(textContent('Add a hero')) } finally { intercept.restore() } @@ -399,9 +450,86 @@ describe('sendAgentMessage — request lifecycle', () => { expect(conversationCalls).toHaveLength(1) expect(chatCalls).toHaveLength(1) expect(useEditorStore.getState().agentConversationId).toBe('conv-1') + expect(result).toEqual({ accepted: true }) + expect(JSON.parse(chatCalls[0]!.body)).toMatchObject({ + conversationId: 'conv-1', + content: [{ kind: 'text', text: 'Add a hero' }], + }) void rootId }) + it('stops a first send while conversation creation is still pending', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const createStarted = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': (_call, init) => { + createStarted.resolve() + return new Promise((_resolve, reject) => { + const signal = init?.signal + const rejectAbort = () => reject( + signal?.reason ?? new DOMException('The operation was aborted.', 'AbortError'), + ) + if (signal?.aborted) rejectAbort() + else signal?.addEventListener('abort', rejectAbort, { once: true }) + }) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const sending = useEditorStore.getState().sendAgentMessage(textContent('Start')) + await createStarted.promise + expect(useEditorStore.getState().isAgentStreaming).toBe(true) + useEditorStore.getState().abortAgent() + + expect(await sending).toEqual({ accepted: false }) + expect(useEditorStore.getState().isAgentStreaming).toBe(false) + expect(useEditorStore.getState().agentConversationId).toBeNull() + expect(useEditorStore.getState().agentMessages).toEqual([]) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + } finally { + intercept.restore() + } + }) + + it('sends and renders an image-only user turn without inventing placeholder text', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const image = { + kind: 'image' as const, + mimeType: 'image/jpeg' as const, + data: 'QUJD', + } + + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': () => conversationCreateResponse('conv-image'), + '/admin/api/ai/chat/site': () => ndjsonResponse([ + { type: 'bridgeReady', bridgeId: 'b-image' }, + { type: 'done' }, + ]), + }) + + let result: { accepted: boolean } | undefined + try { + result = await useEditorStore.getState().sendAgentMessage([image]) + } finally { + intercept.restore() + } + + expect(result).toEqual({ accepted: true }) + const chat = intercept.calls.find((call) => call.url === '/admin/api/ai/chat/site') + expect(chat).toBeDefined() + expect(JSON.parse(chat!.body)).toMatchObject({ + conversationId: 'conv-image', + content: [image], + }) + const userMessage = useEditorStore.getState().agentMessages.find((message) => message.role === 'user') + expect(userMessage?.blocks).toEqual([image]) + }) + it('reuses the same conversation id on follow-up sends', async () => { freshAgentState() useEditorStore.setState({ @@ -420,8 +548,8 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('First message.') - await useEditorStore.getState().sendAgentMessage('Follow-up.') + await useEditorStore.getState().sendAgentMessage(textContent('First message.')) + await useEditorStore.getState().sendAgentMessage(textContent('Follow-up.')) } finally { intercept.restore() } @@ -460,7 +588,7 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Create a pricing card class.') + await useEditorStore.getState().sendAgentMessage(textContent('Create a pricing card class.')) } finally { intercept.restore() } @@ -495,7 +623,7 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Anything.') + await useEditorStore.getState().sendAgentMessage(textContent('Anything.')) } finally { intercept.restore() } @@ -506,9 +634,71 @@ describe('sendAgentMessage — request lifecycle', () => { }) }) +describe('loadAgentConversation — image rehydration', () => { + it('restores persisted user image blocks and advances the composer epoch', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const image = { kind: 'image', mimeType: 'image/jpeg', data: 'QUJD' } + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-image': () => + conversationDetailResponse('conv-image', [image]), + }) + + try { + await useEditorStore.getState().loadAgentConversation('conv-image') + } finally { + intercept.restore() + } + + const state = useEditorStore.getState() + expect(state.agentConversationId).toBe('conv-image') + expect(state.agentComposerEpoch).toBe(1) + expect(state.agentMessages).toHaveLength(1) + expect(state.agentMessages[0]?.blocks).toEqual([image]) + }) + + it('blocks Send until an in-flight conversation load commits', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-old', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + const loadStarted = deferred() + const loadResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-new': () => { + loadStarted.resolve() + return loadResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const loading = useEditorStore.getState().loadAgentConversation('conv-new') + await loadStarted.promise + expect(useEditorStore.getState().isAgentConversationPending).toBe(true) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Wait'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + + loadResponse.resolve(conversationDetailResponse('conv-new', [ + { kind: 'text', text: 'Loaded' }, + ])) + await loading + expect(useEditorStore.getState().agentConversationId).toBe('conv-new') + expect(useEditorStore.getState().isAgentConversationPending).toBe(false) + } finally { + intercept.restore() + } + }) +}) + describe('conversation reset key-set', () => { - // All three reset paths must clear the SAME six keys — agentContextTokens and - // agentError have each silently drifted out of one copy in the past. + // All three reset paths clear the same conversation keys and advance the + // composer epoch so local text/image drafts cannot cross conversations. const RESET_SNAPSHOT = { agentMessages: [], agentError: null, @@ -516,11 +706,13 @@ describe('conversation reset key-set', () => { agentActiveCredentialId: null, agentActiveModelId: null, agentContextTokens: null, + agentComposerEpoch: 1, } function seedDirtyConversation() { freshAgentState() useEditorStore.setState({ + isAgentStreaming: false, agentMessages: [{ id: 'm1', role: 'user', blocks: [{ kind: 'text', text: 'hi' }], timestamp: 1 }], agentError: 'AI server is not running. Start it with: bun run dev', agentConversationId: 'conv-dirty', @@ -539,10 +731,11 @@ describe('conversation reset key-set', () => { agentActiveCredentialId: s.agentActiveCredentialId, agentActiveModelId: s.agentActiveModelId, agentContextTokens: s.agentContextTokens, + agentComposerEpoch: s.agentComposerEpoch, } } - it('startNewAgentConversation resets the full six-key set (incl. agentContextTokens)', () => { + it('startNewAgentConversation resets conversation state and remounts the composer', () => { seedDirtyConversation() useEditorStore.getState().startNewAgentConversation() expect(pickResetKeys()).toEqual(RESET_SNAPSHOT) @@ -601,7 +794,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Go') + await useEditorStore.getState().sendAgentMessage(textContent('Go')) } finally { intercept.restore() } @@ -615,7 +808,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { expect(useEditorStore.getState().isAgentStreaming).toBe(false) }) - it('surfaces a non-ok chat response once — single agentError + single placeholder block', async () => { + it('rejects a non-ok chat response without adding an optimistic turn', async () => { freshAgentState() useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) @@ -625,17 +818,16 @@ describe('sendAgentMessage — streaming + error surfacing', () => { '/admin/api/ai/chat/site': () => new Response('boom', { status: 500 }), }) + let result: { accepted: boolean } | undefined try { - await useEditorStore.getState().sendAgentMessage('Do a thing') + result = await useEditorStore.getState().sendAgentMessage(textContent('Do a thing')) } finally { intercept.restore() } - expect(useEditorStore.getState().agentError).toContain('Agent request failed') - const assistant = useEditorStore.getState().agentMessages.find((m) => m.role === 'assistant')! - // Collapsed double-set (F10): exactly one placeholder block, not two renders. - expect(assistant.blocks).toHaveLength(1) - expect(assistant.blocks[0]).toMatchObject({ kind: 'text', text: '_(agent error)_' }) + expect(result).toEqual({ accepted: false }) + expect(useEditorStore.getState().agentError).toBe('boom') + expect(useEditorStore.getState().agentMessages).toEqual([]) expect(useEditorStore.getState().isAgentStreaming).toBe(false) }) @@ -662,7 +854,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { // Panel-open path stages the default… await useEditorStore.getState().loadScopeDefault() // …first send must reuse it, NOT fetch the default a second time. - await useEditorStore.getState().sendAgentMessage('Hi') + await useEditorStore.getState().sendAgentMessage(textContent('Hi')) } finally { intercept.restore() } @@ -757,6 +949,37 @@ describe('loadScopeDefault', () => { expect(useEditorStore.getState().agentActiveCredentialId).toBeNull() expect(useEditorStore.getState().agentActiveModelId).toBeNull() }) + + it('does not overwrite a model picked while the default request is in flight', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: null, + agentActiveCredentialId: null, + agentActiveModelId: null, + }) + const requestStarted = deferred() + const response = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': () => { + requestStarted.resolve() + return response.promise + }, + }) + + try { + const loading = useEditorStore.getState().loadScopeDefault() + await requestStarted.promise + await useEditorStore.getState().setAgentProvider('cred-picked', 'model-picked') + response.resolve(defaultsResponse()) + await loading + } finally { + intercept.restore() + } + + expect(useEditorStore.getState().agentActiveCredentialId).toBe('cred-picked') + expect(useEditorStore.getState().agentActiveModelId).toBe('model-picked') + }) }) // --------------------------------------------------------------------------- @@ -780,4 +1003,187 @@ describe('setAgentProvider', () => { expect(useEditorStore.getState().agentActiveModelId).toBe('model-9') expect(useEditorStore.getState().agentError).toBeNull() }) + + it('blocks Send until an existing conversation model update finishes', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-switch', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + + const updateStarted = deferred() + const updateResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-switch': (_call, init) => { + expect(init?.method).toBe('PUT') + updateStarted.resolve() + return updateResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const changingModel = useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + await updateStarted.promise + expect(useEditorStore.getState().isAgentProviderPending).toBe(true) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Use the new model'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + + updateResponse.resolve(new Response(JSON.stringify({ + conversation: { + id: 'conv-switch', + scope: 'site', + title: 'Conversation', + credentialId: 'cred-new', + modelId: 'model-new', + promptTokensTotal: 0, + completionTokensTotal: 0, + costUsdTotal: 0, + cacheReadTokensTotal: 0, + cacheCreationTokensTotal: 0, + contextTokens: 0, + createdAt: '2026-07-11T10:00:00.000Z', + updatedAt: '2026-07-11T10:00:00.000Z', + }, + }), { headers: { 'content-type': 'application/json' } })) + + await changingModel + expect(useEditorStore.getState().isAgentProviderPending).toBe(false) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Use the new model'))) + .toEqual({ accepted: true }) + expect(intercept.calls.map((call) => call.url)).toEqual([ + '/admin/api/ai/conversations/conv-switch', + '/admin/api/ai/chat/site', + ]) + } finally { + intercept.restore() + } + }) + + it('does not send when the model update it was waiting for fails', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-fail', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + + const updateStarted = deferred() + const updateResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-fail': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-fail', [], { + credentialId: 'cred-old', + modelId: 'model-old', + }) + } + updateStarted.resolve() + return updateResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const changingModel = useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + await updateStarted.promise + const sending = useEditorStore.getState().sendAgentMessage(textContent('Do not misroute me')) + updateResponse.resolve(new Response(JSON.stringify({ error: 'update failed' }), { + status: 400, + headers: { 'content-type': 'application/json' }, + })) + + const [, result] = await Promise.all([changingModel, sending]) + expect(result).toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + expect(useEditorStore.getState().agentActiveCredentialId).toBe('cred-old') + expect(useEditorStore.getState().agentActiveModelId).toBe('model-old') + } finally { + intercept.restore() + } + }) + + it('reconciles an update whose response was lost after the server committed it', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-committed', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-committed': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-committed', [], { + credentialId: 'cred-new', + modelId: 'model-new', + }) + } + return Promise.reject(new TypeError('Connection reset after commit')) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + await useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + + const state = useEditorStore.getState() + expect(state.agentActiveCredentialId).toBe('cred-new') + expect(state.agentActiveModelId).toBe('model-new') + expect(state.agentError).toBeNull() + expect(state.isAgentProviderPending).toBe(false) + expect(await state.sendAgentMessage(textContent('Use the committed model'))) + .toEqual({ accepted: true }) + expect(intercept.calls.map((call) => call.method)).toEqual(['PUT', 'GET', 'POST']) + } finally { + intercept.restore() + } + }) + + it('keeps Send locked when an ambiguous update can still commit after reconciliation', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-ambiguous', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + let serverSelection = { credentialId: 'cred-old', modelId: 'model-old' } + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-ambiguous': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-ambiguous', [], serverSelection) + } + return new Response(JSON.stringify({ error: 'Upstream response was lost' }), { + status: 502, + headers: { 'content-type': 'application/json' }, + }) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + await useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + // The disconnected PUT commits after the first reconciliation read. + serverSelection = { credentialId: 'cred-new', modelId: 'model-new' } + + const state = useEditorStore.getState() + expect(state.agentActiveCredentialId).toBeNull() + expect(state.agentActiveModelId).toBeNull() + expect(state.agentError).toContain('server state could not be confirmed') + expect(await state.sendAgentMessage(textContent('Never route against stale state'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + } finally { + intercept.restore() + } + }) }) diff --git a/src/__tests__/ai/auditUsagePersistence.test.ts b/src/__tests__/ai/auditUsagePersistence.test.ts index 1cc94d51f..292dc0763 100644 --- a/src/__tests__/ai/auditUsagePersistence.test.ts +++ b/src/__tests__/ai/auditUsagePersistence.test.ts @@ -21,6 +21,7 @@ const fakeTextOnlyDriver: AiProvider = { return { toolCalling: false, visionInput: false, + toolResultImages: false, promptCache: false, streaming: true, } diff --git a/src/__tests__/ai/chatImageHandler.test.ts b/src/__tests__/ai/chatImageHandler.test.ts new file mode 100644 index 000000000..b4c398591 --- /dev/null +++ b/src/__tests__/ai/chatImageHandler.test.ts @@ -0,0 +1,355 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import sharp from 'sharp' +import { AI_CHAT_MAX_REQUEST_BYTES, AI_CONVERSATION_MAX_USER_IMAGES } from '@core/ai' +import { createCapabilityTestHarness, type CapabilityTestHarness } from '../helpers/capabilityHarness' +import { + appendMessage, + createConversationForUser, + listMessagesForConversation, + readConversationForUser, +} from '../../../server/ai/conversations/store' + +let testSerial = 0 + +describe('AI chat user-image boundary', () => { + let harness: CapabilityTestHarness + let cookie: string + let conversationId: string + let credentialId: string + let originalFetch: typeof globalThis.fetch + + beforeEach(async () => { + originalFetch = globalThis.fetch + harness = await createCapabilityTestHarness() + cookie = await harness.setupOwner() + const { rows } = await harness.db<{ id: string }>`select id from users limit 1` + const userId = rows[0]!.id + credentialId = `cred_image_${++testSerial}` + await harness.db` + insert into ai_provider_credentials ( + id, user_id, provider_id, auth_mode, display_label, base_url + ) values ( + ${credentialId}, ${userId}, 'ollama', 'baseUrl', 'Image test', 'http://ollama.test' + ) + ` + const conversation = await createConversationForUser(harness.db, userId, { + scope: 'site', + credentialId, + modelId: 'vision-model', + }) + conversationId = conversation.id + }) + + afterEach(async () => { + globalThis.fetch = originalFetch + await harness.cleanup() + }) + + it('returns 413 when the complete request envelope exceeds its limit', async () => { + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'x'.repeat(AI_CHAT_MAX_REQUEST_BYTES) }], + }, + }) + + expect(response.status).toBe(413) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects malformed JPEG bytes before persistence', async () => { + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'image', mimeType: 'image/jpeg', data: '/9h/' }], + }, + }) + + expect(response.status).toBe(400) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects a non-vision model before persistence', async () => { + const image = await jpegBlock() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/tags') { + return jsonResponse({ models: [{ name: 'vision-model' }] }) + } + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['completion'] }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + + expect(response.status).toBe(422) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects a known non-tool model before persistence or provider streaming', async () => { + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['vision'] }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'Inspect the page.' }], + }, + }) + + expect(response.status).toBe(422) + expect(await response.json()).toEqual({ + error: 'The selected model does not support tool calling. Choose an agent-capable model.', + }) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('enforces the persisted user-image budget before appending another turn', async () => { + const image = await jpegBlock() + for (let index = 0; index < AI_CONVERSATION_MAX_USER_IMAGES; index += 1) { + await appendMessage(harness.db, conversationId, { role: 'user', content: [image] }) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + + expect(response.status).toBe(413) + expect(await listMessagesForConversation(harness.db, conversationId)) + .toHaveLength(AI_CONVERSATION_MAX_USER_IMAGES) + }) + + it('persists an image-only turn and titles a new conversation Image', async () => { + const image = await jpegBlock() + let providerRequest = '' + globalThis.fetch = async (input, init) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/tags') { + return jsonResponse({ models: [{ name: 'vision-model' }] }) + } + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['vision', 'tools'] }) + } + if (url === 'http://ollama.test/v1/chat/completions') { + providerRequest = String(init?.body ?? '') + return new Response([ + 'data: {"choices":[{"delta":{"content":"Looks good."},"finish_reason":null}]}\n\n', + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":2}}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + + expect(response.status).toBe(200) + expect(response.headers.get('cache-control')).toBe('private, no-store') + await response.text() + const { rows } = await harness.db<{ id: string }>`select id from users limit 1` + const conversation = await readConversationForUser(harness.db, rows[0]!.id, conversationId) + const messages = await listMessagesForConversation(harness.db, conversationId) + + expect(conversation?.title).toBe('Image') + expect(messages[0]?.role).toBe('user') + expect(messages[0]?.content).toHaveLength(1) + const persistedImage = messages[0]?.content[0] + expect(persistedImage).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + if (persistedImage?.kind !== 'image') throw new Error('Expected persisted image block') + expect(providerRequest).toContain(`data:image/jpeg;base64,${persistedImage.data}`) + expect((JSON.parse(providerRequest) as { tools?: unknown }).tools).toBeArray() + }) + + it('allows only one concurrent writer at the eight-image boundary', async () => { + const image = await jpegBlock() + for (let index = 0; index < AI_CONVERSATION_MAX_USER_IMAGES - 1; index += 1) { + await appendMessage(harness.db, conversationId, { role: 'user', content: [image] }) + } + + const capabilityStarted = deferred() + const capabilityResponse = deferred() + const providerResponse = deferred() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + capabilityStarted.resolve() + return capabilityResponse.promise + } + if (url === 'http://ollama.test/v1/chat/completions') { + return providerResponse.promise + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const firstRequest = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + await capabilityStarted.promise + const secondRequest = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + capabilityResponse.resolve(jsonResponse({ capabilities: ['vision', 'tools'] })) + + const responses = await Promise.all([firstRequest, secondRequest]) + expect(responses.map((response) => response.status).sort()).toEqual([200, 409]) + + providerResponse.resolve(new Response([ + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } })) + const accepted = responses.find((response) => response.status === 200) + await accepted?.text() + + const messages = await listMessagesForConversation(harness.db, conversationId) + const persistedUserImages = messages + .filter((message) => message.role === 'user') + .flatMap((message) => message.content) + .filter((block) => block.kind === 'image') + expect(persistedUserImages).toHaveLength(AI_CONVERSATION_MAX_USER_IMAGES) + }) + + it('does not persist a turn aborted during capability discovery', async () => { + const image = await jpegBlock() + const capabilityStarted = deferred() + const capabilityResponse = deferred() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url !== 'http://ollama.test/api/show') { + throw new Error(`Unexpected fetch: ${url}`) + } + capabilityStarted.resolve() + return capabilityResponse.promise + } + const controller = new AbortController() + + const request = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + signal: controller.signal, + json: { conversationId, content: [image] }, + }) + await capabilityStarted.promise + controller.abort() + + const response = await request + expect(response.status).toBe(499) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + + // Let the shared lookup settle so it cannot leak into a later test. + capabilityResponse.resolve(jsonResponse({ capabilities: ['vision', 'tools'] })) + }) + + it('re-reads the image budget after a slower request reaches the writer lease', async () => { + const image = await jpegBlock() + for (let index = 0; index < AI_CONVERSATION_MAX_USER_IMAGES - 1; index += 1) { + await appendMessage(harness.db, conversationId, { role: 'user', content: [image] }) + } + const slowCapabilityStarted = deferred() + const slowCapabilityResponse = deferred() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + slowCapabilityStarted.resolve() + return slowCapabilityResponse.promise + } + if (url === 'http://ollama-fast.test/api/show') { + return jsonResponse({ capabilities: ['vision', 'tools'] }) + } + if (url === 'http://ollama-fast.test/v1/chat/completions') { + return new Response([ + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const slowRequest = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + await slowCapabilityStarted.promise + await harness.db` + update ai_provider_credentials + set base_url = 'http://ollama-fast.test' + where id = ${credentialId} + ` + + const fastResponse = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + expect(fastResponse.status).toBe(200) + await fastResponse.text() + + slowCapabilityResponse.resolve(jsonResponse({ capabilities: ['vision', 'tools'] })) + const slowResponse = await slowRequest + expect(slowResponse.status).toBe(413) + + const userImages = (await listMessagesForConversation(harness.db, conversationId)) + .filter((message) => message.role === 'user') + .flatMap((message) => message.content) + .filter((block) => block.kind === 'image') + expect(userImages).toHaveLength(AI_CONVERSATION_MAX_USER_IMAGES) + }) +}) + +async function jpegBlock() { + const data = (await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }).jpeg().toBuffer()).toString('base64') + return { kind: 'image' as const, mimeType: 'image/jpeg' as const, data } +} + +function requestUrl(input: string | URL | Request): string { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + return input.url +} + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + headers: { 'content-type': 'application/json' }, + }) +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} diff --git a/src/__tests__/ai/chatRequest.test.ts b/src/__tests__/ai/chatRequest.test.ts new file mode 100644 index 000000000..0ba833fee --- /dev/null +++ b/src/__tests__/ai/chatRequest.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from 'bun:test' +import { Value } from '@sinclair/typebox/value' +import { + AI_USER_IMAGE_MAX_BASE64_CHARS, + AiChatRequestBodySchema, + isAiUserImageSourceMimeType, +} from '@core/ai' + +const image = { + kind: 'image' as const, + mimeType: 'image/jpeg' as const, + data: 'AAAA', +} + +describe('AiChatRequestBodySchema', () => { + test('accepts text-only, image-only, and mixed user turns', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'text', text: 'Hello' }], + snapshot: { pageId: 'page-1' }, + })).toBe(true) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [image], + })).toBe(true) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'text', text: 'Describe this' }, image], + })).toBe(true) + }) + + test('rejects empty content and assistant/tool block injection', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'toolCall', toolCallId: 'call-1', toolName: 'site_insert_html', input: {} }], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'toolResult', ok: true }], + })).toBe(false) + }) + + test('bounds the user turn to one text block and one normalised JPEG', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'text', text: 'one' }, { kind: 'text', text: 'two' }, image], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ ...image, mimeType: 'image/png' }], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ ...image, data: 'A'.repeat(AI_USER_IMAGE_MAX_BASE64_CHARS + 1) }], + })).toBe(false) + }) +}) + +describe('AI user-image source MIME guard', () => { + test('accepts browser-normalisable raster formats and rejects unsupported input', () => { + expect(isAiUserImageSourceMimeType('image/jpeg')).toBe(true) + expect(isAiUserImageSourceMimeType('image/png')).toBe(true) + expect(isAiUserImageSourceMimeType('image/webp')).toBe(true) + expect(isAiUserImageSourceMimeType('image/gif')).toBe(false) + expect(isAiUserImageSourceMimeType('image/svg+xml')).toBe(false) + }) +}) diff --git a/src/__tests__/ai/conversationRoundtrip.test.ts b/src/__tests__/ai/conversationRoundtrip.test.ts index 4984cc32c..0ed916b74 100644 --- a/src/__tests__/ai/conversationRoundtrip.test.ts +++ b/src/__tests__/ai/conversationRoundtrip.test.ts @@ -1,12 +1,15 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { Value } from '@sinclair/typebox/value' +import sharp from 'sharp' import { createTestDb, type TestDb } from '../helpers/createTestDb' import { appendMessage, createConversationForUser, listMessagesForConversation, readConversationForUser, + replaceDefaultConversationTitle, toConversationDetailView, + updateConversationForUser, } from '../../../server/ai/conversations/store' import { ConversationDetailViewSchema } from '../../admin/ai/api' @@ -64,4 +67,61 @@ describe('conversation detail round-trip', () => { // The dead context field must not reappear on the wire shape. expect('contextJson' in detail).toBe(false) }) + + it('preserves mixed and image-only user content through SQLite and the wire schema', async () => { + const conv = await createConversationForUser(testDb.db, 'user_1', { + scope: 'site', + credentialId: 'cred_1', + modelId: 'model_1', + }) + const imageData = (await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: { r: 20, g: 40, b: 60 }, + }, + }).jpeg().toBuffer()).toString('base64') + const image = { kind: 'image' as const, mimeType: 'image/jpeg', data: imageData } + + await appendMessage(testDb.db, conv.id, { + role: 'user', + content: [{ kind: 'text', text: 'What is this?' }, image], + }) + await appendMessage(testDb.db, conv.id, { + role: 'user', + content: [image], + }) + + const record = await readConversationForUser(testDb.db, 'user_1', conv.id) + expect(record).not.toBeNull() + const messages = await listMessagesForConversation(testDb.db, conv.id) + const detail = toConversationDetailView(record!, messages) + + expect(Value.Check(ConversationDetailViewSchema, detail)).toBe(true) + expect(detail.messages.map((message) => message.content)).toEqual([ + [{ kind: 'text', text: 'What is this?' }, image], + [image], + ]) + }) + + it('does not let an automatic first-turn title overwrite a user rename', async () => { + const conv = await createConversationForUser(testDb.db, 'user_1', { + scope: 'site', + credentialId: 'cred_1', + modelId: 'model_1', + }) + await updateConversationForUser(testDb.db, 'user_1', conv.id, { + title: 'My reference review', + }) + + expect(await replaceDefaultConversationTitle( + testDb.db, + 'user_1', + conv.id, + 'Image', + )).toBe(false) + expect((await readConversationForUser(testDb.db, 'user_1', conv.id))?.title) + .toBe('My reference review') + }) }) diff --git a/src/__tests__/ai/inputImages.test.ts b/src/__tests__/ai/inputImages.test.ts new file mode 100644 index 000000000..9e00a0074 --- /dev/null +++ b/src/__tests__/ai/inputImages.test.ts @@ -0,0 +1,181 @@ +import { beforeAll, describe, expect, test } from 'bun:test' +import sharp from 'sharp' +import { + AI_USER_IMAGE_MAX_BYTES, + AI_USER_IMAGE_MAX_EDGE, + AI_USER_IMAGE_MAX_PIXELS, + type AiUserImageBlock, +} from '@core/ai' +import { + AiImageInputError, + validateAiUserContent, + validateAiUserImage, +} from '../../../server/ai/inputImages' + +let smallJpegBase64 = '' + +beforeAll(async () => { + smallJpegBase64 = (await makeImage(24, 16, 'jpeg')).toString('base64') +}) + +function imageBlock(data = smallJpegBase64): AiUserImageBlock { + return { kind: 'image', mimeType: 'image/jpeg', data } +} + +function makeImage( + width: number, + height: number, + format: 'jpeg' | 'png', +): Promise { + const source = sharp({ + create: { + width, + height, + channels: 3, + background: { r: 45, g: 90, b: 135 }, + }, + }) + return format === 'jpeg' ? source.jpeg().toBuffer() : source.png().toBuffer() +} + +describe('validateAiUserContent', () => { + test('normalises mixed content to trimmed text followed by the image', async () => { + const content = await validateAiUserContent([ + imageBlock(), + { kind: 'text', text: ' Describe this screenshot. ' }, + ]) + + expect(content[0]).toEqual({ kind: 'text', text: 'Describe this screenshot.' }) + expect(content[1]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + expect((content[1] as AiUserImageBlock).data.length).toBeGreaterThan(0) + }) + + test('accepts image-only content and removes whitespace-only text beside it', async () => { + const imageOnly = await validateAiUserContent([imageBlock()]) + const besideWhitespace = await validateAiUserContent([ + { kind: 'text', text: ' \n ' }, + imageBlock(), + ]) + + expect(imageOnly).toHaveLength(1) + expect(imageOnly[0]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + expect(besideWhitespace).toHaveLength(1) + expect(besideWhitespace[0]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + }) + + test('rejects an empty turn, duplicate text, and duplicate images', async () => { + await expect(validateAiUserContent([ + { kind: 'text', text: ' ' }, + ])).rejects.toMatchObject({ + name: 'AiImageInputError', + status: 400, + message: 'Message must contain text or an image.', + }) + + await expect(validateAiUserContent([ + { kind: 'text', text: 'one' }, + { kind: 'text', text: 'two' }, + ])).rejects.toMatchObject({ + status: 400, + message: 'A message can contain at most one text block.', + }) + + await expect(validateAiUserContent([ + imageBlock(), + imageBlock(), + ])).rejects.toMatchObject({ + status: 400, + message: 'A message can contain at most one image.', + }) + }) +}) + +describe('validateAiUserImage', () => { + test('fully decodes and returns a bounded canonical JPEG', async () => { + const canonical = await validateAiUserImage(imageBlock()) + const bytes = Buffer.from(canonical.data, 'base64') + const metadata = await sharp(bytes).metadata() + + expect(canonical.mimeType).toBe('image/jpeg') + expect(bytes.byteLength).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_BYTES) + expect(metadata).toMatchObject({ format: 'jpeg', width: 24, height: 16 }) + }) + + test('rejects malformed and non-canonical base64', async () => { + for (const data of ['!!!!', 'A===', `${smallJpegBase64} `]) { + await expect(validateAiUserImage(imageBlock(data))).rejects.toMatchObject({ + name: 'AiImageInputError', + status: 400, + message: 'Image data must be canonical base64.', + }) + } + }) + + test('rejects bytes whose real format does not match the JPEG contract', async () => { + const png = await makeImage(16, 16, 'png') + await expect(validateAiUserImage(imageBlock(png.toString('base64')))).rejects.toMatchObject({ + status: 400, + message: 'Image data is not a JPEG.', + }) + }) + + test('rejects corrupt data even when it starts with the JPEG signature', async () => { + const corrupt = Buffer.from([0xff, 0xd8, 0xff, 0x00, 0x01, 0x02, 0x03]) + await expect(validateAiUserImage(imageBlock(corrupt.toString('base64')))).rejects.toBeInstanceOf( + AiImageInputError, + ) + }) + + test('rejects a truncated JPEG whose header metadata is still readable', async () => { + const complete = await makeImage(24, 16, 'jpeg') + const truncated = complete.subarray(0, complete.byteLength - 2) + expect((await sharp(truncated).metadata()).format).toBe('jpeg') + + await expect(validateAiUserImage(imageBlock(truncated.toString('base64')))) + .rejects.toMatchObject({ status: 400, message: 'Image data could not be fully decoded.' }) + }) + + test('strips source metadata before persistence', async () => { + const tagged = await sharp({ + create: { + width: 24, + height: 16, + channels: 3, + background: { r: 45, g: 90, b: 135 }, + }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer() + expect((await sharp(tagged).metadata()).exif).toBeDefined() + + const canonical = await validateAiUserImage(imageBlock(tagged.toString('base64'))) + const metadata = await sharp(Buffer.from(canonical.data, 'base64')).metadata() + + expect(metadata.exif).toBeUndefined() + expect(metadata.orientation).toBeUndefined() + }) + + test('rejects decoded bytes over the image budget with 413 semantics', async () => { + const oversized = Buffer.alloc(AI_USER_IMAGE_MAX_BYTES + 1) + oversized[0] = 0xff + oversized[1] = 0xd8 + oversized[2] = 0xff + + await expect(validateAiUserImage(imageBlock(oversized.toString('base64')))).rejects.toMatchObject({ + status: 413, + message: 'Image exceeds the 1.5 MB limit.', + }) + }) + + test('rejects an excessive edge or pixel area with 413 semantics', async () => { + const tooWide = await makeImage(AI_USER_IMAGE_MAX_EDGE + 1, 1, 'jpeg') + await expect(validateAiUserImage(imageBlock(tooWide.toString('base64')))).rejects.toMatchObject({ + status: 413, + }) + + const areaWidth = 1500 + const areaHeight = Math.floor(AI_USER_IMAGE_MAX_PIXELS / areaWidth) + 1 + const tooManyPixels = await makeImage(areaWidth, areaHeight, 'jpeg') + await expect(validateAiUserImage(imageBlock(tooManyPixels.toString('base64')))).rejects.toMatchObject({ + status: 413, + }) + }) +}) diff --git a/src/__tests__/ai/messageHistory.test.ts b/src/__tests__/ai/messageHistory.test.ts index 1f4c0a9ad..3504f7a8f 100644 --- a/src/__tests__/ai/messageHistory.test.ts +++ b/src/__tests__/ai/messageHistory.test.ts @@ -1,7 +1,10 @@ import { describe, test, expect } from 'bun:test' import { buildMessageHistory, + EARLIER_USER_IMAGE_OMITTED, INTERRUPTED_TOOL_RESULT_ERROR, + NON_VISION_USER_IMAGE_OMITTED, + projectUserImagesForModel, } from '../../../server/ai/conversations/history' import type { MessageRecord } from '../../../server/ai/conversations/types' import type { AiContentBlock } from '../../../server/ai/runtime/types' @@ -34,6 +37,12 @@ function rec( function userText(text: string): MessageRecord { return rec('user', [{ kind: 'text', text }]) } +function userImage(data: string, text?: string): MessageRecord { + return rec('user', [ + ...(text ? [{ kind: 'text' as const, text }] : []), + { kind: 'image', mimeType: 'image/jpeg', data }, + ]) +} function assistantToolCall(id: string, name: string, input: unknown): MessageRecord { return rec('assistant', [{ kind: 'toolCall', toolCallId: id, toolName: name, input }], id, name) } @@ -204,3 +213,61 @@ describe('buildMessageHistory', () => { }) }) }) + +describe('user-image history projection', () => { + test('keeps only the newest image-bearing user turn for a vision model', () => { + const history = buildMessageHistory([ + userImage('old-image', 'Old screenshot'), + rec('assistant', [{ kind: 'text', text: 'old answer' }]), + userImage('new-image'), + rec('assistant', [{ kind: 'text', text: 'new answer' }]), + ]) + const original = JSON.stringify(history) + + const projected = projectUserImagesForModel(history, true) + + expect(projected).toEqual([ + { + role: 'user', + content: [ + { kind: 'text', text: 'Old screenshot' }, + { kind: 'text', text: EARLIER_USER_IMAGE_OMITTED }, + ], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'old answer' }] }, + { + role: 'user', + content: [{ kind: 'image', mimeType: 'image/jpeg', data: 'new-image' }], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'new answer' }] }, + ]) + expect(JSON.stringify(history)).toBe(original) + }) + + test('elides every user image for a text-only model while keeping turns valid', () => { + const history = buildMessageHistory([ + userImage('old-image', 'First'), + rec('assistant', [{ kind: 'text', text: 'answer' }]), + userImage('new-image'), + ]) + const original = JSON.stringify(history) + + const projected = projectUserImagesForModel(history, false) + + expect(projected).toEqual([ + { + role: 'user', + content: [ + { kind: 'text', text: 'First' }, + { kind: 'text', text: NON_VISION_USER_IMAGE_OMITTED }, + ], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'answer' }] }, + { + role: 'user', + content: [{ kind: 'text', text: NON_VISION_USER_IMAGE_OMITTED }], + }, + ]) + expect(JSON.stringify(history)).toBe(original) + }) +}) diff --git a/src/__tests__/ai/modelCapabilities.test.ts b/src/__tests__/ai/modelCapabilities.test.ts new file mode 100644 index 000000000..146f77e4e --- /dev/null +++ b/src/__tests__/ai/modelCapabilities.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, spyOn, test } from 'bun:test' +import { resolveModelCapabilities } from '../../../server/ai/drivers/modelCapabilities' +import { listProviderModels } from '../../../server/ai/drivers/modelList' +import type { + AiProvider, + AiProviderCapabilities, + AiResolvedCredential, +} from '../../../server/ai/drivers/types' + +const TEXT_ONLY: AiProviderCapabilities = { + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, +} + +const VISION: AiProviderCapabilities = { + toolCalling: true, + visionInput: true, + toolResultImages: false, + promptCache: false, + streaming: true, +} + +const credentials: AiResolvedCredential = { + id: 'credential-1', + providerId: 'openrouter', + authMode: 'apiKey', + apiKey: 'secret', + baseUrl: null, +} + +function provider( + fallback: AiProviderCapabilities, + resolveCapabilities?: ( + credentials: AiResolvedCredential, + modelId: string, + ) => Promise, +): AiProvider { + return { + id: 'openrouter', + label: 'Test provider', + supportedAuthModes: ['apiKey'], + capabilities: () => fallback, + ...(resolveCapabilities ? { resolveCapabilities } : {}), + async listModels() { + throw new Error('full catalogue should not be queried for capability resolution') + }, + async *stream() { + yield { type: 'done' } + }, + } +} + +describe('resolveModelCapabilities', () => { + test('uses an authoritative vision-capable static default without a catalogue request', async () => { + const driver = provider(VISION) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(VISION) + }) + + test('uses a provider-owned selected-model capability lookup', async () => { + const driver = provider(TEXT_ONLY, async () => VISION) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(VISION) + }) + + test('fails closed to the static capability when live discovery fails', async () => { + const errorLog = spyOn(console, 'error').mockImplementation(() => {}) + try { + const driver = provider(VISION, async () => { + throw new Error('catalogue unavailable') + }) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(TEXT_ONLY) + expect(errorLog).toHaveBeenCalledTimes(1) + } finally { + errorLog.mockRestore() + } + }) + + test('de-duplicates and caches concurrent selected-model lookups', async () => { + let calls = 0 + const driver = provider(TEXT_ONLY, async () => { + calls += 1 + await Promise.resolve() + return VISION + }) + + const [first, second] = await Promise.all([ + resolveModelCapabilities(driver, credentials, 'vision-model'), + resolveModelCapabilities(driver, credentials, 'vision-model'), + ]) + const cached = await resolveModelCapabilities(driver, credentials, 'vision-model') + + expect(first).toEqual(VISION) + expect(second).toEqual(VISION) + expect(cached).toEqual(VISION) + expect(calls).toBe(1) + }) + + test('fails closed for every waiter when a de-duplicated lookup rejects', async () => { + const errorLog = spyOn(console, 'error').mockImplementation(() => {}) + let calls = 0 + try { + const driver = provider(VISION, async () => { + calls += 1 + await Promise.resolve() + throw new Error('catalogue unavailable') + }) + + const results = await Promise.all([ + resolveModelCapabilities(driver, credentials, 'vision-model'), + resolveModelCapabilities(driver, credentials, 'vision-model'), + ]) + + expect(results).toEqual([TEXT_ONLY, TEXT_ONLY]) + expect(calls).toBe(1) + expect(errorLog).toHaveBeenCalledTimes(1) + } finally { + errorLog.mockRestore() + } + }) + + test('invalidates a cached result when credential auth material changes', async () => { + let calls = 0 + const driver = provider(TEXT_ONLY, async () => { + calls += 1 + return calls === 1 ? VISION : TEXT_ONLY + }) + + const first = await resolveModelCapabilities(driver, credentials, 'vision-model') + const rotated = await resolveModelCapabilities(driver, { + ...credentials, + apiKey: 'rotated-secret', + }, 'vision-model') + + expect(first).toEqual(VISION) + expect(rotated).toEqual(TEXT_ONLY) + expect(calls).toBe(2) + }) +}) + +describe('listProviderModels', () => { + test('releases the caller on abort even if a driver forgets to settle', async () => { + let providerSignal: AbortSignal | undefined + const driver: AiProvider = { + ...provider(TEXT_ONLY), + listModels(_credentials, signal) { + providerSignal = signal + return new Promise(() => {}) + }, + } + const controller = new AbortController() + const models = listProviderModels(driver, credentials, controller.signal) + + controller.abort() + + await expect(models).rejects.toHaveProperty('name', 'AbortError') + expect(providerSignal?.aborted).toBe(true) + }) +}) diff --git a/src/__tests__/ai/ollamaMapping.test.ts b/src/__tests__/ai/ollamaMapping.test.ts index 0b4c68126..8daee6fb9 100644 --- a/src/__tests__/ai/ollamaMapping.test.ts +++ b/src/__tests__/ai/ollamaMapping.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, afterEach } from 'bun:test' import { Type } from '@core/utils/typeboxHelpers' import { ollamaDriver } from '../../../server/ai/drivers/ollama' +import { resolveModelCapabilities } from '../../../server/ai/drivers/modelCapabilities' import { ChatCompletionsTurnTranslator, mapChatHistory, @@ -69,15 +70,30 @@ describe('Ollama mapChatHistory', () => { test('maps an image-bearing user turn to OpenAI content parts', () => { const history: AiMessage[] = [ - { role: 'user', content: [{ kind: 'image', mimeType: 'image/png', data: 'B64' }, { kind: 'text', text: 'see' }] }, + { role: 'user', content: [{ kind: 'text', text: 'see' }, { kind: 'image', mimeType: 'image/jpeg', data: 'B64' }] }, ] const mapped = mapChatHistory([], history).flat() expect(mapped).toEqual([ { role: 'user', content: [ - { type: 'image_url', image_url: { url: 'data:image/png;base64,B64' } }, { type: 'text', text: 'see' }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,B64' } }, + ], + }, + ]) + }) + + test('maps an image-only user turn without inventing text', () => { + const history: AiMessage[] = [ + { role: 'user', content: [{ kind: 'image', mimeType: 'image/jpeg', data: 'B64' }] }, + ] + + expect(mapChatHistory([], history).flat()).toEqual([ + { + role: 'user', + content: [ + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,B64' } }, ], }, ]) @@ -144,7 +160,7 @@ function makeRequest(serverCalls: unknown[]): AiStreamRequest { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool], modelId: 'llama3.3', - modelCapabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'ollama', authMode: 'baseUrl', apiKey: null, baseUrl: 'http://localhost:11434' }, signal: new AbortController().signal, bridge, @@ -193,3 +209,91 @@ describe('runToolLoop via ollamaDriver', () => { expect(usage!.completionTokens).toBe(15) }) }) + +describe('Ollama live model capabilities', () => { + test('uses /api/show to distinguish vision and text-only installed models', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString() + requests.push({ url, init }) + + if (url === 'http://localhost:11434/api/tags') { + return Response.json({ + models: [ + { name: 'llava:latest' }, + { model: 'llama3.3:latest' }, + ], + }) + } + + expect(url).toBe('http://localhost:11434/api/show') + const body = JSON.parse(init?.body as string) as { model: string } + return Response.json({ + capabilities: body.model === 'llava:latest' + ? ['completion', 'vision'] + : ['completion'], + }) + }) as typeof fetch + + const models = await ollamaDriver.listModels({ + id: 'credential-1', + providerId: 'ollama', + authMode: 'baseUrl', + apiKey: 'proxy-secret', + baseUrl: 'http://localhost:11434/', + }) + + expect(models).toEqual([ + expect.objectContaining({ + id: 'llava:latest', + catalogueSource: 'live', + capabilities: expect.objectContaining({ visionInput: true }), + }), + expect.objectContaining({ + id: 'llama3.3:latest', + catalogueSource: 'live', + capabilities: expect.objectContaining({ visionInput: false }), + }), + ]) + expect(requests.map(({ url }) => url)).toEqual([ + 'http://localhost:11434/api/tags', + 'http://localhost:11434/api/show', + 'http://localhost:11434/api/show', + ]) + expect(requests.slice(1).map(({ init }) => ({ + method: init?.method, + contentType: (init?.headers as Record)['content-type'], + authorization: (init?.headers as Record).Authorization, + }))).toEqual([ + { method: 'POST', contentType: 'application/json', authorization: 'Bearer proxy-secret' }, + { method: 'POST', contentType: 'application/json', authorization: 'Bearer proxy-secret' }, + ]) + expect(requests[0]?.init?.headers).toEqual({ Authorization: 'Bearer proxy-secret' }) + }) + + test('resolves only the selected model, fails closed over a positive fallback, and authenticates', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString() + requests.push({ url, init }) + return Response.json({ capabilities: ['completion'] }) + }) as typeof fetch + + const capabilities = await resolveModelCapabilities(ollamaDriver, { + id: 'proxy-credential', + providerId: 'ollama', + authMode: 'baseUrl', + apiKey: 'proxy-secret', + baseUrl: 'http://localhost:11434/', + }, 'llama4') + + expect(capabilities.visionInput).toBe(false) + expect(requests).toHaveLength(1) + expect(requests[0]?.url).toBe('http://localhost:11434/api/show') + expect(requests[0]?.init?.method).toBe('POST') + expect(requests[0]?.init?.headers).toEqual({ + 'content-type': 'application/json', + Authorization: 'Bearer proxy-secret', + }) + }) +}) diff --git a/src/__tests__/ai/responsesMapping.test.ts b/src/__tests__/ai/responsesMapping.test.ts index c12e1cb75..64b2b9ddd 100644 --- a/src/__tests__/ai/responsesMapping.test.ts +++ b/src/__tests__/ai/responsesMapping.test.ts @@ -209,7 +209,7 @@ describe('runToolLoop via openaiDriver (Responses)', () => { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool], modelId: 'gpt-5.4', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'openai', authMode: 'apiKey', apiKey: 'sk-test', baseUrl: null }, signal: new AbortController().signal, bridge, @@ -264,7 +264,7 @@ describe('openrouterDriver', () => { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [], modelId: 'openai/gpt-5.4', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'openrouter', authMode: 'apiKey', apiKey: 'sk-or-test', baseUrl: null }, signal: new AbortController().signal, bridge, @@ -338,14 +338,14 @@ describe('openrouterDriver', () => { { id: 'openai/gpt-5.4', label: 'GPT 5.4', - capabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, pricing: { inputPerMTok: 5, outputPerMTok: 25 }, contextWindow: 128_000, }, { id: 'anthropic/claude-opus-4.8', label: 'Claude Opus 4.8', - capabilities: { toolCalling: false, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: false, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, pricing: { inputPerMTok: 10, outputPerMTok: 50 }, contextWindow: 200_000, }, diff --git a/src/__tests__/ai/toolEvidence.test.ts b/src/__tests__/ai/toolEvidence.test.ts index 4e36e9e14..03fea195f 100644 --- a/src/__tests__/ai/toolEvidence.test.ts +++ b/src/__tests__/ai/toolEvidence.test.ts @@ -63,8 +63,9 @@ const ANTHROPIC_DONE = sse( { type: 'message_stop' }, ) -const VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: true, promptCache: true, streaming: true } -const NO_VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: false, promptCache: false, streaming: true } +const VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: true, toolResultImages: true, promptCache: true, streaming: true } +const NO_VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true } +const USER_IMAGES_ONLY_CAPS: AiProviderCapabilities = { ...VISION_CAPS, toolResultImages: false } const renderSnapshotTool: AiTool = { name: 'site_render_snapshot', @@ -157,6 +158,24 @@ describe('multimodal tool output + heavy elision (Anthropic)', () => { expect(browserInputs).toEqual([{ captureScreenshot: false }]) }) + test('vision input without native tool-result images also skips screenshot capture', async () => { + let call = 0 + globalThis.fetch = (async () => { + call += 1 + return sseResponse(call === 1 ? anthropicSnapTurn('t_s1') : ANTHROPIC_DONE) + }) as typeof fetch + const browserInputs: unknown[] = [] + const bridge: AiBrowserBridge = { + async callBrowser(_name, input): Promise { + browserInputs.push(input) + return { ok: true, data: { layout: { warnings: [] } } } + }, + } + + for await (const _ of anthropicDriver.stream(makeRequest(bridge, USER_IMAGES_ONLY_CAPS))) { void _ } + expect(browserInputs).toEqual([{ captureScreenshot: false }]) + }) + test('stubs the earlier render_snapshot once a newer one supersedes it', async () => { const bodies: Array> = [] globalThis.fetch = (async (_url: string, init: RequestInit) => { diff --git a/src/__tests__/ai/toolLoop.test.ts b/src/__tests__/ai/toolLoop.test.ts index 75e23f128..e202c4181 100644 --- a/src/__tests__/ai/toolLoop.test.ts +++ b/src/__tests__/ai/toolLoop.test.ts @@ -79,7 +79,7 @@ function makeRequest(bridge: AiBrowserBridge, serverCalls: unknown[]): AiStreamR messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool, paintTool], modelId: 'claude-sonnet-4-6', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: true, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: true, promptCache: true, streaming: true }, credentials: { id: 'cr', providerId: 'anthropic', authMode: 'apiKey', apiKey: 'sk-test', baseUrl: null }, signal: new AbortController().signal, bridge, diff --git a/src/__tests__/panels/agentPanel.test.tsx b/src/__tests__/panels/agentPanel.test.tsx index 7c10785b7..5c00b8262 100644 --- a/src/__tests__/panels/agentPanel.test.tsx +++ b/src/__tests__/panels/agentPanel.test.tsx @@ -1,13 +1,131 @@ import { afterEach, describe, expect, it, mock } from 'bun:test' -import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { createStore } from 'zustand/vanilla' import { AgentStoreProvider } from '@admin/ai/AgentStoreContext' +import { clearModelListCache } from '@admin/ai/api' import { MemoryRouter, useLocation } from '@admin/lib/routing' +import { AdminSessionProvider } from '@admin/session' import type { AgentSlice } from '@site/agent' +import type { AiUserContentBlock } from '@core/ai' +import type { CmsCurrentUser } from '@core/persistence' import { AgentPanel } from '@site/panels/AgentPanel' const originalFetch = globalThis.fetch +const TEST_CREDENTIAL = { + id: 'cred_1', + providerId: 'openai', + authMode: 'apiKey', + displayLabel: 'OpenAI', + baseUrl: null, + keyFingerprintCurrent: true, + createdAt: '2026-06-01T10:00:00.000Z', + lastUsedAt: null, +} as const + +function installModelFetch(visionInput: boolean, toolCalling = true): void { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) { + return jsonResponse({ credentials: [TEST_CREDENTIAL] }) + } + if (url.includes('/admin/api/ai/providers/')) { + return jsonResponse({ + models: [{ + id: 'model-1', + label: 'Model 1', + capabilities: { + toolCalling, + visionInput, + toolResultImages: false, + promptCache: false, + streaming: true, + }, + contextWindow: 128_000, + }], + }) + } + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch +} + +interface Deferred { + promise: Promise + resolve(value: T): void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +interface ImageBrowserMocks { + bitmap: ImageBitmap + restore(): void +} + +let activeImageMocks: ImageBrowserMocks | null = null + +function installImageBrowserMocks( + bitmapPromise: Promise = Promise.resolve(fakeBitmap()), +): ImageBrowserMocks { + activeImageMocks?.restore() + const canvasPrototype = Object.getPrototypeOf(document.createElement('canvas')) as object + const createBitmapDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + const getContextDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'getContext') + const toBlobDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'toBlob') + const bitmap = fakeBitmap() + + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: mock(() => bitmapPromise), + }) + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value: () => ({ + fillStyle: '', + fillRect: () => {}, + drawImage: () => {}, + }), + }) + Object.defineProperty(canvasPrototype, 'toBlob', { + configurable: true, + value: (callback: BlobCallback) => { + callback(new Blob([new Uint8Array([1, 2, 3])], { type: 'image/jpeg' })) + }, + }) + const installed: ImageBrowserMocks = { + bitmap, + restore() { + restoreProperty(globalThis, 'createImageBitmap', createBitmapDescriptor) + restoreProperty(canvasPrototype, 'getContext', getContextDescriptor) + restoreProperty(canvasPrototype, 'toBlob', toBlobDescriptor) + }, + } + activeImageMocks = installed + return installed +} + +function fakeBitmap(): ImageBitmap { + return { + width: 100, + height: 80, + close: mock(() => {}), + } as unknown as ImageBitmap +} + +function restoreProperty( + target: object, + key: PropertyKey, + descriptor: PropertyDescriptor | undefined, +): void { + if (descriptor) Object.defineProperty(target, key, descriptor) + else Reflect.deleteProperty(target, key) +} + function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, @@ -25,15 +143,27 @@ function createAgentStore(overrides: Partial = {}) { agentActiveCredentialId: null, agentActiveModelId: null, agentConversations: [], + agentContextTokens: null, + isAgentConversationPending: false, + isAgentProviderPending: false, + agentComposerEpoch: 0, openAgent: () => set({ isAgentOpen: true }), closeAgent: () => set({ isAgentOpen: false }), toggleAgent: () => set((state) => ({ isAgentOpen: !state.isAgentOpen })), - sendAgentMessage: async () => {}, + sendAgentMessage: async () => ({ accepted: true }), abortAgent: () => {}, - clearAgentMessages: () => set({ agentMessages: [], agentError: null }), + clearAgentMessages: () => set((state) => ({ + agentMessages: [], + agentError: null, + agentComposerEpoch: state.agentComposerEpoch + 1, + })), loadAgentConversations: async () => {}, loadAgentConversation: async () => {}, - startNewAgentConversation: () => set({ agentMessages: [], agentError: null }), + startNewAgentConversation: () => set((state) => ({ + agentMessages: [], + agentError: null, + agentComposerEpoch: state.agentComposerEpoch + 1, + })), deleteAgentConversation: async () => {}, setAgentProvider: async (credentialId, modelId) => { set({ agentActiveCredentialId: credentialId, agentActiveModelId: modelId, agentError: null }) @@ -45,14 +175,49 @@ function createAgentStore(overrides: Partial = {}) { function renderAgentPanel(overrides: Partial = {}) { const store = createAgentStore(overrides) - return render( - - - - - - , + const view = render( + + + + + + + + , ) + return { ...view, store } +} + +function testUser(): CmsCurrentUser { + return { + id: 'user-agent-panel', + email: 'agent@example.com', + displayName: 'Agent User', + status: 'active', + role: { + id: 'role-admin', + slug: 'admin', + name: 'Admin', + description: 'Agent panel test role', + isSystem: true, + capabilities: ['ai.chat'], + }, + capabilities: ['ai.chat'], + lastLoginAt: null, + failedLoginCount: 0, + lockedUntil: null, + passwordUpdatedAt: null, + mfaEnabled: false, + mfaEnabledAt: null, + mfaRecoveryCodesRemaining: 0, + stepUpAuthMode: 'password', + stepUpWindowMinutes: 15, + avatarMediaId: null, + avatarUrl: null, + gravatarHash: '', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + } } function RouteProbe() { @@ -60,11 +225,33 @@ function RouteProbe() { return {location.pathname} } +function pasteImage(fileName = 'clipboard.png'): void { + const textarea = screen.getByLabelText('Message to AI assistant') + fireEvent.paste(textarea, { + clipboardData: { + files: [new File([pngHeader(100, 80)], fileName, { type: 'image/png' })], + }, + }) +} + +function pngHeader(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(24) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + bytes.set([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52], 8) + const view = new DataView(bytes.buffer) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + describe('AgentPanel', () => { afterEach(() => { cleanup() + activeImageMocks?.restore() + activeImageMocks = null localStorage.clear() globalThis.fetch = originalFetch + clearModelListCache() }) it('surfaces a large setup empty state and header shortcut when no credentials exist', async () => { @@ -231,4 +418,156 @@ describe('AgentPanel', () => { const textarea = screen.getByLabelText('Message to AI assistant') as HTMLTextAreaElement expect(textarea.disabled).toBe(false) }) + + it('blocks same-tick paste + Enter, then sends a prepared image-only turn', async () => { + installModelFetch(true) + const decode = deferred() + const bitmap = fakeBitmap() + installImageBrowserMocks(decode.promise) + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + pasteImage() + fireEvent.keyDown(textarea, { key: 'Enter' }) + + expect(sendAgentMessage).not.toHaveBeenCalled() + expect(screen.getByText('Preparing…')).toBeTruthy() + + await act(async () => { + decode.resolve(bitmap) + await decode.promise + }) + await waitFor(() => expect(screen.getByText('Ready')).toBeTruthy()) + await waitFor(() => { + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBeNull() + }) + + fireEvent.keyDown(textarea, { key: 'Enter' }) + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([{ + kind: 'image', + mimeType: 'image/jpeg', + data: 'AQID', + }]) + await waitFor(() => { + expect(screen.queryByLabelText('Attached image: clipboard.png')).toBeNull() + }) + expect((textarea as HTMLTextAreaElement).value).toBe('') + }) + + it('retains text and the prepared image when the request is rejected before acceptance', async () => { + installModelFetch(true) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: false })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') as HTMLTextAreaElement + pasteImage('reference.png') + await waitFor(() => expect(screen.getByText('Ready')).toBeTruthy()) + await waitFor(() => { + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBeNull() + }) + fireEvent.change(textarea, { target: { value: 'Use this reference' } }) + fireEvent.click(screen.getByRole('button', { name: 'Send' })) + + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([ + { kind: 'text', text: 'Use this reference' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + ]) + expect(textarea.value).toBe('Use this reference') + expect(screen.getByLabelText('Attached image: reference.png')).toBeTruthy() + }) + + it('keeps an attachment visible but disables send for a non-vision model', async () => { + installModelFetch(false) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + await screen.findByLabelText('Message to AI assistant') + pasteImage() + + expect((await screen.findByRole('alert')).textContent).toContain( + 'Choose a vision-capable model or remove the image.', + ) + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBe('true') + expect(screen.getByLabelText('Attached image: clipboard.png')).toBeTruthy() + expect(sendAgentMessage).not.toHaveBeenCalled() + }) + + it('blocks send when the selected model is known not to support agent tools', async () => { + installModelFetch(true, false) + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + fireEvent.change(textarea, { target: { value: 'Inspect this page' } }) + await screen.findByText('Choose an agent-capable model that supports tool calling.') + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBe('true') + fireEvent.keyDown(textarea, { key: 'Enter' }) + expect(sendAgentMessage).not.toHaveBeenCalled() + }) + + it('drops an in-flight attachment when an explicit conversation reset remounts the composer', async () => { + installModelFetch(true) + const decode = deferred() + const bitmap = fakeBitmap() + installImageBrowserMocks(decode.promise) + const { store } = renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + }) + + await screen.findByLabelText('Message to AI assistant') + pasteImage('stale.png') + expect(screen.getByLabelText('Attached image: stale.png')).toBeTruthy() + + act(() => { + store.setState({ agentComposerEpoch: 1 }) + }) + expect(screen.queryByLabelText('Attached image: stale.png')).toBeNull() + await act(async () => { + decode.resolve(bitmap) + await decode.promise + }) + expect(screen.queryByLabelText('Attached image: stale.png')).toBeNull() + }) + + it('renders a rehydrated user image block in conversation history', async () => { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + + renderAgentPanel({ + agentMessages: [{ + id: 'image-message', + role: 'user', + blocks: [{ kind: 'image', mimeType: 'image/jpeg', data: 'QUJD' }], + timestamp: Date.now(), + }], + }) + + const image = await screen.findByAltText('Attachment from you') as HTMLImageElement + expect(image.src).toContain('data:image/jpeg;base64,QUJD') + }) }) diff --git a/src/__tests__/ui/modelPicker.test.tsx b/src/__tests__/ui/modelPicker.test.tsx index e898d438a..d9ee70aab 100644 --- a/src/__tests__/ui/modelPicker.test.tsx +++ b/src/__tests__/ui/modelPicker.test.tsx @@ -1,13 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test' import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { ModelPicker, type ModelChoice } from '@admin/ai/ModelPicker' -import type { CredentialView } from '@admin/ai/api' +import { clearModelListCache, type CredentialView } from '@admin/ai/api' const originalFetch = globalThis.fetch afterEach(() => { cleanup() globalThis.fetch = originalFetch + clearModelListCache() }) function credential(over: Partial = {}): CredentialView { @@ -29,7 +30,7 @@ function credential(over: Partial = {}): CredentialView { const MODELS = Array.from({ length: 12 }, (_, i) => ({ id: `m${i}`, label: i === 3 ? 'Claude Opus' : i === 4 ? 'Claude Sonnet' : `Model ${i}`, - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, })) function mockModelsFetch() { @@ -42,7 +43,10 @@ function mockModelsFetch() { } describe('ModelPicker', () => { - beforeEach(mockModelsFetch) + beforeEach(() => { + clearModelListCache() + mockModelsFetch() + }) it('renders the field trigger with a placeholder when nothing is selected', () => { render( @@ -60,6 +64,39 @@ describe('ModelPicker', () => { expect(trigger.textContent).toContain('Choose a model') }) + it('shares and briefly caches a credential catalogue across picker instances', async () => { + const selected = { credentialId: 'cred-or', modelId: 'm0' } + const { rerender } = render( + {}} + />, + ) + await waitFor(() => expect(screen.getByRole('button').textContent).toContain('Model 0')) + expect(globalThis.fetch).toHaveBeenCalledTimes(1) + + rerender( +
+ {}} + /> + {}} + /> +
, + ) + await waitFor(() => expect(screen.getAllByRole('button')).toHaveLength(2)) + expect(globalThis.fetch).toHaveBeenCalledTimes(1) + }) + it('opens a searchable, grouped menu and filters models by query', async () => { render( oW=^U?W=XL^ za%oW!NMTuGPAO2NSRpe{p|m(vPeDtcOF { await apiRequest(`/admin/api/ai/credentials/${encodeURIComponent(id)}`, { method: 'DELETE' }) + clearModelListCache(id) } export interface TestResult { @@ -231,15 +233,53 @@ export async function testCredential(id: string): Promise { // Endpoints — models // --------------------------------------------------------------------------- +const MODEL_LIST_TIMEOUT_MS = 10_000 +const MODEL_LIST_CACHE_TTL_MS = 5 * 60_000 +const modelListRequests = new Map>() +const modelListCache = new Map() + +/** Invalidate model catalogues after a credential mutation (or between tests). */ +export function clearModelListCache(credentialId?: string): void { + if (!credentialId) { + modelListCache.clear() + return + } + for (const key of modelListCache.keys()) { + if (key.endsWith(`\0${credentialId}`)) modelListCache.delete(key) + } +} + export async function listModels( providerId: 'anthropic' | 'openai' | 'ollama' | 'openrouter' | 'openai-compatible', credentialId?: string, ): Promise { - const body = await apiRequest(`/admin/api/ai/providers/${providerId}/models`, { + const key = `${providerId}\0${credentialId ?? ''}` + const cached = modelListCache.get(key) + if (cached && cached.expiresAt > Date.now()) return cached.models + if (cached) modelListCache.delete(key) + const pending = modelListRequests.get(key) + if (pending) return pending + + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), MODEL_LIST_TIMEOUT_MS) + const request = apiRequest(`/admin/api/ai/providers/${providerId}/models`, { query: { credentialId }, schema: ModelListResponseSchema, + signal: controller.signal, + }).then((body) => { + modelListCache.set(key, { + expiresAt: Date.now() + MODEL_LIST_CACHE_TTL_MS, + models: body.models, + }) + return body.models }) - return body.models + modelListRequests.set(key, request) + try { + return await request + } finally { + clearTimeout(timeoutId) + if (modelListRequests.get(key) === request) modelListRequests.delete(key) + } } // --------------------------------------------------------------------------- @@ -274,9 +314,13 @@ export async function listConversations(scope: 'site' | 'content' | 'data' | 'pl return body.conversations } -export async function getConversation(id: string): Promise { +export async function getConversation( + id: string, + signal?: AbortSignal, +): Promise { const body = await apiRequest(`/admin/api/ai/conversations/${encodeURIComponent(id)}`, { schema: ConversationDetailResponseSchema, + signal, }) return body.conversation } @@ -289,11 +333,13 @@ export async function updateConversationProvider( id: string, credentialId: string, modelId: string, + signal?: AbortSignal, ): Promise { const body = await apiRequest(`/admin/api/ai/conversations/${encodeURIComponent(id)}`, { method: 'PUT', body: { credentialId, modelId }, schema: ConversationItemResponseSchema, + signal, }) return body.conversation } diff --git a/src/admin/pages/site/agent/agentApi.ts b/src/admin/pages/site/agent/agentApi.ts index db27a7d18..77fad3fb4 100644 --- a/src/admin/pages/site/agent/agentApi.ts +++ b/src/admin/pages/site/agent/agentApi.ts @@ -11,8 +11,9 @@ */ import { nanoid } from 'nanoid' -import { Type, type Static } from '@core/utils/typeboxHelpers' -import { apiRequest } from '@core/http' +import { Type, safeParseValue, type Static } from '@core/utils/typeboxHelpers' +import { apiRequest, isAbortError } from '@core/http' +import { AiUserImageBlockSchema } from '@core/ai' import { AI_CONVERSATIONS_PATH, AI_DEFAULTS_PATH, @@ -90,8 +91,13 @@ export function rehydrateMessages( } msg.blocks.push({ kind: 'toolCall', toolCall }) toolCallIndex.set(block.toolCallId, toolCall) + } else if (rec.role === 'user' && block.kind === 'image') { + // Conversation payloads use the broader persisted block vocabulary. + // Only the bounded canonical user-image shape is safe to render as a + // data URL in the composer thread. + const parsedImage = safeParseValue(AiUserImageBlockSchema, block) + if (parsedImage.ok) msg.blocks.push(parsedImage.value) } - // image blocks — skip in v1; could render via later. } out.push(msg) } @@ -110,13 +116,20 @@ const ScopeDefaultsResponseSchema = Type.Object( { additionalProperties: true }, ) -export async function fetchScopeDefault(scope: AgentToolScope): Promise { +export async function fetchScopeDefault( + scope: AgentToolScope, + signal?: AbortSignal, +): Promise { // Soft fetch: any failure (no default set, network, bad shape) just means // "no preselected credential/model" — the caller falls back to the picker. try { - const body = await apiRequest(AI_DEFAULTS_PATH, { schema: ScopeDefaultsResponseSchema }) + const body = await apiRequest(AI_DEFAULTS_PATH, { + schema: ScopeDefaultsResponseSchema, + signal, + }) return body.defaults?.[scope] ?? null } catch (err) { + if (signal?.aborted || isAbortError(err)) throw err console.error(`[AgentSlice] Failed to fetch ${scope} default:`, err) return null } @@ -132,12 +145,14 @@ export async function createConversationForScope( scope: AgentToolScope, credentialId: string, modelId: string, + signal?: AbortSignal, ): Promise { const body = await apiRequest(AI_CONVERSATIONS_PATH, { method: 'POST', body: { scope, credentialId, modelId }, schema: CreatedConversationEnvelopeSchema, fallbackMessage: 'Conversation create failed', + signal, }) return body.conversation } diff --git a/src/admin/pages/site/agent/agentSlice.ts b/src/admin/pages/site/agent/agentSlice.ts index 561cf2b8e..064b9aae1 100644 --- a/src/admin/pages/site/agent/agentSlice.ts +++ b/src/admin/pages/site/agent/agentSlice.ts @@ -18,7 +18,8 @@ import { nanoid } from 'nanoid' import type { EditorStoreSliceCreator } from '@site/store/types' -import { ApiError } from '@core/http' +import { ApiError, responseErrorMessage } from '@core/http' +import type { AiChatRequestBody } from '@core/ai' import { pushToast } from '@ui/components/Toast' import { listConversations, @@ -43,7 +44,6 @@ export type { AgentSlice, AgentSliceConfig } from './agentSliceTypes' import type { AgentBridgeRuntime, AgentMessage, - AgentRequestBody, AgentTextStreamSink, } from './types' import { getErrorMessage } from '@core/utils/errorMessage' @@ -69,6 +69,33 @@ interface ResolvedCredentials { modelId: string } +interface ConfirmedProviderSelection { + conversationId: string + credentialId: string | null + modelId: string | null +} + +const AGENT_PROVIDER_UPDATE_TIMEOUT_MS = 10_000 + +function waitForPromiseUnlessAborted( + promise: Promise, + signal: AbortSignal, +): Promise { + if (signal.aborted) return Promise.resolve(false) + return new Promise((resolve) => { + let settled = false + const finish = (completed: boolean) => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + resolve(completed) + } + const onAbort = () => finish(false) + signal.addEventListener('abort', onAbort, { once: true }) + void promise.then(() => finish(true), () => finish(true)) + }) +} + /** * Resolve the `(credentialId, modelId)` to use: the staged picker selection if * present, otherwise the per-scope default fetched from the server. Shared by @@ -79,11 +106,15 @@ interface ResolvedCredentials { async function resolveScopeCredentials( get: AgentSliceGet, config: AgentSliceConfig, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted() const credentialId = get().agentActiveCredentialId const modelId = get().agentActiveModelId if (credentialId && modelId) return { credentialId, modelId } - return fetchScopeDefault(config.scope) + const credentials = await fetchScopeDefault(config.scope, signal) + signal?.throwIfAborted() + return credentials } /** @@ -97,14 +128,22 @@ async function ensureConversationId( get: AgentSliceGet, set: EditorStoreSet, config: AgentSliceConfig, + signal: AbortSignal, ): Promise { + signal.throwIfAborted() const existing = get().agentConversationId if (existing) return existing - const creds = await resolveScopeCredentials(get, config) + const creds = await resolveScopeCredentials(get, config, signal) if (!creds) return null - const conv = await createConversationForScope(config.scope, creds.credentialId, creds.modelId) + const conv = await createConversationForScope( + config.scope, + creds.credentialId, + creds.modelId, + signal, + ) + signal.throwIfAborted() set((state) => { state.agentConversationId = conv.id state.agentActiveCredentialId = creds.credentialId @@ -125,8 +164,9 @@ type ConversationResetKeys = | 'agentActiveCredentialId' | 'agentActiveModelId' | 'agentContextTokens' + | 'agentComposerEpoch' -function conversationResetState(): Pick { +function conversationResetState(agentComposerEpoch: number): Pick { return { agentMessages: [], agentError: null, @@ -134,6 +174,7 @@ function conversationResetState(): Pick { agentActiveCredentialId: null, agentActiveModelId: null, agentContextTokens: null, + agentComposerEpoch, } } @@ -176,6 +217,11 @@ export function createAgentSlice( return (set, get) => { // AbortController held in closure (not reactive — intentional, not needed in UI) let _abortController: AbortController | null = null + let _conversationLoadEpoch = 0 + // Provider changes for an existing conversation are ordered so rapid picks + // cannot land in the database out of order. Sending awaits the same queue. + let _providerUpdateQueue: Promise = Promise.resolve() + let _confirmedProviderSelection: ConfirmedProviderSelection | null = null // rAF-buffered text accumulation (Guideline #254). Pending deltas are // flushed once per animation frame, OR explicitly before any tool-call @@ -243,6 +289,9 @@ export function createAgentSlice( agentActiveModelId: null, agentConversations: [], agentContextTokens: null, + isAgentConversationPending: false, + isAgentProviderPending: false, + agentComposerEpoch: 0, // ── UI actions ─────────────────────────────────────────────────────────── openAgent() { @@ -260,16 +309,26 @@ export function createAgentSlice( }, abortAgent() { - _abortController?.abort() - _abortController = null - set({ isAgentStreaming: false }) + if (_abortController) _abortController.abort() + else set({ isAgentStreaming: false }) }, clearAgentMessages() { - set(conversationResetState()) + _conversationLoadEpoch += 1 + _confirmedProviderSelection = null + set((state) => { + Object.assign(state, conversationResetState(state.agentComposerEpoch + 1)) + state.isAgentConversationPending = false + state.isAgentProviderPending = false + }) }, startNewAgentConversation() { + if ( + get().isAgentStreaming + || get().isAgentConversationPending + || get().isAgentProviderPending + ) return // Reset to a fresh conversation, then re-apply the scope default so the // composer stays ready (provider + model picked) instead of dropping to // the "choose a model" lock. `loadScopeDefault` only fills the gap when @@ -294,37 +353,64 @@ export function createAgentSlice( }, async loadAgentConversation(id: string) { + if ( + get().isAgentStreaming + || get().isAgentConversationPending + || get().isAgentProviderPending + ) return + const loadEpoch = ++_conversationLoadEpoch + set({ isAgentConversationPending: true }) try { const conv = await getConversation(id) - set({ - agentConversationId: conv.id, - agentActiveCredentialId: conv.credentialId, - agentActiveModelId: conv.modelId, - agentMessages: rehydrateMessages(conv.messages), - agentError: null, + if (loadEpoch !== _conversationLoadEpoch) return + _confirmedProviderSelection = { + conversationId: conv.id, + credentialId: conv.credentialId, + modelId: conv.modelId, + } + set((state) => { + state.agentConversationId = conv.id + state.agentActiveCredentialId = conv.credentialId + state.agentActiveModelId = conv.modelId + state.agentMessages = rehydrateMessages(conv.messages) + state.agentError = null // Restore the meter from the persisted snapshot (0 → null so the // meter reads as "empty" against the window until the next turn). - agentContextTokens: conv.contextTokens > 0 ? conv.contextTokens : null, + state.agentContextTokens = conv.contextTokens > 0 ? conv.contextTokens : null + state.agentComposerEpoch += 1 }) } catch (err) { + if (loadEpoch !== _conversationLoadEpoch) return console.error('[AgentSlice] Failed to load conversation:', err) set({ agentError: err instanceof ApiError ? err.message : 'Failed to load conversation.', }) + } finally { + if (loadEpoch === _conversationLoadEpoch) { + set({ isAgentConversationPending: false }) + } } }, async deleteAgentConversation(id: string) { + if (get().isAgentConversationPending || get().isAgentProviderPending) return + if (get().isAgentStreaming && get().agentConversationId === id) return + set({ isAgentConversationPending: true }) try { await deleteConversation(id) const wasActive = get().agentConversationId === id + if (wasActive) _conversationLoadEpoch += 1 + if (wasActive) _confirmedProviderSelection = null set((state) => { state.agentConversations = state.agentConversations.filter((c) => c.id !== id) // Deleting the active conversation resets it through the same key-set // as clearAgentMessages — including agentError, so a stuck 502/error // banner doesn't survive the delete. if (state.agentConversationId === id) { - Object.assign(state, conversationResetState()) + Object.assign( + state, + conversationResetState(state.agentComposerEpoch + 1), + ) } }) // If the active chat was the one deleted, re-apply the scope default so @@ -338,11 +424,25 @@ export function createAgentSlice( body: getErrorMessage(err, 'Failed to delete the conversation.'), location: 'site-editor', }) + } finally { + set({ isAgentConversationPending: false }) } }, async setAgentProvider(credentialId: string, modelId: string) { + if ( + get().isAgentStreaming + || get().isAgentConversationPending + || get().isAgentProviderPending + ) return const currentId = get().agentConversationId + if (currentId && _confirmedProviderSelection?.conversationId !== currentId) { + _confirmedProviderSelection = { + conversationId: currentId, + credentialId: get().agentActiveCredentialId, + modelId: get().agentActiveModelId, + } + } // Always reflect the picker selection locally so the dropdown's // displayed value updates immediately. Clearing agentError is essential: // a prior send with no configured default leaves a sticky "no provider @@ -357,11 +457,127 @@ export function createAgentSlice( agentError: null, }) if (!currentId) return // staged for the next conversation-create call - try { - await updateConversationProvider(currentId, credentialId, modelId) - } catch (err) { + set({ isAgentProviderPending: true }) + + const update = _providerUpdateQueue.then(async () => { + const controller = new AbortController() + const timeoutId = setTimeout( + () => controller.abort(), + AGENT_PROVIDER_UPDATE_TIMEOUT_MS, + ) + try { + await updateConversationProvider(currentId, credentialId, modelId, controller.signal) + } catch (err) { + if (controller.signal.aborted) { + throw new Error('Model change timed out. Try again.', { cause: err }) + } + throw err + } finally { + clearTimeout(timeoutId) + } + if (get().agentConversationId === currentId) { + _confirmedProviderSelection = { conversationId: currentId, credentialId, modelId } + } + }) + let updateFailed = false + const handledUpdate = update.catch(async (err) => { console.error('[AgentSlice] Failed to update provider:', err) - set({ agentError: 'Failed to update conversation provider.' }) + let message = getErrorMessage(err, 'Failed to update conversation provider.') + // Only a handler-originated 4xx is a definite terminal rejection. + // Network failures, timeouts, and proxy/origin 5xx responses can all + // race a server commit after the browser has received the failure. + const commitWasAmbiguous = !( + err instanceof ApiError + && err.status >= 400 + && err.status < 500 + ) + + // A lost response is an ambiguous commit: the server may have applied + // the PUT even though the browser saw a timeout/network error. Re-read + // the row before allowing Send so picker and chat routing cannot split. + const reconcileController = new AbortController() + const reconcileTimeoutId = setTimeout( + () => reconcileController.abort(), + AGENT_PROVIDER_UPDATE_TIMEOUT_MS, + ) + let authoritative: Awaited> | null = null + try { + authoritative = await getConversation(currentId, reconcileController.signal) + } catch (reconcileErr) { + console.error('[AgentSlice] Failed to reconcile provider update:', reconcileErr) + } finally { + clearTimeout(reconcileTimeoutId) + } + + // A replacement conversation owns the UI now; this request must not + // mutate its selection or pending state. + if (get().agentConversationId !== currentId) return + + if (authoritative) { + if ( + authoritative.credentialId === credentialId + && authoritative.modelId === modelId + ) { + // The response was lost after commit. The requested outcome is + // authoritative, so treat the model change as successful. + _confirmedProviderSelection = { + conversationId: currentId, + credentialId: authoritative.credentialId, + modelId: authoritative.modelId, + } + set({ + agentActiveCredentialId: authoritative.credentialId, + agentActiveModelId: authoritative.modelId, + }) + return + } + + if (!commitWasAmbiguous) { + // An explicit HTTP error means the PUT reached a terminal response; + // the subsequent read is safe to treat as the rollback state. + _confirmedProviderSelection = { + conversationId: currentId, + credentialId: authoritative.credentialId, + modelId: authoritative.modelId, + } + set({ + agentActiveCredentialId: authoritative.credentialId, + agentActiveModelId: authoritative.modelId, + }) + } else { + authoritative = null + } + } + + if (!authoritative) { + _confirmedProviderSelection = null + message = `${message} The server state could not be confirmed; choose a model again.` + set({ agentActiveCredentialId: null, agentActiveModelId: null }) + } + + updateFailed = true + set({ agentError: message }) + pushToast({ + kind: 'error', + title: "Couldn't change model", + body: message, + location: 'site-editor', + }) + }) + // Later selections and Send wait until rollback/error handling finishes, + // while this action still resolves after surfacing the operation failure. + _providerUpdateQueue = handledUpdate + await handledUpdate + if (get().agentConversationId === currentId) { + set({ isAgentProviderPending: false }) + } + if ( + !updateFailed + && get().agentConversationId === currentId + && get().agentActiveCredentialId === credentialId + && get().agentActiveModelId === modelId + ) { + set({ agentError: null }) } }, @@ -380,6 +596,11 @@ export function createAgentSlice( console.error('[AgentSlice] Failed to load scope default:', err) return } + // The request may have been in flight while the user picked a model or + // opened a conversation. A late default must never overwrite that newer + // explicit state. + if (get().agentConversationId) return + if (get().agentActiveCredentialId && get().agentActiveModelId) return // No default configured for this scope: leave the picker empty (shows // its "Choose a model" placeholder) and let the user pick one. if (!creds) return @@ -392,12 +613,21 @@ export function createAgentSlice( // ── sendAgentMessage ───────────────────────────────────────────────────── async sendAgentMessage(content) { - if (get().isAgentStreaming) return // one request at a time + if ( + get().isAgentStreaming + || get().isAgentConversationPending + || get().isAgentProviderPending + || content.length === 0 + ) return { accepted: false } + + const intendedConversationId = get().agentConversationId + const intendedCredentialId = get().agentActiveCredentialId + const intendedModelId = get().agentActiveModelId const userMsg: AgentMessage = { id: nanoid(), role: 'user', - blocks: [{ kind: 'text', text: content }], + blocks: content.map((block) => ({ ...block })), timestamp: Date.now(), } @@ -409,55 +639,74 @@ export function createAgentSlice( timestamp: Date.now(), } - set((state) => { - state.agentMessages.push(userMsg) - state.agentMessages.push(assistantMsg) - state.agentError = null - state.isAgentStreaming = true - }) + set({ agentError: null, isAgentStreaming: true }) - _abortController = new AbortController() + const controller = new AbortController() + _abortController = controller const bridge: AgentBridgeRuntime = { bridgeId: null } + let accepted = false try { + // A model picked immediately before Send must reach the conversation + // row before the chat handler resolves its capability. + const providerReady = await waitForPromiseUnlessAborted( + _providerUpdateQueue, + controller.signal, + ) + if (!providerReady) return { accepted: false } + if ( + intendedConversationId + && ( + _confirmedProviderSelection?.conversationId !== intendedConversationId + || _confirmedProviderSelection.credentialId !== intendedCredentialId + || _confirmedProviderSelection.modelId !== intendedModelId + ) + ) return { accepted: false } const snapshot = config.buildSnapshot() // Lazily create the conversation row (staged picker values or scope // default). Null means no provider is configured for this scope. - const conversationId = await ensureConversationId(get, set, config) + const conversationId = await ensureConversationId( + get, + set, + config, + controller.signal, + ) if (!conversationId) { - surfaceAssistantError( - set, - assistantId, - config.noProviderMessage - ?? `No AI provider configured for the "${config.scope}" scope. Open /admin/ai/providers to add a credential, then /admin/ai/defaults to pick one.`, - '_(no AI provider configured)_', - ) - return + const message = config.noProviderMessage + ?? `No AI provider configured for the "${config.scope}" scope. Open /admin/ai/providers to add a credential, then /admin/ai/defaults to pick one.` + set({ agentError: message }) + pushToast({ kind: 'error', title: "Couldn't send message", body: message }) + return { accepted: false } + } + if (_confirmedProviderSelection?.conversationId !== conversationId) { + _confirmedProviderSelection = { + conversationId, + credentialId: get().agentActiveCredentialId, + modelId: get().agentActiveModelId, + } } - const body: AgentRequestBody = { conversationId, prompt: content, snapshot } + const body: AiChatRequestBody = { conversationId, content: [...content], snapshot } const res = await fetch(`/admin/api/ai/chat/${config.scope}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), - signal: _abortController.signal, + signal: controller.signal, }) if (!res.ok) { - if (res.status === 502) { - console.error('[AgentSlice] 502 — agent server unreachable') - surfaceAssistantError( - set, - assistantId, - 'AI server is not running. Start it with: bun run dev', - '_(agent error)_', - ) - return - } - throw new Error(`Agent request failed: ${res.status} ${res.statusText}`) + const fallback = res.status === 502 + ? 'AI server is not running. Start it with: bun run dev' + : `Agent request failed: ${res.status} ${res.statusText}` + throw new ApiError(await responseErrorMessage(res, fallback), res.status) } + accepted = true + set((state) => { + state.agentMessages.push(userMsg) + state.agentMessages.push(assistantMsg) + }) if (!res.body) throw new Error('Agent response has no body') for await (const event of readNdjsonStream(res.body.getReader(), ServerStreamEventSchema)) { @@ -467,21 +716,22 @@ export function createAgentSlice( textSink, set, bridge, - _abortController?.signal ?? null, + controller.signal, config.dispatchTool, config.buildSnapshot, ) } flushPendingText() + return { accepted: true } } catch (err) { // Abort the fetch so any in-flight MCP tool handler on the server // rejects cleanly (via destroyBridge in the stream's finally block) // instead of waiting forever for a tool-result that won't arrive. - _abortController?.abort() + controller.abort() if (err instanceof Error && err.name === 'AbortError') { - flushPendingText() + if (accepted) flushPendingText() } else { // Admin-only surface (capability gated) — show the actual // failure cause so the operator can act. Network / unexpected @@ -489,11 +739,19 @@ export function createAgentSlice( // server-classified driver errors. const detail = getErrorMessage(err, String(err)) console.error('[AgentSlice] sendAgentMessage error:', err) - surfaceAssistantError(set, assistantId, `Agent request failed: ${detail}`, '_(agent error)_') + if (accepted) { + surfaceAssistantError(set, assistantId, `Agent request failed: ${detail}`, '_(agent error)_') + } else { + set({ agentError: detail }) + pushToast({ kind: 'error', title: "Couldn't send message", body: detail }) + } } + return { accepted } } finally { - _abortController = null - set({ isAgentStreaming: false }) + if (_abortController === controller) { + _abortController = null + set({ isAgentStreaming: false }) + } } }, } diff --git a/src/admin/pages/site/agent/agentSliceTypes.ts b/src/admin/pages/site/agent/agentSliceTypes.ts index 91efa50ac..997099173 100644 --- a/src/admin/pages/site/agent/agentSliceTypes.ts +++ b/src/admin/pages/site/agent/agentSliceTypes.ts @@ -1,5 +1,5 @@ import type { EditorStoreSliceCreator } from '@site/store/types' -import type { AiToolOutput } from '@core/ai' +import type { AiToolOutput, AiUserContentBlock } from '@core/ai' import type { ConversationView } from '@admin/ai/api' import type { AgentMessage, AgentToolScope } from './types' @@ -38,11 +38,17 @@ export interface AgentSlice { agentActiveModelId: string | null agentConversations: ConversationView[] agentContextTokens: number | null + /** True while a history load/delete can replace the active conversation. */ + isAgentConversationPending: boolean + /** True while an existing conversation's provider/model update is pending. */ + isAgentProviderPending: boolean + /** Remounts local composer drafts on explicit conversation replacement. */ + agentComposerEpoch: number openAgent(): void closeAgent(): void toggleAgent(): void - sendAgentMessage(content: string): Promise + sendAgentMessage(content: AiUserContentBlock[]): Promise<{ accepted: boolean }> abortAgent(): void clearAgentMessages(): void loadAgentConversations(): Promise diff --git a/src/admin/pages/site/agent/types.ts b/src/admin/pages/site/agent/types.ts index 50a944f04..d2f12994a 100644 --- a/src/admin/pages/site/agent/types.ts +++ b/src/admin/pages/site/agent/types.ts @@ -9,7 +9,7 @@ * JSON). Each line is a `ServerStreamEvent` value, JSON-serialised. */ -import type { AiToolOutput } from '@core/ai' +import type { AiContentBlock, AiToolOutput, AiUserImageBlock } from '@core/ai' // --------------------------------------------------------------------------- // Execution result @@ -168,7 +168,8 @@ export interface AgentToolCall { * tools" (which mis-orders late text in front of earlier tool calls). */ type AgentMessageBlock = - | { kind: 'text'; text: string } + | Extract + | AiUserImageBlock | { kind: 'toolCall'; toolCall: AgentToolCall } export interface AgentMessage { @@ -178,24 +179,6 @@ export interface AgentMessage { timestamp: number } -// --------------------------------------------------------------------------- -// Browser → Server request body -// --------------------------------------------------------------------------- - -export interface AgentRequestBody { - /** Per-conversation id; the chat handler loads its credential + history. */ - conversationId: string - /** The user's new message. */ - prompt: string - /** - * Scope-specific snapshot handed to the read tools via - * `ToolContext.snapshot`. Loose `unknown` here because the body now - * crosses every scope (site → SiteAgentSnapshot, content → ContentSnapshot, - * …); each scope's tool handlers cast at the boundary. - */ - snapshot: unknown -} - export interface AgentLayoutRect { x: number y: number diff --git a/src/admin/pages/site/panels/AgentPanel/AgentComposer.tsx b/src/admin/pages/site/panels/AgentPanel/AgentComposer.tsx new file mode 100644 index 000000000..d7787eb0d --- /dev/null +++ b/src/admin/pages/site/panels/AgentPanel/AgentComposer.tsx @@ -0,0 +1,296 @@ +import { useEffect, useRef, useState } from 'react' +import { useAgentStore } from '@admin/ai/useAgentStore' +import { useAsyncResource } from '@admin/lib/useAsyncResource' +import { listModels, type CredentialView } from '@admin/ai/api' +import type { AiUserContentBlock } from '@core/ai' +import { Button } from '@ui/components/Button' +import { Textarea } from '@ui/components/Input' +import { pushToast } from '@ui/components/Toast' +import { CloseIcon } from 'pixel-art-icons/icons/close' +import { SendSolidIcon } from 'pixel-art-icons/icons/send-solid' +import { SquareSolidIcon } from 'pixel-art-icons/icons/square-solid' +import { ContextMeter } from './ContextMeter' +import { ModelPicker } from './ModelPicker' +import { usePendingImageAttachment } from './usePendingImageAttachment' +import styles from './AgentPanel.module.css' + +export type ComposerLockReason = 'setup' | 'chooseModel' + +interface AgentComposerProps { + composerLocked: boolean + lockReason: ComposerLockReason | null + credentials: CredentialView[] + credentialsLoaded: boolean + onRefreshCredentials(): void +} + +export function AgentComposer({ + composerLocked, + lockReason, + credentials, + credentialsLoaded, + onRefreshCredentials, +}: AgentComposerProps) { + const isStreaming = useAgentStore((state) => state.isAgentStreaming) + const conversationPending = useAgentStore((state) => state.isAgentConversationPending) + const providerPending = useAgentStore((state) => state.isAgentProviderPending) + const isOpen = useAgentStore((state) => state.isAgentOpen) + const sendAgentMessage = useAgentStore((state) => state.sendAgentMessage) + const abortAgent = useAgentStore((state) => state.abortAgent) + const activeCredentialId = useAgentStore((state) => state.agentActiveCredentialId) + const activeModelId = useAgentStore((state) => state.agentActiveModelId) + const [draft, setDraft] = useState('') + const [submitting, setSubmitting] = useState(false) + const attachment = usePendingImageAttachment() + const inputRef = useRef(null) + + useEffect(() => { + if (!isOpen) return + const id = setTimeout(() => inputRef.current?.focus(), 50) + return () => clearTimeout(id) + }, [isOpen]) + + const activeProviderId = + credentials.find((credential) => credential.id === activeCredentialId)?.providerId ?? null + const activeModelResource = useAsyncResource( + async () => { + if (!activeProviderId || !activeCredentialId || !activeModelId) return null + const models = await listModels(activeProviderId, activeCredentialId) + return { + credentialId: activeCredentialId, + modelId: activeModelId, + model: models.find((model) => model.id === activeModelId) ?? null, + } + }, + [activeProviderId, activeCredentialId, activeModelId], + { fallbackError: 'Could not verify image support for this model.' }, + ) + const resolvedSelection = activeModelResource.data + const activeModel = + resolvedSelection?.credentialId === activeCredentialId + && resolvedSelection.modelId === activeModelId + ? resolvedSelection.model + : null + const modelCannotRunAgent = activeModel?.capabilities.toolCalling === false + + const imageStatus = !attachment.pending + ? 'none' + : attachment.pending.status === 'processing' + ? 'processing' + : attachment.pending.status === 'error' + ? 'error' + : activeModelResource.loading + ? 'checking-model' + : activeModelResource.error || !resolvedSelection + ? 'model-error' + : activeModel?.capabilities.visionInput + ? 'ready' + : 'unsupported-model' + + async function submit(): Promise { + if ( + isStreaming + || conversationPending + || providerPending + || submitting + || modelCannotRunAgent + ) return + const text = draft.trim() + const pending = attachment.current() + if (!text && !pending) return + if (pending?.status === 'processing') { + pushToast({ kind: 'error', title: 'Image is still processing', body: 'Wait a moment, then send again.' }) + return + } + if (pending?.status === 'error' || (pending && !pending.block)) return + if (pending && imageStatus !== 'ready') return + + const content: AiUserContentBlock[] = [] + if (text) content.push({ kind: 'text', text }) + if (pending?.block) content.push(pending.block) + + setSubmitting(true) + try { + const result = await sendAgentMessage(content) + if (result.accepted) { + setDraft('') + attachment.clear() + } + } finally { + setSubmitting(false) + } + } + + function handlePaste(event: React.ClipboardEvent): void { + const imageFiles = Array.from(event.clipboardData.files).filter((file) => + file.type.startsWith('image/'), + ) + if (imageFiles.length === 0) return + event.preventDefault() + attachment.queueFile(imageFiles[0]!) + if (imageFiles.length > 1) { + pushToast({ + kind: 'error', + title: 'One image per message', + body: 'Only the first pasted image was attached.', + }) + } + } + + function handleKeyDown(event: React.KeyboardEvent): void { + if (event.key !== 'Enter' || event.shiftKey) return + event.preventDefault() + void submit() + } + + const imageBlocksSend = imageStatus !== 'none' && imageStatus !== 'ready' + const sendDisabled = + composerLocked + || conversationPending + || providerPending + || submitting + || imageBlocksSend + || modelCannotRunAgent + let sendTooltip = 'Send' + if (lockReason === 'setup') sendTooltip = 'Add AI credentials first' + else if (lockReason === 'chooseModel') sendTooltip = 'Choose a model first' + else if (modelCannotRunAgent) sendTooltip = 'Choose an agent-capable model' + else if (imageStatus === 'processing') sendTooltip = 'Preparing image' + else if (imageStatus === 'checking-model') sendTooltip = 'Checking image support' + else if (imageStatus === 'model-error') sendTooltip = 'Could not verify image support' + else if (imageStatus === 'unsupported-model') sendTooltip = 'Choose a vision-capable model' + else if (imageStatus === 'error') sendTooltip = 'Remove the failed image' + + return ( +
+ + {attachment.pending && ( +
+ {attachment.pending.previewUrl ? ( + + ) : ( + + )} + {attachment.pending && imageStatus === 'checking-model' && ( +

Checking whether this model accepts images…

+ )} + {attachment.pending && imageStatus === 'unsupported-model' && ( +

+ Choose a vision-capable model or remove the image. +

+ )} + {attachment.pending && imageStatus === 'model-error' && ( +

+ Could not verify image support for this model. Choose another model or remove the image. +

+ )} + {modelCannotRunAgent && ( +

+ Choose an agent-capable model that supports tool calling. +

+ )} +
{ + event.preventDefault() + void submit() + }} + className={styles.inputForm} + > + {!isStreaming && ( +