diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..490e25e6f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/admin/ai/ModelPicker/ModelPicker.tsx diff diff --git a/CLAUDE.md b/CLAUDE.md index b086b4217..61d190068 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -184,8 +184,9 @@ Detailed patterns: [`docs/reference/typebox-patterns.md`](docs/reference/typebox Every untyped boundary uses TypeBox. Inside the boundary, code trusts the parsed value. -- **HTTP responses (client):** `@core/http` is a single three-layer stack — there is exactly ONE way to validate a response, expressed at the altitude you need: +- **HTTP responses (client):** `@core/http` is the single client stack — use the entry matching the response kind and altitude: - **`apiRequest(path, { schema, … })`** — the canonical entry. Does the `fetch` itself: sets `credentials`, serializes a JSON body (FormData passes through untouched), validates the success body against `schema`, and throws a single `ApiError` (carrying the HTTP status) on failure. Detect cancellation with `isAbortError(err)`. **Default to this** — do NOT hand-roll `fetch` + `res.ok` + `res.json()` in admin code. + - **`apiBlobRequest(path, …)`** — the binary-response counterpart for authenticated image/file reads. It shares `apiRequest`'s credential, cancellation, and `ApiError` behavior, then returns a `Blob`; the caller validates the MIME type before use. - **`readEnvelope(res, Schema, fallbackMessage)`** — for the persistence layer, which performs its own injectable `fetch` (test seam) and then hands the `Response` here. Checks `res.ok` (throws `ApiError` with status + the `{ error }` envelope message), then validates the body. Its no-body sibling is `assertOk(res, fallback)` for `void`/Blob/streaming/text responses. - **`parseJsonResponse(res, Schema)`** — the low-level body-validation primitive that `apiRequest` and `readEnvelope` are *built on*. It validates a body with NO HTTP-status semantics. Reserved for genuine primitives only: the `@core/http` internals, the XHR upload path (`useUploadQueue`), and server-side fetches of external APIs. Do NOT reach for it in admin/persistence code — `assertOk(res, m); parseJsonResponse(res, S)` is exactly `readEnvelope(res, S, m)`; always write the latter. - **`JSON.parse` of persisted data:** `safeParseJson(raw, Schema)` for hard, `parseJsonWithFallback(raw, Schema, default)` for soft. diff --git a/docs/architecture.md b/docs/architecture.md index 2838d99dc..3c4b67c7a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -326,7 +326,8 @@ The codebase enforces "validate, then trust": every untyped input goes through a | Boundary | Helper | Lives in | |--------------------------------------|-------------------------------------------------------|---------------------------------------| -| HTTP request (client, canonical) | `apiRequest(path, { schema, … })` → throws `ApiError` | `src/core/http/apiClient.ts` | +| HTTP request (client, canonical JSON) | `apiRequest(path, { schema, … })` → throws `ApiError` | `src/core/http/apiClient.ts` | +| HTTP request (client, binary body) | `apiBlobRequest(path, …)` → `Blob` / throws `ApiError` | `src/core/http/apiClient.ts` | | HTTP response from a held `Response` | `readEnvelope(res, Schema, fallbackMessage)` | `src/core/http/apiClient.ts` | | Raw JSON response validation | `parseJsonResponse(res, Schema)` | `src/core/utils/jsonValidate.ts` | | `JSON.parse` of persisted strings | `safeParseJson(raw, Schema)` / `parseJsonWithFallback`| `src/core/utils/jsonValidate.ts` | diff --git a/docs/features/agent.md b/docs/features/agent.md index 855aef6fe..46d32aa68 100644 --- a/docs/features/agent.md +++ b/docs/features/agent.md @@ -87,6 +87,7 @@ src/admin/ai/ src/admin/pages/site/agent/ ├── index.ts — public barrel (all external imports go through here) ├── agentSlice.ts — scope-agnostic Zustand slice factory (createAgentSlice(config)) +├── agentProviderUpdate.ts — timed provider/model persistence and fail-closed reconciliation ├── agentSliceConfig.site.ts— site-editor config: scope, snapshot builder, executor wiring ├── agentConfig.ts — conversation/default API path constants ├── agentApi.ts — conversation bootstrap and message rehydration @@ -108,7 +109,16 @@ src/admin/pages/content/agent/ └── useContentToolBridge.ts — always-mounted handle + content-scope MCP relay src/admin/pages/site/panels/AgentPanel/ -├── AgentPanel.tsx — main panel; resolves active model's contextWindow from the models endpoint +├── AgentPanel.tsx — panel shell, persisted message thread, and image-gallery orchestration +├── AgentComposer.tsx — controlled draft, paste/send lifecycle, active-model capability check +├── AgentImageGallery.tsx — compact shared thumbnails for user and agent-tool images +├── AgentImagePreview.tsx — modeless draggable full-image preview +├── AgentImageContextMenu.tsx — shared copy/download/Media actions for every image surface +├── PendingImageAttachmentGrid.tsx — compact pending tiles and per-image actions +├── agentImageActions.ts — authenticated blob reads, clipboard/download, and Media upload +├── agentImageTypes.ts — shared preview/menu image contracts and keyboard positioning +├── agentImageAttachment.ts — browser decode, resize, JPEG normalisation, and base64 encoding +├── usePendingImageAttachments.ts — ref-backed sequential image queue and per-item cancellation ├── ModelPicker.tsx — credential + model selector used in the input bar ├── ConversationHistory.tsx — history popover (browse, restore, delete past threads) ├── ContextMeter.tsx — "context used / window" progress indicator (display only) @@ -139,7 +149,27 @@ While credentials are still loading, `lockReason` stays `null` so the panel does When the panel opens, `AgentPanel` calls `loadScopeDefault()` so the model picker immediately shows the configured scope default — no "Default" placeholder, no send-time no-provider surprise. `composerLocked` is gated by `hasActiveProvider` (`Boolean(activeCredentialId && activeModelId)`), meaning a stale "No AI provider configured" error string never locks out the UI once a credential + model is staged; picking a model via `setAgentProvider` clears `agentError` immediately, re-enabling the composer. -The composer area includes a `` that shows "context used / window" as a progress bar. `AgentPanel` resolves the active model's `contextWindow` from `GET /admin/api/ai/providers/:id/models?credentialId=…` (the same catalogue-enriched response the picker uses), so the meter appears as soon as a model is selected — before the first turn. The "used" half comes from `agentContextTokens` in the store (see slice state below). The meter is hidden when no context window is known (Ollama, uncatalogued models). +The composer area includes a `` that shows "context used / window" as a progress bar. `AgentComposer` resolves the full active-model descriptor from `GET /admin/api/ai/providers/:id/models?credentialId=…` (the same catalogue-enriched response the picker uses), then uses its `contextWindow`, `capabilities.visionInput`, and `capabilities.toolCalling`. A model known not to support tools is blocked with an inline "choose an agent-capable model" message; the server repeats that gate authoritatively. The meter appears as soon as a model is selected — before the first turn. The "used" half comes from `agentContextTokens` in the store (see slice state below). The meter is hidden when no context window is known (Ollama, uncatalogued models). + +### Attaching user images + +The composer accepts up to eight local or clipboard images alongside optional text. The icon button beside Send uses the shared `FileUpload` primitive and supports multi-selection; picking the same file again works after removal. Pasting or picking reserves every accepted attachment slot synchronously, then normalises the files sequentially so one selection cannot fan out into eight large browser decoders. Send stays disabled while any image is processing, while model support is being checked, after a processing failure, when the selected model is not vision-capable, or when it is known not to support agent tools. Pending attachments use compact thumbnails in a responsive grid. They never point an `` at the potentially huge source file: a placeholder is shown during decoding, then replaced with the bounded normalised JPEG. Each primitive `Button` removes only its image. Removing an attachment or replacing the composer aborts its queued preparation and stops all downstream resize/encode work after the browser's current decode returns (`createImageBitmap` itself has no cancellation API). A message may contain only images: the server persists the turn normally and titles a new conversation `Image` or `Images`. + +Prepared attachments, persisted user images, and session-only images returned by agent tools use the same compact 2/3-column gallery. Each thumbnail is a keyboard-accessible button that opens the original fitted inside a modeless draggable preview window. The window uses the shared admin `FloatingWindow` shell, has only a title and close action, closes on Escape without also closing the Agent Panel, and restores focus to the thumbnail that opened it. + +Right-clicking an image in a pending tile, conversation gallery, or preview opens the same point-anchored `ContextMenu`; keyboard users can use the Context Menu key or Shift+F10. Actions copy the image bytes as PNG, start a MIME-correct browser download, or explicitly upload the bytes to Media. The latter uses the canonical `uploadCmsMediaAsset` pipeline (magic-byte validation, storage adapter selection, variants), requires `media.write`, primes the editor media cache, and upserts an already-mounted Media explorer. Escape closes the menu before the preview, and the preview before the Agent Panel. + +The provider-neutral v1 policy is defined once in `src/core/ai/userImage.ts` and enforced on both sides of the boundary: + +- accepted clipboard sources: PNG, JPEG, or WebP; +- maximum source file: 12 MiB; +- source-header guard: at most 16,384 px on either edge and 40,000,000 encoded pixels; PNG/JPEG/WebP dimensions (including JPEG EXIF orientation) are read before allocating a decoder, and `createImageBitmap` is asked for the bounded output size; +- at most eight images per message; there is no per-conversation image-count quota; +- browser output: metadata-stripped `image/jpeg`, transparent pixels composited over white; +- maximum output: 1,500,000 bytes, 1568 px on either edge, and 1,500,000 total pixels; +- complete chat-request envelope: eight maximum base64 images plus a further 16 MiB reserve for JSON framing and the scope snapshot (about 32.8 MB total). + +`agentImageAttachment.ts` fits both the edge and pixel budgets, tries progressively lower JPEG qualities, then reduces dimensions when necessary. The server never trusts that browser work: `server/ai/inputImages.ts` checks canonical base64, decoded byte length, JPEG magic bytes and dimensions, then fully decodes and re-encodes the JPEG through Sharp before appending the message. That second canonicalisation rejects truncated pixel data and strips EXIF/XMP/ICC metadata even when a direct authenticated client bypasses the browser. Unsupported, malformed, or oversized content is rejected before it can enter conversation history. The conversation single-writer lease is acquired before Sharp work, and the request signal is checked around every sequential decode: a competing request returns 409 without decoding, while a disconnected request finishes only its active Sharp pipeline and never starts the remaining images. --- @@ -152,24 +182,26 @@ Each entry in **Settings → AI → Providers** stores one credential. The provi | `anthropic` | Anthropic (Claude) | `apiKey` | API key (`sk-ant-…`) | — | Static `claude-*` catalogue enriched with OpenRouter prices + context windows | | `openai` | OpenAI | `apiKey` | API key (`sk-…`) | — | Static `gpt-*` / `o*` catalogue enriched with OpenRouter prices + context windows | | `openrouter` | OpenRouter | `apiKey` | API key (`sk-or-…`) | — | Live `GET /api/v1/models` (cross-provider; native cost reporting) | -| `ollama` | Ollama (local) | `baseUrl` | Base URL (e.g. `http://localhost:11434`) | API key (bearer, for proxied deployments) | Live `GET {baseUrl}/api/tags`; static fallback list when unreachable | +| `ollama` | Ollama (local) | `baseUrl` | Base URL (e.g. `http://localhost:11434`) | API key (bearer, for proxied deployments) | Live `GET {baseUrl}/api/tags`, with `POST {baseUrl}/api/show` capability lookup per model; static fallback list when unreachable | | `openai-compatible` | Custom Provider | `baseUrl` | Base URL — any host serving the OpenAI `/v1/chat/completions` wire protocol | API key (bearer; cloud services need one, local servers often don't) | Live `GET {baseUrl}/v1/models` (standard OpenAI list shape); model `id` used as label | -**Custom Provider** (id `openai-compatible`) is the generic adapter for any endpoint that speaks the OpenAI chat/completions wire protocol — Groq (`https://api.groq.com/openai`), Together, DeepSeek, Mistral, Fireworks, self-hosted vLLM, LM Studio, and others. Capabilities default to `{ toolCalling: true, visionInput: false, promptCache: false, streaming: true }`; the operator is responsible for selecting a model that actually supports tool calling. Because arbitrary endpoints are not in the OpenRouter catalogue, no context-window enrichment is available and the context meter stays hidden for these models. +**Custom Provider** (id `openai-compatible`) is the generic adapter for any endpoint that speaks the OpenAI chat/completions wire protocol — Groq (`https://api.groq.com/openai`), Together, DeepSeek, Mistral, Fireworks, self-hosted vLLM, LM Studio, and others. Capabilities default to `{ toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }`; the operator is responsible for selecting a model that actually supports tool calling. Because arbitrary endpoints are not in the OpenRouter catalogue, no context-window enrichment is available and the context meter stays hidden for these models. + +**The server is the model-capability authority.** The composer catalogue flags are early UX gates, but they are not trusted for persistence, provider calls, or tool screenshots. `chat.ts` resolves the selected model on every turn through `resolveModelCapabilities`. Providers with stable capabilities (Anthropic, OpenAI, Custom Provider) use their driver default. Model-specific providers own a selected-model lookup: OpenRouter resolves the exact entry's `architecture.input_modalities`, while Ollama sends an authenticated `POST /api/show` for only the selected model. The shared resolver de-duplicates concurrent lookups, applies a ten-second provider timeout, includes credential/backend revisions in its cache key, and caches successful results for five minutes. Missing or unavailable model-specific metadata fails closed for vision input; Custom Provider also remains fail-closed in v1. An image targeting a non-vision model, or an editor-agent turn targeting a model known not to support tools, receives 422 before the user message is stored. --- ## Flow ```text -User types prompt → Agent Panel +User types text and/or pastes images → Agent Panel │ ▼ -agentSlice.sendAgentMessage(content) +agentSlice.sendAgentMessage(contentBlocks) │ ├─→ buildSnapshot() → SiteAgentSnapshot or ContentSnapshot ├─→ ensure conversation row (lazily created from AI defaults on first call) - ├─→ POST /admin/api/ai/chat/ { conversationId, prompt, snapshot } + ├─→ POST /admin/api/ai/chat/ { conversationId, content, snapshot } │ ▼ Server: chat.ts @@ -177,13 +209,19 @@ Server: chat.ts ├─→ CSRF + requireCapability('ai.chat') ├─→ load conversation row (credentialId, modelId) + full message history ├─→ decrypt credential; resolveDriver(credential.providerId) + ├─→ preflight text/image blocks + encoded bytes; enforce the per-message bound + ├─→ resolve/cache the selected model's capabilities (also gates tool screenshots) + ├─→ acquire the conversation's single-writer stream lease + ├─→ fully decode/canonicalise images sequentially (request-cancellable) + ├─→ project persisted images for the selected model ├─→ selectToolsForScope(scope, capabilities) │ — write tools excluded unless caller has ai.tools.write ├─→ build scope system prompt → [staticPrefix, BOUNDARY, dynamicSuffix] ├─→ createBridge(emit) → { bridgeId, bridge, destroy } ├─→ emit { type: 'bridgeReady', bridgeId } └─→ runChat({ driver, request, persister, emit }) — streaming begins - │ request carries the FULL conversation history as req.messages. + │ request carries the conversation history as req.messages; user + │ images use the provider replay policy described below. │ Direct HTTP drivers have no server-side session — every turn │ replays the whole log, mapped into the provider's message array. │ @@ -459,7 +497,7 @@ The agent works **design-system-first**: it establishes or reuses tokens, then r | Tool | Input | Success `data` | What it does | |-------------------|-----------------------|----------------|------------------------------------------------------------------| -| `site_render_snapshot` | `{ breakpointId?, nodeId? }` | `{ breakpointId, nodeId?, label, width, capturedAt, layout, screenshot }` + optional `images[]` | Inspect the rendered canvas: always returns a layout report (viewport, per-node bounding boxes, overflow / broken-image / invisible warnings); on a vision-capable model a PNG is attached via the tool-output **image channel**. `breakpointId` picks the frame (defaults to active); `nodeId` scopes the capture to that node's subtree — image and report cover only that section, with coordinates relative to its box, and the report carries the same `nodeId`. Omit `nodeId` for the whole page; an unknown `nodeId` returns an `aiToolError` | +| `site_render_snapshot` | `{ breakpointId?, nodeId? }` | `{ breakpointId, nodeId?, label, width, capturedAt, layout, screenshot }` + optional `images[]` | Inspect the rendered canvas: always returns a layout report (viewport, per-node bounding boxes, overflow / broken-image / invisible warnings); when the active provider supports native image-bearing tool results, a PNG is attached via the tool-output **image channel**. `breakpointId` picks the frame (defaults to active); `nodeId` scopes the capture to that node's subtree — image and report cover only that section, with coordinates relative to its box, and the report carries the same `nodeId`. Omit `nodeId` for the whole page; an unknown `nodeId` returns an `aiToolError` | ### Auto-navigation @@ -507,7 +545,7 @@ The content system prompt is markdown-native: it tells the model to exchange bod `site_render_snapshot` (and `site_read_document` / `site_get_node_html`) return large payloads. Five rules keep them from exploding context (a screenshot inlined as base64 JSON text once pushed a single turn past 1M tokens): 1. **Image channel, not text.** `AiToolOutput` carries an optional `images: { mimeType, data }[]` (`src/core/ai/toolOutput.ts`). `site_render_snapshot` puts the PNG there — never in `data`. The Anthropic driver forwards it as a **native `image` block** inside the `tool_result` (billed at the rendered image's token cost). Text-only tool channels (Ollama / OpenAI-compatible `function_call_output`) **drop** the image and append a one-line `[N screenshot(s) omitted…]` note. The capture caps the screenshot's long edge at `MAX_IMAGE_EDGE` (1568px in `renderEvidence.ts`) — a tall landing page would otherwise exceed Anthropic's hard 8000px-per-dimension limit (400 error), and the model downsizes the long edge to ~1568px anyway. -2. **Capture is vision-gated.** The chat handler resolves `driver.capabilities(modelId)` into `AiStreamRequest.modelCapabilities`. The shared tool loop injects `captureScreenshot: visionInput` into every `site_render_snapshot` call, so a non-vision model never pays the html-to-image cost — it gets the layout report only. (The model never sets `captureScreenshot` itself.) +2. **Capture is provider-channel-gated.** The chat handler resolves the selected model on every turn and places the result in `AiStreamRequest.modelCapabilities`. `visionInput` means pasted user images are accepted; the separate `toolResultImages` flag means that provider's tool-result wire shape can actually carry a native image. The shared tool loop injects `captureScreenshot: visionInput && toolResultImages` into `site_render_snapshot`. Today Anthropic supports both; Responses and chat/completions providers accept user images but have text-only function/tool results, so they get the layout report without paying for a screenshot that would be discarded. (The model never sets `captureScreenshot` itself.) 3. **`site_read_document` CSS is document-relevant, not the public full-site CSS bundle.** Public pages can share page-invariant CSS files, but `site_read_document` inlines CSS into model context. It keeps framework variables/utilities, font token variables, target-document module CSS, used class rules, ambient selectors whose class tokens all exist on the target document, classless/global ambient selectors, and document-targeted user stylesheets. It omits browser-only `@font-face` file declarations and ambient selectors from unrelated imported pages. 4. **`site_read_document` is cleaned and paged before it reaches the model.** `renderAgentDocument` strips pathological strings from the broad read surface: long base64/data URLs become `data:;base64,[omitted N chars]`, and very long URLs are middle-truncated. The returned object always includes `pageInfo` with `part`, `totalParts`, `nextPart`, `ranges`, `serializedChars`, and cleanup counts. The hard budget is measured against `JSON.stringify({ html, css, pageInfo }).length`, because that is the text providers receive as the tool result. If `nextPart` is not `null`, the agent calls `site_read_document({ document, part: nextPart })` to continue. For exact node-level markup, use the `uid` with `site_get_node_html`. 5. **Stale evidence is elided.** Within one tool loop, only the **most recent** heavy result per tool name (`site_render_snapshot`, `site_read_document`, `site_get_node_html`, or anything with an image) is replayed at full fidelity; earlier ones are rewritten to a one-line breadcrumb (`"Earlier output removed… Call again…"`). Older snapshots describe page state the model has since mutated, so they carry no value. See `applyHeavyElision` in `server/ai/drivers/http/toolLoop.ts`. @@ -583,6 +621,8 @@ export const siteAgentSliceConfig: AgentSliceConfig = { The content workspace uses the same factory with `contentAgentSliceConfig` mounted in a standalone per-page store (`contentAgentStore.ts`). +`agentProviderUpdate.ts` owns the existing-conversation provider/model PUT and its failure reconciliation. A definite 4xx can roll the picker back to the re-read row; a timeout, network failure, or 5xx stays fail-closed unless the re-read already proves that the requested selection committed. `agentSlice.ts` keeps the ordering queue and Send lock because those coordinate store actions rather than HTTP persistence. + Key slice state and actions: ```ts @@ -606,12 +646,18 @@ interface AgentSlice { * on loadAgentConversation; updated live from each turn's `usage` event. */ agentContextTokens: number | null + /** Blocks Send/navigation while a history load or delete may replace the active chat. */ + isAgentConversationPending: boolean + /** Blocks Send/navigation while an existing chat's model PUT is pending. */ + isAgentProviderPending: boolean + /** Incremented when a conversation is replaced so local text/image drafts remount cleanly. */ + agentComposerEpoch: number // ── Actions ─────────────────────────────────────────────────────────── openAgent(): void closeAgent(): void toggleAgent(): void - sendAgentMessage(content: string): Promise + sendAgentMessage(content: AiUserContentBlock[]): Promise<{ accepted: boolean }> abortAgent(): void clearAgentMessages(): void startNewAgentConversation(): void @@ -627,7 +673,35 @@ interface AgentSlice { Conversations and their message history are persisted server-side in `ai_conversations` + `ai_messages`. `loadAgentConversation(id)` rehydrates a past thread into `agentMessages` without re-running the conversation. -**Content blocks are one schema.** Every message body is an `AiContentBlock[]` — a discriminated union of `text` / `image` / `toolCall` / `toolResult` kinds defined once as a TypeBox schema in `@core/ai` (`src/core/ai/contentBlock.ts`). The server runtime type (`AiContentBlock`), the read boundary (`ContentBlocksSchema` in `conversations/store.ts`, which validates every block out of `content_json`), and the client wire schema (`MessageViewSchema` in `src/admin/ai/api.ts`) all derive from it. Add a kind there and every reader/writer sees it. +**Content blocks have one persisted vocabulary and one safe browser projection.** Every stored/provider message body is an `AiContentBlock[]` — a discriminated union of `text` / base64 `image` / `toolCall` / `toolResult` kinds defined once as a TypeBox schema in `@core/ai` (`src/core/ai/contentBlock.ts`). The server runtime type and the `content_json` read boundary derive from it. Conversation-detail responses derive from the sibling `AiContentViewBlockSchema`: non-image blocks keep the same schemas, while an image carries an authenticated lazy `url` instead of inline `data`. The client validates that view schema before rehydrating its render model. + +**User turns use the same canonical blocks at the HTTP boundary.** `AiChatRequestBodySchema` in `src/core/ai/chatRequest.ts` accepts `{ conversationId, content, snapshot? }`, where `content` contains at most one trimmed text block plus up to eight canonical JPEG blocks. The server canonicalises a mixed turn as text followed by the images in paste order and removes whitespace-only text. It does not accept `toolCall` or `toolResult` blocks from the browser. + +```ts +{ + conversationId, + content: [ + { kind: 'text', text: 'Use this mockup as the reference.' }, + { kind: 'image', mimeType: 'image/jpeg', data: '' }, + { kind: 'image', mimeType: 'image/jpeg', data: '' }, + ], + snapshot, +} +``` + +**Persisted images, browser history, and provider replay are deliberately different views.** Every accepted user JPEG is stored inline in `ai_messages.content_json`; conversations have no image-count quota. A conversation-detail response replaces each base64 block with `GET /admin/api/ai/conversations/:conversationId/messages/:messageId/images/:blockIndex`. The ownership-guarded endpoint returns only a canonical JPEG with `private, no-store`; native lazy image loading means reopening a large collection does not embed all bytes in one JSON response. Before a provider call, `projectUserImagesForModel` creates a non-mutating outbound projection: + +- a vision model first receives every persisted image in conversation order; there is no Instatic replay count cap; +- a non-vision model receives no image bytes at all; every persisted image becomes a text breadcrumb, so switching models cannot poison the conversation; +- the database rows are never rewritten by projection, so the UI history remains intact and switching back to a vision model restores the complete persisted image history. + +Providers may enforce a physical request, context, or routed image limit before accepting that full replay. The shared HTTP tool loop classifies only those explicit overflow responses (`413`, or a matching provider `400`) and retries once before any SSE or tool side effect: images on older user turns become one breadcrumb per turn, while every image on the newest/current user turn remains. Generic 400s, authentication, quota, rate-limit, and service failures are never retried. If the reduced request still fails—or only the current turn has images—the error explains that history remains saved and suggests a new conversation or a larger-context model. This is provider-triggered fallback, not a stored-image quota or an arbitrary app-side count. + +User attachments are private chat data by default, not media-library assets: the normalised base64 bytes live in the database until the conversation is deleted and purged, are exposed to the owning authorised user only through the lazy conversation-image endpoint, and are sent to the configured AI provider whenever they survive the outbound replay projection. They enter public Media storage only when a user with `media.write` explicitly chooses **Save to Media** from the image context menu; saving creates a separate media asset and does not change or delete the private conversation copy. + +The server admits only one active writer per conversation. A concurrent tab receives a retryable 409 before appending, which keeps message positions ordered. In the browser, model changes for an existing conversation are serialized; Send waits for the provider/model update to reach the server, and conversation/model controls stay disabled while a turn streams. If a model PUT times out after an ambiguous commit, the browser re-reads the conversation before re-enabling Send, so the picker cannot disagree with server routing. Stop owns the whole first-send lifecycle, including default lookup and lazy conversation creation, so an aborted bootstrap cannot leave the composer locked. + +**User attachments are not tool screenshots.** A pasted image is a persisted `kind:'image'` block on a user message. Conversation-detail responses, authenticated image responses, and chat streams use `Cache-Control: private, no-store`; the database remains the intentional durable copy. Images returned by `site_render_snapshot` or another browser tool instead travel transiently on the plural `AiToolOutput.images` channel, are subject to the heavy-evidence rules above, and remain session-only even though the panel exposes every returned image through the same gallery and draggable preview. The two paths share provider-native image mapping but have different storage and replay lifecycles. **Tool outcomes are first-class.** A `role:'tool'` row records its result as a `{ kind: 'toolResult', ok, error? }` block — `ok` is an explicit boolean, never inferred from the emptiness of a text block. The persister writes it (`appendToolResult`), `buildMessageHistory` reads `ok`/`error` straight off the block to reconstruct the replay `AiToolOutput`, and the client folds it back into the matching tool-call badge (`rehydrateMessages`). The heavy successful `data` an `AiToolOutput` may carry is intentionally **not** persisted: the model already consumed it in the round that produced the result, so replay only needs `{ ok, error }` — re-feeding large tool payloads every turn would bloat the context for no benefit. @@ -648,6 +722,8 @@ The `` shows how much of the active model's context window the cur ### Live model catalogue +The browser de-duplicates concurrent requests for the same credential, applies a ten-second timeout, and retains successful catalogues for five minutes so the composer capability check and model picker share one result across conversation switches. Credential deletion invalidates its cached catalogue. The server independently applies the same ten-second deadline and forwards request cancellation into provider fetches; Ollama resolves `/api/show` metadata in batches of six rather than launching an unbounded fan-out. + `server/ai/pricing/` is the single source for per-model prices **and context windows**. It sources from OpenRouter's public `/api/v1/models` endpoint (no key required), which publishes list prices and `context_length` for Anthropic and OpenAI models. The module lifecycle: - **Cold start**: loads the DB cache from `ai_model_pricing` (durable fallback) and kicks a background refresh. The first turn prices immediately off the last-known data. @@ -707,6 +783,9 @@ unblocks deletion of the credential that had been protected by the default FK. - `docs/features/content-workspace.md` — content workspace UI and content-scope Agent Panel mount - Source-of-truth files: - `src/core/ai/toolOutput.ts` — `AiToolOutput` type, `AiToolOutputSchema`, `aiToolOk`, `aiToolError` (canonical bridge result) + - `src/core/ai/chatRequest.ts` — canonical browser-to-server chat envelope and computed multi-image request ceiling + - `src/core/ai/contentBlock.ts` — persisted/provider content blocks plus the lazy-URL conversation-detail view schema + - `src/core/ai/userImage.ts` — accepted source formats, normalised JPEG schema, byte/dimension limits, and eight-image per-message bound - `src/core/ai/toolSchemas.ts` — all site browser-tool input schemas (single source of truth; imported by both the server registry and the browser executor) - `src/core/ai/documentRefs.ts` — document refs/descriptors for pages, templates, and visual components - `src/core/ai/readSurface.ts` — runtime-agnostic `renderAgentDocument` annotated HTML + compact CSS renderer @@ -718,10 +797,16 @@ unblocks deletion of the credential that had been protected by the default FK. - `server/ai/tools/site/snapshot.ts` — `SiteAgentSnapshotSchema` + `SiteAgentSnapshot` re-export + catalog output types (`ModuleInfo`, `SnapshotTokens`, …) - `server/ai/tools/content/readTools.ts` — 7 server-side content read tool definitions - `server/ai/tools/content/writeTools.ts` — 8 browser-bridged content write/navigation tool definitions + - `server/ai/inputImages.ts` — server-side base64, JPEG, byte, and dimension validation before persistence + - `server/ai/drivers/modelCapabilities.ts` — cached, timed, authoritative/fail-closed selected-model capability resolution on every turn + - `server/ai/drivers/modelList.ts` — bounded provider catalogue lookup with caller cancellation + - `server/ai/drivers/http/toolLoop.ts` — provider loop, heavy tool-result elision, and one provider-triggered historical-image fallback + - `server/ai/conversations/history.ts` — interrupted-tool healing plus outbound user-image replay projection - `server/ai/tools/content/systemPrompt.ts` — markdown-native content system prompt - `server/ai/tools/content/snapshot.ts` — `ContentSnapshot` shape consumed by the content prompt and tool context - `src/admin/pages/site/agent/siteAgentSnapshot.ts` — `SiteAgentSnapshotSchema` (TypeBox source of truth) + `SiteAgentSnapshot` (derived type) + `buildSiteAgentSnapshot` - `server/ai/handlers/chat.ts` — `POST /admin/api/ai/chat/:scope` endpoint + - `server/ai/handlers/conversations.ts` — conversation CRUD plus the ownership-guarded lazy image endpoint - `server/ai/handlers/toolResult.ts` — `POST /admin/api/ai/tool-result` endpoint - `server/ai/conversations/history.ts` — `buildMessageHistory()` + `INTERRUPTED_TOOL_RESULT_ERROR` (heals interrupted tool calls) - `server/ai/conversations/store.ts` — `appendMessage`, `listMessagesForConversation`, `readConversationForUser` @@ -741,9 +826,15 @@ unblocks deletion of the credential that had been protected by the default FK. - `src/admin/pages/ai/tabs/UsageTablePanel.tsx` — shared table scaffolding for audit rollups - `src/admin/pages/ai/tabs/usageFormat.ts` — `formatNumber` / `formatCost` formatting helpers - `src/admin/pages/site/agent/agentSlice.ts` — scope-agnostic slice factory (`createAgentSlice`) + - `src/admin/pages/site/agent/agentProviderUpdate.ts` — timed provider/model update and ambiguous-commit reconciliation - `src/admin/pages/site/agent/agentSliceConfig.site.ts` — site-editor scope config - `src/admin/pages/site/agent/agentApi.ts` — tool-result POST, conversation bootstrap, message rehydration - `src/admin/pages/site/agent/streamEvents.ts` — `ServerStreamEventSchema` + `processStreamEvent` + - `src/admin/pages/site/panels/AgentPanel/AgentImageGallery.tsx` — shared compact gallery for persisted and session-only images + - `src/admin/pages/site/panels/AgentPanel/AgentImagePreview.tsx` — draggable modeless image preview + - `src/admin/pages/site/panels/AgentPanel/AgentImageContextMenu.tsx` — shared image actions menu + - `src/admin/pages/site/panels/AgentPanel/agentImageActions.ts` — clipboard, download, and Media-save pipeline + - `src/admin/shared/FloatingWindow/` — shared portal, panel header, and persisted drag behavior for admin floating windows - `src/admin/pages/site/agent/pageContext.ts` — `buildCurrentPageContext` - `src/admin/pages/site/agent/executor.ts` — write-tool browser dispatcher + auto-navigation - `src/admin/pages/site/agent/tokenRunners.ts` — design-system token tool runners (`site_set_color_tokens`, `site_set_font_tokens`, `site_set_type_scale`, `site_set_spacing_scale`) diff --git a/docs/features/media.md b/docs/features/media.md index cb4780a36..de69cceeb 100644 --- a/docs/features/media.md +++ b/docs/features/media.md @@ -13,7 +13,7 @@ The workspace is canvas-style: it uses `AdminWorkspaceCanvasLayout`, the lighter - **State:** one hook — `useMediaWorkspace()` — orchestrates folders, assets, selection, filters, upload queue, and folder moves. The editor store doesn't grow new slices; the Media page is self-contained. - **Folders:** folders render as first-class grid/list items in the canvas. Opening a folder filters the canvas to its contents, and nested folders show a parent-folder entry to navigate back. - **Drag/drop:** assets can be dragged into folders from the canvas or folder tree. A drop replaces the asset's folder memberships with the target folder, matching desktop file-manager move semantics. -- **Floating windows:** Asset viewer, upload queue, bulk edit. Each is `useDraggablePanel('mediaViewer' | 'mediaUploadQueue' | 'mediaBulkEdit')`. Position survives reload via `workspaceLayoutStorage`. +- **Floating windows:** Asset viewer, upload queue, bulk edit. Each uses the shared `@admin/shared/FloatingWindow` shell or its `useDraggablePanel` hook with a unique id (`mediaDetachedInspector`, `mediaUploadQueue`, `mediaBulkEdit`). Position survives reload via `workspaceLayoutStorage`. - **Auto-open behavior:** upload queue opens when uploads start; bulk-edit opens at 2+ selected; viewer opens on primary selection. - **Server side:** `media_assets`, `media_folders`, `media_asset_folders` tables. Handlers under `/admin/api/cms/media`, `/admin/api/cms/media/folders`, `/admin/api/cms/media/storage`. Repositories at `server/repositories/media*.ts`. - **Storage adapters:** built-in local-disk plus plugin-registered adapters. Non-public-url adapters route through `/_instatic/media//` for signed redirects. @@ -25,6 +25,7 @@ The workspace is canvas-style: it uses `AdminWorkspaceCanvasLayout`, the lighter ```text src/admin/pages/media/ ├── MediaPage.tsx — top-level component +├── mediaAssetEvents.ts — typed cross-workspace asset-created notifications ├── components/ │ ├── MediaSidebar/ — folder tree + storage + smart folders │ ├── MediaCanvas/ — file grid / list with FilterBar @@ -37,7 +38,6 @@ src/admin/pages/media/ │ ├── MediaPickerField/ — embedded picker control │ ├── MediaFolderPanel/ — folder list panel │ ├── MediaStoragePanel/ — storage adapter management -│ ├── FloatingWindow/ — shared floating-window shell │ └── viewers/ — per-media-type viewers (image, video, pdf, etc.) ├── hooks/ │ ├── useMediaWorkspace.ts — orchestrates server state (folders, assets, filters) @@ -54,6 +54,8 @@ src/admin/pages/media/ ├── mediaDragDrop.ts — TypeBox-validated drag/drop payload helpers ├── smartFolders.ts — smart folder IDs, type guard, per-ID predicates └── variants.ts — image variant URL helpers + +src/admin/shared/FloatingWindow/ — shared portal shell + persisted drag hook used across admin workspaces ``` --- @@ -144,7 +146,7 @@ The count badge shown next to each smart folder in the sidebar is computed clien ### Floating windows -Each floating window has a unique `FloatingPanelId` (`'mediaViewer' | 'mediaUploadQueue' | 'mediaBulkEdit'`), uses `useDraggablePanel(id)` for position, and gets its position persisted via `workspaceLayoutStorage.ts`. Visibility differs by window: +Each floating window has a unique `FloatingPanelId` (`'mediaDetachedInspector' | 'mediaUploadQueue' | 'mediaBulkEdit'`), uses `useDraggablePanel(id)` for position, and gets its position persisted via `workspaceLayoutStorage.ts`. Visibility differs by window: | Window | How visibility is determined | |-----------------|-------------------------------------------------------------------------------------------------| @@ -154,7 +156,7 @@ Each floating window has a unique `FloatingPanelId` (`'mediaViewer' | 'mediaUplo The viewer and bulk-edit are derived rather than stored because "closed" is identical to "no selection" — every close path calls `workspace.clearSelection()`. Deriving avoids an extra render commit and the one-frame open lag that appeared with the old `setState`-in-effect approach. -The shared `FloatingWindow` shell at `components/FloatingWindow/` provides the chrome: drag handle, close button, position bounding. +The shared `FloatingWindow` shell at `src/admin/shared/FloatingWindow/` provides the portal, chrome, drag handle, close button, and position bounding without pulling media-specific editor code into other workspaces. Its dimension-aware clamp measures each panel and keeps at least a 50px header strip reachable on every viewport edge; stored positions are re-clamped when a window mounts, changes size, or the viewport resizes. --- @@ -250,8 +252,10 @@ Folder routes (`/admin/api/cms/media/folders/...`) are matched **before** asset ### Upload pipeline +Uploads initiated outside the Media page use the same pipeline. In particular, the Agent Panel's explicit **Save to Media** image action resolves the private chat image, wraps it in a MIME-correct `File`, and calls `uploadCmsMediaAsset`; it does not create an AI-specific storage route. On success, `mediaAssetEvents.ts` upserts the new row into an already-mounted Site → Media explorer while the normal media cache is primed for canvas consumers. + ```text -POST /admin/api/cms/media/upload +POST /admin/api/cms/media │ ▼ mediaUpload.ts ← validates upload (size + magic-byte MIME sniff) @@ -325,8 +329,8 @@ The redirect handler is `tryServeMediaRedirect` in `server/router.ts`. The redir ### Add a new floating window 1. Add a unique id to `FloatingPanelId` in the layout storage module. -2. Create `components//` with the window React component using the shared `FloatingWindow` shell. -3. Use `useDraggablePanel('newWindowId')` for position; track `open` in `MediaPage` local state. +2. Create `components//` with the window React component using `FloatingWindow` from `@admin/shared/FloatingWindow`. +3. Pass the id to `FloatingWindow`; track `open` in `MediaPage` local state. Use the exported `useDraggablePanel` directly only for a custom shell such as `MediaViewerWindow`. 4. Add the auto-open rule (selection threshold, upload event, etc.) inside `MediaPage`. ### Add a new sidebar panel diff --git a/docs/reference/design-tokens.md b/docs/reference/design-tokens.md index 887e2291a..856b98e5a 100644 --- a/docs/reference/design-tokens.md +++ b/docs/reference/design-tokens.md @@ -418,7 +418,7 @@ The visual editor uses additional raw z-index values that are **not** tokenised. | 30 | Main toolbar | | 50 | Floating panels: PropertiesPanel, AgentPanel, DomPanel | | 55 | LeftSidebar, RightSidebar, PanelRail | -| 70 | Media floating windows (FloatingWindow, MediaViewerWindow) | +| 90 | Shared admin floating windows (`FloatingWindow`, `MediaViewerWindow`, agent image preview) | | 80 | CodeEditorPanel | | 201 | Toolbar popovers / dropdowns | | 400–401 | PreviewOverlay | diff --git a/docs/reference/typebox-patterns.md b/docs/reference/typebox-patterns.md index a461699bb..ace4e2a50 100644 --- a/docs/reference/typebox-patterns.md +++ b/docs/reference/typebox-patterns.md @@ -13,7 +13,7 @@ Schemas are the **source of truth**. Domain types come from `Static` — never a hand-rolled interface beside the schema. - **Soft fallbacks** for corrupted local storage / optional config use `withFallback(schema, default)` + `parseJsonWithFallback`. - **Hard fallbacks** for required documents throw and bubble to an error boundary. @@ -94,10 +94,11 @@ const prefs = parseJsonWithFallback( | Helper | Purpose | |-------------------------------------------------|------------------------------------------------------------------| | `apiRequest(path, { method, body, schema, query, signal, … })` | **The default for browser→server calls.** Sets `credentials`, JSON-serializes `body`, validates the success body against `schema` (returns `Static`; `void` without one), and throws `ApiError` on a non-OK status. | +| `apiBlobRequest(path, { signal, … })` | Binary-response counterpart for authenticated image/file reads. Uses the same transport and `ApiError` envelope handling, then returns a `Blob`; the caller validates the MIME type before use. | | `readEnvelope(res, schema, fallbackMessage)` | For code that already holds a `Response` (the persistence layer, which injects its own `fetch`): check `res.ok` (throw `ApiError` with `responseErrorMessage(res, fallback)` if not), then validate body against `schema` | | `assertOk(res, fallbackMessage)` | No-body counterpart to `readEnvelope`: throw `ApiError` if the `Response` is not OK, otherwise return (for void mutations / bodies parsed separately) | | `responseErrorMessage(res, fallback)` | Extract a useful error message from a failed `Response` (reads `{ error: string }` envelope if present, then raw text, otherwise the fallback) | -| `ApiError` | The single error type thrown by `apiRequest`/`readEnvelope`; carries `.status` so UI can branch (403, 404, …) | +| `ApiError` | The single error type thrown by `apiRequest`/`apiBlobRequest`/`readEnvelope`; carries `.status` so UI can branch (403, 404, …) | | `isAbortError(err)` | True for an aborted fetch (user cancellation / superseded request) — the uniform replacement for `(err as Error).name === 'AbortError'` | --- @@ -286,7 +287,8 @@ Common boundaries already wrapped — extend the same pattern when you add a new | Boundary | Helper | Lives in | |--------------------------------------------|-----------------------------------------------------|-----------------------------------------| -| HTTP request (client, canonical) | `apiRequest(path, { schema, … })` | `src/core/http/apiClient.ts` | +| HTTP request (client, canonical JSON) | `apiRequest(path, { schema, … })` | `src/core/http/apiClient.ts` | +| HTTP request (client, binary body) | `apiBlobRequest(path, { signal, … })` | `src/core/http/apiClient.ts` | | HTTP response from a held `Response` | `readEnvelope(res, Schema, fallback)` | `src/core/http/apiClient.ts` | | HTTP body-validation primitive (no status semantics; `@core/http` internals, XHR, server-side external APIs) | `parseJsonResponse(res, Schema)` | `src/core/utils/jsonValidate.ts` | | Request body (server handler) | `readValidatedBody(req, Schema)` → typed value or `null` (return `badRequest` on null) | `server/http.ts` + per-handler | @@ -333,7 +335,7 @@ Common boundaries already wrapped — extend the same pattern when you add a new - `src/core/utils/typeboxHelpers.ts` — helper layer (`parseValue`, `withFallback`, `filterArray`, etc.) - `src/core/utils/typeboxCompiler.ts` — cached TypeCompiler layer for hot validation execution - `src/core/utils/jsonValidate.ts` — JSON boundary helpers - - `src/core/http/apiClient.ts` — `apiRequest`, `ApiError`, `isAbortError`, `readEnvelope`, `assertOk`, `responseErrorMessage`, `ErrorEnvelopeSchema` + - `src/core/http/apiClient.ts` — `apiRequest`, `apiBlobRequest`, `ApiError`, `isAbortError`, `readEnvelope`, `assertOk`, `responseErrorMessage`, `ErrorEnvelopeSchema` - `src/core/persistence/responseSchemas.ts` — shared CMS HTTP response schemas - `src/core/persistence/validate.ts` — `validateSite`, `validatePages`, `ValidatePagesOptions`, `SiteValidationError` - `src/core/plugins/manifest.ts` — `parsePluginManifest` diff --git a/server/ai/conversations/history.ts b/server/ai/conversations/history.ts index a15bbdc99..08f744b83 100644 --- a/server/ai/conversations/history.ts +++ b/server/ai/conversations/history.ts @@ -23,7 +23,7 @@ * preceding tool call in the replayed history. */ -import type { AiMessage, AiToolOutput } from '../runtime/types' +import type { AiContentBlock, AiMessage, AiToolOutput } from '../runtime/types' import type { MessageRecord } from './types' /** @@ -33,6 +33,9 @@ import type { MessageRecord } from './types' export const INTERRUPTED_TOOL_RESULT_ERROR = 'Tool call did not complete — the previous turn was interrupted before a result was produced.' +export const NON_VISION_USER_IMAGE_OMITTED = + '[Attached image omitted because the selected model does not support image input.]' + /** * Reconstruct `AiMessage` history from persisted `MessageRecord` rows. * @@ -100,3 +103,36 @@ export function buildMessageHistory(records: MessageRecord[]): AiMessage[] { return out } + +/** + * Adapt persisted user images to the selected model without mutating history. + * + * Vision models retain every image-bearing turn. Text-only models receive one + * breadcrumb per image-bearing turn, which keeps a conversation usable after + * switching away from a vision model. + */ +export function projectUserImagesForModel( + messages: readonly AiMessage[], + visionInput: boolean, +): AiMessage[] { + if (visionInput) return [...messages] + + return messages.map((message) => { + if (message.role !== 'user' || !message.content.some((block) => block.kind === 'image')) { + return message + } + let breadcrumbAdded = false + const content: AiContentBlock[] = [] + for (const block of message.content) { + if (block.kind !== 'image') { + content.push(block) + continue + } + if (!breadcrumbAdded) { + content.push({ kind: 'text', text: NON_VISION_USER_IMAGE_OMITTED }) + breadcrumbAdded = true + } + } + return { role: 'user', content } + }) +} diff --git a/server/ai/conversations/store.ts b/server/ai/conversations/store.ts index 1609f4a8b..10e6d1b1c 100644 --- a/server/ai/conversations/store.ts +++ b/server/ai/conversations/store.ts @@ -11,7 +11,7 @@ import { nanoid } from 'nanoid' import { Type, safeParseValue } from '@core/utils/typeboxHelpers' -import { AiContentBlockSchema } from '@core/ai' +import { AiContentBlockSchema, type AiContentViewBlock } from '@core/ai' import type { DbClient } from '../../db/client' import { isoDateOrNull } from '@core/utils/isoDate' import type { AiContentBlock, ToolScope } from '../runtime/types' @@ -147,12 +147,27 @@ export function toConversationView(record: ConversationRecord): ConversationView } } -function toMessageView(record: MessageRecord): MessageView { +type ImageUrlFor = (messageId: string, blockIndex: number) => string + +const UNSUPPORTED_STORED_IMAGE = + '[Stored image omitted because its format is not supported by conversation preview.]' + +function toMessageView(record: MessageRecord, imageUrlFor: ImageUrlFor): MessageView { return { id: record.id, position: record.position, role: record.role, - content: record.content, + content: record.content.map((block, blockIndex): AiContentViewBlock => { + if (block.kind !== 'image') return block + if (block.mimeType !== 'image/jpeg') { + return { kind: 'text', text: UNSUPPORTED_STORED_IMAGE } + } + return { + kind: 'image', + mimeType: 'image/jpeg', + url: imageUrlFor(record.id, blockIndex), + } + }), toolCallId: record.toolCallId, toolName: record.toolName, createdAt: record.createdAt, @@ -162,10 +177,11 @@ function toMessageView(record: MessageRecord): MessageView { export function toConversationDetailView( conversation: ConversationRecord, messages: MessageRecord[], + imageUrlFor: ImageUrlFor, ): ConversationDetailView { return { ...toConversationView(conversation), - messages: messages.map(toMessageView), + messages: messages.map((message) => toMessageView(message, imageUrlFor)), } } @@ -239,6 +255,29 @@ export async function listMessagesForConversation( return rows.map(messageRowToRecord) } +/** Read one message through its owning, non-deleted user conversation. */ +export async function readMessageForUser( + db: DbClient, + userId: string, + conversationId: string, + messageId: string, +): Promise { + const { rows } = await db` + select m.id, m.conversation_id, m.position, m.role, m.content_json, + m.tool_call_id, m.tool_name, + m.prompt_tokens, m.completion_tokens, m.cost_usd, + m.cache_read_tokens, m.cache_creation_tokens, m.created_at + from ai_messages m + join ai_conversations c on c.id = m.conversation_id + where m.id = ${messageId} + and m.conversation_id = ${conversationId} + and c.user_id = ${userId} + and c.deleted_at is null + limit 1 + ` + return rows[0] ? messageRowToRecord(rows[0]) : null +} + // --------------------------------------------------------------------------- // Writes // --------------------------------------------------------------------------- @@ -320,6 +359,31 @@ export async function updateConversationForUser( return rows[0] ? conversationRowToRecord(rows[0]) : null } +/** + * Give a first turn its derived title without overwriting a rename that won + * the race in another tab. The placeholder predicate belongs in the UPDATE, + * not in a preceding read. + */ +export async function replaceDefaultConversationTitle( + db: DbClient, + userId: string, + conversationId: string, + title: string, +): Promise { + const nextTitle = title.trim() + if (!nextTitle) return false + const result = await db` + update ai_conversations + set title = ${nextTitle}, + updated_at = current_timestamp + where id = ${conversationId} + and user_id = ${userId} + and deleted_at is null + and title = ${DEFAULT_CONVERSATION_TITLE} + ` + return result.rowCount > 0 +} + /** * Soft-delete by setting `deleted_at`. Idempotent — calling on an * already-deleted row sets deleted_at to the current time again. diff --git a/server/ai/conversations/types.ts b/server/ai/conversations/types.ts index 86f68003c..aee967905 100644 --- a/server/ai/conversations/types.ts +++ b/server/ai/conversations/types.ts @@ -7,9 +7,11 @@ * Wire shapes are separate from records: * - `ConversationView` is what /admin/api/ai/conversations returns — * summary fields only; full message history fetched on open. - * - `MessageView` is the per-message wire shape, AiContentBlock[] inline. + * - `MessageView` keeps non-image blocks and uses lazy image URLs; base64 + * remains only on `MessageRecord` for durable provider replay. */ +import type { AiContentViewBlock } from '@core/ai' import type { AiContentBlock, ToolScope } from '../runtime/types' // --------------------------------------------------------------------------- @@ -84,7 +86,7 @@ export interface MessageView { readonly id: string readonly position: number readonly role: MessageRole - readonly content: AiContentBlock[] + readonly content: AiContentViewBlock[] readonly toolCallId: string | null readonly toolName: string | null readonly createdAt: string diff --git a/server/ai/drivers/anthropic.ts b/server/ai/drivers/anthropic.ts index fbad77562..57176251e 100644 --- a/server/ai/drivers/anthropic.ts +++ b/server/ai/drivers/anthropic.ts @@ -60,13 +60,14 @@ export const anthropicDriver: AiProvider = { return { toolCalling: true, visionInput: true, + toolResultImages: true, promptCache: true, streaming: true, } }, - async listModels(creds) { - return fetchAnthropicModels(creds) + async listModels(creds, signal) { + return fetchAnthropicModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -122,7 +123,10 @@ const AnthropicModelsResponseSchema = Type.Object( * - a failed request or unparseable body throws, so the caller surfaces the * error rather than masking it with a stale hardcoded list. */ -async function fetchAnthropicModels(creds: AiResolvedCredential): Promise { +async function fetchAnthropicModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { if (creds.authMode !== 'apiKey' || !creds.apiKey) return [] const res = await fetch(`${ANTHROPIC_MODELS_ENDPOINT}?limit=1000`, { @@ -130,6 +134,7 @@ async function fetchAnthropicModels(creds: AiResolvedCredential): Promise { + it('turns provider request limits into a recoverable conversation action', () => { + expect(classifyHttpError('Anthropic', 413, '')).toBe( + "Anthropic could not accept this conversation because it exceeds the provider's request or context limit. Your history is still saved; start a new conversation or choose a model with a larger context window.", + ) + expect(classifyHttpError( + 'OpenRouter', + 400, + JSON.stringify({ error: { message: 'maximum context length exceeded' } }), + )).toContain('Your history is still saved; start a new conversation') + expect(classifyHttpFailure( + 'OpenRouter', + 400, + JSON.stringify({ error: { code: 'too_many_images', message: 'Bad request' } }), + ).kind).toBe('replayOverflow') + }) + + it('does not relabel unrelated bad requests as context exhaustion', () => { + expect(classifyHttpError( + 'OpenAI', + 400, + JSON.stringify({ error: { message: 'Invalid tool schema' } }), + )).toBe('OpenAI error (400): Invalid tool schema.') + }) +}) diff --git a/server/ai/drivers/http/errors.ts b/server/ai/drivers/http/errors.ts index 9a4de30f1..9b8f436b6 100644 --- a/server/ai/drivers/http/errors.ts +++ b/server/ai/drivers/http/errors.ts @@ -29,18 +29,61 @@ export function classifyHttpError( status: number, bodyText: string, ): string { + return classifyHttpFailure(providerLabel, status, bodyText).message +} + +export interface ProviderHttpFailure { + kind: 'replayOverflow' | 'generic' + message: string +} + +/** Structured classification lets the tool loop retry only replay overflows. */ +export function classifyHttpFailure( + providerLabel: string, + status: number, + bodyText: string, +): ProviderHttpFailure { const detail = extractErrorMessage(bodyText) if (status === 401 || status === 403) { - return `${providerLabel} authentication failed. Check your API key in /admin/ai/providers.` + return { + kind: 'generic', + message: `${providerLabel} authentication failed. Check your API key in /admin/ai/providers.`, + } } if (status === 402 || status === 429) { - return `${providerLabel} quota or rate limit reached${detail ? `: ${detail}` : ''}. Check your account balance.` + return { + kind: 'generic', + message: `${providerLabel} quota or rate limit reached${detail ? `: ${detail}` : ''}. Check your account balance.`, + } + } + if (requestExceedsProviderContext(status, bodyText, detail)) { + return { + kind: 'replayOverflow', + message: `${providerLabel} could not accept this conversation because it exceeds the provider's request or context limit${detail ? `: ${detail}` : ''}. Your history is still saved; start a new conversation or choose a model with a larger context window.`, + } } if (status >= 500) { - return `${providerLabel} service error (${status})${detail ? `: ${detail}` : ''}. Please try again.` + return { + kind: 'generic', + message: `${providerLabel} service error (${status})${detail ? `: ${detail}` : ''}. Please try again.`, + } + } + return { + kind: 'generic', + message: `${providerLabel} error (${status})${detail ? `: ${detail}` : ''}.`, } - return `${providerLabel} error (${status})${detail ? `: ${detail}` : ''}.` +} + +function requestExceedsProviderContext( + status: number, + bodyText: string, + detail: string | null, +): boolean { + if (status === 413) return true + if (status !== 400) return false + const providerSignal = `${detail ?? ''} ${bodyText}` + return /(?:context.{0,24}(?:length|limit|window|exceed)|maximum.{0,16}tokens|too[_ ]many[_ ]tokens|request[_ ].{0,16}(?:too[_ ]large|exceed)|input[_ ]too[_ ]long|too[_ ]many[_ ]images|image.{0,16}(?:count|limit|maximum))/i.test(providerSignal) } /** diff --git a/server/ai/drivers/http/toolLoop.ts b/server/ai/drivers/http/toolLoop.ts index 31f2d8a13..a09c8fd2e 100644 --- a/server/ai/drivers/http/toolLoop.ts +++ b/server/ai/drivers/http/toolLoop.ts @@ -24,6 +24,8 @@ */ import type { + AiContentBlock, + AiMessage, AiStreamEvent, AiTool, AiToolOutput, @@ -31,7 +33,10 @@ import type { import type { AiStreamRequest } from '../types' import { parseSseStream, type SseFrame } from './sse' import { executeAiTool } from './execTool' -import { isAbortError, classifyHttpError } from './errors' +import { isAbortError, classifyHttpFailure } from './errors' + +export const PROVIDER_RETRY_IMAGE_OMITTED = + '[Earlier attached images omitted after the provider rejected the full conversation context.]' /** A resolved tool call the model issued this turn. */ export interface TurnToolCall { @@ -106,8 +111,10 @@ export async function* runToolLoop( req: AiStreamRequest, ): AsyncIterable { const toolsByName = new Map(req.tools.map((t) => [t.name, t])) - const messages = adapter.mapHistory(req) + let messages = adapter.mapHistory(req) const headers = adapter.buildHeaders(req) + let initialProviderRound = true + let replayOverflowRetried = false // Track tool-result messages that carry heavy evidence (screenshots, // full-page HTML/CSS). Once superseded they describe stale page state and are @@ -147,7 +154,16 @@ export async function* runToolLoop( if (!res.ok) { const bodyText = await res.text().catch(() => '') console.error(`[ai/${adapter.label.toLowerCase()}] HTTP ${res.status}:`, bodyText.slice(0, 500)) - yield { type: 'error', message: classifyHttpError(adapter.label, res.status, bodyText) } + const failure = classifyHttpFailure(adapter.label, res.status, bodyText) + if (initialProviderRound && !replayOverflowRetried && failure.kind === 'replayOverflow') { + const projected = elideHistoricalUserImages(req.messages) + if (projected) { + replayOverflowRetried = true + messages = adapter.mapHistory({ ...req, messages: projected }) + continue + } + } + yield { type: 'error', message: failure.message } return } @@ -170,6 +186,7 @@ export async function* runToolLoop( if (req.signal.aborted) return const turn = translator.finish() + initialProviderRound = false if (turn.usage) { promptTokens += turn.usage.promptTokens completionTokens += turn.usage.completionTokens @@ -233,6 +250,42 @@ export async function* runToolLoop( } } +/** + * One provider-directed retry projection: retain the newest/current user + * turn verbatim and replace images on earlier user turns with one breadcrumb + * per turn. Persistence and the caller-owned history remain untouched. + */ +function elideHistoricalUserImages(messages: readonly AiMessage[]): AiMessage[] | null { + let newestUserIndex = -1 + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index]?.role === 'user') { + newestUserIndex = index + break + } + } + if (newestUserIndex <= 0) return null + + let changed = false + const projected = messages.map((message, messageIndex): AiMessage => { + if (messageIndex >= newestUserIndex || message.role !== 'user') return message + if (!message.content.some((block) => block.kind === 'image')) return message + + changed = true + let breadcrumbAdded = false + const content: AiContentBlock[] = [] + for (const block of message.content) { + if (block.kind !== 'image') { + content.push(block) + } else if (!breadcrumbAdded) { + content.push({ kind: 'text', text: PROVIDER_RETRY_IMAGE_OMITTED }) + breadcrumbAdded = true + } + } + return { role: 'user', content } + }) + return changed ? projected : null +} + // --------------------------------------------------------------------------- // Per-tool input preparation // --------------------------------------------------------------------------- @@ -246,7 +299,11 @@ export async function* runToolLoop( function prepareToolInput(call: TurnToolCall, req: AiStreamRequest): unknown { if (call.name === 'site_render_snapshot') { const base = call.input && typeof call.input === 'object' ? call.input : {} - return { ...base, captureScreenshot: req.modelCapabilities.visionInput } + return { + ...base, + captureScreenshot: + req.modelCapabilities.visionInput && req.modelCapabilities.toolResultImages, + } } return call.input } diff --git a/server/ai/drivers/modelCapabilities.ts b/server/ai/drivers/modelCapabilities.ts new file mode 100644 index 000000000..e6825594c --- /dev/null +++ b/server/ai/drivers/modelCapabilities.ts @@ -0,0 +1,120 @@ +import type { + AiProvider, + AiProviderCapabilities, + AiResolvedCredential, +} from './types' + +const CAPABILITY_CACHE_TTL_MS = 5 * 60 * 1000 +const CAPABILITY_LOOKUP_TIMEOUT_MS = 10_000 +const MAX_CACHE_ENTRIES_PER_DRIVER = 256 + +interface CapabilityCacheEntry { + expiresAt: number + value: AiProviderCapabilities +} + +const capabilityCache = new WeakMap>() +const capabilityLookups = new WeakMap>>() + +/** + * Resolve the selected model's runtime capabilities. Providers with + * model-specific capability metadata own the authoritative lookup; the shared + * layer bounds it with a short cache and de-duplicates concurrent requests. + */ +export async function resolveModelCapabilities( + driver: AiProvider, + credentials: AiResolvedCredential, + modelId: string, +): Promise { + const fallback = driver.capabilities(modelId) + if (!driver.resolveCapabilities) return fallback + + const key = `${credentialRevisionKey(credentials)}\0${modelId}` + const cache = mapFor(capabilityCache, driver) + const cached = cache.get(key) + if (cached && cached.expiresAt > Date.now()) return cached.value + if (cached) cache.delete(key) + + const lookups = mapFor(capabilityLookups, driver) + const existingLookup = lookups.get(key) + if (existingLookup) return existingLookup + + // Store the handled promise, not the raw provider request. Every concurrent + // waiter must receive the same fail-closed result when discovery rejects. + const lookup = (async () => { + try { + const value = await resolveWithTimeout(driver, credentials, modelId) + cache.set(key, { expiresAt: Date.now() + CAPABILITY_CACHE_TTL_MS, value }) + if (cache.size > MAX_CACHE_ENTRIES_PER_DRIVER) { + const oldestKey = cache.keys().next().value + if (oldestKey !== undefined) cache.delete(oldestKey) + } + return value + } catch (err) { + console.error(`[ai/${driver.id}] model capability lookup failed:`, err) + // A provider-specific lookup exists because its static vision flag is not + // authoritative. Network/schema failures therefore fail closed for image + // input while retaining the driver's safe defaults for other features. + return { ...fallback, visionInput: false } + } finally { + lookups.delete(key) + } + })() + lookups.set(key, lookup) + return lookup +} + +function mapFor( + root: WeakMap>, + driver: AiProvider, +): Map { + const existing = root.get(driver) + if (existing) return existing + const created = new Map() + root.set(driver, created) + return created +} + +function credentialRevisionKey(credentials: AiResolvedCredential): string { + // Credentials are edited in place. Include backend/auth material so a base + // URL or key rotation cannot reuse a positive result from the old backend; + // hash the secret rather than retaining it in a Map key. + return [ + credentials.id, + credentials.authMode, + credentials.baseUrl ?? '', + stableHash(credentials.apiKey ?? ''), + ].join('\0') +} + +function stableHash(value: string): string { + let hash = 2166136261 + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index) + hash = Math.imul(hash, 16777619) + } + return (hash >>> 0).toString(16) +} + +async function resolveWithTimeout( + driver: AiProvider, + credentials: AiResolvedCredential, + modelId: string, +): Promise { + const controller = new AbortController() + let timeoutId: ReturnType | null = null + const timeout = new Promise((_resolve, reject) => { + timeoutId = setTimeout(() => { + controller.abort() + reject(new Error(`Model capability lookup timed out after ${CAPABILITY_LOOKUP_TIMEOUT_MS}ms.`)) + }, CAPABILITY_LOOKUP_TIMEOUT_MS) + }) + try { + return await Promise.race([ + driver.resolveCapabilities!(credentials, modelId, controller.signal), + timeout, + ]) + } finally { + if (timeoutId) clearTimeout(timeoutId) + } +} diff --git a/server/ai/drivers/modelList.ts b/server/ai/drivers/modelList.ts new file mode 100644 index 000000000..25e15982e --- /dev/null +++ b/server/ai/drivers/modelList.ts @@ -0,0 +1,42 @@ +import type { AiProvider, AiProviderModel, AiResolvedCredential } from './types' + +const MODEL_LIST_TIMEOUT_MS = 10_000 + +/** + * Resolve a provider catalogue with both caller cancellation and a server-side + * deadline. The race releases the HTTP handler even if a future driver forgets + * to honour the signal; current fetch-based drivers also stop their upstream + * request when the composed controller aborts. + */ +export async function listProviderModels( + driver: AiProvider, + credentials: AiResolvedCredential, + parentSignal?: AbortSignal, +): Promise { + const controller = new AbortController() + const abortFromParent = () => controller.abort(parentSignal?.reason) + if (parentSignal?.aborted) abortFromParent() + else parentSignal?.addEventListener('abort', abortFromParent, { once: true }) + + const timeoutId = setTimeout(() => { + controller.abort(new Error('Model catalogue request timed out.')) + }, MODEL_LIST_TIMEOUT_MS) + let rejectAborted!: (reason?: unknown) => void + const aborted = new Promise((_resolve, reject) => { rejectAborted = reject }) + const rejectAbort = () => rejectAborted( + controller.signal.reason ?? new DOMException('The operation was aborted.', 'AbortError'), + ) + if (controller.signal.aborted) rejectAbort() + else controller.signal.addEventListener('abort', rejectAbort, { once: true }) + + try { + return await Promise.race([ + driver.listModels(credentials, controller.signal), + aborted, + ]) + } finally { + clearTimeout(timeoutId) + parentSignal?.removeEventListener('abort', abortFromParent) + controller.signal.removeEventListener('abort', rejectAbort) + } +} diff --git a/server/ai/drivers/ollama.ts b/server/ai/drivers/ollama.ts index 3e4767020..3e3ac017c 100644 --- a/server/ai/drivers/ollama.ts +++ b/server/ai/drivers/ollama.ts @@ -16,6 +16,7 @@ */ import { Type, parseValue } from '@core/utils/typeboxHelpers' +import { isAbortError } from '@core/http' import { type AiAuthMode, type AiProviderId, @@ -23,6 +24,7 @@ import { } from '../runtime/types' import type { AiProvider, + AiProviderCapabilities, AiProviderModel, AiResolvedCredential, AiStreamRequest, @@ -31,6 +33,7 @@ import { runToolLoop } from './http/toolLoop' import { makeChatCompletionsAdapter, trimSlash } from './http/chatCompletions' const SUPPORTED_AUTH_MODES: AiAuthMode[] = ['baseUrl'] +const OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY = 6 // Ollama models vary per-install. Defaults are common picks as of May 2026 and // only surface when the `/api/tags` catalogue fetch fails. @@ -40,21 +43,21 @@ const FALLBACK_MODELS: AiProviderModel[] = [ label: 'Llama 4', tier: 'smart', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, }, { id: 'llama3.3', label: 'Llama 3.3', tier: 'balanced', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, }, { id: 'qwen3', label: 'Qwen 3', tier: 'balanced', catalogueSource: 'fallback', - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, }, ] @@ -64,18 +67,23 @@ export const ollamaDriver: AiProvider = { supportedAuthModes: SUPPORTED_AUTH_MODES, capabilities(modelId: string) { - const model = FALLBACK_MODELS.find((m) => m.id === modelId) - return model?.capabilities ?? { - toolCalling: true, - visionInput: false, - promptCache: false, - streaming: true, + return fallbackCapabilities(modelId) + }, + + async resolveCapabilities(creds: AiResolvedCredential, modelId: string, signal: AbortSignal) { + const declared = await fetchOllamaDeclaredCapabilities(creds, modelId, signal) + const fallback = fallbackCapabilities(modelId) + if (!declared) return { ...fallback, visionInput: false } + return { + ...fallback, + toolCalling: declared.includes('tools'), + visionInput: declared.includes('vision'), } }, - async listModels(creds: AiResolvedCredential) { + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { if (!creds.baseUrl) return FALLBACK_MODELS - return fetchOllamaModels(creds.baseUrl) + return fetchOllamaModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -112,23 +120,92 @@ const OllamaTagsSchema = Type.Object({ ), }) -async function fetchOllamaModels(baseUrl: string): Promise { +const OllamaShowSchema = Type.Object( + { capabilities: Type.Optional(Type.Array(Type.String())) }, + { additionalProperties: true }, +) + +function fallbackCapabilities(modelId: string): AiProviderCapabilities { + const model = FALLBACK_MODELS.find((candidate) => candidate.id === modelId) + return model?.capabilities ?? { + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, + } +} + +async function fetchOllamaModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { + const baseUrl = creds.baseUrl + if (!baseUrl) return FALLBACK_MODELS try { - const res = await fetch(`${trimSlash(baseUrl)}/api/tags`) + const res = await fetch(`${trimSlash(baseUrl)}/api/tags`, { + headers: ollamaHeaders(creds.apiKey), + signal, + }) if (!res.ok) return FALLBACK_MODELS const parsed = parseValue(OllamaTagsSchema, await res.json()) - const models = (parsed.models ?? []) + const modelIds = (parsed.models ?? []) .map((m) => m.name ?? m.model) .filter((id): id is string => typeof id === 'string' && id.length > 0) - .map((id) => ({ - id, - label: id, - catalogueSource: 'live' as const, - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + const models: AiProviderModel[] = [] + for (let offset = 0; offset < modelIds.length; offset += OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY) { + signal?.throwIfAborted() + const batch = modelIds.slice(offset, offset + OLLAMA_CAPABILITY_LOOKUP_CONCURRENCY) + const resolvedBatch = await Promise.all(batch.map(async (id) => { + const declared = await fetchOllamaDeclaredCapabilities(creds, id, signal).catch((err) => { + if (signal?.aborted || isAbortError(err)) throw err + return null + }) + return { + id, + label: id, + catalogueSource: 'live' as const, + capabilities: { + toolCalling: declared ? declared.includes('tools') : true, + visionInput: declared?.includes('vision') ?? false, + toolResultImages: false, + promptCache: false, + streaming: true, + }, + } satisfies AiProviderModel })) + models.push(...resolvedBatch) + } return models.length > 0 ? models : FALLBACK_MODELS } catch (err) { + if (signal?.aborted || isAbortError(err)) throw err console.error('[ai/ollama] models request failed:', err) return FALLBACK_MODELS } } + +async function fetchOllamaDeclaredCapabilities( + creds: AiResolvedCredential, + modelId: string, + signal?: AbortSignal, +): Promise { + if (!creds.baseUrl) return null + const res = await fetch(`${trimSlash(creds.baseUrl)}/api/show`, { + method: 'POST', + headers: ollamaHeaders(creds.apiKey, true), + body: JSON.stringify({ model: modelId }), + signal, + }) + if (!res.ok) { + throw new Error(`[ai/ollama] model capability request failed: ${res.status} ${res.statusText}`) + } + const parsed = parseValue(OllamaShowSchema, await res.json()) + return parsed.capabilities ?? null +} + +function ollamaHeaders(apiKey: string | null, json = false): Record { + const headers: Record = {} + if (json) headers['content-type'] = 'application/json' + if (apiKey) headers.Authorization = `Bearer ${apiKey}` + return headers +} diff --git a/server/ai/drivers/openai.ts b/server/ai/drivers/openai.ts index c3071c3ed..dda2049fa 100644 --- a/server/ai/drivers/openai.ts +++ b/server/ai/drivers/openai.ts @@ -65,20 +65,21 @@ export const openaiDriver: AiProvider = { capabilities(_modelId: string) { // OpenAI's catalogue reports no capability flags, so we can't key off the - // model id. Current GPT/o-series chat models tool-call and accept images; - // the tool loop only reads `visionInput` here, so a permissive default is - // correct. (Prompt caching is automatic on OpenAI, not the Anthropic-style + // model id. Current GPT/o-series chat models tool-call and accept user + // images, so a permissive input default is correct; Responses tool outputs + // remain text-only. (Prompt caching is automatic on OpenAI, not the Anthropic-style // cache_control prefix this flag controls, so it stays false.) return { toolCalling: true, visionInput: true, + toolResultImages: false, promptCache: false, streaming: true, } }, - async listModels(creds: AiResolvedCredential) { - return fetchOpenAiModels(creds) + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { + return fetchOpenAiModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -136,11 +137,15 @@ function stableHash(value: string): string { * moderation, …) with no metadata, so we filter to the chat/reasoning families * and derive the label + tier from the id — heuristic, not authoritative. */ -async function fetchOpenAiModels(creds: AiResolvedCredential): Promise { +async function fetchOpenAiModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { if (creds.authMode !== 'apiKey' || !creds.apiKey) return [] const res = await fetch(OPENAI_MODELS_ENDPOINT, { headers: { Authorization: `Bearer ${creds.apiKey}` }, + signal, }) if (!res.ok) { throw new Error(`[ai/openai] models request failed: ${res.status} ${res.statusText}`) @@ -157,6 +162,7 @@ async function fetchOpenAiModels(creds: AiResolvedCredential): Promise { @@ -86,11 +88,12 @@ const ModelsResponseSchema = Type.Object( async function fetchOpenAiCompatibleModels( baseUrl: string, apiKey: string | null, + signal?: AbortSignal, ): Promise { try { const headers: Record = {} if (apiKey) headers.Authorization = `Bearer ${apiKey}` - const res = await fetch(`${normalizeOpenAiBaseUrl(baseUrl)}/v1/models`, { headers }) + const res = await fetch(`${normalizeOpenAiBaseUrl(baseUrl)}/v1/models`, { headers, signal }) if (!res.ok) return [] const parsed = parseValue(ModelsResponseSchema, await res.json()) return parsed.data.map((m) => ({ @@ -100,6 +103,7 @@ async function fetchOpenAiCompatibleModels( capabilities: { ...GENERIC_CAPABILITIES }, })) } catch (err) { + if (signal?.aborted || isAbortError(err)) throw err console.error('[ai/openai-compatible] models request failed:', err) return [] } diff --git a/server/ai/drivers/openrouter.ts b/server/ai/drivers/openrouter.ts index 4ffcb5d3a..e656e1cee 100644 --- a/server/ai/drivers/openrouter.ts +++ b/server/ai/drivers/openrouter.ts @@ -43,6 +43,7 @@ const OPENROUTER_ENDPOINT = `${OPENROUTER_BASE_URL}/responses` const DEFAULT_CAPABILITIES: AiProviderCapabilities = { toolCalling: true, visionInput: false, + toolResultImages: false, promptCache: false, streaming: true, } @@ -67,8 +68,14 @@ export const openrouterDriver: AiProvider = { return DEFAULT_CAPABILITIES }, - async listModels(creds: AiResolvedCredential) { - return fetchOpenRouterModels(creds) + async resolveCapabilities(creds: AiResolvedCredential, modelId: string, signal: AbortSignal) { + const models = await fetchOpenRouterModels(creds, signal) + return models.find((model) => model.id === modelId)?.capabilities + ?? DEFAULT_CAPABILITIES + }, + + async listModels(creds: AiResolvedCredential, signal?: AbortSignal) { + return fetchOpenRouterModels(creds, signal) }, async *stream(req: AiStreamRequest): AsyncIterable { @@ -125,13 +132,16 @@ function perMTok(value: string | undefined): number | null { return perToken * 1_000_000 } -async function fetchOpenRouterModels(creds: AiResolvedCredential): Promise { +async function fetchOpenRouterModels( + creds: AiResolvedCredential, + signal?: AbortSignal, +): Promise { const headers: Record = {} // The catalogue endpoint is public, but sending the bearer lets per-key // availability (e.g. BYOK-only models) reflect in the list. if (creds.apiKey) headers.Authorization = `Bearer ${creds.apiKey}` - const res = await fetch(`${OPENROUTER_BASE_URL}/models`, { headers }) + const res = await fetch(`${OPENROUTER_BASE_URL}/models`, { headers, signal }) if (!res.ok) { throw new Error(`[ai/openrouter] models request failed: ${res.status} ${res.statusText}`) } @@ -156,6 +166,7 @@ async function fetchOpenRouterModels(creds: AiResolvedCredential): Promise + /** + * Optional selected-model lookup for providers whose vision/tool support is + * model-specific. The chat boundary caches and de-duplicates this result; + * drivers should query the narrowest authoritative provider endpoint they + * expose. A failed lookup is treated as non-vision by the caller. + */ + resolveCapabilities?( + credentials: AiResolvedCredential, + modelId: string, + signal: AbortSignal, + ): Promise + + listModels( + credentials: AiResolvedCredential, + signal?: AbortSignal, + ): Promise /** * Run one agent turn. Yields canonical AiStreamEvents as the model diff --git a/server/ai/handlers/chat.ts b/server/ai/handlers/chat.ts index 06b3f5dc8..5485ba2d8 100644 --- a/server/ai/handlers/chat.ts +++ b/server/ai/handlers/chat.ts @@ -4,7 +4,7 @@ * Opens an NDJSON stream against a chat. Body: * { * conversationId: string, - * prompt: string, + * content: Array<{ kind: 'text' | 'image', ... }>, * snapshot?: unknown // scope-specific per-request context * } * @@ -19,8 +19,20 @@ * 6. Streams NDJSON events back as the driver produces them. */ -import { Type, safeParseValue } from '@core/utils/typeboxHelpers' -import { jsonResponse, readValidatedBody, badRequest } from '../../http' +import { safeParseValue } from '@core/utils/typeboxHelpers' +import { + AI_CHAT_MAX_REQUEST_BYTES, + AiChatRequestBodySchema, + type AiChatRequestBody, + type AiContentBlock, +} from '@core/ai' +import { + RequestBodyTooLargeError, + badRequest, + jsonResponse, + payloadTooLarge, + readValidatedBody, +} from '../../http' import { requireCapability } from '../../auth/authz' import type { DbClient } from '../../db/client' import { createAuditEvent } from '../../repositories/audit' @@ -28,17 +40,26 @@ import { appendMessage, listMessagesForConversation, readConversationForUser, - updateConversationForUser, + replaceDefaultConversationTitle, deriveConversationTitle, DEFAULT_CONVERSATION_TITLE, } from '../conversations/store' -import { buildMessageHistory } from '../conversations/history' +import { + buildMessageHistory, + projectUserImagesForModel, +} from '../conversations/history' import { readCredentialForUser, resolveCredentialForDriver, touchCredentialLastUsed, } from '../credentials/store' import { resolveDriver } from '../drivers' +import { resolveModelCapabilities } from '../drivers/modelCapabilities' +import { + AiImageInputError, + canonicaliseAiUserContent, + preflightAiUserContent, +} from '../inputImages' import { selectToolsForScope } from '../tools' import { buildSiteSystemPrompt, @@ -62,16 +83,9 @@ import type { } from '../runtime/types' import type { AiStreamRequest } from '../drivers/types' -const ChatRequestBodySchema = Type.Object({ - conversationId: Type.String({ minLength: 1 }), - prompt: Type.String({ minLength: 1 }), - // snapshot stays loose here — scope-specific shape; tools cast it inside - // their handlers. The handler narrows below based on the conversation's - // scope before passing to the system-prompt builder. - snapshot: Type.Optional(Type.Unknown()), -}) - const VALID_SCOPES: ToolScope[] = ['site', 'content', 'data', 'plugin'] +const activeChatConversations = new Set() +const REQUEST_ABORTED = Symbol('request-aborted') /** * Match `/admin/api/ai/chat/:scope`. Returns `null` if path doesn't match. @@ -105,9 +119,19 @@ async function handleAiChat( if (userOrResponse instanceof Response) return userOrResponse const user = userOrResponse - const chatBody = await readValidatedBody(req, ChatRequestBodySchema) + let chatBody: AiChatRequestBody | null + try { + chatBody = await readValidatedBody(req, AiChatRequestBodySchema, { + maxBytes: AI_CHAT_MAX_REQUEST_BYTES, + }) + } catch (err) { + if (err instanceof RequestBodyTooLargeError) { + return payloadTooLarge('Chat request is too large.') + } + throw err + } if (!chatBody) return badRequest('Invalid request body.') - const { conversationId, prompt, snapshot } = chatBody + const { conversationId, content, snapshot } = chatBody const conversation = await readConversationForUser(db, user.id, conversationId) if (!conversation) { @@ -142,54 +166,159 @@ async function handleAiChat( } const driver = resolveDriver(credential.providerId) - // Capability-filtered toolset. Callers without `ai.tools.write` only see - // read tools registered with the driver — the model has no way to - // emit a write call. See B6 in the capabilities review. + let preflight: ReturnType + try { + preflight = preflightAiUserContent(content) + } catch (err) { + if (err instanceof AiImageInputError) { + return err.status === 413 ? payloadTooLarge(err.message) : badRequest(err.message) + } + throw err + } + const requestedImage = preflight.images.length > 0 + + // Resolve every selected model, not only image-bearing turns: the same + // authoritative flag also gates browser-tool screenshots. Model-specific + // drivers are cached/de-duplicated by the shared resolver. + const modelCapabilities = await waitForRequest( + resolveModelCapabilities(driver, resolvedCredential, conversation.modelId), + req.signal, + ) + if (modelCapabilities === REQUEST_ABORTED) return clientClosedRequest() const tools = selectToolsForScope(scope, user.capabilities) + if (requestedImage && !modelCapabilities.visionInput) { + return jsonResponse( + { error: 'The selected model does not support image input. Choose a vision-capable model.' }, + { status: 422 }, + ) + } + if (tools.length > 0 && !modelCapabilities.toolCalling) { + return jsonResponse( + { error: 'The selected model does not support tool calling. Choose an agent-capable model.' }, + { status: 422 }, + ) + } + if (req.signal.aborted) return clientClosedRequest() - // Append the user's message BEFORE streaming so it's persisted even if - // the stream aborts mid-response. - await appendMessage(db, conversation.id, { - role: 'user', - content: [{ kind: 'text', text: prompt }], - }) + // One provider stream may write a conversation at a time so concurrent tabs + // cannot interleave assistant/tool rows. Acquire admission before the + // expensive Sharp boundary: the retryable loser must not decode eight images + // only to discover that another request already owns the conversation. + const releaseConversation = acquireConversationStream(conversation.id) + if (!releaseConversation) { + return jsonResponse( + { error: 'This conversation is already generating a response. Wait for it to finish.' }, + { status: 409 }, + ) + } + if (req.signal.aborted) { + releaseConversation() + return clientClosedRequest() + } - // The first prompt names the conversation: replace the placeholder title - // with an excerpt of what the user asked for. Only fires while the title is - // still the default, so a user-renamed chat is never overwritten. - if (conversation.title === DEFAULT_CONVERSATION_TITLE) { - const derivedTitle = deriveConversationTitle(prompt) - if (derivedTitle) { - await updateConversationForUser(db, user.id, conversation.id, { title: derivedTitle }) - .catch((err) => { console.error('[ai/chat] auto-title failed:', err) }) + // Full decode/re-encode is deliberately after the capability gates so an + // incompatible selected model cannot force needless Sharp work. + let userContent: AiContentBlock[] + try { + userContent = await canonicaliseAiUserContent(preflight, req.signal) + } catch (err) { + releaseConversation() + if (req.signal.aborted) return clientClosedRequest() + if (err instanceof AiImageInputError) { + return err.status === 413 ? payloadTooLarge(err.message) : badRequest(err.message) } + throw err + } + if (req.signal.aborted) { + releaseConversation() + return clientClosedRequest() } - const existingMessages = await listMessagesForConversation(db, conversation.id) - const messages = buildMessageHistory(existingMessages) + let existingRecords: Awaited> + let latestConversation: NonNullable>> + try { + const refreshedConversation = await readConversationForUser(db, user.id, conversation.id) + if (!refreshedConversation) { + releaseConversation() + return jsonResponse({ error: 'Conversation not found' }, { status: 404 }) + } + latestConversation = refreshedConversation + if ( + latestConversation.credentialId !== conversation.credentialId + || latestConversation.modelId !== conversation.modelId + ) { + releaseConversation() + return jsonResponse( + { error: 'The conversation model changed while this message was being prepared. Send again.' }, + { status: 409 }, + ) + } + existingRecords = await listMessagesForConversation(db, conversation.id) + } catch (err) { + releaseConversation() + throw err + } + if (req.signal.aborted) { + releaseConversation() + return clientClosedRequest() + } + const prepared = await (async () => { + try { + // Append the user's message BEFORE streaming so it's persisted even if + // the stream aborts mid-response. + const appendedMessage = await appendMessage(db, conversation.id, { + role: 'user', + content: userContent, + }) - const systemPrompt = buildSystemPromptForScope(scope, snapshot) + // The first prompt names the conversation: replace the placeholder title + // with an excerpt of what the user asked for. Only fires while the title + // is still the default, so a user-renamed chat is never overwritten. + if (latestConversation.title === DEFAULT_CONVERSATION_TITLE) { + const text = userContent.find((block) => block.kind === 'text') + const imageCount = userContent.filter((block) => block.kind === 'image').length + const derivedTitle = text?.kind === 'text' + ? deriveConversationTitle(text.text) + : imageCount === 1 ? 'Image' : 'Images' + if (derivedTitle) { + await replaceDefaultConversationTitle(db, user.id, conversation.id, derivedTitle) + .catch((err) => { console.error('[ai/chat] auto-title failed:', err) }) + } + } - // Capture totals reported by the persister so the audit row can hold - // them when the stream completes (we read them off the conversation row - // diff post-stream — see the post-loop block). - const tokensAtStart = { - prompt: conversation.promptTokensTotal, - completion: conversation.completionTokensTotal, - cost: conversation.costUsdTotal, - } + const messages = projectUserImagesForModel( + buildMessageHistory([...existingRecords, appendedMessage]), + modelCapabilities.visionInput, + ) + const systemPrompt = buildSystemPromptForScope(scope, snapshot) - await createAuditEvent(db, { - actorUserId: user.id, - action: 'ai.chat.started', - targetType: 'ai_conversation', - targetId: conversation.id, - metadata: { - scope, - providerId: credential.providerId, - modelId: conversation.modelId, - }, - }) + // Capture totals reported by the persister so the audit row can hold + // them when the stream completes (we read them off the conversation row + // diff post-stream — see the post-loop block). + const tokensAtStart = { + prompt: latestConversation.promptTokensTotal, + completion: latestConversation.completionTokensTotal, + cost: latestConversation.costUsdTotal, + } + + await createAuditEvent(db, { + actorUserId: user.id, + action: 'ai.chat.started', + targetType: 'ai_conversation', + targetId: conversation.id, + metadata: { + scope, + providerId: credential.providerId, + modelId: conversation.modelId, + }, + }) + return { messages, systemPrompt, tokensAtStart } + } catch (err) { + releaseConversation() + throw err + } + })() + const { messages, systemPrompt, tokensAtStart } = prepared const stream = new ReadableStream({ async start(controller) { @@ -251,7 +380,7 @@ async function handleAiChat( messages, tools, modelId: conversation.modelId, - modelCapabilities: driver.capabilities(conversation.modelId), + modelCapabilities, credentials: resolvedCredential, signal: req.signal, bridge, @@ -274,7 +403,6 @@ async function handleAiChat( emit({ type: 'error', message: `AI chat failed: ${detail}` }) } finally { if (destroyBridge) destroyBridge() - closeStream() // Emit the terminal audit event. Re-read the conversation row to // capture the deltas the persister just committed. try { @@ -301,6 +429,9 @@ async function handleAiChat( // Audit failures must never break the user-visible stream — the // request already finished by the time we hit this branch. console.error('[ai/chat] audit emit failed:', auditErr) + } finally { + releaseConversation() + closeStream() } } }, @@ -310,7 +441,7 @@ async function handleAiChat( status: 200, headers: { 'Content-Type': 'application/x-ndjson', - 'Cache-Control': 'no-cache', + 'Cache-Control': 'private, no-store', 'X-Accel-Buffering': 'no', }, }) @@ -320,6 +451,30 @@ async function handleAiChat( // Helpers // --------------------------------------------------------------------------- +function acquireConversationStream(conversationId: string): (() => void) | null { + if (activeChatConversations.has(conversationId)) return null + activeChatConversations.add(conversationId) + let released = false + return () => { + if (released) return + released = true + activeChatConversations.delete(conversationId) + } +} + +function clientClosedRequest(): Response { + return new Response(null, { status: 499, statusText: 'Client Closed Request' }) +} + +function waitForRequest(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(REQUEST_ABORTED) + return new Promise((resolve, reject) => { + const onAbort = () => resolve(REQUEST_ABORTED) + signal.addEventListener('abort', onAbort, { once: true }) + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + export function buildSystemPromptForScope( scope: ToolScope, snapshot: unknown, diff --git a/server/ai/handlers/conversations.ts b/server/ai/handlers/conversations.ts index c5a21bebc..846a94906 100644 --- a/server/ai/handlers/conversations.ts +++ b/server/ai/handlers/conversations.ts @@ -11,7 +11,12 @@ * return 404). */ +import { + AI_USER_IMAGE_MAX_BASE64_CHARS, + AI_USER_IMAGE_MAX_BYTES, +} from '@core/ai' import { Type } from '@core/utils/typeboxHelpers' +import { binaryResponse } from '../../binary' import { jsonResponse, readValidatedBody, badRequest } from '../../http' import { requireCapability } from '../../auth/authz' import type { DbClient } from '../../db/client' @@ -19,6 +24,7 @@ import { createConversationForUser, listConversationsForUserScope, listMessagesForConversation, + readMessageForUser, readConversationForUser, softDeleteConversationForUser, toConversationDetailView, @@ -51,6 +57,18 @@ export function tryHandleAiConversations( if (pathname === '/admin/api/ai/conversations') { return dispatchCollection(req, db, url) } + const imageMatch = pathname.match( + /^\/admin\/api\/ai\/conversations\/([^/]+)\/messages\/([^/]+)\/images\/(\d+)$/, + ) + if (imageMatch) { + return handleMessageImage( + req, + db, + imageMatch[1]!, + imageMatch[2]!, + Number(imageMatch[3]), + ) + } const match = pathname.match(/^\/admin\/api\/ai\/conversations\/([^/]+)$/) if (match) { return dispatchItem(req, db, match[1]!) @@ -58,6 +76,58 @@ export function tryHandleAiConversations( return null } +async function handleMessageImage( + req: Request, + db: DbClient, + conversationId: string, + messageId: string, + blockIndex: number, +): Promise { + if (req.method !== 'GET') { + return jsonResponse({ error: 'Method not allowed' }, { status: 405 }) + } + const userOrResponse = await requireCapability(req, db, 'ai.chat') + if (userOrResponse instanceof Response) return userOrResponse + + const message = await readMessageForUser( + db, + userOrResponse.id, + conversationId, + messageId, + ) + const block = message?.content[blockIndex] + if (block?.kind !== 'image' || block.mimeType !== 'image/jpeg') return imageNotFound() + + const bytes = decodeStoredJpeg(block.data) + if (!bytes) return imageNotFound() + return binaryResponse(bytes, { + headers: { + 'Content-Type': 'image/jpeg', + 'Content-Length': String(bytes.byteLength), + 'Cache-Control': 'private, no-store', + 'X-Content-Type-Options': 'nosniff', + }, + }) +} + +function decodeStoredJpeg(data: string): Buffer | null { + if (!data || data.length > AI_USER_IMAGE_MAX_BASE64_CHARS || data.length % 4 !== 0) return null + const bytes = Buffer.from(data, 'base64') + if ( + bytes.byteLength === 0 + || bytes.byteLength > AI_USER_IMAGE_MAX_BYTES + || bytes.toString('base64') !== data + || bytes[0] !== 0xff + || bytes[1] !== 0xd8 + || bytes[2] !== 0xff + ) return null + return bytes +} + +function imageNotFound(): Response { + return jsonResponse({ error: 'Conversation image not found' }, { status: 404 }) +} + // --------------------------------------------------------------------------- // Collection // --------------------------------------------------------------------------- @@ -117,7 +187,24 @@ async function handleRead(req: Request, db: DbClient, id: string): Promise conversationImageUrl(id, messageId, blockIndex), + ), + }, + { headers: { 'Cache-Control': 'private, no-store' } }, + ) +} + +function conversationImageUrl( + conversationId: string, + messageId: string, + blockIndex: number, +): string { + return `/admin/api/ai/conversations/${encodeURIComponent(conversationId)}/messages/${encodeURIComponent(messageId)}/images/${blockIndex}` } async function handleUpdate(req: Request, db: DbClient, id: string): Promise { diff --git a/server/ai/handlers/credentials.ts b/server/ai/handlers/credentials.ts index a25d7e270..be264ca6d 100644 --- a/server/ai/handlers/credentials.ts +++ b/server/ai/handlers/credentials.ts @@ -24,6 +24,7 @@ import { updateCredentialForUser, } from '../credentials/store' import { resolveDriver } from '../drivers' +import { listProviderModels } from '../drivers/modelList' import type { AiProviderModel } from '../drivers/types' import type { CredentialRecord } from '../credentials/types' import { listDefaults, setDefaultForScope } from '../defaults/store' @@ -126,7 +127,7 @@ async function handleCreate(req: Request, db: DbClient): Promise { // credential. Never overwrites an existing choice; failures here must not // fail credential creation. try { - await seedEmptyDefaults(db, record, userOrResponse.id) + await seedEmptyDefaults(db, record, userOrResponse.id, req.signal) } catch (err) { console.warn( '[ai/credentials] auto-default skipped - default seeding failed:', @@ -160,6 +161,7 @@ async function seedEmptyDefaults( db: DbClient, record: CredentialRecord, userId: string, + signal?: AbortSignal, ): Promise { const existing = await listDefaults(db) const filled = new Set(existing.map((d) => d.scope)) @@ -172,7 +174,7 @@ async function seedEmptyDefaults( const resolved = await resolveCredentialForDriver(record) apiKeyForRedaction = resolved.apiKey const driver = resolveDriver(record.providerId) - const models = await driver.listModels(resolved) + const models = await listProviderModels(driver, resolved, signal) const liveModels = models.filter((model) => model.catalogueSource !== 'fallback') const top = liveModels.find((m) => m.tier === 'smartest') ?? liveModels[0] topModelId = top?.id ?? null @@ -300,7 +302,7 @@ async function dispatchTest(req: Request, db: DbClient, id: string): Promise { + return canonicaliseAiUserContent(preflightAiUserContent(content), signal) +} + +/** Cheap structural/byte checks that never invoke an image decoder. */ +export function preflightAiUserContent( + content: readonly AiUserContentBlock[], +): AiUserContentPreflight { + const textBlocks = content.filter((block) => block.kind === 'text') + const imageBlocks = content.filter((block) => block.kind === 'image') + + if (textBlocks.length > 1) { + throw new AiImageInputError('A message can contain at most one text block.') + } + if (imageBlocks.length > AI_USER_IMAGE_MAX_PER_MESSAGE) { + throw new AiImageInputError( + `A message can contain at most ${AI_USER_IMAGE_MAX_PER_MESSAGE} images.`, + ) + } + + const text = textBlocks[0]?.text.trim() ?? '' + if (!text && imageBlocks.length === 0) { + throw new AiImageInputError('Message must contain text or an image.') + } + + const imageBytes = imageBlocks.map(preflightAiUserImage) + return { text, images: imageBlocks, imageBytes } +} + +/** Full decoder boundary + canonical block reconstruction after admission gates. */ +export async function canonicaliseAiUserContent( + preflight: AiUserContentPreflight, + signal?: AbortSignal, +): Promise { + // Text precedes the image in the canonical provider-facing order. This also + // removes whitespace-only text so no driver emits an empty text part and + // reconstructs the image so request-only extra fields cannot be persisted. + const canonical: AiContentBlock[] = [] + if (preflight.text) canonical.push({ kind: 'text', text: preflight.text }) + // Decode sequentially. Eight concurrent Sharp pipelines would multiply the + // per-request memory peak for no user-visible benefit. + for (const bytes of preflight.imageBytes) { + signal?.throwIfAborted() + canonical.push(await canonicaliseAiUserImage(bytes, signal)) + // Sharp cannot be interrupted safely mid-pipeline. Check again after the + // active decode so a disconnected request never starts the next image. + signal?.throwIfAborted() + } + return canonical +} + +export async function validateAiUserImage( + image: AiUserImageBlock, + signal?: AbortSignal, +): Promise { + return canonicaliseAiUserImage(preflightAiUserImage(image), signal) +} + +function preflightAiUserImage(image: AiUserImageBlock): Buffer { + const bytes = decodeCanonicalBase64(image.data) + if (bytes.byteLength > AI_USER_IMAGE_MAX_BYTES) { + throw new AiImageInputError( + `Image exceeds the ${formatMegabytes(AI_USER_IMAGE_MAX_BYTES)} MB limit.`, + 413, + ) + } + + if (bytes[0] !== 0xff || bytes[1] !== 0xd8 || bytes[2] !== 0xff) { + throw new AiImageInputError('Image data is not a JPEG.') + } + return bytes +} + +async function canonicaliseAiUserImage( + bytes: Buffer, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + let metadata: sharp.Metadata + try { + metadata = await sharp(bytes).metadata() + } catch (err) { + throw new AiImageInputError('Image data could not be decoded.', 400, { cause: err }) + } + signal?.throwIfAborted() + + if (metadata.format !== 'jpeg' || !metadata.width || !metadata.height) { + throw new AiImageInputError('Image data is not a valid JPEG.') + } + if ( + metadata.width > AI_USER_IMAGE_MAX_EDGE + || metadata.height > AI_USER_IMAGE_MAX_EDGE + || metadata.width * metadata.height > AI_USER_IMAGE_MAX_PIXELS + ) { + throw new AiImageInputError( + `Image dimensions exceed the ${AI_USER_IMAGE_MAX_EDGE}px / ${AI_USER_IMAGE_MAX_PIXELS.toLocaleString()}px limit.`, + 413, + ) + } + + const canonicalBytes = await canonicaliseJpeg( + bytes, + metadata.width, + metadata.height, + signal, + ) + return { + kind: 'image', + mimeType: 'image/jpeg', + data: canonicalBytes.toString('base64'), + } +} + +/** + * Fully decode and re-encode the image before persistence. Sharp strips source + * metadata by default; `failOn: 'warning'` also rejects truncated JPEGs whose + * headers are readable but whose pixel data is incomplete. + */ +async function canonicaliseJpeg( + bytes: Buffer, + sourceWidth: number, + sourceHeight: number, + signal?: AbortSignal, +): Promise { + let width = sourceWidth + let height = sourceHeight + let lastSize = bytes.byteLength + + try { + for (let resizeAttempt = 0; resizeAttempt < 3; resizeAttempt += 1) { + for (const quality of SERVER_JPEG_QUALITIES) { + signal?.throwIfAborted() + let pipeline = sharp(bytes, { + failOn: 'warning', + limitInputPixels: AI_USER_IMAGE_MAX_PIXELS, + }) + .rotate() + .flatten({ background: { r: 255, g: 255, b: 255 } }) + + if (resizeAttempt > 0) { + pipeline = pipeline.resize({ + width, + height, + fit: 'inside', + withoutEnlargement: true, + }) + } + + const output = await pipeline.jpeg({ quality }).toBuffer() + // Sharp does not expose an abortable encode. Stop immediately after + // the active pipeline instead of beginning another quality/resize. + signal?.throwIfAborted() + lastSize = output.byteLength + if (output.byteLength <= AI_USER_IMAGE_MAX_BYTES) return output + } + + const scale = Math.max( + 0.5, + Math.sqrt(AI_USER_IMAGE_MAX_BYTES / lastSize) * 0.9, + ) + width = Math.max(1, Math.floor(width * scale)) + height = Math.max(1, Math.floor(height * scale)) + } + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === 'AbortError')) throw err + throw new AiImageInputError('Image data could not be fully decoded.', 400, { cause: err }) + } + + throw new AiImageInputError( + `Image could not be reduced below the ${formatMegabytes(AI_USER_IMAGE_MAX_BYTES)} MB limit.`, + 413, + ) +} + +function decodeCanonicalBase64(data: string): Buffer { + if (!data || data.length % 4 !== 0 || !CANONICAL_BASE64.test(data)) { + throw new AiImageInputError('Image data must be canonical base64.') + } + const bytes = Buffer.from(data, 'base64') + if (bytes.toString('base64') !== data) { + throw new AiImageInputError('Image data must be canonical base64.') + } + return bytes +} + +function formatMegabytes(bytes: number): string { + return (bytes / 1_000_000).toFixed(1) +} diff --git a/server/ai/tools/site/writeTools.ts b/server/ai/tools/site/writeTools.ts index 2f7b04199..212b67cbe 100644 --- a/server/ai/tools/site/writeTools.ts +++ b/server/ai/tools/site/writeTools.ts @@ -400,7 +400,7 @@ const renderSnapshotTool: AiTool = { scope: 'site', execution: 'browser', description: - "Inspect the rendered canvas. Returns a layout report: viewport size, per-node bounding boxes, image-load status, and warnings (overflow / broken-image / invisible-node) — enough to catch most layout bugs in text. On a vision-capable model a screenshot is also attached as an image. Pass `breakpointId` to choose which breakpoint frame (defaults to active). Pass `nodeId` to capture just that node's subtree — a sharper, cheaper image than the whole page, and a report scoped to that section with coordinates relative to the node; omit `nodeId` to capture the full page.", + "Inspect the rendered canvas. Returns a layout report: viewport size, per-node bounding boxes, image-load status, and warnings (overflow / broken-image / invisible-node) — enough to catch most layout bugs in text. When the active provider supports native image-bearing tool results, a screenshot is also attached as an image. Pass `breakpointId` to choose which breakpoint frame (defaults to active). Pass `nodeId` to capture just that node's subtree — a sharper, cheaper image than the whole page, and a report scoped to that section with coordinates relative to the node; omit `nodeId` to capture the full page.", inputSchema: RenderSnapshotInputSchema, } diff --git a/src/__tests__/agent/agentImageActions.test.ts b/src/__tests__/agent/agentImageActions.test.ts new file mode 100644 index 000000000..dd68d1535 --- /dev/null +++ b/src/__tests__/agent/agentImageActions.test.ts @@ -0,0 +1,209 @@ +import { afterEach, describe, expect, it, mock } from 'bun:test' +import { + agentImageFilename, + copyAgentImageToClipboard, + downloadAgentImage, + isAgentImageMediaSavePending, + readAgentImageBlob, + saveAgentImageToMedia, +} from '@site/panels/AgentPanel/agentImageActions' +import type { AgentPreviewImage } from '@site/panels/AgentPanel/agentImageTypes' + +const originalFetch = globalThis.fetch + +afterEach(() => { + globalThis.fetch = originalFetch +}) + +function image(overrides: Partial = {}): AgentPreviewImage { + return { + id: 'image-1', + src: '/image-1', + alt: 'Test image', + filename: 'Reference source.png', + ...overrides, + } +} + +function restoreProperty( + target: object, + key: PropertyKey, + descriptor: PropertyDescriptor | undefined, +): void { + if (descriptor) Object.defineProperty(target, key, descriptor) + else Reflect.deleteProperty(target, key) +} + +describe('agent image actions', () => { + it('reads only non-empty image responses and derives a MIME-correct safe filename', async () => { + const blob = await readAgentImageBlob(image(), { + fetchImpl: async () => new Response(new Uint8Array([1, 2, 3]), { + headers: { 'content-type': 'image/jpeg' }, + }), + }) + + expect(blob.type).toBe('image/jpeg') + expect(agentImageFilename(image(), blob)).toBe('Reference-source.jpg') + + await expect(readAgentImageBlob(image(), { + fetchImpl: async () => new Response('not an image', { + headers: { 'content-type': 'text/plain' }, + }), + })).rejects.toThrow('not an image') + }) + + it('passes promised PNG bytes to the clipboard in the initiating call stack', async () => { + const clipboardDescriptor = Object.getOwnPropertyDescriptor(navigator, 'clipboard') + const clipboardItemDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'ClipboardItem') + const canvasPrototype = Object.getPrototypeOf(document.createElement('canvas')) as object + const bitmapDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + const contextDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'getContext') + const toBlobDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'toBlob') + const itemPayloads: Array>> = [] + const write = mock(async (_items: ClipboardItem[]) => {}) + const closeBitmap = mock(() => {}) + class TestClipboardItem { + constructor(payload: Record>) { + itemPayloads.push(payload) + } + } + Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { write }, + }) + Object.defineProperty(globalThis, 'ClipboardItem', { + configurable: true, + value: TestClipboardItem, + }) + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: async () => ({ width: 40, height: 30, close: closeBitmap }), + }) + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value: () => ({ drawImage: () => {} }), + }) + Object.defineProperty(canvasPrototype, 'toBlob', { + configurable: true, + value: (callback: BlobCallback) => { + callback(new Blob([new Uint8Array([4, 5, 6])], { type: 'image/png' })) + }, + }) + globalThis.fetch = mock(async () => new Response(new Uint8Array([1, 2, 3]), { + headers: { 'content-type': 'image/jpeg' }, + })) as typeof fetch + + try { + await copyAgentImageToClipboard(image()) + expect(write).toHaveBeenCalledTimes(1) + expect(itemPayloads).toHaveLength(1) + const png = await itemPayloads[0]?.['image/png'] + expect(png).toBeInstanceOf(Blob) + expect((png as Blob).type).toBe('image/png') + expect(closeBitmap).toHaveBeenCalledTimes(1) + } finally { + restoreProperty(navigator, 'clipboard', clipboardDescriptor) + restoreProperty(globalThis, 'ClipboardItem', clipboardItemDescriptor) + restoreProperty(globalThis, 'createImageBitmap', bitmapDescriptor) + restoreProperty(canvasPrototype, 'getContext', contextDescriptor) + restoreProperty(canvasPrototype, 'toBlob', toBlobDescriptor) + } + }) + + it('downloads with a MIME-correct filename and releases the object URL', async () => { + const createDescriptor = Object.getOwnPropertyDescriptor(URL, 'createObjectURL') + const revokeDescriptor = Object.getOwnPropertyDescriptor(URL, 'revokeObjectURL') + const clickDescriptor = Object.getOwnPropertyDescriptor(HTMLAnchorElement.prototype, 'click') + const timeoutDescriptor = Object.getOwnPropertyDescriptor(window, 'setTimeout') + const revoke = mock((_url: string) => {}) + let downloadName = '' + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + value: () => 'blob:agent-image', + }) + Object.defineProperty(URL, 'revokeObjectURL', { + configurable: true, + value: revoke, + }) + Object.defineProperty(HTMLAnchorElement.prototype, 'click', { + configurable: true, + value: function click(this: HTMLAnchorElement) { + downloadName = this.download + }, + }) + Object.defineProperty(window, 'setTimeout', { + configurable: true, + value: (handler: TimerHandler) => { + if (typeof handler === 'function') handler() + return 1 + }, + }) + globalThis.fetch = mock(async () => new Response(new Uint8Array([1, 2, 3]), { + headers: { 'content-type': 'image/webp' }, + })) as typeof fetch + + try { + await downloadAgentImage(image()) + expect(downloadName).toBe('Reference-source.webp') + expect(revoke).toHaveBeenCalledWith('blob:agent-image') + } finally { + restoreProperty(URL, 'createObjectURL', createDescriptor) + restoreProperty(URL, 'revokeObjectURL', revokeDescriptor) + restoreProperty(HTMLAnchorElement.prototype, 'click', clickDescriptor) + restoreProperty(window, 'setTimeout', timeoutDescriptor) + } + }) + + it('deduplicates concurrent saves of the same image to Media', async () => { + let resolveUpload!: (response: Response) => void + const uploadResponse = new Promise((resolve) => { + resolveUpload = resolve + }) + let sourceReads = 0 + let uploads = 0 + globalThis.fetch = mock(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input.toString() + if (url === '/image-1') { + sourceReads += 1 + return new Response(new Uint8Array([0xff, 0xd8, 0xff, 0xd9]), { + headers: { 'content-type': 'image/jpeg' }, + }) + } + if (url === '/admin/api/cms/media' && init?.method === 'POST') { + uploads += 1 + return uploadResponse + } + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + + const target = image() + const first = saveAgentImageToMedia(target) + const second = saveAgentImageToMedia(target) + expect(first).toBe(second) + expect(isAgentImageMediaSavePending(target)).toBe(true) + expect(isAgentImageMediaSavePending({ ...target, id: 'another-image' })).toBe(false) + + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(sourceReads).toBe(1) + expect(uploads).toBe(1) + resolveUpload(new Response(JSON.stringify({ + asset: { + id: 'saved-once', + filename: 'Reference-source.jpg', + mimeType: 'image/jpeg', + sizeBytes: 4, + publicPath: '/uploads/reference-source.jpg', + uploadedByUserId: null, + createdAt: '2026-07-11T10:00:00.000Z', + }, + }), { + status: 201, + headers: { 'content-type': 'application/json' }, + })) + + const [firstAsset, secondAsset] = await Promise.all([first, second]) + expect(firstAsset.id).toBe('saved-once') + expect(secondAsset).toBe(firstAsset) + expect(isAgentImageMediaSavePending(target)).toBe(false) + }) +}) diff --git a/src/__tests__/agent/agentImageAttachment.test.ts b/src/__tests__/agent/agentImageAttachment.test.ts new file mode 100644 index 000000000..ee6d00189 --- /dev/null +++ b/src/__tests__/agent/agentImageAttachment.test.ts @@ -0,0 +1,230 @@ +import { afterEach, describe, expect, it, mock } from 'bun:test' +import sharp from 'sharp' +import { + AI_USER_IMAGE_MAX_EDGE, + AI_USER_IMAGE_MAX_PIXELS, +} from '@core/ai' +import { + fitAgentImageSize, + normaliseAgentImage, + readAgentImageSourceSize, +} from '@site/panels/AgentPanel/agentImageAttachment' + +interface BrowserImageMocks { + close: ReturnType + createImageBitmap: ReturnType + drawImage: ReturnType + fillRect: ReturnType + restore(): void +} + +let activeMocks: BrowserImageMocks | null = null + +function installBrowserImageMocks(blob: Blob | null = new Blob( + [new Uint8Array([1, 2, 3])], + { type: 'image/jpeg' }, +)): BrowserImageMocks { + activeMocks?.restore() + const canvasPrototype = Object.getPrototypeOf(document.createElement('canvas')) as object + const createBitmapDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + const getContextDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'getContext') + const toBlobDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'toBlob') + const close = mock(() => {}) + const drawImage = mock(() => {}) + const fillRect = mock(() => {}) + const createImageBitmap = mock(async () => ({ + width: 2000, + height: 1000, + close, + } as unknown as ImageBitmap)) + + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: createImageBitmap, + }) + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value: () => ({ fillStyle: '', fillRect, drawImage }), + }) + Object.defineProperty(canvasPrototype, 'toBlob', { + configurable: true, + value: (callback: BlobCallback) => callback(blob), + }) + + const installed: BrowserImageMocks = { + close, + createImageBitmap, + drawImage, + fillRect, + restore() { + restoreProperty(globalThis, 'createImageBitmap', createBitmapDescriptor) + restoreProperty(canvasPrototype, 'getContext', getContextDescriptor) + restoreProperty(canvasPrototype, 'toBlob', toBlobDescriptor) + }, + } + activeMocks = installed + return installed +} + +function restoreProperty( + target: object, + key: PropertyKey, + descriptor: PropertyDescriptor | undefined, +): void { + if (descriptor) Object.defineProperty(target, key, descriptor) + else Reflect.deleteProperty(target, key) +} + +describe('agent image attachment preparation', () => { + afterEach(() => { + activeMocks?.restore() + activeMocks = null + }) + + it('does not upscale images that already fit the policy', () => { + expect(fitAgentImageSize(640, 480)).toEqual({ width: 640, height: 480 }) + }) + + it('fits wide images inside both the long-edge and pixel limits', () => { + const size = fitAgentImageSize(4000, 2000) + expect(size).toEqual({ width: AI_USER_IMAGE_MAX_EDGE, height: AI_USER_IMAGE_MAX_EDGE / 2 }) + expect(size.width * size.height).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_PIXELS) + }) + + it('never rounds a pixel-limited image back above the pixel budget', () => { + const size = fitAgentImageSize(2000, 2000) + expect(size.width * size.height).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_PIXELS) + }) + + it('rejects invalid dimensions before allocating a canvas', () => { + expect(() => fitAgentImageSize(0, 100)).toThrow('Image has invalid dimensions.') + expect(() => fitAgentImageSize(Number.NaN, 100)).toThrow('Image has invalid dimensions.') + }) + + it('normalises a pasted image to a bounded JPEG and closes the decoded bitmap', async () => { + const browser = installBrowserImageMocks() + const block = await normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + ) + + expect(block).toEqual({ kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }) + expect(browser.fillRect).toHaveBeenCalledTimes(1) + expect(browser.drawImage).toHaveBeenCalledTimes(1) + expect(browser.close).toHaveBeenCalledTimes(1) + expect(browser.createImageBitmap.mock.calls[0]?.[1]).toMatchObject({ + imageOrientation: 'from-image', + resizeWidth: 1568, + resizeHeight: 784, + resizeQuality: 'high', + }) + }) + + it('closes the decoded bitmap when browser encoding fails', async () => { + const browser = installBrowserImageMocks(null) + await expect(normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + )).rejects.toThrow('This browser could not encode the pasted image.') + expect(browser.close).toHaveBeenCalledTimes(1) + }) + + it('closes a decoded bitmap and skips encoding when preparation is cancelled', async () => { + const browser = installBrowserImageMocks() + const decoded = deferred() + browser.createImageBitmap.mockImplementation(() => decoded.promise) + const controller = new AbortController() + const preparing = normaliseAgentImage( + new File([pngHeader(2000, 1000)], 'reference.png', { type: 'image/png' }), + controller.signal, + ) + + while (browser.createImageBitmap.mock.calls.length === 0) await Promise.resolve() + controller.abort() + decoded.resolve({ + width: 1568, + height: 784, + close: browser.close, + } as unknown as ImageBitmap) + + await expect(preparing).rejects.toHaveProperty('name', 'AbortError') + expect(browser.close).toHaveBeenCalledTimes(1) + expect(browser.drawImage).not.toHaveBeenCalled() + }) + + it('reads bounded PNG dimensions before decoding and rejects decompression bombs', async () => { + expect(readAgentImageSourceSize(pngHeader(640, 480), 'image/png')) + .toEqual({ width: 640, height: 480 }) + const browser = installBrowserImageMocks() + + await expect(normaliseAgentImage( + new File([pngHeader(20_000, 20_000)], 'bomb.png', { type: 'image/png' }), + )).rejects.toThrow('Source image dimensions exceed') + expect(browser.createImageBitmap).not.toHaveBeenCalled() + }) + + it('reads real JPEG and WebP dimensions without a full browser decode', async () => { + const source = sharp({ + create: { + width: 321, + height: 123, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }) + const [jpeg, webp] = await Promise.all([ + source.clone().jpeg().toBuffer(), + source.clone().webp().toBuffer(), + ]) + + expect(readAgentImageSourceSize(jpeg, 'image/jpeg')).toEqual({ width: 321, height: 123 }) + expect(readAgentImageSourceSize(webp, 'image/webp')).toEqual({ width: 321, height: 123 }) + }) + + it('uses EXIF display orientation when sizing a JPEG decode target', async () => { + const oriented = await sharp({ + create: { + width: 400, + height: 300, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer() + + expect(readAgentImageSourceSize(oriented, 'image/jpeg')) + .toEqual({ width: 300, height: 400 }) + }) + + it('rejects unsupported clipboard formats before decoding', async () => { + const createImageBitmap = mock(async () => { + throw new Error('should not decode') + }) + const descriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: createImageBitmap, + }) + try { + await expect(normaliseAgentImage( + new File([new Uint8Array([1])], 'animation.gif', { type: 'image/gif' }), + )).rejects.toThrow('Use a PNG, JPEG, or WebP image.') + expect(createImageBitmap).not.toHaveBeenCalled() + } finally { + restoreProperty(globalThis, 'createImageBitmap', descriptor) + } + }) +}) + +function pngHeader(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(24) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + bytes.set([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52], 8) + const view = new DataView(bytes.buffer) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} diff --git a/src/__tests__/agent/agentSlice.test.ts b/src/__tests__/agent/agentSlice.test.ts index cb91026e3..79c0add52 100644 --- a/src/__tests__/agent/agentSlice.test.ts +++ b/src/__tests__/agent/agentSlice.test.ts @@ -9,6 +9,7 @@ import { type AgentToolCall, } from '@site/agent' import type { ConversationView } from '@admin/ai/api' +import type { AiUserContentBlock } from '@core/ai' import '@modules/base' // --------------------------------------------------------------------------- @@ -34,6 +35,9 @@ function freshAgentState() { agentActiveCredentialId: null, agentActiveModelId: null, agentContextTokens: null, + isAgentConversationPending: false, + isAgentProviderPending: false, + agentComposerEpoch: 0, agentConversations: [], hasUnsavedChanges: false, }) @@ -78,7 +82,7 @@ interface InterceptedFetch { * any unexpected call instead of hanging. */ function captureFetchByRoute( - routes: Record Response>, + routes: Record Response | Promise>, ): { restore: () => void; calls: InterceptedFetch[] } { const original = globalThis.fetch const calls: InterceptedFetch[] = [] @@ -129,12 +133,58 @@ const conversationCreateResponse = (id: string) => { status: 201, headers: { 'Content-Type': 'application/json' } }, ) +const conversationDetailResponse = ( + id: string, + content: unknown[], + selection: { credentialId: string; modelId: string } = { + credentialId: 'cred-1', + modelId: 'claude-sonnet-4-6', + }, +) => + new Response(JSON.stringify({ + conversation: { + id, + scope: 'site', + title: 'Image', + credentialId: selection.credentialId, + modelId: selection.modelId, + promptTokensTotal: 0, + completionTokensTotal: 0, + costUsdTotal: 0, + cacheReadTokensTotal: 0, + cacheCreationTokensTotal: 0, + contextTokens: 0, + createdAt: '2026-07-11T10:00:00.000Z', + updatedAt: '2026-07-11T10:00:00.000Z', + messages: [{ + id: 'message-image', + position: 0, + role: 'user', + content, + toolCallId: null, + toolName: null, + createdAt: '2026-07-11T10:00:00.000Z', + }], + }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + const toolResultAckResponse = () => new Response(JSON.stringify({ ok: true }), { status: 200, headers: { 'Content-Type': 'application/json' }, }) +const textContent = (text: string): AiUserContentBlock[] => [{ kind: 'text', text }] + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} + // --------------------------------------------------------------------------- // processStreamEvent — bridge handshake + tool requests // --------------------------------------------------------------------------- @@ -231,6 +281,67 @@ describe('processStreamEvent — toolRequest dispatches to executor', () => { expect(body.result.ok).toBe(false) expect(body.result.error).toContain('not found') }) + + it('retains every image returned by a browser tool for the conversation gallery', async () => { + const { assistantId } = freshAgentState() + const bridge: AgentBridgeRuntime = { bridgeId: 'bridge-images' } + const intercept = captureFetchByRoute({ + '/admin/api/ai/tool-result': toolResultAckResponse, + }) + + try { + await processStreamEvent( + { + type: 'toolCall', + toolCallId: 'tool-images', + toolName: 'site_render_snapshot', + input: {}, + status: 'pending', + }, + assistantId, + noopTextSink, + useEditorStore.setState, + bridge, + null, + executeAgentTool, + ) + await processStreamEvent( + { + type: 'toolRequest', + requestId: 'request-images', + toolName: 'site_render_snapshot', + input: {}, + }, + assistantId, + noopTextSink, + useEditorStore.setState, + bridge, + null, + async () => ({ + ok: true, + images: [ + { mimeType: 'image/png', data: 'QUJD' }, + { mimeType: 'image/jpeg', data: 'REVG' }, + ], + }), + ) + } finally { + intercept.restore() + } + + const message = useEditorStore.getState().agentMessages[0]! + expect(getToolCallBlocks(message)[0]?.previewImages).toEqual([ + 'data:image/png;base64,QUJD', + 'data:image/jpeg;base64,REVG', + ]) + const posted = JSON.parse(intercept.calls[0]!.body) as { + result: { images?: Array<{ mimeType: string; data: string }> } + } + expect(posted.result.images).toEqual([ + { mimeType: 'image/png', data: 'QUJD' }, + { mimeType: 'image/jpeg', data: 'REVG' }, + ]) + }) }) // --------------------------------------------------------------------------- @@ -385,8 +496,9 @@ describe('sendAgentMessage — request lifecycle', () => { ]), }) + let result: { accepted: boolean } | undefined try { - await useEditorStore.getState().sendAgentMessage('Add a hero') + result = await useEditorStore.getState().sendAgentMessage(textContent('Add a hero')) } finally { intercept.restore() } @@ -399,9 +511,90 @@ describe('sendAgentMessage — request lifecycle', () => { expect(conversationCalls).toHaveLength(1) expect(chatCalls).toHaveLength(1) expect(useEditorStore.getState().agentConversationId).toBe('conv-1') + expect(result).toEqual({ accepted: true }) + expect(JSON.parse(chatCalls[0]!.body)).toMatchObject({ + conversationId: 'conv-1', + content: [{ kind: 'text', text: 'Add a hero' }], + }) void rootId }) + it('stops a first send while conversation creation is still pending', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const createStarted = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': (_call, init) => { + createStarted.resolve() + return new Promise((_resolve, reject) => { + const signal = init?.signal + const rejectAbort = () => reject( + signal?.reason ?? new DOMException('The operation was aborted.', 'AbortError'), + ) + if (signal?.aborted) rejectAbort() + else signal?.addEventListener('abort', rejectAbort, { once: true }) + }) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const sending = useEditorStore.getState().sendAgentMessage(textContent('Start')) + await createStarted.promise + expect(useEditorStore.getState().isAgentStreaming).toBe(true) + useEditorStore.getState().abortAgent() + + expect(await sending).toEqual({ accepted: false }) + expect(useEditorStore.getState().isAgentStreaming).toBe(false) + expect(useEditorStore.getState().agentConversationId).toBeNull() + expect(useEditorStore.getState().agentMessages).toEqual([]) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + } finally { + intercept.restore() + } + }) + + it('sends and renders an image-only user turn without inventing placeholder text', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const image = { + kind: 'image' as const, + mimeType: 'image/jpeg' as const, + data: 'QUJD', + } + + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': defaultsResponse, + '/admin/api/ai/conversations': () => conversationCreateResponse('conv-image'), + '/admin/api/ai/chat/site': () => ndjsonResponse([ + { type: 'bridgeReady', bridgeId: 'b-image' }, + { type: 'done' }, + ]), + }) + + let result: { accepted: boolean } | undefined + try { + result = await useEditorStore.getState().sendAgentMessage([image]) + } finally { + intercept.restore() + } + + expect(result).toEqual({ accepted: true }) + const chat = intercept.calls.find((call) => call.url === '/admin/api/ai/chat/site') + expect(chat).toBeDefined() + expect(JSON.parse(chat!.body)).toMatchObject({ + conversationId: 'conv-image', + content: [image], + }) + const userMessage = useEditorStore.getState().agentMessages.find((message) => message.role === 'user') + expect(userMessage?.blocks).toEqual([{ + kind: 'image', + mimeType: 'image/jpeg', + src: 'data:image/jpeg;base64,QUJD', + }]) + }) + it('reuses the same conversation id on follow-up sends', async () => { freshAgentState() useEditorStore.setState({ @@ -420,8 +613,8 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('First message.') - await useEditorStore.getState().sendAgentMessage('Follow-up.') + await useEditorStore.getState().sendAgentMessage(textContent('First message.')) + await useEditorStore.getState().sendAgentMessage(textContent('Follow-up.')) } finally { intercept.restore() } @@ -460,7 +653,7 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Create a pricing card class.') + await useEditorStore.getState().sendAgentMessage(textContent('Create a pricing card class.')) } finally { intercept.restore() } @@ -495,7 +688,7 @@ describe('sendAgentMessage — request lifecycle', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Anything.') + await useEditorStore.getState().sendAgentMessage(textContent('Anything.')) } finally { intercept.restore() } @@ -506,9 +699,79 @@ describe('sendAgentMessage — request lifecycle', () => { }) }) +describe('loadAgentConversation — image rehydration', () => { + it('restores persisted user image blocks and advances the composer epoch', async () => { + freshAgentState() + useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) + const image = { + kind: 'image', + mimeType: 'image/jpeg', + url: '/admin/api/ai/conversations/conv-image/messages/message-image/images/0', + } + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-image': () => + conversationDetailResponse('conv-image', [image]), + }) + + try { + await useEditorStore.getState().loadAgentConversation('conv-image') + } finally { + intercept.restore() + } + + const state = useEditorStore.getState() + expect(state.agentConversationId).toBe('conv-image') + expect(state.agentComposerEpoch).toBe(1) + expect(state.agentMessages).toHaveLength(1) + expect(state.agentMessages[0]?.blocks).toEqual([{ + kind: 'image', + mimeType: 'image/jpeg', + src: image.url, + }]) + }) + + it('blocks Send until an in-flight conversation load commits', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-old', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + const loadStarted = deferred() + const loadResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-new': () => { + loadStarted.resolve() + return loadResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const loading = useEditorStore.getState().loadAgentConversation('conv-new') + await loadStarted.promise + expect(useEditorStore.getState().isAgentConversationPending).toBe(true) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Wait'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + + loadResponse.resolve(conversationDetailResponse('conv-new', [ + { kind: 'text', text: 'Loaded' }, + ])) + await loading + expect(useEditorStore.getState().agentConversationId).toBe('conv-new') + expect(useEditorStore.getState().isAgentConversationPending).toBe(false) + } finally { + intercept.restore() + } + }) +}) + describe('conversation reset key-set', () => { - // All three reset paths must clear the SAME six keys — agentContextTokens and - // agentError have each silently drifted out of one copy in the past. + // All three reset paths clear the same conversation keys and advance the + // composer epoch so local text/image drafts cannot cross conversations. const RESET_SNAPSHOT = { agentMessages: [], agentError: null, @@ -516,11 +779,13 @@ describe('conversation reset key-set', () => { agentActiveCredentialId: null, agentActiveModelId: null, agentContextTokens: null, + agentComposerEpoch: 1, } function seedDirtyConversation() { freshAgentState() useEditorStore.setState({ + isAgentStreaming: false, agentMessages: [{ id: 'm1', role: 'user', blocks: [{ kind: 'text', text: 'hi' }], timestamp: 1 }], agentError: 'AI server is not running. Start it with: bun run dev', agentConversationId: 'conv-dirty', @@ -539,10 +804,11 @@ describe('conversation reset key-set', () => { agentActiveCredentialId: s.agentActiveCredentialId, agentActiveModelId: s.agentActiveModelId, agentContextTokens: s.agentContextTokens, + agentComposerEpoch: s.agentComposerEpoch, } } - it('startNewAgentConversation resets the full six-key set (incl. agentContextTokens)', () => { + it('startNewAgentConversation resets conversation state and remounts the composer', () => { seedDirtyConversation() useEditorStore.getState().startNewAgentConversation() expect(pickResetKeys()).toEqual(RESET_SNAPSHOT) @@ -601,7 +867,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { }) try { - await useEditorStore.getState().sendAgentMessage('Go') + await useEditorStore.getState().sendAgentMessage(textContent('Go')) } finally { intercept.restore() } @@ -615,7 +881,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { expect(useEditorStore.getState().isAgentStreaming).toBe(false) }) - it('surfaces a non-ok chat response once — single agentError + single placeholder block', async () => { + it('rejects a non-ok chat response without adding an optimistic turn', async () => { freshAgentState() useEditorStore.setState({ isAgentStreaming: false, agentMessages: [] }) @@ -625,17 +891,16 @@ describe('sendAgentMessage — streaming + error surfacing', () => { '/admin/api/ai/chat/site': () => new Response('boom', { status: 500 }), }) + let result: { accepted: boolean } | undefined try { - await useEditorStore.getState().sendAgentMessage('Do a thing') + result = await useEditorStore.getState().sendAgentMessage(textContent('Do a thing')) } finally { intercept.restore() } - expect(useEditorStore.getState().agentError).toContain('Agent request failed') - const assistant = useEditorStore.getState().agentMessages.find((m) => m.role === 'assistant')! - // Collapsed double-set (F10): exactly one placeholder block, not two renders. - expect(assistant.blocks).toHaveLength(1) - expect(assistant.blocks[0]).toMatchObject({ kind: 'text', text: '_(agent error)_' }) + expect(result).toEqual({ accepted: false }) + expect(useEditorStore.getState().agentError).toBe('boom') + expect(useEditorStore.getState().agentMessages).toEqual([]) expect(useEditorStore.getState().isAgentStreaming).toBe(false) }) @@ -662,7 +927,7 @@ describe('sendAgentMessage — streaming + error surfacing', () => { // Panel-open path stages the default… await useEditorStore.getState().loadScopeDefault() // …first send must reuse it, NOT fetch the default a second time. - await useEditorStore.getState().sendAgentMessage('Hi') + await useEditorStore.getState().sendAgentMessage(textContent('Hi')) } finally { intercept.restore() } @@ -757,6 +1022,37 @@ describe('loadScopeDefault', () => { expect(useEditorStore.getState().agentActiveCredentialId).toBeNull() expect(useEditorStore.getState().agentActiveModelId).toBeNull() }) + + it('does not overwrite a model picked while the default request is in flight', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: null, + agentActiveCredentialId: null, + agentActiveModelId: null, + }) + const requestStarted = deferred() + const response = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/defaults': () => { + requestStarted.resolve() + return response.promise + }, + }) + + try { + const loading = useEditorStore.getState().loadScopeDefault() + await requestStarted.promise + await useEditorStore.getState().setAgentProvider('cred-picked', 'model-picked') + response.resolve(defaultsResponse()) + await loading + } finally { + intercept.restore() + } + + expect(useEditorStore.getState().agentActiveCredentialId).toBe('cred-picked') + expect(useEditorStore.getState().agentActiveModelId).toBe('model-picked') + }) }) // --------------------------------------------------------------------------- @@ -780,4 +1076,187 @@ describe('setAgentProvider', () => { expect(useEditorStore.getState().agentActiveModelId).toBe('model-9') expect(useEditorStore.getState().agentError).toBeNull() }) + + it('blocks Send until an existing conversation model update finishes', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-switch', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + + const updateStarted = deferred() + const updateResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-switch': (_call, init) => { + expect(init?.method).toBe('PUT') + updateStarted.resolve() + return updateResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const changingModel = useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + await updateStarted.promise + expect(useEditorStore.getState().isAgentProviderPending).toBe(true) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Use the new model'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + + updateResponse.resolve(new Response(JSON.stringify({ + conversation: { + id: 'conv-switch', + scope: 'site', + title: 'Conversation', + credentialId: 'cred-new', + modelId: 'model-new', + promptTokensTotal: 0, + completionTokensTotal: 0, + costUsdTotal: 0, + cacheReadTokensTotal: 0, + cacheCreationTokensTotal: 0, + contextTokens: 0, + createdAt: '2026-07-11T10:00:00.000Z', + updatedAt: '2026-07-11T10:00:00.000Z', + }, + }), { headers: { 'content-type': 'application/json' } })) + + await changingModel + expect(useEditorStore.getState().isAgentProviderPending).toBe(false) + expect(await useEditorStore.getState().sendAgentMessage(textContent('Use the new model'))) + .toEqual({ accepted: true }) + expect(intercept.calls.map((call) => call.url)).toEqual([ + '/admin/api/ai/conversations/conv-switch', + '/admin/api/ai/chat/site', + ]) + } finally { + intercept.restore() + } + }) + + it('does not send when the model update it was waiting for fails', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-fail', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + + const updateStarted = deferred() + const updateResponse = deferred() + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-fail': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-fail', [], { + credentialId: 'cred-old', + modelId: 'model-old', + }) + } + updateStarted.resolve() + return updateResponse.promise + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + const changingModel = useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + await updateStarted.promise + const sending = useEditorStore.getState().sendAgentMessage(textContent('Do not misroute me')) + updateResponse.resolve(new Response(JSON.stringify({ error: 'update failed' }), { + status: 400, + headers: { 'content-type': 'application/json' }, + })) + + const [, result] = await Promise.all([changingModel, sending]) + expect(result).toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + expect(useEditorStore.getState().agentActiveCredentialId).toBe('cred-old') + expect(useEditorStore.getState().agentActiveModelId).toBe('model-old') + } finally { + intercept.restore() + } + }) + + it('reconciles an update whose response was lost after the server committed it', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-committed', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-committed': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-committed', [], { + credentialId: 'cred-new', + modelId: 'model-new', + }) + } + return Promise.reject(new TypeError('Connection reset after commit')) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + await useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + + const state = useEditorStore.getState() + expect(state.agentActiveCredentialId).toBe('cred-new') + expect(state.agentActiveModelId).toBe('model-new') + expect(state.agentError).toBeNull() + expect(state.isAgentProviderPending).toBe(false) + expect(await state.sendAgentMessage(textContent('Use the committed model'))) + .toEqual({ accepted: true }) + expect(intercept.calls.map((call) => call.method)).toEqual(['PUT', 'GET', 'POST']) + } finally { + intercept.restore() + } + }) + + it('keeps Send locked when an ambiguous update can still commit after reconciliation', async () => { + freshAgentState() + useEditorStore.setState({ + isAgentStreaming: false, + agentConversationId: 'conv-ambiguous', + agentActiveCredentialId: 'cred-old', + agentActiveModelId: 'model-old', + agentMessages: [], + }) + let serverSelection = { credentialId: 'cred-old', modelId: 'model-old' } + const intercept = captureFetchByRoute({ + '/admin/api/ai/conversations/conv-ambiguous': (_call, init) => { + if ((init?.method ?? 'GET').toUpperCase() === 'GET') { + return conversationDetailResponse('conv-ambiguous', [], serverSelection) + } + return new Response(JSON.stringify({ error: 'Upstream response was lost' }), { + status: 502, + headers: { 'content-type': 'application/json' }, + }) + }, + '/admin/api/ai/chat/site': () => ndjsonResponse([{ type: 'done' }]), + }) + + try { + await useEditorStore.getState().setAgentProvider('cred-new', 'model-new') + // The disconnected PUT commits after the first reconciliation read. + serverSelection = { credentialId: 'cred-new', modelId: 'model-new' } + + const state = useEditorStore.getState() + expect(state.agentActiveCredentialId).toBeNull() + expect(state.agentActiveModelId).toBeNull() + expect(state.agentError).toContain('server state could not be confirmed') + expect(await state.sendAgentMessage(textContent('Never route against stale state'))) + .toEqual({ accepted: false }) + expect(intercept.calls.some((call) => call.url === '/admin/api/ai/chat/site')).toBe(false) + } finally { + intercept.restore() + } + }) }) diff --git a/src/__tests__/ai/anthropicMapping.test.ts b/src/__tests__/ai/anthropicMapping.test.ts index c31d4ffa0..834590d1e 100644 --- a/src/__tests__/ai/anthropicMapping.test.ts +++ b/src/__tests__/ai/anthropicMapping.test.ts @@ -127,15 +127,23 @@ describe('Anthropic mapHistory', () => { test('maps base64 image blocks to Anthropic image source', () => { const history: AiMessage[] = [ - { role: 'user', content: [{ kind: 'image', mimeType: 'image/png', data: 'BASE64' }, { kind: 'text', text: 'look' }] }, + { + role: 'user', + content: [ + { kind: 'image', mimeType: 'image/png', data: 'BASE64-1' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'BASE64-2' }, + { kind: 'text', text: 'compare' }, + ], + }, ] const mapped = mapHistory(history) expect(mapped).toEqual([ { role: 'user', content: [ - { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'BASE64' } }, - { type: 'text', text: 'look' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'BASE64-1' } }, + { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: 'BASE64-2' } }, + { type: 'text', text: 'compare' }, ], }, ]) diff --git a/src/__tests__/ai/auditUsagePersistence.test.ts b/src/__tests__/ai/auditUsagePersistence.test.ts index 1cc94d51f..292dc0763 100644 --- a/src/__tests__/ai/auditUsagePersistence.test.ts +++ b/src/__tests__/ai/auditUsagePersistence.test.ts @@ -21,6 +21,7 @@ const fakeTextOnlyDriver: AiProvider = { return { toolCalling: false, visionInput: false, + toolResultImages: false, promptCache: false, streaming: true, } diff --git a/src/__tests__/ai/chatImageHandler.test.ts b/src/__tests__/ai/chatImageHandler.test.ts new file mode 100644 index 000000000..bd1c7aca3 --- /dev/null +++ b/src/__tests__/ai/chatImageHandler.test.ts @@ -0,0 +1,315 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import sharp from 'sharp' +import { AI_CHAT_MAX_REQUEST_BYTES, AI_USER_IMAGE_MAX_PER_MESSAGE } from '@core/ai' +import { createCapabilityTestHarness, type CapabilityTestHarness } from '../helpers/capabilityHarness' +import { + appendMessage, + createConversationForUser, + listMessagesForConversation, + readConversationForUser, +} from '../../../server/ai/conversations/store' + +let testSerial = 0 + +describe('AI chat user-image boundary', () => { + let harness: CapabilityTestHarness + let cookie: string + let conversationId: string + let credentialId: string + let originalFetch: typeof globalThis.fetch + + beforeEach(async () => { + originalFetch = globalThis.fetch + harness = await createCapabilityTestHarness() + cookie = await harness.setupOwner() + const { rows } = await harness.db<{ id: string }>`select id from users limit 1` + const userId = rows[0]!.id + credentialId = `cred_image_${++testSerial}` + await harness.db` + insert into ai_provider_credentials ( + id, user_id, provider_id, auth_mode, display_label, base_url + ) values ( + ${credentialId}, ${userId}, 'ollama', 'baseUrl', 'Image test', 'http://ollama.test' + ) + ` + const conversation = await createConversationForUser(harness.db, userId, { + scope: 'site', + credentialId, + modelId: 'vision-model', + }) + conversationId = conversation.id + }) + + afterEach(async () => { + globalThis.fetch = originalFetch + await harness.cleanup() + }) + + it('returns 413 when the complete request envelope exceeds its limit', async () => { + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'x'.repeat(AI_CHAT_MAX_REQUEST_BYTES) }], + }, + }) + + expect(response.status).toBe(413) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects malformed JPEG bytes before persistence', async () => { + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'image', mimeType: 'image/jpeg', data: '/9h/' }], + }, + }) + + expect(response.status).toBe(400) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects a non-vision model before persistence', async () => { + const image = await jpegBlock() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/tags') { + return jsonResponse({ models: [{ name: 'vision-model' }] }) + } + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['completion'] }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + + expect(response.status).toBe(422) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('rejects a known non-tool model before persistence or provider streaming', async () => { + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['vision'] }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { + conversationId, + content: [{ kind: 'text', text: 'Inspect the page.' }], + }, + }) + + expect(response.status).toBe(422) + expect(await response.json()).toEqual({ + error: 'The selected model does not support tool calling. Choose an agent-capable model.', + }) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + }) + + it('persists a multi-image turn, forwards every image, and titles it Images', async () => { + const image = await jpegBlock() + let providerRequest = '' + globalThis.fetch = async (input, init) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/tags') { + return jsonResponse({ models: [{ name: 'vision-model' }] }) + } + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['vision', 'tools'] }) + } + if (url === 'http://ollama.test/v1/chat/completions') { + providerRequest = String(init?.body ?? '') + return new Response([ + 'data: {"choices":[{"delta":{"content":"Looks good."},"finish_reason":null}]}\n\n', + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":2}}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image, image] }, + }) + + expect(response.status).toBe(200) + expect(response.headers.get('cache-control')).toBe('private, no-store') + await response.text() + const { rows } = await harness.db<{ id: string }>`select id from users limit 1` + const conversation = await readConversationForUser(harness.db, rows[0]!.id, conversationId) + const messages = await listMessagesForConversation(harness.db, conversationId) + + expect(conversation?.title).toBe('Images') + expect(messages[0]?.role).toBe('user') + expect(messages[0]?.content).toHaveLength(2) + const persistedImages = messages[0]?.content.filter((block) => block.kind === 'image') ?? [] + expect(persistedImages).toHaveLength(2) + expect(providerRequest.match(/data:image\/jpeg;base64,/g)).toHaveLength(2) + expect((JSON.parse(providerRequest) as { tools?: unknown }).tools).toBeArray() + }) + + it('allows only one concurrent writer', async () => { + const image = await jpegBlock() + const capabilityStarted = deferred() + const capabilityResponse = deferred() + const providerResponse = deferred() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + capabilityStarted.resolve() + return capabilityResponse.promise + } + if (url === 'http://ollama.test/v1/chat/completions') { + return providerResponse.promise + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const firstRequest = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + await capabilityStarted.promise + const secondRequest = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image] }, + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + capabilityResponse.resolve(jsonResponse({ capabilities: ['vision', 'tools'] })) + + const responses = await Promise.all([firstRequest, secondRequest]) + expect(responses.map((response) => response.status).sort()).toEqual([200, 409]) + + providerResponse.resolve(new Response([ + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } })) + const accepted = responses.find((response) => response.status === 200) + await accepted?.text() + + const messages = await listMessagesForConversation(harness.db, conversationId) + const persistedUserImages = messages + .filter((message) => message.role === 'user') + .flatMap((message) => message.content) + .filter((block) => block.kind === 'image') + expect(persistedUserImages).toHaveLength(1) + }) + + it('does not persist a turn aborted during capability discovery', async () => { + const image = await jpegBlock() + const capabilityStarted = deferred() + const capabilityResponse = deferred() + globalThis.fetch = async (input) => { + const url = requestUrl(input) + if (url !== 'http://ollama.test/api/show') { + throw new Error(`Unexpected fetch: ${url}`) + } + capabilityStarted.resolve() + return capabilityResponse.promise + } + const controller = new AbortController() + + const request = harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + signal: controller.signal, + json: { conversationId, content: [image] }, + }) + await capabilityStarted.promise + controller.abort() + + const response = await request + expect(response.status).toBe(499) + expect(await listMessagesForConversation(harness.db, conversationId)).toHaveLength(0) + + // Let the shared lookup settle so it cannot leak into a later test. + capabilityResponse.resolve(jsonResponse({ capabilities: ['vision', 'tools'] })) + }) + + it('retains images across turns beyond the per-message limit', async () => { + const image = await jpegBlock() + for (let index = 0; index < AI_USER_IMAGE_MAX_PER_MESSAGE; index += 1) { + await appendMessage(harness.db, conversationId, { role: 'user', content: [image] }) + } + let providerRequest = '' + globalThis.fetch = async (input, init) => { + const url = requestUrl(input) + if (url === 'http://ollama.test/api/show') { + return jsonResponse({ capabilities: ['vision', 'tools'] }) + } + if (url === 'http://ollama.test/v1/chat/completions') { + providerRequest = String(init?.body ?? '') + return new Response([ + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: [DONE]\n\n', + ].join(''), { headers: { 'content-type': 'text/event-stream' } }) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const response = await harness.ai('/admin/api/ai/chat/site', { + method: 'POST', + cookie, + json: { conversationId, content: [image, image] }, + }) + expect(response.status).toBe(200) + await response.text() + + const userImages = (await listMessagesForConversation(harness.db, conversationId)) + .filter((message) => message.role === 'user') + .flatMap((message) => message.content) + .filter((block) => block.kind === 'image') + expect(userImages).toHaveLength(AI_USER_IMAGE_MAX_PER_MESSAGE + 2) + expect(providerRequest.match(/data:image\/jpeg;base64,/g)) + .toHaveLength(AI_USER_IMAGE_MAX_PER_MESSAGE + 2) + }) +}) + +async function jpegBlock() { + const data = (await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: { r: 30, g: 60, b: 90 }, + }, + }).jpeg().toBuffer()).toString('base64') + return { kind: 'image' as const, mimeType: 'image/jpeg' as const, data } +} + +function requestUrl(input: string | URL | Request): string { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + return input.url +} + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + headers: { 'content-type': 'application/json' }, + }) +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { resolve = done }) + return { promise, resolve } +} diff --git a/src/__tests__/ai/chatRequest.test.ts b/src/__tests__/ai/chatRequest.test.ts new file mode 100644 index 000000000..69ff399aa --- /dev/null +++ b/src/__tests__/ai/chatRequest.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from 'bun:test' +import { Value } from '@sinclair/typebox/value' +import { + AI_CHAT_MAX_REQUEST_BYTES, + AI_USER_IMAGE_MAX_BASE64_CHARS, + AI_USER_IMAGE_MAX_PER_MESSAGE, + AiChatRequestBodySchema, + isAiUserImageSourceMimeType, +} from '@core/ai' + +const image = { + kind: 'image' as const, + mimeType: 'image/jpeg' as const, + data: 'AAAA', +} + +describe('AiChatRequestBodySchema', () => { + test('accepts text-only, image-only, and multi-image user turns', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'text', text: 'Hello' }], + snapshot: { pageId: 'page-1' }, + })).toBe(true) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [image], + })).toBe(true) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [image, { kind: 'text', text: 'Compare these' }, image], + })).toBe(true) + }) + + test('rejects empty content and assistant/tool block injection', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'toolCall', toolCallId: 'call-1', toolName: 'site_insert_html', input: {} }], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'toolResult', ok: true }], + })).toBe(false) + }) + + test('bounds the user turn to one text block and eight normalised JPEGs', () => { + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ kind: 'text', text: 'one' }, { kind: 'text', text: 'two' }, image], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ ...image, mimeType: 'image/png' }], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: [{ ...image, data: 'A'.repeat(AI_USER_IMAGE_MAX_BASE64_CHARS + 1) }], + })).toBe(false) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: Array.from({ length: AI_USER_IMAGE_MAX_PER_MESSAGE }, () => image), + })).toBe(true) + + expect(Value.Check(AiChatRequestBodySchema, { + conversationId: 'conversation-1', + content: Array.from({ length: AI_USER_IMAGE_MAX_PER_MESSAGE + 1 }, () => image), + })).toBe(false) + }) + + test('reserves snapshot overhead above eight maximum base64 image blocks', () => { + expect(AI_CHAT_MAX_REQUEST_BYTES).toBeGreaterThan( + AI_USER_IMAGE_MAX_PER_MESSAGE * AI_USER_IMAGE_MAX_BASE64_CHARS, + ) + }) +}) + +describe('AI user-image source MIME guard', () => { + test('accepts browser-normalisable raster formats and rejects unsupported input', () => { + expect(isAiUserImageSourceMimeType('image/jpeg')).toBe(true) + expect(isAiUserImageSourceMimeType('image/png')).toBe(true) + expect(isAiUserImageSourceMimeType('image/webp')).toBe(true) + expect(isAiUserImageSourceMimeType('image/gif')).toBe(false) + expect(isAiUserImageSourceMimeType('image/svg+xml')).toBe(false) + }) +}) diff --git a/src/__tests__/ai/conversationImageHandler.test.ts b/src/__tests__/ai/conversationImageHandler.test.ts new file mode 100644 index 000000000..23770ebbf --- /dev/null +++ b/src/__tests__/ai/conversationImageHandler.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import sharp from 'sharp' +import { createCapabilityTestHarness, type CapabilityTestHarness } from '../helpers/capabilityHarness' +import { + appendMessage, + createConversationForUser, + softDeleteConversationForUser, +} from '../../../server/ai/conversations/store' + +describe('conversation image delivery', () => { + let harness: CapabilityTestHarness + let ownerCookie: string + let ownerId: string + + beforeEach(async () => { + harness = await createCapabilityTestHarness() + ownerCookie = await harness.setupOwner() + const { rows } = await harness.db<{ id: string }>`select id from users limit 1` + ownerId = rows[0]!.id + }) + + afterEach(async () => { + await harness.cleanup() + }) + + it('projects lazy URLs and serves only an owned live JPEG block', async () => { + await harness.db` + insert into ai_provider_credentials ( + id, user_id, provider_id, auth_mode, display_label, base_url + ) values ('cred-image-view', ${ownerId}, 'ollama', 'baseUrl', 'Images', 'http://local') + ` + const conversation = await createConversationForUser(harness.db, ownerId, { + scope: 'site', + credentialId: 'cred-image-view', + modelId: 'vision-model', + }) + const bytes = await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: { r: 12, g: 34, b: 56 }, + }, + }).jpeg().toBuffer() + const message = await appendMessage(harness.db, conversation.id, { + role: 'user', + content: [ + { kind: 'text', text: 'Reference' }, + { kind: 'image', mimeType: 'image/jpeg', data: bytes.toString('base64') }, + ], + }) + + const detailResponse = await harness.ai( + `/admin/api/ai/conversations/${conversation.id}`, + { cookie: ownerCookie }, + ) + expect(detailResponse.status).toBe(200) + const detailText = await detailResponse.text() + expect(detailText).not.toContain(bytes.toString('base64')) + const detail = JSON.parse(detailText) as { + conversation: { messages: Array<{ content: Array> }> } + } + const image = detail.conversation.messages[0]!.content[1]! + expect(image).toEqual({ + kind: 'image', + mimeType: 'image/jpeg', + url: `/admin/api/ai/conversations/${conversation.id}/messages/${message.id}/images/1`, + }) + + const response = await harness.ai(String(image.url), { cookie: ownerCookie }) + expect(response.status).toBe(200) + expect(response.headers.get('content-type')).toBe('image/jpeg') + expect(response.headers.get('cache-control')).toBe('private, no-store') + expect(response.headers.get('x-content-type-options')).toBe('nosniff') + expect(Buffer.from(await response.arrayBuffer())).toEqual(bytes) + + const textBlock = await harness.ai( + `/admin/api/ai/conversations/${conversation.id}/messages/${message.id}/images/0`, + { cookie: ownerCookie }, + ) + expect(textBlock.status).toBe(404) + + const outsider = await harness.createRoleUser({ + name: 'Chat reader', + slug: 'chat-reader', + capabilities: ['ai.chat'], + }) + expect((await harness.ai(String(image.url), { cookie: outsider.cookie })).status).toBe(404) + + await softDeleteConversationForUser(harness.db, ownerId, conversation.id) + expect((await harness.ai(String(image.url), { cookie: ownerCookie })).status).toBe(404) + }) +}) diff --git a/src/__tests__/ai/conversationRoundtrip.test.ts b/src/__tests__/ai/conversationRoundtrip.test.ts index 4984cc32c..111612b53 100644 --- a/src/__tests__/ai/conversationRoundtrip.test.ts +++ b/src/__tests__/ai/conversationRoundtrip.test.ts @@ -1,12 +1,15 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { Value } from '@sinclair/typebox/value' +import sharp from 'sharp' import { createTestDb, type TestDb } from '../helpers/createTestDb' import { appendMessage, createConversationForUser, listMessagesForConversation, readConversationForUser, + replaceDefaultConversationTitle, toConversationDetailView, + updateConversationForUser, } from '../../../server/ai/conversations/store' import { ConversationDetailViewSchema } from '../../admin/ai/api' @@ -55,7 +58,7 @@ describe('conversation detail round-trip', () => { expect(record).not.toBeNull() const messages = await listMessagesForConversation(testDb.db, conv.id) - const detail = toConversationDetailView(record!, messages) + const detail = toConversationDetailView(record!, messages, () => '/unused-image') // The exact validator that threw "Expected union value" on reopen. expect(Value.Check(ConversationDetailViewSchema, detail)).toBe(true) @@ -64,4 +67,81 @@ describe('conversation detail round-trip', () => { // The dead context field must not reappear on the wire shape. expect('contextJson' in detail).toBe(false) }) + + it('preserves mixed and image-only user content through SQLite and the wire schema', async () => { + const conv = await createConversationForUser(testDb.db, 'user_1', { + scope: 'site', + credentialId: 'cred_1', + modelId: 'model_1', + }) + const imageData = (await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: { r: 20, g: 40, b: 60 }, + }, + }).jpeg().toBuffer()).toString('base64') + const image = { kind: 'image' as const, mimeType: 'image/jpeg', data: imageData } + + await appendMessage(testDb.db, conv.id, { + role: 'user', + content: [{ kind: 'text', text: 'What is this?' }, image], + }) + await appendMessage(testDb.db, conv.id, { + role: 'user', + content: [image], + }) + + const record = await readConversationForUser(testDb.db, 'user_1', conv.id) + expect(record).not.toBeNull() + const messages = await listMessagesForConversation(testDb.db, conv.id) + const detail = toConversationDetailView( + record!, + messages, + (messageId, blockIndex) => `/images/${messageId}/${blockIndex}`, + ) + + expect(Value.Check(ConversationDetailViewSchema, detail)).toBe(true) + expect(messages.map((message) => message.content)).toEqual([ + [{ kind: 'text', text: 'What is this?' }, image], + [image], + ]) + expect(detail.messages.map((message) => message.content)).toEqual([ + [ + { kind: 'text', text: 'What is this?' }, + { + kind: 'image', + mimeType: 'image/jpeg', + url: `/images/${messages[0]!.id}/1`, + }, + ], + [{ + kind: 'image', + mimeType: 'image/jpeg', + url: `/images/${messages[1]!.id}/0`, + }], + ]) + expect(JSON.stringify(detail)).not.toContain(imageData) + }) + + it('does not let an automatic first-turn title overwrite a user rename', async () => { + const conv = await createConversationForUser(testDb.db, 'user_1', { + scope: 'site', + credentialId: 'cred_1', + modelId: 'model_1', + }) + await updateConversationForUser(testDb.db, 'user_1', conv.id, { + title: 'My reference review', + }) + + expect(await replaceDefaultConversationTitle( + testDb.db, + 'user_1', + conv.id, + 'Image', + )).toBe(false) + expect((await readConversationForUser(testDb.db, 'user_1', conv.id))?.title) + .toBe('My reference review') + }) }) diff --git a/src/__tests__/ai/inputImages.test.ts b/src/__tests__/ai/inputImages.test.ts new file mode 100644 index 000000000..34362f5b2 --- /dev/null +++ b/src/__tests__/ai/inputImages.test.ts @@ -0,0 +1,209 @@ +import { beforeAll, describe, expect, test } from 'bun:test' +import sharp from 'sharp' +import { + AI_USER_IMAGE_MAX_PER_MESSAGE, + AI_USER_IMAGE_MAX_BYTES, + AI_USER_IMAGE_MAX_EDGE, + AI_USER_IMAGE_MAX_PIXELS, + type AiUserImageBlock, +} from '@core/ai' +import { + AiImageInputError, + canonicaliseAiUserContent, + preflightAiUserContent, + validateAiUserContent, + validateAiUserImage, +} from '../../../server/ai/inputImages' + +let smallJpegBase64 = '' + +beforeAll(async () => { + smallJpegBase64 = (await makeImage(24, 16, 'jpeg')).toString('base64') +}) + +function imageBlock(data = smallJpegBase64): AiUserImageBlock { + return { kind: 'image', mimeType: 'image/jpeg', data } +} + +function makeImage( + width: number, + height: number, + format: 'jpeg' | 'png', +): Promise { + const source = sharp({ + create: { + width, + height, + channels: 3, + background: { r: 45, g: 90, b: 135 }, + }, + }) + return format === 'jpeg' ? source.jpeg().toBuffer() : source.png().toBuffer() +} + +describe('validateAiUserContent', () => { + test('normalises mixed content to trimmed text followed by every image', async () => { + const content = await validateAiUserContent([ + imageBlock(), + { kind: 'text', text: ' Describe this screenshot. ' }, + imageBlock(), + ]) + + expect(content[0]).toEqual({ kind: 'text', text: 'Describe this screenshot.' }) + expect(content[1]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + expect((content[1] as AiUserImageBlock).data.length).toBeGreaterThan(0) + expect(content[2]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + }) + + test('accepts image-only content and removes whitespace-only text beside it', async () => { + const imageOnly = await validateAiUserContent([imageBlock()]) + const besideWhitespace = await validateAiUserContent([ + { kind: 'text', text: ' \n ' }, + imageBlock(), + ]) + + expect(imageOnly).toHaveLength(1) + expect(imageOnly[0]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + expect(besideWhitespace).toHaveLength(1) + expect(besideWhitespace[0]).toMatchObject({ kind: 'image', mimeType: 'image/jpeg' }) + }) + + test('stops before the next decode when cancellation lands between images', async () => { + const corruptButPreflightValid = Buffer.from([0xff, 0xd8, 0xff, 0x00]) + const preflight = preflightAiUserContent([ + imageBlock(), + imageBlock(corruptButPreflightValid.toString('base64')), + ]) + let abortChecks = 0 + const signal = { + throwIfAborted() { + abortChecks += 1 + // Outer per-image check, metadata checks, then the first JPEG encode's + // before/after checks. Abort immediately after that active pipeline. + if (abortChecks === 5) throw new DOMException('Request aborted', 'AbortError') + }, + } as AbortSignal + + await expect(canonicaliseAiUserContent(preflight, signal)).rejects.toMatchObject({ + name: 'AbortError', + }) + // Reaching either another quality attempt or the corrupt second image + // would throw AiImageInputError instead. + expect(abortChecks).toBe(5) + }) + + test('rejects an empty turn, duplicate text, and more than eight images', async () => { + await expect(validateAiUserContent([ + { kind: 'text', text: ' ' }, + ])).rejects.toMatchObject({ + name: 'AiImageInputError', + status: 400, + message: 'Message must contain text or an image.', + }) + + await expect(validateAiUserContent([ + { kind: 'text', text: 'one' }, + { kind: 'text', text: 'two' }, + ])).rejects.toMatchObject({ + status: 400, + message: 'A message can contain at most one text block.', + }) + + await expect(validateAiUserContent([ + ...Array.from({ length: AI_USER_IMAGE_MAX_PER_MESSAGE + 1 }, () => imageBlock()), + ])).rejects.toMatchObject({ + status: 400, + message: `A message can contain at most ${AI_USER_IMAGE_MAX_PER_MESSAGE} images.`, + }) + }) +}) + +describe('validateAiUserImage', () => { + test('fully decodes and returns a bounded canonical JPEG', async () => { + const canonical = await validateAiUserImage(imageBlock()) + const bytes = Buffer.from(canonical.data, 'base64') + const metadata = await sharp(bytes).metadata() + + expect(canonical.mimeType).toBe('image/jpeg') + expect(bytes.byteLength).toBeLessThanOrEqual(AI_USER_IMAGE_MAX_BYTES) + expect(metadata).toMatchObject({ format: 'jpeg', width: 24, height: 16 }) + }) + + test('rejects malformed and non-canonical base64', async () => { + for (const data of ['!!!!', 'A===', `${smallJpegBase64} `]) { + await expect(validateAiUserImage(imageBlock(data))).rejects.toMatchObject({ + name: 'AiImageInputError', + status: 400, + message: 'Image data must be canonical base64.', + }) + } + }) + + test('rejects bytes whose real format does not match the JPEG contract', async () => { + const png = await makeImage(16, 16, 'png') + await expect(validateAiUserImage(imageBlock(png.toString('base64')))).rejects.toMatchObject({ + status: 400, + message: 'Image data is not a JPEG.', + }) + }) + + test('rejects corrupt data even when it starts with the JPEG signature', async () => { + const corrupt = Buffer.from([0xff, 0xd8, 0xff, 0x00, 0x01, 0x02, 0x03]) + await expect(validateAiUserImage(imageBlock(corrupt.toString('base64')))).rejects.toBeInstanceOf( + AiImageInputError, + ) + }) + + test('rejects a truncated JPEG whose header metadata is still readable', async () => { + const complete = await makeImage(24, 16, 'jpeg') + const truncated = complete.subarray(0, complete.byteLength - 2) + expect((await sharp(truncated).metadata()).format).toBe('jpeg') + + await expect(validateAiUserImage(imageBlock(truncated.toString('base64')))) + .rejects.toMatchObject({ status: 400, message: 'Image data could not be fully decoded.' }) + }) + + test('strips source metadata before persistence', async () => { + const tagged = await sharp({ + create: { + width: 24, + height: 16, + channels: 3, + background: { r: 45, g: 90, b: 135 }, + }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer() + expect((await sharp(tagged).metadata()).exif).toBeDefined() + + const canonical = await validateAiUserImage(imageBlock(tagged.toString('base64'))) + const metadata = await sharp(Buffer.from(canonical.data, 'base64')).metadata() + + expect(metadata.exif).toBeUndefined() + expect(metadata.orientation).toBeUndefined() + }) + + test('rejects decoded bytes over the image budget with 413 semantics', async () => { + const oversized = Buffer.alloc(AI_USER_IMAGE_MAX_BYTES + 1) + oversized[0] = 0xff + oversized[1] = 0xd8 + oversized[2] = 0xff + + await expect(validateAiUserImage(imageBlock(oversized.toString('base64')))).rejects.toMatchObject({ + status: 413, + message: 'Image exceeds the 1.5 MB limit.', + }) + }) + + test('rejects an excessive edge or pixel area with 413 semantics', async () => { + const tooWide = await makeImage(AI_USER_IMAGE_MAX_EDGE + 1, 1, 'jpeg') + await expect(validateAiUserImage(imageBlock(tooWide.toString('base64')))).rejects.toMatchObject({ + status: 413, + }) + + const areaWidth = 1500 + const areaHeight = Math.floor(AI_USER_IMAGE_MAX_PIXELS / areaWidth) + 1 + const tooManyPixels = await makeImage(areaWidth, areaHeight, 'jpeg') + await expect(validateAiUserImage(imageBlock(tooManyPixels.toString('base64')))).rejects.toMatchObject({ + status: 413, + }) + }) +}) diff --git a/src/__tests__/ai/messageHistory.test.ts b/src/__tests__/ai/messageHistory.test.ts index 1f4c0a9ad..0fc9a8e2e 100644 --- a/src/__tests__/ai/messageHistory.test.ts +++ b/src/__tests__/ai/messageHistory.test.ts @@ -2,6 +2,8 @@ import { describe, test, expect } from 'bun:test' import { buildMessageHistory, INTERRUPTED_TOOL_RESULT_ERROR, + NON_VISION_USER_IMAGE_OMITTED, + projectUserImagesForModel, } from '../../../server/ai/conversations/history' import type { MessageRecord } from '../../../server/ai/conversations/types' import type { AiContentBlock } from '../../../server/ai/runtime/types' @@ -34,6 +36,12 @@ function rec( function userText(text: string): MessageRecord { return rec('user', [{ kind: 'text', text }]) } +function userImage(data: string, text?: string): MessageRecord { + return rec('user', [ + ...(text ? [{ kind: 'text' as const, text }] : []), + { kind: 'image', mimeType: 'image/jpeg', data }, + ]) +} function assistantToolCall(id: string, name: string, input: unknown): MessageRecord { return rec('assistant', [{ kind: 'toolCall', toolCallId: id, toolName: name, input }], id, name) } @@ -204,3 +212,67 @@ describe('buildMessageHistory', () => { }) }) }) + +describe('user-image history projection', () => { + test('keeps every image-bearing user turn for a vision model', () => { + const history = buildMessageHistory([ + userImage('old-image', 'Old screenshot'), + rec('assistant', [{ kind: 'text', text: 'old answer' }]), + rec('user', [ + { kind: 'image', mimeType: 'image/jpeg', data: 'new-image-1' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'new-image-2' }, + ]), + rec('assistant', [{ kind: 'text', text: 'new answer' }]), + ]) + const original = JSON.stringify(history) + + const projected = projectUserImagesForModel(history, true) + + expect(projected).toEqual([ + { + role: 'user', + content: [ + { kind: 'text', text: 'Old screenshot' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'old-image' }, + ], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'old answer' }] }, + { + role: 'user', + content: [ + { kind: 'image', mimeType: 'image/jpeg', data: 'new-image-1' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'new-image-2' }, + ], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'new answer' }] }, + ]) + expect(JSON.stringify(history)).toBe(original) + }) + + test('elides every user image for a text-only model while keeping turns valid', () => { + const history = buildMessageHistory([ + userImage('old-image', 'First'), + rec('assistant', [{ kind: 'text', text: 'answer' }]), + userImage('new-image'), + ]) + const original = JSON.stringify(history) + + const projected = projectUserImagesForModel(history, false) + + expect(projected).toEqual([ + { + role: 'user', + content: [ + { kind: 'text', text: 'First' }, + { kind: 'text', text: NON_VISION_USER_IMAGE_OMITTED }, + ], + }, + { role: 'assistant', content: [{ kind: 'text', text: 'answer' }] }, + { + role: 'user', + content: [{ kind: 'text', text: NON_VISION_USER_IMAGE_OMITTED }], + }, + ]) + expect(JSON.stringify(history)).toBe(original) + }) +}) diff --git a/src/__tests__/ai/modelCapabilities.test.ts b/src/__tests__/ai/modelCapabilities.test.ts new file mode 100644 index 000000000..146f77e4e --- /dev/null +++ b/src/__tests__/ai/modelCapabilities.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, spyOn, test } from 'bun:test' +import { resolveModelCapabilities } from '../../../server/ai/drivers/modelCapabilities' +import { listProviderModels } from '../../../server/ai/drivers/modelList' +import type { + AiProvider, + AiProviderCapabilities, + AiResolvedCredential, +} from '../../../server/ai/drivers/types' + +const TEXT_ONLY: AiProviderCapabilities = { + toolCalling: true, + visionInput: false, + toolResultImages: false, + promptCache: false, + streaming: true, +} + +const VISION: AiProviderCapabilities = { + toolCalling: true, + visionInput: true, + toolResultImages: false, + promptCache: false, + streaming: true, +} + +const credentials: AiResolvedCredential = { + id: 'credential-1', + providerId: 'openrouter', + authMode: 'apiKey', + apiKey: 'secret', + baseUrl: null, +} + +function provider( + fallback: AiProviderCapabilities, + resolveCapabilities?: ( + credentials: AiResolvedCredential, + modelId: string, + ) => Promise, +): AiProvider { + return { + id: 'openrouter', + label: 'Test provider', + supportedAuthModes: ['apiKey'], + capabilities: () => fallback, + ...(resolveCapabilities ? { resolveCapabilities } : {}), + async listModels() { + throw new Error('full catalogue should not be queried for capability resolution') + }, + async *stream() { + yield { type: 'done' } + }, + } +} + +describe('resolveModelCapabilities', () => { + test('uses an authoritative vision-capable static default without a catalogue request', async () => { + const driver = provider(VISION) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(VISION) + }) + + test('uses a provider-owned selected-model capability lookup', async () => { + const driver = provider(TEXT_ONLY, async () => VISION) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(VISION) + }) + + test('fails closed to the static capability when live discovery fails', async () => { + const errorLog = spyOn(console, 'error').mockImplementation(() => {}) + try { + const driver = provider(VISION, async () => { + throw new Error('catalogue unavailable') + }) + + expect(await resolveModelCapabilities(driver, credentials, 'vision-model')).toEqual(TEXT_ONLY) + expect(errorLog).toHaveBeenCalledTimes(1) + } finally { + errorLog.mockRestore() + } + }) + + test('de-duplicates and caches concurrent selected-model lookups', async () => { + let calls = 0 + const driver = provider(TEXT_ONLY, async () => { + calls += 1 + await Promise.resolve() + return VISION + }) + + const [first, second] = await Promise.all([ + resolveModelCapabilities(driver, credentials, 'vision-model'), + resolveModelCapabilities(driver, credentials, 'vision-model'), + ]) + const cached = await resolveModelCapabilities(driver, credentials, 'vision-model') + + expect(first).toEqual(VISION) + expect(second).toEqual(VISION) + expect(cached).toEqual(VISION) + expect(calls).toBe(1) + }) + + test('fails closed for every waiter when a de-duplicated lookup rejects', async () => { + const errorLog = spyOn(console, 'error').mockImplementation(() => {}) + let calls = 0 + try { + const driver = provider(VISION, async () => { + calls += 1 + await Promise.resolve() + throw new Error('catalogue unavailable') + }) + + const results = await Promise.all([ + resolveModelCapabilities(driver, credentials, 'vision-model'), + resolveModelCapabilities(driver, credentials, 'vision-model'), + ]) + + expect(results).toEqual([TEXT_ONLY, TEXT_ONLY]) + expect(calls).toBe(1) + expect(errorLog).toHaveBeenCalledTimes(1) + } finally { + errorLog.mockRestore() + } + }) + + test('invalidates a cached result when credential auth material changes', async () => { + let calls = 0 + const driver = provider(TEXT_ONLY, async () => { + calls += 1 + return calls === 1 ? VISION : TEXT_ONLY + }) + + const first = await resolveModelCapabilities(driver, credentials, 'vision-model') + const rotated = await resolveModelCapabilities(driver, { + ...credentials, + apiKey: 'rotated-secret', + }, 'vision-model') + + expect(first).toEqual(VISION) + expect(rotated).toEqual(TEXT_ONLY) + expect(calls).toBe(2) + }) +}) + +describe('listProviderModels', () => { + test('releases the caller on abort even if a driver forgets to settle', async () => { + let providerSignal: AbortSignal | undefined + const driver: AiProvider = { + ...provider(TEXT_ONLY), + listModels(_credentials, signal) { + providerSignal = signal + return new Promise(() => {}) + }, + } + const controller = new AbortController() + const models = listProviderModels(driver, credentials, controller.signal) + + controller.abort() + + await expect(models).rejects.toHaveProperty('name', 'AbortError') + expect(providerSignal?.aborted).toBe(true) + }) +}) diff --git a/src/__tests__/ai/ollamaMapping.test.ts b/src/__tests__/ai/ollamaMapping.test.ts index 0b4c68126..bd9543cd2 100644 --- a/src/__tests__/ai/ollamaMapping.test.ts +++ b/src/__tests__/ai/ollamaMapping.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, afterEach } from 'bun:test' import { Type } from '@core/utils/typeboxHelpers' import { ollamaDriver } from '../../../server/ai/drivers/ollama' +import { resolveModelCapabilities } from '../../../server/ai/drivers/modelCapabilities' import { ChatCompletionsTurnTranslator, mapChatHistory, @@ -69,15 +70,38 @@ describe('Ollama mapChatHistory', () => { test('maps an image-bearing user turn to OpenAI content parts', () => { const history: AiMessage[] = [ - { role: 'user', content: [{ kind: 'image', mimeType: 'image/png', data: 'B64' }, { kind: 'text', text: 'see' }] }, + { + role: 'user', + content: [ + { kind: 'text', text: 'compare' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'B64-1' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'B64-2' }, + ], + }, ] const mapped = mapChatHistory([], history).flat() expect(mapped).toEqual([ { role: 'user', content: [ - { type: 'image_url', image_url: { url: 'data:image/png;base64,B64' } }, - { type: 'text', text: 'see' }, + { type: 'text', text: 'compare' }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,B64-1' } }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,B64-2' } }, + ], + }, + ]) + }) + + test('maps an image-only user turn without inventing text', () => { + const history: AiMessage[] = [ + { role: 'user', content: [{ kind: 'image', mimeType: 'image/jpeg', data: 'B64' }] }, + ] + + expect(mapChatHistory([], history).flat()).toEqual([ + { + role: 'user', + content: [ + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,B64' } }, ], }, ]) @@ -144,7 +168,7 @@ function makeRequest(serverCalls: unknown[]): AiStreamRequest { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool], modelId: 'llama3.3', - modelCapabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'ollama', authMode: 'baseUrl', apiKey: null, baseUrl: 'http://localhost:11434' }, signal: new AbortController().signal, bridge, @@ -193,3 +217,91 @@ describe('runToolLoop via ollamaDriver', () => { expect(usage!.completionTokens).toBe(15) }) }) + +describe('Ollama live model capabilities', () => { + test('uses /api/show to distinguish vision and text-only installed models', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString() + requests.push({ url, init }) + + if (url === 'http://localhost:11434/api/tags') { + return Response.json({ + models: [ + { name: 'llava:latest' }, + { model: 'llama3.3:latest' }, + ], + }) + } + + expect(url).toBe('http://localhost:11434/api/show') + const body = JSON.parse(init?.body as string) as { model: string } + return Response.json({ + capabilities: body.model === 'llava:latest' + ? ['completion', 'vision'] + : ['completion'], + }) + }) as typeof fetch + + const models = await ollamaDriver.listModels({ + id: 'credential-1', + providerId: 'ollama', + authMode: 'baseUrl', + apiKey: 'proxy-secret', + baseUrl: 'http://localhost:11434/', + }) + + expect(models).toEqual([ + expect.objectContaining({ + id: 'llava:latest', + catalogueSource: 'live', + capabilities: expect.objectContaining({ visionInput: true }), + }), + expect.objectContaining({ + id: 'llama3.3:latest', + catalogueSource: 'live', + capabilities: expect.objectContaining({ visionInput: false }), + }), + ]) + expect(requests.map(({ url }) => url)).toEqual([ + 'http://localhost:11434/api/tags', + 'http://localhost:11434/api/show', + 'http://localhost:11434/api/show', + ]) + expect(requests.slice(1).map(({ init }) => ({ + method: init?.method, + contentType: (init?.headers as Record)['content-type'], + authorization: (init?.headers as Record).Authorization, + }))).toEqual([ + { method: 'POST', contentType: 'application/json', authorization: 'Bearer proxy-secret' }, + { method: 'POST', contentType: 'application/json', authorization: 'Bearer proxy-secret' }, + ]) + expect(requests[0]?.init?.headers).toEqual({ Authorization: 'Bearer proxy-secret' }) + }) + + test('resolves only the selected model, fails closed over a positive fallback, and authenticates', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString() + requests.push({ url, init }) + return Response.json({ capabilities: ['completion'] }) + }) as typeof fetch + + const capabilities = await resolveModelCapabilities(ollamaDriver, { + id: 'proxy-credential', + providerId: 'ollama', + authMode: 'baseUrl', + apiKey: 'proxy-secret', + baseUrl: 'http://localhost:11434/', + }, 'llama4') + + expect(capabilities.visionInput).toBe(false) + expect(requests).toHaveLength(1) + expect(requests[0]?.url).toBe('http://localhost:11434/api/show') + expect(requests[0]?.init?.method).toBe('POST') + expect(requests[0]?.init?.headers).toEqual({ + 'content-type': 'application/json', + Authorization: 'Bearer proxy-secret', + }) + }) +}) diff --git a/src/__tests__/ai/responsesMapping.test.ts b/src/__tests__/ai/responsesMapping.test.ts index c12e1cb75..faaa63b13 100644 --- a/src/__tests__/ai/responsesMapping.test.ts +++ b/src/__tests__/ai/responsesMapping.test.ts @@ -122,7 +122,14 @@ describe('Responses mapHistory', () => { test('maps base64 image blocks to a Responses input_image data URL', () => { const history: AiMessage[] = [ - { role: 'user', content: [{ kind: 'image', mimeType: 'image/png', data: 'BASE64' }, { kind: 'text', text: 'look' }] }, + { + role: 'user', + content: [ + { kind: 'image', mimeType: 'image/png', data: 'BASE64-1' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'BASE64-2' }, + { kind: 'text', text: 'compare' }, + ], + }, ] const mapped = mapResponsesHistory(history).flat() expect(mapped).toEqual([ @@ -130,8 +137,9 @@ describe('Responses mapHistory', () => { type: 'message', role: 'user', content: [ - { type: 'input_image', image_url: 'data:image/png;base64,BASE64' }, - { type: 'input_text', text: 'look' }, + { type: 'input_image', image_url: 'data:image/png;base64,BASE64-1' }, + { type: 'input_image', image_url: 'data:image/jpeg;base64,BASE64-2' }, + { type: 'input_text', text: 'compare' }, ], }, ]) @@ -209,7 +217,7 @@ describe('runToolLoop via openaiDriver (Responses)', () => { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool], modelId: 'gpt-5.4', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'openai', authMode: 'apiKey', apiKey: 'sk-test', baseUrl: null }, signal: new AbortController().signal, bridge, @@ -264,7 +272,7 @@ describe('openrouterDriver', () => { messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [], modelId: 'openai/gpt-5.4', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, credentials: { id: 'cr', providerId: 'openrouter', authMode: 'apiKey', apiKey: 'sk-or-test', baseUrl: null }, signal: new AbortController().signal, bridge, @@ -338,14 +346,14 @@ describe('openrouterDriver', () => { { id: 'openai/gpt-5.4', label: 'GPT 5.4', - capabilities: { toolCalling: true, visionInput: true, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: true, toolResultImages: false, promptCache: false, streaming: true }, pricing: { inputPerMTok: 5, outputPerMTok: 25 }, contextWindow: 128_000, }, { id: 'anthropic/claude-opus-4.8', label: 'Claude Opus 4.8', - capabilities: { toolCalling: false, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: false, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, pricing: { inputPerMTok: 10, outputPerMTok: 50 }, contextWindow: 200_000, }, diff --git a/src/__tests__/ai/toolEvidence.test.ts b/src/__tests__/ai/toolEvidence.test.ts index 4e36e9e14..03fea195f 100644 --- a/src/__tests__/ai/toolEvidence.test.ts +++ b/src/__tests__/ai/toolEvidence.test.ts @@ -63,8 +63,9 @@ const ANTHROPIC_DONE = sse( { type: 'message_stop' }, ) -const VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: true, promptCache: true, streaming: true } -const NO_VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: false, promptCache: false, streaming: true } +const VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: true, toolResultImages: true, promptCache: true, streaming: true } +const NO_VISION_CAPS: AiProviderCapabilities = { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true } +const USER_IMAGES_ONLY_CAPS: AiProviderCapabilities = { ...VISION_CAPS, toolResultImages: false } const renderSnapshotTool: AiTool = { name: 'site_render_snapshot', @@ -157,6 +158,24 @@ describe('multimodal tool output + heavy elision (Anthropic)', () => { expect(browserInputs).toEqual([{ captureScreenshot: false }]) }) + test('vision input without native tool-result images also skips screenshot capture', async () => { + let call = 0 + globalThis.fetch = (async () => { + call += 1 + return sseResponse(call === 1 ? anthropicSnapTurn('t_s1') : ANTHROPIC_DONE) + }) as typeof fetch + const browserInputs: unknown[] = [] + const bridge: AiBrowserBridge = { + async callBrowser(_name, input): Promise { + browserInputs.push(input) + return { ok: true, data: { layout: { warnings: [] } } } + }, + } + + for await (const _ of anthropicDriver.stream(makeRequest(bridge, USER_IMAGES_ONLY_CAPS))) { void _ } + expect(browserInputs).toEqual([{ captureScreenshot: false }]) + }) + test('stubs the earlier render_snapshot once a newer one supersedes it', async () => { const bodies: Array> = [] globalThis.fetch = (async (_url: string, init: RequestInit) => { diff --git a/src/__tests__/ai/toolLoop.test.ts b/src/__tests__/ai/toolLoop.test.ts index 75e23f128..bbc4a5c6a 100644 --- a/src/__tests__/ai/toolLoop.test.ts +++ b/src/__tests__/ai/toolLoop.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, afterEach } from 'bun:test' import { Type } from '@core/utils/typeboxHelpers' import { anthropicDriver } from '../../../server/ai/drivers/anthropic' +import { PROVIDER_RETRY_IMAGE_OMITTED } from '../../../server/ai/drivers/http/toolLoop' import type { AiStreamRequest } from '../../../server/ai/drivers/types' import type { AiBrowserBridge, AiStreamEvent, AiTool, AiToolOutput } from '../../../server/ai/runtime/types' @@ -79,7 +80,7 @@ function makeRequest(bridge: AiBrowserBridge, serverCalls: unknown[]): AiStreamR messages: [{ role: 'user', content: [{ kind: 'text', text: 'go' }] }], tools: [echoTool, paintTool], modelId: 'claude-sonnet-4-6', - modelCapabilities: { toolCalling: true, visionInput: true, promptCache: true, streaming: true }, + modelCapabilities: { toolCalling: true, visionInput: true, toolResultImages: true, promptCache: true, streaming: true }, credentials: { id: 'cr', providerId: 'anthropic', authMode: 'apiKey', apiKey: 'sk-test', baseUrl: null }, signal: new AbortController().signal, bridge, @@ -163,4 +164,56 @@ describe('runToolLoop via anthropicDriver', () => { expect(events[0]!.type).toBe('error') expect((events[0] as { message: string }).message).toContain('authentication failed') }) + + test('retries a provider overflow once with only historical images elided', async () => { + const requestBodies: Array> = [] + globalThis.fetch = (async (_url: string, init: RequestInit) => { + requestBodies.push(JSON.parse(init.body as string)) + if (requestBodies.length === 1) { + return new Response(JSON.stringify({ + error: { type: 'request_too_large', message: 'Request exceeds the context limit' }, + }), { status: 413 }) + } + return sseResponse(TURN2) + }) as typeof fetch + + const req = makeRequest({ async callBrowser() { return { ok: true } } }, []) + const image = { kind: 'image' as const, mimeType: 'image/jpeg', data: '/9j/' } + req.messages.splice(0, req.messages.length, + { role: 'user', content: [{ kind: 'text', text: 'Earlier turn' }, image] }, + { role: 'assistant', content: [{ kind: 'text', text: 'Earlier reply' }] }, + { role: 'user', content: [{ kind: 'text', text: 'Current turn' }, image] }, + ) + + const events: AiStreamEvent[] = [] + for await (const event of anthropicDriver.stream(req)) events.push(event) + + expect(requestBodies).toHaveLength(2) + expect(JSON.stringify(requestBodies[0]).match(/"type":"image"/g)).toHaveLength(2) + expect(JSON.stringify(requestBodies[1]).match(/"type":"image"/g)).toHaveLength(1) + expect(JSON.stringify(requestBodies[1])).toContain(PROVIDER_RETRY_IMAGE_OMITTED) + expect(req.messages[0]?.content.some((block) => block.kind === 'image')).toBe(true) + expect(events.some((event) => event.type === 'error')).toBe(false) + }) + + test('does not retry when only the current user turn contains images', async () => { + let requests = 0 + globalThis.fetch = (async () => { + requests += 1 + return new Response('', { status: 413 }) + }) as typeof fetch + const req = makeRequest({ async callBrowser() { return { ok: true } } }, []) + req.messages[0] = { + role: 'user', + content: [{ kind: 'image', mimeType: 'image/jpeg', data: '/9j/' }], + } + + const events: AiStreamEvent[] = [] + for await (const event of anthropicDriver.stream(req)) events.push(event) + + expect(requests).toBe(1) + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ type: 'error' }) + expect((events[0] as { message: string }).message).toContain('Your history is still saved') + }) }) diff --git a/src/__tests__/http/apiClient.test.ts b/src/__tests__/http/apiClient.test.ts index edb40a301..e023eedf4 100644 --- a/src/__tests__/http/apiClient.test.ts +++ b/src/__tests__/http/apiClient.test.ts @@ -1,6 +1,14 @@ import { describe, expect, it } from 'bun:test' import { Type } from '@core/utils/typeboxHelpers' -import { apiRequest, ApiError, assertOk, isAbortError, readEnvelope, responseErrorMessage } from '@core/http' +import { + apiBlobRequest, + apiRequest, + ApiError, + assertOk, + isAbortError, + readEnvelope, + responseErrorMessage, +} from '@core/http' const BodySchema = Type.Object({ value: Type.Number() }) @@ -94,6 +102,34 @@ describe('apiRequest', () => { }) }) +describe('apiBlobRequest', () => { + it('returns binary bodies through the shared authenticated transport', async () => { + let seen: RequestInit | undefined + const blob = await apiBlobRequest('/image', { + fetchImpl: async (_input, init) => { + seen = init + return new Response(new Uint8Array([1, 2, 3]), { + headers: { 'content-type': 'image/png' }, + }) + }, + }) + + expect(blob.type).toBe('image/png') + expect(new Uint8Array(await blob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])) + expect(seen?.credentials).toBe('include') + }) + + it('throws the same ApiError envelope as JSON requests', async () => { + const err = await apiBlobRequest('/image', { + fetchImpl: async () => jsonResponse({ error: 'image unavailable' }, 404), + }).catch((error) => error) + + expect(err).toBeInstanceOf(ApiError) + expect((err as ApiError).status).toBe(404) + expect((err as ApiError).message).toBe('image unavailable') + }) +}) + describe('readEnvelope', () => { it('throws ApiError on a non-OK response', async () => { const err = await readEnvelope(jsonResponse({ error: 'bad' }, 400), BodySchema, 'fallback').catch((e) => e) diff --git a/src/__tests__/layout/floatingWindowPosition.test.ts b/src/__tests__/layout/floatingWindowPosition.test.ts new file mode 100644 index 000000000..5018315b5 --- /dev/null +++ b/src/__tests__/layout/floatingWindowPosition.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'bun:test' +import { clampFloatingPanelPosition } from '@admin/shared/FloatingWindow' + +describe('floating window viewport clamp', () => { + it('keeps a reachable header strip after a far-left drag', () => { + expect(clampFloatingPanelPosition( + { x: -5_000, y: 80 }, + { viewportWidth: 1_000, viewportHeight: 700, panelWidth: 820 }, + )).toEqual({ x: -770, y: 80 }) + }) + + it('reclamps a stored wide-window position against a narrower reopened panel', () => { + expect(clampFloatingPanelPosition( + { x: -770, y: 900 }, + { viewportWidth: 600, viewportHeight: 500, panelWidth: 520 }, + )).toEqual({ x: -470, y: 450 }) + }) +}) diff --git a/src/__tests__/panels/agentPanel.test.tsx b/src/__tests__/panels/agentPanel.test.tsx index 7c10785b7..215d050d0 100644 --- a/src/__tests__/panels/agentPanel.test.tsx +++ b/src/__tests__/panels/agentPanel.test.tsx @@ -1,13 +1,131 @@ import { afterEach, describe, expect, it, mock } from 'bun:test' -import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { createStore } from 'zustand/vanilla' import { AgentStoreProvider } from '@admin/ai/AgentStoreContext' +import { clearModelListCache } from '@admin/ai/api' import { MemoryRouter, useLocation } from '@admin/lib/routing' +import { AdminSessionProvider } from '@admin/session' import type { AgentSlice } from '@site/agent' +import type { AiUserContentBlock } from '@core/ai' +import type { CmsCurrentUser } from '@core/persistence' import { AgentPanel } from '@site/panels/AgentPanel' const originalFetch = globalThis.fetch +const TEST_CREDENTIAL = { + id: 'cred_1', + providerId: 'openai', + authMode: 'apiKey', + displayLabel: 'OpenAI', + baseUrl: null, + keyFingerprintCurrent: true, + createdAt: '2026-06-01T10:00:00.000Z', + lastUsedAt: null, +} as const + +function installModelFetch(visionInput: boolean, toolCalling = true): void { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) { + return jsonResponse({ credentials: [TEST_CREDENTIAL] }) + } + if (url.includes('/admin/api/ai/providers/')) { + return jsonResponse({ + models: [{ + id: 'model-1', + label: 'Model 1', + capabilities: { + toolCalling, + visionInput, + toolResultImages: false, + promptCache: false, + streaming: true, + }, + contextWindow: 128_000, + }], + }) + } + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch +} + +interface Deferred { + promise: Promise + resolve(value: T): void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +interface ImageBrowserMocks { + bitmap: ImageBitmap + restore(): void +} + +let activeImageMocks: ImageBrowserMocks | null = null + +function installImageBrowserMocks( + bitmapPromise: Promise = Promise.resolve(fakeBitmap()), +): ImageBrowserMocks { + activeImageMocks?.restore() + const canvasPrototype = Object.getPrototypeOf(document.createElement('canvas')) as object + const createBitmapDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'createImageBitmap') + const getContextDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'getContext') + const toBlobDescriptor = Object.getOwnPropertyDescriptor(canvasPrototype, 'toBlob') + const bitmap = fakeBitmap() + + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: mock(() => bitmapPromise), + }) + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value: () => ({ + fillStyle: '', + fillRect: () => {}, + drawImage: () => {}, + }), + }) + Object.defineProperty(canvasPrototype, 'toBlob', { + configurable: true, + value: (callback: BlobCallback) => { + callback(new Blob([new Uint8Array([1, 2, 3])], { type: 'image/jpeg' })) + }, + }) + const installed: ImageBrowserMocks = { + bitmap, + restore() { + restoreProperty(globalThis, 'createImageBitmap', createBitmapDescriptor) + restoreProperty(canvasPrototype, 'getContext', getContextDescriptor) + restoreProperty(canvasPrototype, 'toBlob', toBlobDescriptor) + }, + } + activeImageMocks = installed + return installed +} + +function fakeBitmap(): ImageBitmap { + return { + width: 100, + height: 80, + close: mock(() => {}), + } as unknown as ImageBitmap +} + +function restoreProperty( + target: object, + key: PropertyKey, + descriptor: PropertyDescriptor | undefined, +): void { + if (descriptor) Object.defineProperty(target, key, descriptor) + else Reflect.deleteProperty(target, key) +} + function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, @@ -25,15 +143,27 @@ function createAgentStore(overrides: Partial = {}) { agentActiveCredentialId: null, agentActiveModelId: null, agentConversations: [], + agentContextTokens: null, + isAgentConversationPending: false, + isAgentProviderPending: false, + agentComposerEpoch: 0, openAgent: () => set({ isAgentOpen: true }), closeAgent: () => set({ isAgentOpen: false }), toggleAgent: () => set((state) => ({ isAgentOpen: !state.isAgentOpen })), - sendAgentMessage: async () => {}, + sendAgentMessage: async () => ({ accepted: true }), abortAgent: () => {}, - clearAgentMessages: () => set({ agentMessages: [], agentError: null }), + clearAgentMessages: () => set((state) => ({ + agentMessages: [], + agentError: null, + agentComposerEpoch: state.agentComposerEpoch + 1, + })), loadAgentConversations: async () => {}, loadAgentConversation: async () => {}, - startNewAgentConversation: () => set({ agentMessages: [], agentError: null }), + startNewAgentConversation: () => set((state) => ({ + agentMessages: [], + agentError: null, + agentComposerEpoch: state.agentComposerEpoch + 1, + })), deleteAgentConversation: async () => {}, setAgentProvider: async (credentialId, modelId) => { set({ agentActiveCredentialId: credentialId, agentActiveModelId: modelId, agentError: null }) @@ -43,16 +173,54 @@ function createAgentStore(overrides: Partial = {}) { })) } -function renderAgentPanel(overrides: Partial = {}) { +function renderAgentPanel( + overrides: Partial = {}, + user: CmsCurrentUser = testUser(), +) { const store = createAgentStore(overrides) - return render( - - - - - - , + const view = render( + + + + + + + + , ) + return { ...view, store } +} + +function testUser(capabilities: CmsCurrentUser['capabilities'] = ['ai.chat']): CmsCurrentUser { + return { + id: 'user-agent-panel', + email: 'agent@example.com', + displayName: 'Agent User', + status: 'active', + role: { + id: 'role-admin', + slug: 'admin', + name: 'Admin', + description: 'Agent panel test role', + isSystem: true, + capabilities, + }, + capabilities, + lastLoginAt: null, + failedLoginCount: 0, + lockedUntil: null, + passwordUpdatedAt: null, + mfaEnabled: false, + mfaEnabledAt: null, + mfaRecoveryCodesRemaining: 0, + stepUpAuthMode: 'password', + stepUpWindowMinutes: 15, + avatarMediaId: null, + avatarUrl: null, + gravatarHash: '', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + } } function RouteProbe() { @@ -60,11 +228,38 @@ function RouteProbe() { return {location.pathname} } +function pasteImage(fileName = 'clipboard.png'): void { + pasteImages([fileName]) +} + +function pasteImages(fileNames: string[]): void { + const textarea = screen.getByLabelText('Message to AI assistant') + fireEvent.paste(textarea, { + clipboardData: { + files: fileNames.map((fileName) => + new File([pngHeader(100, 80)], fileName, { type: 'image/png' })), + }, + }) +} + +function pngHeader(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(24) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + bytes.set([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52], 8) + const view = new DataView(bytes.buffer) + view.setUint32(16, width) + view.setUint32(20, height) + return bytes +} + describe('AgentPanel', () => { afterEach(() => { cleanup() + activeImageMocks?.restore() + activeImageMocks = null localStorage.clear() globalThis.fetch = originalFetch + clearModelListCache() }) it('surfaces a large setup empty state and header shortcut when no credentials exist', async () => { @@ -231,4 +426,451 @@ describe('AgentPanel', () => { const textarea = screen.getByLabelText('Message to AI assistant') as HTMLTextAreaElement expect(textarea.disabled).toBe(false) }) + + it('blocks same-tick paste + Enter, then sends a prepared image-only turn', async () => { + installModelFetch(true) + const decode = deferred() + const bitmap = fakeBitmap() + installImageBrowserMocks(decode.promise) + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + pasteImage() + fireEvent.keyDown(textarea, { key: 'Enter' }) + + expect(sendAgentMessage).not.toHaveBeenCalled() + expect(screen.getByText('Preparing…')).toBeTruthy() + + await act(async () => { + decode.resolve(bitmap) + await decode.promise + }) + await waitFor(() => expect(screen.getByText('Ready')).toBeTruthy()) + await waitFor(() => { + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBeNull() + }) + + fireEvent.keyDown(textarea, { key: 'Enter' }) + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([{ + kind: 'image', + mimeType: 'image/jpeg', + data: 'AQID', + }]) + await waitFor(() => { + expect(screen.queryByLabelText('Attached image: clipboard.png')).toBeNull() + }) + expect((textarea as HTMLTextAreaElement).value).toBe('') + }) + + it('retains text and the prepared image when the request is rejected before acceptance', async () => { + installModelFetch(true) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: false })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') as HTMLTextAreaElement + pasteImage('reference.png') + await waitFor(() => expect(screen.getByText('Ready')).toBeTruthy()) + await waitFor(() => { + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBeNull() + }) + fireEvent.change(textarea, { target: { value: 'Use this reference' } }) + fireEvent.click(screen.getByRole('button', { name: 'Send' })) + + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([ + { kind: 'text', text: 'Use this reference' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + ]) + expect(textarea.value).toBe('Use this reference') + expect(screen.getByLabelText('Attached image: reference.png')).toBeTruthy() + }) + + it('normalizes pasted images sequentially and removes attachments independently', async () => { + installModelFetch(true) + const firstDecode = deferred() + const secondDecode = deferred() + const imageMocks = installImageBrowserMocks() + let decodeIndex = 0 + Object.defineProperty(globalThis, 'createImageBitmap', { + configurable: true, + value: mock(() => [firstDecode.promise, secondDecode.promise][decodeIndex++]!), + }) + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + pasteImages(['first.png', 'second.png']) + expect(screen.getByLabelText('Attached image: first.png')).toBeTruthy() + expect(screen.getByLabelText('Attached image: second.png')).toBeTruthy() + await waitFor(() => expect(globalThis.createImageBitmap).toHaveBeenCalledTimes(1)) + + await act(async () => { + firstDecode.resolve(imageMocks.bitmap) + await firstDecode.promise + }) + await waitFor(() => expect(globalThis.createImageBitmap).toHaveBeenCalledTimes(2)) + await act(async () => { + secondDecode.resolve(fakeBitmap()) + await secondDecode.promise + }) + await waitFor(() => expect(screen.getAllByText('Ready')).toHaveLength(2)) + + fireEvent.click(screen.getByRole('button', { name: 'Remove attached image: first.png' })) + expect(screen.queryByLabelText('Attached image: first.png')).toBeNull() + expect(screen.getByLabelText('Attached image: second.png')).toBeTruthy() + + fireEvent.keyDown(textarea, { key: 'Enter' }) + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([{ + kind: 'image', + mimeType: 'image/jpeg', + data: 'AQID', + }]) + }) + + it('sends every prepared image in its original paste order', async () => { + installModelFetch(true) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + pasteImages(['first.png', 'second.png']) + await waitFor(() => expect(screen.getAllByText('Ready')).toHaveLength(2)) + fireEvent.keyDown(textarea, { key: 'Enter' }) + + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([ + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + ]) + }) + + it('attaches multiple local images through the picker beside Send', async () => { + installModelFetch(true) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + const { container } = renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + await screen.findByLabelText('Message to AI assistant') + const picker = screen.getByRole('button', { name: 'Attach images' }) + expect(picker).toBeTruthy() + const input = container.querySelector( + 'input[type="file"][accept="image/png,image/jpeg,image/webp"]', + ) + expect(input?.multiple).toBe(true) + + fireEvent.change(input!, { + target: { + files: [ + new File([pngHeader(100, 80)], 'picked-first.png', { type: 'image/png' }), + new File([pngHeader(100, 80)], 'picked-second.png', { type: 'image/png' }), + ], + }, + }) + + expect(input?.value).toBe('') + await waitFor(() => expect(screen.getAllByText('Ready')).toHaveLength(2)) + const pendingPreview = screen.getByRole('button', { + name: 'Preview attached image: picked-first.png', + }) + fireEvent.contextMenu(pendingPreview, { clientX: 40, clientY: 50 }) + const menu = await screen.findByRole('menu', { name: 'Image actions' }) + fireEvent.keyDown(menu, { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('menu', { name: 'Image actions' })).toBeNull()) + await waitFor(() => expect(document.activeElement).toBe(pendingPreview)) + fireEvent.click(screen.getByRole('button', { name: 'Send' })) + await waitFor(() => expect(sendAgentMessage).toHaveBeenCalledTimes(1)) + expect(sendAgentMessage.mock.calls[0]?.[0]).toEqual([ + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + { kind: 'image', mimeType: 'image/jpeg', data: 'AQID' }, + ]) + }) + + it('keeps an attachment visible but disables send for a non-vision model', async () => { + installModelFetch(false) + installImageBrowserMocks() + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + await screen.findByLabelText('Message to AI assistant') + pasteImage() + + expect((await screen.findByRole('alert')).textContent).toContain( + 'Choose a vision-capable model or remove the image.', + ) + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBe('true') + expect(screen.getByLabelText('Attached image: clipboard.png')).toBeTruthy() + expect(sendAgentMessage).not.toHaveBeenCalled() + }) + + it('blocks send when the selected model is known not to support agent tools', async () => { + installModelFetch(true, false) + const sendAgentMessage = mock(async (_content: AiUserContentBlock[]) => ({ accepted: true })) + renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + sendAgentMessage, + }) + + const textarea = await screen.findByLabelText('Message to AI assistant') + fireEvent.change(textarea, { target: { value: 'Inspect this page' } }) + await screen.findByText('Choose an agent-capable model that supports tool calling.') + expect(screen.getByRole('button', { name: 'Send' }).getAttribute('aria-disabled')).toBe('true') + fireEvent.keyDown(textarea, { key: 'Enter' }) + expect(sendAgentMessage).not.toHaveBeenCalled() + }) + + it('drops an in-flight attachment when an explicit conversation reset remounts the composer', async () => { + installModelFetch(true) + const decode = deferred() + const bitmap = fakeBitmap() + installImageBrowserMocks(decode.promise) + const { store } = renderAgentPanel({ + agentActiveCredentialId: TEST_CREDENTIAL.id, + agentActiveModelId: 'model-1', + }) + + await screen.findByLabelText('Message to AI assistant') + pasteImage('stale.png') + expect(screen.getByLabelText('Attached image: stale.png')).toBeTruthy() + + act(() => { + store.setState({ agentComposerEpoch: 1 }) + }) + expect(screen.queryByLabelText('Attached image: stale.png')).toBeNull() + await act(async () => { + decode.resolve(bitmap) + await decode.promise + }) + expect(screen.queryByLabelText('Attached image: stale.png')).toBeNull() + }) + + it('renders a rehydrated user image block in conversation history', async () => { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + + renderAgentPanel({ + agentMessages: [{ + id: 'image-message', + role: 'user', + blocks: [{ kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/0' }], + timestamp: Date.now(), + }], + }) + + const image = await screen.findByAltText('Attachment from you') as HTMLImageElement + expect(image.getAttribute('src')).toBe('/conversation-image/0') + }) + + it('coalesces images into compact galleries and opens a focus-restoring preview', async () => { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + const { store } = renderAgentPanel({ + agentMessages: [{ + id: 'image-message', + role: 'user', + blocks: [ + { kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/0' }, + { kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/1' }, + { kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/2' }, + ], + timestamp: Date.now(), + }], + }) + + const gallery = await screen.findByRole('group', { name: 'Images from you' }) + expect(gallery.querySelectorAll('button')).toHaveLength(3) + const trigger = screen.getByRole('button', { name: 'Open image preview: Attachment 1 of 3 from you' }) + trigger.focus() + fireEvent.click(trigger) + + const preview = await screen.findByRole('dialog', { name: 'Your attachment' }) + expect(preview.querySelector('img')?.getAttribute('alt')).toBe('Attachment 1 of 3 from you') + await waitFor(() => expect(document.activeElement).toBe(preview)) + + fireEvent.keyDown(document, { key: 'Escape' }) + await waitFor(() => expect(screen.queryByTestId('agent-image-preview')).toBeNull()) + expect(store.getState().isAgentOpen).toBe(true) + await waitFor(() => expect(document.activeElement).toBe(trigger)) + + fireEvent.click(trigger) + await screen.findByRole('dialog', { name: 'Your attachment' }) + fireEvent.click(screen.getByRole('button', { name: 'Close Your attachment panel' })) + await waitFor(() => expect(screen.queryByTestId('agent-image-preview')).toBeNull()) + await waitFor(() => expect(document.activeElement).toBe(trigger)) + + fireEvent.click(trigger) + await screen.findByRole('dialog', { name: 'Your attachment' }) + act(() => { + store.setState((state) => ({ + agentComposerEpoch: state.agentComposerEpoch + 1, + agentMessages: [], + })) + }) + await waitFor(() => expect(screen.queryByTestId('agent-image-preview')).toBeNull()) + expect(store.getState().isAgentOpen).toBe(true) + }) + + it('opens the same image action menu from chat, keyboard, and the preview', async () => { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + renderAgentPanel({ + agentMessages: [{ + id: 'image-message', + role: 'user', + blocks: [{ kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/0' }], + timestamp: Date.now(), + }], + }) + + const trigger = await screen.findByRole('button', { + name: 'Open image preview: Attachment from you', + }) + fireEvent.contextMenu(trigger, { clientX: 80, clientY: 90 }) + let menu = await screen.findByRole('menu', { name: 'Image actions' }) + expect(screen.getByRole('menuitem', { name: 'Copy image' })).toBeTruthy() + expect(screen.getByRole('menuitem', { name: 'Save to desktop' })).toBeTruthy() + expect(screen.getByRole('menuitem', { name: 'Save to Media' }).getAttribute('aria-disabled')) + .toBe('true') + + fireEvent.keyDown(menu, { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('menu', { name: 'Image actions' })).toBeNull()) + + trigger.focus() + fireEvent.keyDown(trigger, { key: 'F10', shiftKey: true }) + menu = await screen.findByRole('menu', { name: 'Image actions' }) + fireEvent.keyDown(menu, { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('menu', { name: 'Image actions' })).toBeNull()) + await waitFor(() => expect(document.activeElement).toBe(trigger)) + + fireEvent.click(trigger) + const preview = await screen.findByRole('dialog', { name: 'Your attachment' }) + const previewImage = preview.querySelector('img')! + fireEvent.contextMenu(previewImage, { clientX: 120, clientY: 130 }) + menu = await screen.findByRole('menu', { name: 'Image actions' }) + fireEvent.keyDown(screen.getByRole('menuitem', { name: 'Save to desktop' }), { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('menu', { name: 'Image actions' })).toBeNull()) + expect(screen.getByRole('dialog', { name: 'Your attachment' })).toBeTruthy() + await waitFor(() => expect(document.activeElement).toBe(preview)) + }) + + it('saves a lazy conversation image through the canonical Media upload', async () => { + let uploadedFile: File | null = null + globalThis.fetch = mock(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + if (url === '/conversation-image/0') { + return new Response(new Uint8Array([0xff, 0xd8, 0xff, 0xd9]), { + headers: { 'content-type': 'image/jpeg' }, + }) + } + if (url === '/admin/api/cms/media' && init?.method === 'POST') { + uploadedFile = (init.body as FormData).get('file') as File + return jsonResponse({ + asset: { + id: 'saved-image', + filename: uploadedFile.name, + mimeType: uploadedFile.type, + sizeBytes: uploadedFile.size, + publicPath: '/uploads/saved-image.jpg', + uploadedByUserId: 'user-agent-panel', + createdAt: '2026-07-11T10:00:00.000Z', + }, + }, 201) + } + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + renderAgentPanel({ + agentMessages: [{ + id: 'image-message', + role: 'user', + blocks: [{ kind: 'image', mimeType: 'image/jpeg', src: '/conversation-image/0' }], + timestamp: Date.now(), + }], + }, testUser(['ai.chat', 'media.write'])) + + const trigger = await screen.findByRole('button', { + name: 'Open image preview: Attachment from you', + }) + fireEvent.contextMenu(trigger, { clientX: 80, clientY: 90 }) + const save = await screen.findByRole('menuitem', { name: 'Save to Media' }) + expect(save.getAttribute('aria-disabled')).toBeNull() + fireEvent.click(save) + + await waitFor(() => expect(uploadedFile).not.toBeNull()) + expect(uploadedFile?.name).toBe('your-attachment-1.jpg') + expect(uploadedFile?.type).toBe('image/jpeg') + }) + + it('uses the same gallery for assistant and plural tool-result images', async () => { + globalThis.fetch = mock(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/admin/api/ai/credentials')) return jsonResponse({ credentials: [] }) + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + renderAgentPanel({ + agentMessages: [{ + id: 'assistant-images', + role: 'assistant', + blocks: [ + { kind: 'image', mimeType: 'image/jpeg', src: '/assistant-image/0' }, + { kind: 'image', mimeType: 'image/jpeg', src: '/assistant-image/1' }, + { + kind: 'toolCall', + toolCall: { + id: 'tool-1', + actionType: 'site_render_snapshot', + params: {}, + result: { ok: true }, + status: 'success', + previewImages: ['data:image/png;base64,R0hJ', 'data:image/png;base64,SktM'], + }, + }, + ], + timestamp: Date.now(), + }], + }) + + expect((await screen.findByRole('group', { name: 'Images from assistant' })) + .querySelectorAll('button')).toHaveLength(2) + expect(screen.getByRole('group', { name: 'Images captured by assistant tools' }) + .querySelectorAll('button')).toHaveLength(2) + }) }) diff --git a/src/__tests__/site-explorer/siteExplorerPanel.test.tsx b/src/__tests__/site-explorer/siteExplorerPanel.test.tsx index 8d2cf7b1b..d43837590 100644 --- a/src/__tests__/site-explorer/siteExplorerPanel.test.tsx +++ b/src/__tests__/site-explorer/siteExplorerPanel.test.tsx @@ -4,6 +4,8 @@ import { act, cleanup, fireEvent, render, screen, waitFor, within } from '@testi import { readFileSync } from 'fs' import { SiteExplorerPanel } from '@site/panels/SiteExplorerPanel' import { MediaExplorerPanel } from '@site/panels/MediaExplorerPanel' +import { publishCmsMediaAssetCreated } from '@admin/pages/media/mediaAssetEvents' +import { normalizeCmsMediaAsset } from '@core/persistence/cmsMedia' import { useEditorStore } from '@site/store/store' import { makeNode, makePage, makeSite } from '../fixtures' import type { VisualComponent } from '@core/visualComponents' @@ -533,6 +535,42 @@ describe('SiteExplorerPanel', () => { } }) + it('keeps an Agent-created media asset when an older list request finishes later', async () => { + loadSite() + const originalFetch = globalThis.fetch + let resolveList!: (response: Response) => void + globalThis.fetch = (() => new Promise((resolve) => { + resolveList = resolve + })) as typeof fetch + + try { + render() + const created = normalizeCmsMediaAsset({ + id: 'agent-created-image', + filename: 'agent-reference.jpg', + mimeType: 'image/jpeg', + sizeBytes: 42, + publicPath: '/uploads/agent-reference.jpg', + uploadedByUserId: null, + createdAt: '2026-07-11T10:00:00.000Z', + }) + act(() => publishCmsMediaAssetCreated(created)) + const panel = screen.getByTestId('media-explorer-panel') + expect(await within(panel).findByRole('button', { name: /open media agent-reference\.jpg/i })) + .toBeDefined() + + await act(async () => { + resolveList(new Response(JSON.stringify({ assets: [] }), { status: 200 })) + }) + await waitFor(() => { + expect(within(panel).getByRole('button', { name: /open media agent-reference\.jpg/i })) + .toBeDefined() + }) + } finally { + globalThis.fetch = originalFetch + } + }) + it('filters media by search text and defaults to the grid view, togglable to list', async () => { loadSite() const originalFetch = globalThis.fetch diff --git a/src/__tests__/ui/modelPicker.test.tsx b/src/__tests__/ui/modelPicker.test.tsx index e898d438a..d9ee70aab 100644 --- a/src/__tests__/ui/modelPicker.test.tsx +++ b/src/__tests__/ui/modelPicker.test.tsx @@ -1,13 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test' import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' import { ModelPicker, type ModelChoice } from '@admin/ai/ModelPicker' -import type { CredentialView } from '@admin/ai/api' +import { clearModelListCache, type CredentialView } from '@admin/ai/api' const originalFetch = globalThis.fetch afterEach(() => { cleanup() globalThis.fetch = originalFetch + clearModelListCache() }) function credential(over: Partial = {}): CredentialView { @@ -29,7 +30,7 @@ function credential(over: Partial = {}): CredentialView { const MODELS = Array.from({ length: 12 }, (_, i) => ({ id: `m${i}`, label: i === 3 ? 'Claude Opus' : i === 4 ? 'Claude Sonnet' : `Model ${i}`, - capabilities: { toolCalling: true, visionInput: false, promptCache: false, streaming: true }, + capabilities: { toolCalling: true, visionInput: false, toolResultImages: false, promptCache: false, streaming: true }, })) function mockModelsFetch() { @@ -42,7 +43,10 @@ function mockModelsFetch() { } describe('ModelPicker', () => { - beforeEach(mockModelsFetch) + beforeEach(() => { + clearModelListCache() + mockModelsFetch() + }) it('renders the field trigger with a placeholder when nothing is selected', () => { render( @@ -60,6 +64,39 @@ describe('ModelPicker', () => { expect(trigger.textContent).toContain('Choose a model') }) + it('shares and briefly caches a credential catalogue across picker instances', async () => { + const selected = { credentialId: 'cred-or', modelId: 'm0' } + const { rerender } = render( + {}} + />, + ) + await waitFor(() => expect(screen.getByRole('button').textContent).toContain('Model 0')) + expect(globalThis.fetch).toHaveBeenCalledTimes(1) + + rerender( +
+ {}} + /> + {}} + /> +
, + ) + await waitFor(() => expect(screen.getAllByRole('button')).toHaveLength(2)) + expect(globalThis.fetch).toHaveBeenCalledTimes(1) + }) + it('opens a searchable, grouped menu and filters models by query', async () => { render( { await apiRequest(`/admin/api/ai/credentials/${encodeURIComponent(id)}`, { method: 'DELETE' }) + clearModelListCache(id) } export interface TestResult { @@ -231,15 +232,53 @@ export async function testCredential(id: string): Promise { // Endpoints — models // --------------------------------------------------------------------------- +const MODEL_LIST_TIMEOUT_MS = 10_000 +const MODEL_LIST_CACHE_TTL_MS = 5 * 60_000 +const modelListRequests = new Map>() +const modelListCache = new Map() + +/** Invalidate model catalogues after a credential mutation (or between tests). */ +export function clearModelListCache(credentialId?: string): void { + if (!credentialId) { + modelListCache.clear() + return + } + for (const key of modelListCache.keys()) { + if (key.endsWith(`\0${credentialId}`)) modelListCache.delete(key) + } +} + export async function listModels( providerId: 'anthropic' | 'openai' | 'ollama' | 'openrouter' | 'openai-compatible', credentialId?: string, ): Promise { - const body = await apiRequest(`/admin/api/ai/providers/${providerId}/models`, { + const key = `${providerId}\0${credentialId ?? ''}` + const cached = modelListCache.get(key) + if (cached && cached.expiresAt > Date.now()) return cached.models + if (cached) modelListCache.delete(key) + const pending = modelListRequests.get(key) + if (pending) return pending + + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), MODEL_LIST_TIMEOUT_MS) + const request = apiRequest(`/admin/api/ai/providers/${providerId}/models`, { query: { credentialId }, schema: ModelListResponseSchema, + signal: controller.signal, + }).then((body) => { + modelListCache.set(key, { + expiresAt: Date.now() + MODEL_LIST_CACHE_TTL_MS, + models: body.models, + }) + return body.models }) - return body.models + modelListRequests.set(key, request) + try { + return await request + } finally { + clearTimeout(timeoutId) + if (modelListRequests.get(key) === request) modelListRequests.delete(key) + } } // --------------------------------------------------------------------------- @@ -274,9 +313,13 @@ export async function listConversations(scope: 'site' | 'content' | 'data' | 'pl return body.conversations } -export async function getConversation(id: string): Promise { +export async function getConversation( + id: string, + signal?: AbortSignal, +): Promise { const body = await apiRequest(`/admin/api/ai/conversations/${encodeURIComponent(id)}`, { schema: ConversationDetailResponseSchema, + signal, }) return body.conversation } @@ -289,11 +332,13 @@ export async function updateConversationProvider( id: string, credentialId: string, modelId: string, + signal?: AbortSignal, ): Promise { const body = await apiRequest(`/admin/api/ai/conversations/${encodeURIComponent(id)}`, { method: 'PUT', body: { credentialId, modelId }, schema: ConversationItemResponseSchema, + signal, }) return body.conversation } diff --git a/src/admin/ai/useAgentStore.ts b/src/admin/ai/useAgentStore.ts index 146e3370f..59a1987a7 100644 --- a/src/admin/ai/useAgentStore.ts +++ b/src/admin/ai/useAgentStore.ts @@ -39,12 +39,8 @@ export interface AgentStoreApi { */ export const AgentStoreContext = createContext(null) -/** - * Read agent state from the host store. Throws when called outside an - * AgentStoreProvider — the panel components rely on a host being - * mounted; an unprovided render is a wiring bug, not a missing-data case. - */ -export function useAgentStore(selector: (slice: AgentSlice) => U): U { +/** Raw injected store API for effects that need an external-store subscription. */ +export function useAgentStoreApi(): AgentStoreApi { const api = useContext(AgentStoreContext) if (!api) { throw new Error( @@ -52,5 +48,15 @@ export function useAgentStore(selector: (slice: AgentSlice) => U): U { 'Wrap the AgentPanel mount in .', ) } + return api +} + +/** + * Read agent state from the host store. Throws when called outside an + * AgentStoreProvider — the panel components rely on a host being + * mounted; an unprovided render is a wiring bug, not a missing-data case. + */ +export function useAgentStore(selector: (slice: AgentSlice) => U): U { + const api = useAgentStoreApi() return useZustandStore(api, selector) } diff --git a/src/admin/pages/media/components/BulkEditWindow/BulkEditWindow.tsx b/src/admin/pages/media/components/BulkEditWindow/BulkEditWindow.tsx index 5f450e742..496894e19 100644 --- a/src/admin/pages/media/components/BulkEditWindow/BulkEditWindow.tsx +++ b/src/admin/pages/media/components/BulkEditWindow/BulkEditWindow.tsx @@ -20,7 +20,7 @@ import { FolderGlyphIcon } from 'pixel-art-icons/icons/folder-glyph' import { CloseIcon } from 'pixel-art-icons/icons/close' import { ReloadIcon } from 'pixel-art-icons/icons/reload' import type { CmsMediaFolder } from '@core/persistence/cmsMedia' -import { FloatingWindow } from '../FloatingWindow/FloatingWindow' +import { FloatingWindow } from '@admin/shared/FloatingWindow' import { TagEditor } from '../TagEditor/TagEditor' import type { UseMediaWorkspaceResult } from '../../hooks/useMediaWorkspace' import styles from './BulkEditWindow.module.css' diff --git a/src/admin/pages/media/components/FloatingWindow/FloatingWindow.tsx b/src/admin/pages/media/components/FloatingWindow/FloatingWindow.tsx deleted file mode 100644 index fb31809b8..000000000 --- a/src/admin/pages/media/components/FloatingWindow/FloatingWindow.tsx +++ /dev/null @@ -1,102 +0,0 @@ -/** - * FloatingWindow — shared shell for the Media page's three draggable windows - * (Upload Queue, Detached Inspector, Bulk Edit). Wraps `useDraggablePanel` - * with a `PanelHeader` and a scrollable body so each window can focus on - * its own contents. - * - * Position is persisted automatically by `useDraggablePanel` via the - * `workspaceLayoutStorage` module (each `FloatingPanelId` gets its own key). - * Visibility is owned by the caller — pass `open` from your component - * state. Closing fires `onClose`. - */ -import { useImperativeHandle, type CSSProperties, type ReactNode, type Ref } from 'react' -import { createPortal } from 'react-dom' -import { PanelHeader } from '@admin/shared/PanelHeader' -import { useDraggablePanel } from '@site/hooks/useDraggablePanel' -import type { FloatingPanelId, PanelPosition } from '@admin/state/workspaceLayoutStorage' -import { cn } from '@ui/cn' -import styles from './FloatingWindow.module.css' - -interface FloatingWindowProps { - panelId: FloatingPanelId - /** Visibility — when false the window unmounts (drag state stays in storage). */ - open: boolean - title: string - /** Default position when no stored position exists. */ - defaultPosition: PanelPosition - /** Header right-slot actions (e.g. "Clear" buttons on the upload queue). */ - headerActions?: ReactNode - /** Width in pixels — driven by a CSS var, allows simple resizing later. */ - width?: number - /** Optional max height in pixels — body scrolls when exceeded. */ - maxHeight?: number - /** Extra class on the root container. */ - className?: string - ariaLabel?: string - testId?: string - onClose: () => void - children?: ReactNode - /** React 19: ref is a regular prop on function components. */ - ref?: Ref -} - -export function FloatingWindow({ - panelId, - open, - title, - defaultPosition, - headerActions, - width = 320, - maxHeight, - className, - ariaLabel, - testId, - onClose, - children, - ref: forwardedRef, -}: FloatingWindowProps) { - const { panelRef, headerDragProps, panelPositionStyle } = useDraggablePanel( - panelId, - () => defaultPosition, - ) - - // Forward the same DOM node that `panelRef` (returned from useDraggablePanel) - // attaches to. `useImperativeHandle` re-runs every render and React guarantees - // `panelRef.current` is up to date by then, so we can route the forwarded - // contract through it without a parallel local ref. Avoids the React Compiler - // bailout from writing `panelRef.current = node` inside a ref callback. - useImperativeHandle(forwardedRef, () => panelRef.current as HTMLDivElement) - - if (!open) return null - - const style = { - '--floating-window-w': `${width}px`, - ...(maxHeight ? { '--floating-window-max-h': `${maxHeight}px` } : {}), - ...panelPositionStyle, - } as CSSProperties - - // Portal into so the window can float above ANY ancestor — including - // sidebars with `overflow: hidden` and modal backdrops. - return createPortal( - , - document.body, - ) -} diff --git a/src/admin/pages/media/components/MediaViewerWindow/MediaViewerWindow.tsx b/src/admin/pages/media/components/MediaViewerWindow/MediaViewerWindow.tsx index b25309c80..27b3a6a3b 100644 --- a/src/admin/pages/media/components/MediaViewerWindow/MediaViewerWindow.tsx +++ b/src/admin/pages/media/components/MediaViewerWindow/MediaViewerWindow.tsx @@ -36,7 +36,7 @@ import { ReloadIcon } from 'pixel-art-icons/icons/reload' import { TrashSolidIcon } from 'pixel-art-icons/icons/trash-solid' import { VideoSolidIcon } from 'pixel-art-icons/icons/video-solid' import { PanelHeader } from '@admin/shared/PanelHeader' -import { useDraggablePanel } from '@site/hooks/useDraggablePanel' +import { useDraggablePanel } from '@admin/shared/FloatingWindow' import type { CmsMediaAsset, CmsMediaFolder, UpdateCmsMediaAssetInput } from '@core/persistence/cmsMedia' import { bucketForMime } from '../../utils/filters' import { useDebouncedSave } from '../../hooks/useDebouncedSave' @@ -108,7 +108,7 @@ function ViewerForAsset({ editor, onClose }: ViewerForAssetProps) { // Persistent window position — same key the old detached inspector used, // so saved positions carry over for users who already moved it. - const { panelRef, headerDragProps, panelPositionStyle } = useDraggablePanel( + const { setPanelRef, headerDragProps, panelPositionStyle } = useDraggablePanel( 'mediaDetachedInspector', () => ({ x: window.innerWidth - 880, y: 80 }), ) @@ -162,7 +162,7 @@ function ViewerForAsset({ editor, onClose }: ViewerForAssetProps) { return createPortal(