From 6e91da31c27e22c40316db7f3b975b1c9b6a5c7a Mon Sep 17 00:00:00 2001 From: David Moore Date: Wed, 14 Oct 2026 16:35:09 +1100 Subject: [PATCH] feat(core): let bots read attached files and see attached images --- bun.lock | 1 + packages/contracts/src/attachments.ts | 5 + packages/core/package.json | 1 + .../conversations/tools/browser/browser.ts | 3 +- .../tools/browser/viewer.test.ts | 2 +- .../core/src/conversations/tools/images.ts | 81 +++++++++ .../tools/read-thread-file/tool.test.ts | 156 +++++++++++++++--- .../tools/read-thread-file/tool.ts | 95 +++++++++-- .../src/conversations/tools/sandbox.test.ts | 2 +- .../core/src/conversations/tools/sandbox.ts | 12 +- .../src/conversations/tools/sandbox/tools.ts | 62 ++----- .../src/conversations/turns/context.test.ts | 1 + .../core/src/conversations/turns/execution.ts | 5 + .../turns/tool-calls/large-results.ts | 4 +- .../turns/tool-calls/repository.test.ts | 13 ++ .../turns/tool-calls/repository.ts | 6 +- .../core/src/conversations/turns/tools.ts | 7 +- .../conversations/turns/turn.steps.test.ts | 1 + .../src/conversations/turns/turn.steps.ts | 2 +- .../model-providers/dialects/openrouter.ts | 21 ++- .../model-providers/dialects/tool-images.ts | 63 +++++++ .../providers/models/models.recording.test.ts | 43 +++++ .../server/src/routes/chats/file-response.ts | 8 +- 23 files changed, 482 insertions(+), 112 deletions(-) create mode 100644 packages/core/src/conversations/tools/images.ts create mode 100644 packages/core/src/providers/model-providers/dialects/tool-images.ts diff --git a/bun.lock b/bun.lock index 29ce1e0f..ee18e001 100644 --- a/bun.lock +++ b/bun.lock @@ -65,6 +65,7 @@ "@ai-sdk/google": "^4.0.80", "@ai-sdk/mcp": "^2.0.49", "@ai-sdk/openai": "^4.0.65", + "@ai-sdk/provider-utils": "^5.0.47", "@alibaba-group/opensandbox": "1.1.0", "@effect/platform-node": "catalog:", "@effect/sql-pg": "catalog:", diff --git a/packages/contracts/src/attachments.ts b/packages/contracts/src/attachments.ts index 4dedf3f2..3f42cd8e 100644 --- a/packages/contracts/src/attachments.ts +++ b/packages/contracts/src/attachments.ts @@ -27,6 +27,11 @@ export const mediaTypeSchema = Schema.Trim.check( Schema.isPattern(/^[\w.+-]+\/[\w.+-]+(\s*;\s*[\w.+-]+=("[^"\r\n]*"|[\w.+-]+))*$/), ); +/** A media type without its parameters, lowercased: `image/png` from `image/PNG; charset=binary`. */ +export function mediaTypeEssence(mediaType: string): string { + return (mediaType.split(";")[0] ?? "").trim().toLowerCase(); +} + export const uploadAttachmentQuerySchema = Schema.Struct({ name: attachmentNameSchema, /** The file's media type, when the uploader knows it; otherwise it is taken from the name. */ diff --git a/packages/core/package.json b/packages/core/package.json index bc933e06..7200c03e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,6 +19,7 @@ "@ai-sdk/google": "^4.0.80", "@ai-sdk/mcp": "^2.0.49", "@ai-sdk/openai": "^4.0.65", + "@ai-sdk/provider-utils": "^5.0.47", "@alibaba-group/opensandbox": "1.1.0", "@effect/platform-node": "catalog:", "@effect/sql-pg": "catalog:", diff --git a/packages/core/src/conversations/tools/browser/browser.ts b/packages/core/src/conversations/tools/browser/browser.ts index ec422c82..2f49e776 100644 --- a/packages/core/src/conversations/tools/browser/browser.ts +++ b/packages/core/src/conversations/tools/browser/browser.ts @@ -5,7 +5,8 @@ import { jsonSchema, type ToolSet, tool } from "ai"; import { Data, Effect } from "effect"; import { Sandboxes } from "../../../sandboxes/sandboxes.ts"; import type { UserFacing } from "../../../user-message.ts"; -import { type Image, type OpenSandbox, type Place, withImages } from "../sandbox/tools.ts"; +import { type Image, withImages } from "../images.ts"; +import type { OpenSandbox, Place } from "../sandbox/tools.ts"; import playwrightTools from "./playwright-tools.json" with { type: "json" }; /** diff --git a/packages/core/src/conversations/tools/browser/viewer.test.ts b/packages/core/src/conversations/tools/browser/viewer.test.ts index 230d9b83..a929efc2 100644 --- a/packages/core/src/conversations/tools/browser/viewer.test.ts +++ b/packages/core/src/conversations/tools/browser/viewer.test.ts @@ -170,7 +170,7 @@ describe.skipIf(!configured)("the desktop viewer, against Postgres and OpenSandb turnId: "turn-1", threadId, agentId, - model: "no-such-model", + acceptsImages: false, }), ), ).then((offered) => offered.tools); diff --git a/packages/core/src/conversations/tools/images.ts b/packages/core/src/conversations/tools/images.ts new file mode 100644 index 00000000..03c0eb21 --- /dev/null +++ b/packages/core/src/conversations/tools/images.ts @@ -0,0 +1,81 @@ +import { detectMediaType } from "@ai-sdk/provider-utils"; + +/** + * Images the tools show the model: a screenshot, an image in the sandbox, or + * one a person attached to the thread. + */ + +/** An image a tool's result carries for the model to look at. */ +export interface Image { + /** Base 64. */ + data: string; + mediaType: string; +} + +/** The formats shown, as the tools' descriptions name them. */ +export const SHOWN_IMAGE_FORMATS = "PNG, JPEG, GIF or WebP"; + +/** + * The largest image shown, in bytes: 5 MiB once base 64 encoded, which is how + * model providers measure the images they accept. + */ +export const MAX_IMAGE_BYTES = Math.floor((5 * 1024 * 1024 * 3) / 4); + +/** The media types of the formats shown. Others, such as HEIC or TIFF, models refuse. */ +const SHOWN_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); + +/** + * The media type `bytes` begin as, when they are an image in a format shown: + * read from the bytes rather than taken from a name or a declared type, as a + * provider refuses an image whose declared type its bytes contradict. The AI + * SDK reads them the same way when it corrects a part's type. + */ +export function shownImageType(bytes: Uint8Array): string | undefined { + const detected = detectMediaType({ data: bytes, topLevelType: "image" }); + return detected !== undefined && SHOWN_IMAGE_TYPES.has(detected) ? detected : undefined; +} + +/** Whether a tool's result carries images for the model. */ +export function hasImages( + output: unknown, +): output is { images: readonly Image[] } & Record { + return typeof output === "object" && output !== null && "images" in output; +} + +/** + * A tool's result as the model is given it: one with images as text and + * images, when the model can take them; anything else as it is. + */ +export function withImages(output: unknown, acceptsImages: boolean) { + if (!hasImages(output)) { + return { type: "json" as const, value: output as never }; + } + const { images, ...rest } = output; + const text = + typeof rest.text === "string" + ? [rest.note, rest.text].filter((part) => typeof part === "string").join("\n\n") + : JSON.stringify(rest); + if (!acceptsImages) return { type: "text" as const, value: text }; + return { + type: "content" as const, + value: [ + { type: "text" as const, text }, + ...images.map((image) => ({ + type: "file" as const, + data: { type: "data" as const, data: image.data }, + mediaType: image.mediaType, + })), + ], + }; +} + +/** + * A tool's result as it is kept, with its images counted rather than kept: + * the model has been shown them, and their data, read back as text in a later + * turn's history, would be noise. + */ +export function withImagesCounted(output: unknown): unknown { + if (!hasImages(output)) return output; + const { images, ...rest } = output; + return { ...rest, images: `${images.length} shown` }; +} diff --git a/packages/core/src/conversations/tools/read-thread-file/tool.test.ts b/packages/core/src/conversations/tools/read-thread-file/tool.test.ts index 15978bc2..d634f315 100644 --- a/packages/core/src/conversations/tools/read-thread-file/tool.test.ts +++ b/packages/core/src/conversations/tools/read-thread-file/tool.test.ts @@ -2,45 +2,65 @@ import { Effect, Stream } from "effect"; import { describe, expect, it } from "vitest"; import { BlobStore } from "../../../blob-store/blob-store.ts"; import { ThreadFiles } from "../../thread-files/thread-files.ts"; +import { MAX_IMAGE_BYTES } from "../images.ts"; import { MAX_READ_CHARACTERS, type ReadThreadFileResult, readThreadFileTool } from "./tool.ts"; const THREAD = "0199a3a0-0000-7000-8000-000000000001"; const FILE = "0199a3a0-0000-7000-8000-0000000000f1"; -/** `read_thread_file` over one file in `THREAD`, called with `input`. */ -async function read( - text: string, - input: { file?: string; offset?: number; limit?: number; pattern?: string }, +/** The first bytes of every PNG file. */ +const PNG = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +/** A file in `THREAD`: its content, and the name and type it was kept with. */ +interface Kept { + content: string | Uint8Array; + name?: string; + contentType?: string; +} + +/** The thread files `read_thread_file` reads `kept` through. */ +function holding({ + content, + name = "notes.txt", contentType = "text/plain; charset=utf-8", -): Promise { - return readFrom( - { - read: ({ threadId, fileId }) => - threadId === THREAD && fileId === FILE - ? Effect.succeed({ - id: FILE, - name: undefined, - contentType, - size: text.length, - stream: Stream.succeed(new TextEncoder().encode(text)), - }) - : new ThreadFiles.FileNotFound({ fileId }), - }, - input, - ); +}: Kept): Pick { + const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content; + return { + read: ({ threadId, fileId }) => + threadId === THREAD && fileId === FILE + ? Effect.succeed({ + id: FILE, + name, + contentType, + size: bytes.byteLength, + stream: Stream.succeed(bytes), + }) + : new ThreadFiles.FileNotFound({ fileId }), + }; +} + +function toolOver(files: Pick, acceptsImages: boolean) { + return readThreadFileTool({ threadId: THREAD, files, acceptsImages, run: Effect.runPromise }); } async function readFrom( files: Pick, input: { file?: string; offset?: number; limit?: number; pattern?: string }, + acceptsImages = false, ): Promise { - const tool = readThreadFileTool({ threadId: THREAD, files, run: Effect.runPromise }); + const tool = toolOver(files, acceptsImages); const options = { toolCallId: "call", messages: [] } as unknown as Parameters< NonNullable >[1]; return tool.execute?.({ file: FILE, ...input }, options) as Promise; } +/** `read_thread_file` over one text file holding `text`, called with `input`. */ +const read = ( + text: string, + input: { file?: string; offset?: number; limit?: number; pattern?: string }, +) => readFrom(holding({ content: text }), input); + const numbered = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n"); @@ -75,14 +95,104 @@ describe("read_thread_file", () => { expect(result.continueFrom).toBe(result.lines.split("\n").length + 1); }); - it("refuses a file from another thread, an offset past the end, and a file that isn't text", async () => { + it("refuses a file from another thread, and an offset past the end", async () => { expect(await read("text", { file: "0199a3a0-0000-7000-8000-0000000000f2" })).toHaveProperty( "refused", ); expect(await read(numbered(3), { offset: 4 })).toEqual({ refused: "The file has 3 lines, so there is no line 4.", }); - expect(await read("PNG", {}, "image/png")).toHaveProperty("refused"); + }); + + it("reads text whatever type it was given, and text it was told is text in any encoding", async () => { + // A TypeScript file, which extension lookup calls an MPEG video. + const source = holding({ + content: "const a = 1;\n", + name: "main.ts", + contentType: "video/mp2t", + }); + expect(await readFrom(source, {})).toEqual({ lines: "1\tconst a = 1;", totalLines: 1 }); + // A CSV saved by Excel in Windows-1252: "café", with é as the byte E9. + const csv = holding({ + content: new Uint8Array([0x63, 0x61, 0x66, 0xe9]), + name: "menu.csv", + contentType: "text/csv", + }); + expect(await readFrom(csv, {})).toEqual({ lines: "1\tcaf�", totalLines: 1 }); + }); + + it("refuses a file that is neither text nor an image", async () => { + const zip = holding({ + content: new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0x00]), + name: "archive.zip", + contentType: "application/zip", + }); + const notUtf8 = holding({ + content: new Uint8Array([0xc3, 0x28]), + name: "data.bin", + contentType: "application/octet-stream", + }); + + expect(await readFrom(zip, {})).toEqual({ + refused: "The file is application/zip, not text, so it has no lines to read.", + }); + expect(await readFrom(notUtf8, {})).toHaveProperty("refused"); + }); + + it("shows an image as the type its bytes are, whatever type it was given", async () => { + const jpeg = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); + const png = new Uint8Array(PNG); + const shown = (content: Uint8Array, contentType: string) => + readFrom(holding({ content, name: "chart.png", contentType }), {}, true); + + expect(await shown(jpeg, "image/png")).toEqual({ + text: "The image chart.png.", + images: [{ data: Buffer.from(jpeg).toString("base64"), mediaType: "image/jpeg" }], + }); + expect(await shown(png, "application/octet-stream")).toMatchObject({ + images: [{ mediaType: "image/png" }], + }); + }); + + it("refuses an image to a model that can't see one, or one too large to show", async () => { + const large = new Uint8Array(MAX_IMAGE_BYTES + 1); + large.set(PNG); + + expect( + await readFrom( + holding({ content: new Uint8Array(PNG), contentType: "image/png" }), + {}, + false, + ), + ).toEqual({ refused: "The file is an image, and you can't be shown images." }); + expect(await readFrom(holding({ content: large, contentType: "image/png" }), {}, true)).toEqual( + { + refused: `The image is ${MAX_IMAGE_BYTES + 1} bytes, larger than the ${MAX_IMAGE_BYTES} that can be shown.`, + }, + ); + }); + + it("gives a model that can see images the image itself", () => { + const tool = toolOver( + holding({ content: new Uint8Array(PNG), contentType: "image/png" }), + true, + ); + const output = { + text: "The image chart.png.", + images: [{ data: "iVBORw0KGgo=", mediaType: "image/png" }], + }; + + expect(tool.toModelOutput?.({ toolCallId: "call", input: { file: FILE }, output })).toEqual({ + type: "content", + value: [ + { type: "text", text: "The image chart.png." }, + { + type: "file", + data: { type: "data", data: "iVBORw0KGgo=" }, + mediaType: "image/png", + }, + ], + }); }); it("refuses, so the model can try again, while the blob store can't be reached", async () => { diff --git a/packages/core/src/conversations/tools/read-thread-file/tool.ts b/packages/core/src/conversations/tools/read-thread-file/tool.ts index 1f30c394..286a1056 100644 --- a/packages/core/src/conversations/tools/read-thread-file/tool.ts +++ b/packages/core/src/conversations/tools/read-thread-file/tool.ts @@ -1,7 +1,15 @@ +import { mediaTypeEssence } from "@sugabots/contracts"; import { tool } from "ai"; -import { Effect, Schema, Stream } from "effect"; +import { Effect, Option, Schema, Stream } from "effect"; import type { RunEffect } from "../../../database/database.ts"; import type { ThreadFiles } from "../../thread-files/thread-files.ts"; +import { + type Image, + MAX_IMAGE_BYTES, + SHOWN_IMAGE_FORMATS, + shownImageType, + withImages, +} from "../images.ts"; export const READ_THREAD_FILE_TOOL = "read_thread_file"; @@ -40,24 +48,29 @@ export type ReadThreadFileResult = /** Where to start the next call to read on, when there is more. */ continueFrom?: number; } + | { text: string; images: Image[] } | { refused: string }; /** - * The `read_thread_file` tool: reads part of a file in this thread, such as a tool - * result too large to have been given whole, by line or by searching it. - * Offered on every turn, so the tools a thread sends never change for it. + * The `read_thread_file` tool: reads part of a file in this thread, such as a + * file a person attached or a tool result too large to have been given whole, + * by line or by searching it, or shows an image to a model that takes them. + * Offered on every turn, so the tools a thread sends change only when its + * agent's model does. */ export function readThreadFileTool({ threadId, files, + acceptsImages, run, }: { threadId: string; files: Pick; + acceptsImages: boolean; run: RunEffect; }) { return tool({ - description: `Read a file in this thread, such as a tool result too large to have been given to you whole, which named its file and how to read on. Returns up to ${DEFAULT_LINES} lines from \`offset\`, each after its line number, with the file's total lines and, when there is more, the line to continue from. Give \`pattern\` to get only the lines containing it instead.`, + description: `Read a file in this thread: one a person attached, or a tool result too large to have been given to you whole, which named its file and how to read on. For a text file, returns up to ${DEFAULT_LINES} lines from \`offset\`, each after its line number, with the file's total lines and, when there is more, the line to continue from. Give \`pattern\` to get only the lines containing it instead.${acceptsImages ? ` A ${SHOWN_IMAGE_FORMATS} image is shown to you instead.` : ""}`, inputSchema: Schema.Struct({ file: Schema.String.annotate({ description: "The file's id." }), offset: Schema.optional( @@ -84,12 +97,41 @@ export function readThreadFileTool({ run( Effect.gen(function* () { const found = yield* files.read({ threadId, fileId: file }); - if (!isText(found.contentType)) { + const notText = { + refused: `The file is ${found.contentType}, not text, so it has no lines to read.`, + }; + // The file's start says what it is, so one that is neither text nor + // an image is refused without reading the rest of it. + const start = Option.getOrElse( + yield* Stream.runHead(found.stream), + () => new Uint8Array(), + ); + const wholeFile = Effect.flatMap(files.read({ threadId, fileId: file }), ({ stream }) => + collect(stream), + ); + const imageType = shownImageType(start); + if (imageType !== undefined) { + if (!acceptsImages) { + return { refused: "The file is an image, and you can't be shown images." }; + } + if (found.size > MAX_IMAGE_BYTES) { + return { + refused: `The image is ${found.size} bytes, larger than the ${MAX_IMAGE_BYTES} that can be shown.`, + }; + } return { - refused: `The file is ${found.contentType}, not text, so it has no lines to read.`, + text: `The image ${found.name ?? file}.`, + images: [{ data: (yield* wholeFile).toString("base64"), mediaType: imageType }], }; } - const text = yield* Stream.mkString(Stream.decodeText(found.stream)); + if (declaredText(found.contentType)) { + // As declared text, in whatever encoding, with what isn't UTF-8 replaced. + const text = new TextDecoder().decode(yield* wholeFile); + return readLines(text, { offset: offset ?? 1, limit: limit ?? DEFAULT_LINES, pattern }); + } + if (!startsAsText(start)) return notText; + const text = strictText(yield* wholeFile); + if (text === undefined) return notText; return readLines(text, { offset: offset ?? 1, limit: limit ?? DEFAULT_LINES, pattern }); }).pipe( Effect.catchTags({ @@ -103,11 +145,44 @@ export function readThreadFileTool({ }), ), ), + toModelOutput: ({ output }) => withImages(output, acceptsImages), }); } -function isText(contentType: string): boolean { - return contentType.startsWith("text/") || contentType.startsWith("application/json"); +/** Whether a file was given a text type: read as text, whatever its bytes. */ +function declaredText(contentType: string): boolean { + const essence = mediaTypeEssence(contentType); + return essence.startsWith("text/") || essence === "application/json"; +} + +/** + * Whether a file's first bytes read as text: UTF-8 without a NUL byte. Many + * text files, such as source code, arrive with a type that says otherwise or + * with none, so a file not declared text is judged by its bytes. + */ +function startsAsText(bytes: Uint8Array): boolean { + if (bytes.includes(0)) return false; + try { + // Streaming, so a character cut off at the end of the chunk is not an error. + new TextDecoder("utf-8", { fatal: true }).decode(bytes, { stream: true }); + return true; + } catch { + return false; + } +} + +/** `bytes` as UTF-8 text without a NUL byte, or undefined when they are not that. */ +function strictText(bytes: Uint8Array): string | undefined { + if (bytes.includes(0)) return undefined; + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + return undefined; + } +} + +function collect(stream: Stream.Stream): Effect.Effect { + return Effect.map(Stream.runCollect(stream), (chunks) => Buffer.concat(chunks)); } /** diff --git a/packages/core/src/conversations/tools/sandbox.test.ts b/packages/core/src/conversations/tools/sandbox.test.ts index d9106a57..f4faf1b7 100644 --- a/packages/core/src/conversations/tools/sandbox.test.ts +++ b/packages/core/src/conversations/tools/sandbox.test.ts @@ -107,7 +107,7 @@ describe.skipIf(!configured)("sandbox tools, against Postgres and OpenSandbox", turnId, threadId, agentId, - model: "no-such-model", + acceptsImages: false, }), ), ).then((offered) => offered.tools); diff --git a/packages/core/src/conversations/tools/sandbox.ts b/packages/core/src/conversations/tools/sandbox.ts index ee36c772..43dcb182 100644 --- a/packages/core/src/conversations/tools/sandbox.ts +++ b/packages/core/src/conversations/tools/sandbox.ts @@ -4,7 +4,6 @@ import type { UserText } from "@sugabots/errors"; import type { Tool, ToolSet } from "ai"; import { Cause, Context, Effect, Exit, Layer, type Scope } from "effect"; import { type RunEffect, serviceOperations } from "../../database/database.ts"; -import { modelAcceptsImages } from "../../providers/model-providers/model-provider-reads.ts"; import { allowedHostsOf, blockedHostsOf } from "../../sandboxes/allowed-hosts.ts"; import { PodSandboxes } from "../../sandboxes/pod-sandboxes.ts"; import { SandboxNetwork } from "../../sandboxes/sandbox-network.ts"; @@ -92,14 +91,9 @@ export const make = Effect.gen(function* () { }, }); return Service.of({ - forTurn: ({ pod, turnId, threadId, agentId, model, usesSandbox }) => + forTurn: ({ pod, turnId, threadId, agentId, acceptsImages, usesSandbox }) => Effect.gen(function* (): Effect.fn.Return { const place = placeOf({ threadId, agentId }); - // Whether a model takes images shapes the tools, so it is asked even without a sandbox. - const acceptsImages = yield* operation( - "forTurn", - modelAcceptsImages(pod.workspaceId, model), - ); const provider = usesSandbox ? yield* providers.enabled(pod.workspaceId) : undefined; if (!provider) { return { @@ -202,8 +196,8 @@ export interface Turn { readonly turnId: string; readonly threadId: string; readonly agentId: string; - /** The agent's model, which decides whether results may carry images. */ - readonly model: string; + /** Whether the agent's model takes images, which results then may carry. */ + readonly acceptsImages: boolean; } /** No sandbox tools, for cases that offer none. */ diff --git a/packages/core/src/conversations/tools/sandbox/tools.ts b/packages/core/src/conversations/tools/sandbox/tools.ts index eaf67241..bec951c0 100644 --- a/packages/core/src/conversations/tools/sandbox/tools.ts +++ b/packages/core/src/conversations/tools/sandbox/tools.ts @@ -4,6 +4,7 @@ import { tool } from "ai"; import { Effect, Schema } from "effect"; import type { PodSandboxes } from "../../../sandboxes/pod-sandboxes.ts"; import { Sandboxes } from "../../../sandboxes/sandboxes.ts"; +import { MAX_IMAGE_BYTES, SHOWN_IMAGE_FORMATS, shownImageType, withImages } from "../images.ts"; import { REQUEST_NETWORK_ACCESS_TOOL } from "../network-access/tool.ts"; export const RUN_COMMAND_TOOL = "run_command"; @@ -171,36 +172,36 @@ export function runCommandTool(openSandbox: OpenSandbox, place: Place) { }); } -/** Images read_file shows the model, by extension. */ -const IMAGE_TYPES: Readonly> = { - png: "image/png", - jpg: "image/jpeg", - jpeg: "image/jpeg", - gif: "image/gif", - webp: "image/webp", -}; -const MAX_IMAGE_BYTES = 5_000_000; +/** The extensions of images read_file shows the model. */ +const IMAGE_EXTENSIONS = new Set(["png", "jpg", "jpeg", "gif", "webp"]); export function readFileTool(openSandbox: OpenSandbox, place: Place, acceptsImages: boolean) { return tool({ description: acceptsImages - ? `Read a file in the pod's sandbox: a text file as text, or an image (PNG, JPEG, GIF or WebP) to look at, such as a screenshot. Long text files are cut off; use run_command with head, tail or grep for parts of them.` + ? `Read a file in the pod's sandbox: a text file as text, or an image (${SHOWN_IMAGE_FORMATS}) to look at, such as a screenshot. Long text files are cut off; use run_command with head, tail or grep for parts of them.` : `Read a text file in the pod's sandbox. Long files are cut off; use run_command with head, tail or grep for parts of them.`, inputSchema: Schema.Struct({ path: pathIn(place) }).pipe( Schema.toStandardSchemaV1, Schema.toStandardJSONSchemaV1, ), execute: ({ path }) => { - const mediaType = IMAGE_TYPES[posix.extname(path).slice(1).toLowerCase()]; - const asImage = mediaType !== undefined && acceptsImages; + const asImage = + acceptsImages && IMAGE_EXTENSIONS.has(posix.extname(path).slice(1).toLowerCase()); return inSandbox(openSandbox, (sandbox) => sandbox.readFile(absolute(place, path), asImage ? MAX_IMAGE_BYTES : MAX_READ_BYTES).pipe( Effect.map(({ content, sizeBytes }): Record => { if (asImage) { - return sizeBytes > MAX_IMAGE_BYTES + if (sizeBytes > MAX_IMAGE_BYTES) { + return { + status: "failed", + error: `The image is larger than ${MAX_IMAGE_BYTES} bytes.`, + }; + } + const mediaType = shownImageType(content); + return mediaType === undefined ? { status: "failed", - error: `The image is larger than ${MAX_IMAGE_BYTES} bytes.`, + error: `The file isn't a ${SHOWN_IMAGE_FORMATS} image, whatever its name says.`, } : { text: `The image at ${absolute(place, path)}.`, @@ -248,36 +249,3 @@ export function writeFileTool(openSandbox: OpenSandbox, place: Place) { export function allowedHostsNote(allowedHosts: readonly string[]): string { return `The pod's sandbox, its browser included, connects only to these hosts (*. covers a domain's subdomains): ${allowedHosts.join(", ")}. A connection to any other host fails as if it weren't there: a name that doesn't resolve, a TLS error or an empty reply. To reach one the task needs, call ${REQUEST_NETWORK_ACCESS_TOOL}.`; } - -export interface Image { - /** Base 64. */ - data: string; - mediaType: string; -} - -/** - * A tool's output as the model is given it: a result with images as text and - * images, when the model can take them; anything else as it is. - */ -export function withImages(output: unknown, acceptsImages: boolean) { - if (!output || typeof output !== "object" || !("images" in output)) { - return { type: "json" as const, value: output as never }; - } - const { images, ...rest } = output as { images: readonly Image[] } & Record; - const text = - typeof rest.text === "string" - ? [rest.note, rest.text].filter((part) => typeof part === "string").join("\n\n") - : JSON.stringify(rest); - if (!acceptsImages) return { type: "text" as const, value: text }; - return { - type: "content" as const, - value: [ - { type: "text" as const, text }, - ...images.map((image) => ({ - type: "image-data" as const, - data: image.data, - mediaType: image.mediaType, - })), - ], - }; -} diff --git a/packages/core/src/conversations/turns/context.test.ts b/packages/core/src/conversations/turns/context.test.ts index b4a5584e..60db0622 100644 --- a/packages/core/src/conversations/turns/context.test.ts +++ b/packages/core/src/conversations/turns/context.test.ts @@ -422,6 +422,7 @@ function context(): TurnContext { disabledTools: [], interviewing: false, usesSandbox: false, + acceptsImages: false, }, reason: "default", routing: { facilitator: false }, diff --git a/packages/core/src/conversations/turns/execution.ts b/packages/core/src/conversations/turns/execution.ts index 33b514a2..b9072269 100644 --- a/packages/core/src/conversations/turns/execution.ts +++ b/packages/core/src/conversations/turns/execution.ts @@ -13,6 +13,7 @@ import { transaction, } from "../../database/database.ts"; import { type TurnReason, threadCompaction } from "../../database/schema.ts"; +import { modelAcceptsImages } from "../../providers/model-providers/model-provider-reads.ts"; import { INTERVIEW_PROMPT } from "../../workspaces/agents/interview-prompt.ts"; import { ConversationEvents } from "../conversation-events.ts"; import { ConversationEvent } from "../events.ts"; @@ -163,6 +164,7 @@ export const make = Effect.gen(function* () { const windowTokens = yield* query((db) => loadContextWindow(db, loaded.workspaceId, model), ); + const acceptsImages = yield* modelAcceptsImages(loaded.workspaceId, model); const messages = loaded.messages .reverse() .map((stored) => messageFromRelations(stored)); @@ -189,6 +191,7 @@ export const make = Effect.gen(function* () { name: speaker.name, handle: speaker.handle, model, + acceptsImages, prompt: speaker.prompt, disabledTools: speaker.disabledTools, usesSandbox: speaker.usesSandbox, @@ -285,6 +288,8 @@ export interface TurnContext { name: string; handle: string; model: string; + /** Whether its model takes images, which tools can then show it. */ + acceptsImages: boolean; prompt: string; /** Built-in tools an admin switched off for this agent, by key. */ disabledTools: string[]; diff --git a/packages/core/src/conversations/turns/tool-calls/large-results.ts b/packages/core/src/conversations/turns/tool-calls/large-results.ts index 7c4c2b60..3ebf2f57 100644 --- a/packages/core/src/conversations/turns/tool-calls/large-results.ts +++ b/packages/core/src/conversations/turns/tool-calls/large-results.ts @@ -1,5 +1,6 @@ import { Effect, Stream } from "effect"; import type { ThreadFiles } from "../../thread-files/thread-files.ts"; +import { hasImages } from "../../tools/images.ts"; import { READ_THREAD_FILE_TOOL, splitLines } from "../../tools/read-thread-file/tool.ts"; /** @@ -49,9 +50,6 @@ interface Keeping { } /** A result with images for the model, such as a screenshot, which the tool's `toModelOutput` gives it as image parts. */ -function hasImages(output: unknown): output is { images: unknown } & Record { - return typeof output === "object" && output !== null && "images" in output; -} /** What the model is given in place of `output`, or `undefined` when `output` is short enough to give whole. */ const longResultFor = ( diff --git a/packages/core/src/conversations/turns/tool-calls/repository.test.ts b/packages/core/src/conversations/turns/tool-calls/repository.test.ts index 3026c1e4..e7e41447 100644 --- a/packages/core/src/conversations/turns/tool-calls/repository.test.ts +++ b/packages/core/src/conversations/turns/tool-calls/repository.test.ts @@ -207,6 +207,19 @@ describe.skipIf(!process.env.DATABASE_URL)("tool calls, against Postgres", async expect(closed?.finishedAt).not.toBeNull(); }); + it("keeps the images a call showed the model counted, not their data", async () => { + const opened = await calls.open(from(0)); + + const closed = await calls.close(opened.id, { + output: { + text: "The image chart.png.", + images: [{ data: "iVBORw0KGgo".repeat(10_000), mediaType: "image/png" }], + }, + }); + + expect(closed?.output).toEqual({ text: "The image chart.png.", images: "1 shown" }); + }); + it("closes a failed call with the error and no output", async () => { const opened = await calls.open(from(0)); diff --git a/packages/core/src/conversations/turns/tool-calls/repository.ts b/packages/core/src/conversations/turns/tool-calls/repository.ts index 847fbe41..fc621e5e 100644 --- a/packages/core/src/conversations/turns/tool-calls/repository.ts +++ b/packages/core/src/conversations/turns/tool-calls/repository.ts @@ -24,6 +24,7 @@ import { import { ConversationEvents } from "../../conversation-events.ts"; import { ConversationEvent, type ToolCallChange } from "../../events.ts"; import { toToolCallPart } from "../../threads/tool-calls.ts"; +import { withImagesCounted } from "../../tools/images.ts"; import { mayRunTools } from "../lifecycle.ts"; import { WorkAdmission } from "../work-admission.ts"; import { @@ -188,7 +189,10 @@ export const make = Effect.gen(function* () { const closed = yield* apply( row, ToolCallEvent.Close({ - outcome: "error" in outcome ? outcome : { output: boundedJson(outcome.output) }, + outcome: + "error" in outcome + ? outcome + : { output: boundedJson(withImagesCounted(outcome.output)) }, }), ); if (!closed) return undefined; diff --git a/packages/core/src/conversations/turns/tools.ts b/packages/core/src/conversations/turns/tools.ts index 873f9066..3e020bb6 100644 --- a/packages/core/src/conversations/turns/tools.ts +++ b/packages/core/src/conversations/turns/tools.ts @@ -224,7 +224,12 @@ export function toolsForTurn(prepared: PreparedTurn, deps: ToolDependencies): Tu ); tools[READ_THREAD_FILE_TOOL] = recorded( READ_THREAD_FILE_TOOL, - readThreadFileTool({ threadId: prepared.context.thread.id, files: deps.files, run: deps.run }), + readThreadFileTool({ + threadId: prepared.context.thread.id, + files: deps.files, + acceptsImages: prepared.context.agent.acceptsImages, + run: deps.run, + }), recording, ); const routinesOff = prepared.context.agent.disabledTools.includes(ROUTINES_KEY); diff --git a/packages/core/src/conversations/turns/turn.steps.test.ts b/packages/core/src/conversations/turns/turn.steps.test.ts index 7c71111e..1a1f38a8 100644 --- a/packages/core/src/conversations/turns/turn.steps.test.ts +++ b/packages/core/src/conversations/turns/turn.steps.test.ts @@ -92,6 +92,7 @@ const prepared: PreparedTurn = { disabledTools: [], interviewing: false, usesSandbox: false, + acceptsImages: false, }, reason: "default", routing: { facilitator: false }, diff --git a/packages/core/src/conversations/turns/turn.steps.ts b/packages/core/src/conversations/turns/turn.steps.ts index 7ca28a1e..ee38dbb5 100644 --- a/packages/core/src/conversations/turns/turn.steps.ts +++ b/packages/core/src/conversations/turns/turn.steps.ts @@ -434,7 +434,7 @@ const streamReply = ( turnId: prepared.turnId, threadId: prepared.context.thread.id, agentId: prepared.context.agent.id, - model: prepared.context.agent.model, + acceptsImages: prepared.context.agent.acceptsImages, }); // The connections' sessions live as long as the turn. const connections = yield* Effect.acquireRelease( diff --git a/packages/core/src/providers/model-providers/dialects/openrouter.ts b/packages/core/src/providers/model-providers/dialects/openrouter.ts index 314ab300..4a9cac02 100644 --- a/packages/core/src/providers/model-providers/dialects/openrouter.ts +++ b/packages/core/src/providers/model-providers/dialects/openrouter.ts @@ -1,5 +1,6 @@ import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import type { ProviderModelCapability } from "@sugabots/contracts"; +import { wrapLanguageModel } from "ai"; import { Effect, Schema, SchemaGetter } from "effect"; import { recognizeAnthropicFailure } from "./anthropic.ts"; import { @@ -9,6 +10,7 @@ import { type ProviderModelFields, } from "./dialect.ts"; import { openaiCompatible, recognizeOpenAIFailure } from "./openai-compatible.ts"; +import { toolImagesFromUser } from "./tool-images.ts"; const names = Schema.Array(Schema.String).pipe( Schema.withDecodingDefault(Effect.succeed([])), @@ -32,14 +34,17 @@ export const openrouter: ProviderDialect = { // OpenRouter serves both OpenAI-style and Anthropic models, so check for either's errors. recognizeFailure: (report) => recognizeOpenAIFailure(report) ?? recognizeAnthropicFailure(report), languageModel: (endpoint, modelId, fetch) => - createOpenRouter({ - apiKey: endpoint.apiKey ?? "", - baseURL: endpoint.baseUrl, - headers: endpoint.headers, - fetch, - // Include usage at the end of each stream; costs are recorded from it. - compatibility: "strict", - })(modelId), + wrapLanguageModel({ + model: createOpenRouter({ + apiKey: endpoint.apiKey ?? "", + baseURL: endpoint.baseUrl, + headers: endpoint.headers, + fetch, + // Include usage at the end of each stream; costs are recorded from it. + compatibility: "strict", + })(modelId), + middleware: toolImagesFromUser, + }), cacheBreakpoint: (modelId) => modelId.startsWith(ANTHROPIC_MODELS) ? { openrouter: { cacheControl: { type: "ephemeral" } } } diff --git a/packages/core/src/providers/model-providers/dialects/tool-images.ts b/packages/core/src/providers/model-providers/dialects/tool-images.ts new file mode 100644 index 00000000..1f86dd21 --- /dev/null +++ b/packages/core/src/providers/model-providers/dialects/tool-images.ts @@ -0,0 +1,63 @@ +import type { LanguageModelMiddleware } from "ai"; + +type Prompt = Parameters< + NonNullable +>[0]["params"]["prompt"]; +type PromptMessage = Prompt[number]; +type UserContent = Extract["content"]; + +/** Where an image a tool returned was, in the result it was moved out of. */ +export const IMAGE_MOVED_NOTE = "(The image is in the next message.)"; + +/** + * Moves the images in tool results into a message from the user after them, + * leaving a note where each was. Gemini through OpenRouter refuses a request + * with an image in a tool result once its reasoning is in the request, saying + * the request ends with a model turn, but takes the same image from the user. + * The message starts with text, as one with only an image is refused the same + * way. + * + * Every model through OpenRouter reads an image sent this way, so this keeps + * working once OpenRouter takes images in Gemini's tool results. It can go + * then: the test of this in `models.recording.test.ts` shows what is sent, and + * a Gemini model with reasoning answering a tool result with an image, sent + * without this, shows whether it is still refused. + */ +export const toolImagesFromUser: LanguageModelMiddleware = { + transformParams: async ({ params }) => ({ + ...params, + prompt: params.prompt.flatMap(withImagesFromUser), + }), +}; + +function withImagesFromUser(message: PromptMessage): PromptMessage[] { + if (message.role !== "tool") return [message]; + const moved: UserContent = []; + const content = message.content.map((part) => { + if (part.type !== "tool-result" || part.output.type !== "content") return part; + const images = part.output.value.filter(isImage); + if (images.length === 0) return part; + moved.push({ type: "text", text: `What ${part.toolName} returned:` }, ...images); + const value = part.output.value.map((item) => + isImage(item) ? { type: "text" as const, text: IMAGE_MOVED_NOTE } : item, + ); + return { ...part, output: { ...part.output, value } }; + }); + if (moved.length === 0) return [message]; + return [ + { ...message, content }, + { role: "user", content: moved }, + ]; +} + +type ToolResultContent = Extract< + Extract< + Extract["content"][number], + { type: "tool-result" } + >["output"], + { type: "content" } +>["value"][number]; + +function isImage(part: ToolResultContent): part is Extract { + return part.type === "file" && part.mediaType.startsWith("image/"); +} diff --git a/packages/core/src/providers/models/models.recording.test.ts b/packages/core/src/providers/models/models.recording.test.ts index 3f3ce850..ea186c2d 100644 --- a/packages/core/src/providers/models/models.recording.test.ts +++ b/packages/core/src/providers/models/models.recording.test.ts @@ -2,9 +2,11 @@ import { jsonSchema, tool } from "ai"; import { Effect, ManagedRuntime } from "effect"; import { describe, expect, it } from "vitest"; import type { ModelRequests } from "../../accounting/model-requests.ts"; +import { withImages } from "../../conversations/tools/images.ts"; import { effectRunner } from "../../database/database.ts"; import { noDatabase } from "../../database/testing.ts"; import type { ProviderIdentity } from "../model-providers/dialects/index.ts"; +import { IMAGE_MOVED_NOTE } from "../model-providers/dialects/tool-images.ts"; import { Models } from "./models.ts"; import { resolvedModel } from "./testing.ts"; @@ -223,6 +225,47 @@ describe("a response at its step limit", () => { }); }); +describe("images a tool returns", () => { + it("reach a model through OpenRouter in a message after the tool's result", async () => { + const { model, sent } = modelAnswering([toolCall, reply], { + preset: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + apiFormat: "openai", + }); + const showing = tool({ + description: "Looks it up.", + inputSchema: jsonSchema>({ type: "object", properties: {} }), + execute: async () => ({ + text: "The image chart.png.", + images: [{ data: "iVBORw0KGgo=", mediaType: "image/png" }], + }), + toModelOutput: ({ output }) => withImages(output, true), + }); + + await answer(model, { ...input, model: "google/gemini-test", tools: { lookup: showing } }); + + const [, second = {}] = sent; + expect((second.messages as unknown[]).slice(-2)).toEqual([ + { + role: "tool", + tool_call_id: "call-1", + name: "lookup", + content: [ + { type: "text", text: "The image chart.png." }, + { type: "text", text: IMAGE_MOVED_NOTE }, + ], + }, + { + role: "user", + content: [ + { type: "text", text: "What lookup returned:" }, + { type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } }, + ], + }, + ]); + }); +}); + describe("prompt caching", () => { const marked = { type: "ephemeral" }; const turn: Models.StreamRequest = { diff --git a/packages/server/src/routes/chats/file-response.ts b/packages/server/src/routes/chats/file-response.ts index c21782b3..b3aa53e3 100644 --- a/packages/server/src/routes/chats/file-response.ts +++ b/packages/server/src/routes/chats/file-response.ts @@ -1,3 +1,4 @@ +import { mediaTypeEssence } from "@sugabots/contracts"; import type { ThreadFiles } from "@sugabots/core/conversations/thread-files/thread-files"; import { HttpServerResponse } from "effect/unstable/http"; @@ -21,7 +22,7 @@ const SHOWN_IN_PLACE = new Set([ * its copy. */ export function fileResponse(file: ThreadFiles.StoredFile): HttpServerResponse.HttpServerResponse { - const shownInPlace = SHOWN_IN_PLACE.has(essenceOf(file.contentType)); + const shownInPlace = SHOWN_IN_PLACE.has(mediaTypeEssence(file.contentType)); return HttpServerResponse.stream(file.stream, { contentType: file.contentType, contentLength: file.size, @@ -34,11 +35,6 @@ export function fileResponse(file: ThreadFiles.StoredFile): HttpServerResponse.H }); } -/** `image/png` from `image/png; charset=binary`. */ -function essenceOf(mediaType: string): string { - return (mediaType.split(";")[0] ?? "").trim().toLowerCase(); -} - /** * RFC 6266: an ASCII `filename` for old browsers, and the exact name, in * UTF-8, as `filename*` for the rest.