feat(core): let bots read attached files and see attached images

packages/contracts/src/attachments.ts

@@ -27,6 +27,11 @@ export const mediaTypeSchema = Schema.Trim.check(
 	Schema.isPattern(/^[\w.+-]+\/[\w.+-]+(\s*;\s*[\w.+-]+=("[^"\r\n]*"|[\w.+-]+))*$/),
 );
 
+/** A media type without its parameters, lowercased: `image/png` from `image/PNG; charset=binary`. */
+export function mediaTypeEssence(mediaType: string): string {
+	return (mediaType.split(";")[0] ?? "").trim().toLowerCase();
+}
+
 export const uploadAttachmentQuerySchema = Schema.Struct({
 	name: attachmentNameSchema,
 	/** The file's media type, when the uploader knows it; otherwise it is taken from the name. */

packages/core/package.json

@@ -19,6 +19,7 @@
 		"@ai-sdk/google": "^4.0.80",
 		"@ai-sdk/mcp": "^2.0.49",
 		"@ai-sdk/openai": "^4.0.65",
+		"@ai-sdk/provider-utils": "^5.0.47",
 		"@alibaba-group/opensandbox": "1.1.0",
 		"@effect/platform-node": "catalog:",
 		"@effect/sql-pg": "catalog:",

packages/core/src/conversations/tools/browser/browser.ts

@@ -5,7 +5,8 @@ import { jsonSchema, type ToolSet, tool } from "ai";
 import { Data, Effect } from "effect";
 import { Sandboxes } from "../../../sandboxes/sandboxes.ts";
 import type { UserFacing } from "../../../user-message.ts";
-import { type Image, type OpenSandbox, type Place, withImages } from "../sandbox/tools.ts";
+import { type Image, withImages } from "../images.ts";
+import type { OpenSandbox, Place } from "../sandbox/tools.ts";
 import playwrightTools from "./playwright-tools.json" with { type: "json" };
 
 /**

packages/core/src/conversations/tools/browser/viewer.test.ts

@@ -170,7 +170,7 @@ describe.skipIf(!configured)("the desktop viewer, against Postgres and OpenSandb
 					turnId: "turn-1",
 					threadId,
 					agentId,
-					model: "no-such-model",
+					acceptsImages: false,
 				}),
 			),
 		).then((offered) => offered.tools);

packages/core/src/conversations/tools/images.ts

@@ -0,0 +1,81 @@
+import { detectMediaType } from "@ai-sdk/provider-utils";
+
+/**
+ * Images the tools show the model: a screenshot, an image in the sandbox, or
+ * one a person attached to the thread.
+ */
+
+/** An image a tool's result carries for the model to look at. */
+export interface Image {
+	/** Base 64. */
+	data: string;
+	mediaType: string;
+}
+
+/** The formats shown, as the tools' descriptions name them. */
+export const SHOWN_IMAGE_FORMATS = "PNG, JPEG, GIF or WebP";
+
+/**
+ * The largest image shown, in bytes: 5 MiB once base 64 encoded, which is how
+ * model providers measure the images they accept.
+ */
+export const MAX_IMAGE_BYTES = Math.floor((5 * 1024 * 1024 * 3) / 4);
+
+/** The media types of the formats shown. Others, such as HEIC or TIFF, models refuse. */
+const SHOWN_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
+
+/**
+ * The media type `bytes` begin as, when they are an image in a format shown:
+ * read from the bytes rather than taken from a name or a declared type, as a
+ * provider refuses an image whose declared type its bytes contradict. The AI
+ * SDK reads them the same way when it corrects a part's type.
+ */
+export function shownImageType(bytes: Uint8Array): string | undefined {
+	const detected = detectMediaType({ data: bytes, topLevelType: "image" });
+	return detected !== undefined && SHOWN_IMAGE_TYPES.has(detected) ? detected : undefined;
+}
+
+/** Whether a tool's result carries images for the model. */
+export function hasImages(
+	output: unknown,
+): output is { images: readonly Image[] } & Record<string, unknown> {
+	return typeof output === "object" && output !== null && "images" in output;
+}
+
+/**
+ * A tool's result as the model is given it: one with images as text and
+ * images, when the model can take them; anything else as it is.
+ */
+export function withImages(output: unknown, acceptsImages: boolean) {
+	if (!hasImages(output)) {
+		return { type: "json" as const, value: output as never };
+	}
+	const { images, ...rest } = output;
+	const text =
+		typeof rest.text === "string"
+			? [rest.note, rest.text].filter((part) => typeof part === "string").join("\n\n")
+			: JSON.stringify(rest);
+	if (!acceptsImages) return { type: "text" as const, value: text };
+	return {
+		type: "content" as const,
+		value: [
+			{ type: "text" as const, text },
+			...images.map((image) => ({
+				type: "file" as const,
+				data: { type: "data" as const, data: image.data },
+				mediaType: image.mediaType,
+			})),
+		],
+	};
+}
+
+/**
+ * A tool's result as it is kept, with its images counted rather than kept:
+ * the model has been shown them, and their data, read back as text in a later
+ * turn's history, would be noise.
+ */
+export function withImagesCounted(output: unknown): unknown {
+	if (!hasImages(output)) return output;
+	const { images, ...rest } = output;
+	return { ...rest, images: `${images.length} shown` };
+}

packages/core/src/conversations/tools/read-thread-file/tool.test.ts

@@ -2,45 +2,65 @@ import { Effect, Stream } from "effect";
 import { describe, expect, it } from "vitest";
 import { BlobStore } from "../../../blob-store/blob-store.ts";
 import { ThreadFiles } from "../../thread-files/thread-files.ts";
+import { MAX_IMAGE_BYTES } from "../images.ts";
 import { MAX_READ_CHARACTERS, type ReadThreadFileResult, readThreadFileTool } from "./tool.ts";
 
 const THREAD = "0199a3a0-0000-7000-8000-000000000001";
 const FILE = "0199a3a0-0000-7000-8000-0000000000f1";
 
-/** `read_thread_file` over one file in `THREAD`, called with `input`. */
-async function read(
-	text: string,
-	input: { file?: string; offset?: number; limit?: number; pattern?: string },
+/** The first bytes of every PNG file. */
+const PNG = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+
+/** A file in `THREAD`: its content, and the name and type it was kept with. */
+interface Kept {
+	content: string | Uint8Array;
+	name?: string;
+	contentType?: string;
+}
+
+/** The thread files `read_thread_file` reads `kept` through. */
+function holding({
+	content,
+	name = "notes.txt",
 	contentType = "text/plain; charset=utf-8",
-): Promise<ReadThreadFileResult> {
-	return readFrom(
-		{
-			read: ({ threadId, fileId }) =>
-				threadId === THREAD && fileId === FILE
-					? Effect.succeed({
-							id: FILE,
-							name: undefined,
-							contentType,
-							size: text.length,
-							stream: Stream.succeed(new TextEncoder().encode(text)),
-						})
-					: new ThreadFiles.FileNotFound({ fileId }),
-		},
-		input,
-	);
+}: Kept): Pick<ThreadFiles.Interface, "read"> {
+	const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
+	return {
+		read: ({ threadId, fileId }) =>
+			threadId === THREAD && fileId === FILE
+				? Effect.succeed({
+						id: FILE,
+						name,
+						contentType,
+						size: bytes.byteLength,
+						stream: Stream.succeed(bytes),
+					})
+				: new ThreadFiles.FileNotFound({ fileId }),
+	};
+}
+
+function toolOver(files: Pick<ThreadFiles.Interface, "read">, acceptsImages: boolean) {
+	return readThreadFileTool({ threadId: THREAD, files, acceptsImages, run: Effect.runPromise });
 }
 
 async function readFrom(
 	files: Pick<ThreadFiles.Interface, "read">,
 	input: { file?: string; offset?: number; limit?: number; pattern?: string },
+	acceptsImages = false,
 ): Promise<ReadThreadFileResult> {
-	const tool = readThreadFileTool({ threadId: THREAD, files, run: Effect.runPromise });
+	const tool = toolOver(files, acceptsImages);
 	const options = { toolCallId: "call", messages: [] } as unknown as Parameters<
 		NonNullable<typeof tool.execute>
 	>[1];
 	return tool.execute?.({ file: FILE, ...input }, options) as Promise<ReadThreadFileResult>;
 }
 
+/** `read_thread_file` over one text file holding `text`, called with `input`. */
+const read = (
+	text: string,
+	input: { file?: string; offset?: number; limit?: number; pattern?: string },
+) => readFrom(holding({ content: text }), input);
+
 const numbered = (count: number) =>
 	Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n");
 
@@ -75,14 +95,104 @@ describe("read_thread_file", () => {
 		expect(result.continueFrom).toBe(result.lines.split("\n").length + 1);
 	});
 
-	it("refuses a file from another thread, an offset past the end, and a file that isn't text", async () => {
+	it("refuses a file from another thread, and an offset past the end", async () => {
 		expect(await read("text", { file: "0199a3a0-0000-7000-8000-0000000000f2" })).toHaveProperty(
 			"refused",
 		);
 		expect(await read(numbered(3), { offset: 4 })).toEqual({
 			refused: "The file has 3 lines, so there is no line 4.",
 		});
-		expect(await read("PNG", {}, "image/png")).toHaveProperty("refused");
+	});
+
+	it("reads text whatever type it was given, and text it was told is text in any encoding", async () => {
+		// A TypeScript file, which extension lookup calls an MPEG video.
+		const source = holding({
+			content: "const a = 1;\n",
+			name: "main.ts",
+			contentType: "video/mp2t",
+		});
+		expect(await readFrom(source, {})).toEqual({ lines: "1\tconst a = 1;", totalLines: 1 });
+		// A CSV saved by Excel in Windows-1252: "café", with é as the byte E9.
+		const csv = holding({
+			content: new Uint8Array([0x63, 0x61, 0x66, 0xe9]),
+			name: "menu.csv",
+			contentType: "text/csv",
+		});
+		expect(await readFrom(csv, {})).toEqual({ lines: "1\tcaf�", totalLines: 1 });
+	});
+
+	it("refuses a file that is neither text nor an image", async () => {
+		const zip = holding({
+			content: new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0x00]),
+			name: "archive.zip",
+			contentType: "application/zip",
+		});
+		const notUtf8 = holding({
+			content: new Uint8Array([0xc3, 0x28]),
+			name: "data.bin",
+			contentType: "application/octet-stream",
+		});
+
+		expect(await readFrom(zip, {})).toEqual({
+			refused: "The file is application/zip, not text, so it has no lines to read.",
+		});
+		expect(await readFrom(notUtf8, {})).toHaveProperty("refused");
+	});
+
+	it("shows an image as the type its bytes are, whatever type it was given", async () => {
+		const jpeg = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]);
+		const png = new Uint8Array(PNG);
+		const shown = (content: Uint8Array, contentType: string) =>
+			readFrom(holding({ content, name: "chart.png", contentType }), {}, true);
+
+		expect(await shown(jpeg, "image/png")).toEqual({
+			text: "The image chart.png.",
+			images: [{ data: Buffer.from(jpeg).toString("base64"), mediaType: "image/jpeg" }],
+		});
+		expect(await shown(png, "application/octet-stream")).toMatchObject({
+			images: [{ mediaType: "image/png" }],
+		});
+	});
+
+	it("refuses an image to a model that can't see one, or one too large to show", async () => {
+		const large = new Uint8Array(MAX_IMAGE_BYTES + 1);
+		large.set(PNG);
+
+		expect(
+			await readFrom(
+				holding({ content: new Uint8Array(PNG), contentType: "image/png" }),
+				{},
+				false,
+			),
+		).toEqual({ refused: "The file is an image, and you can't be shown images." });
+		expect(await readFrom(holding({ content: large, contentType: "image/png" }), {}, true)).toEqual(
+			{
+				refused: `The image is ${MAX_IMAGE_BYTES + 1} bytes, larger than the ${MAX_IMAGE_BYTES} that can be shown.`,
+			},
+		);
+	});
+
+	it("gives a model that can see images the image itself", () => {
+		const tool = toolOver(
+			holding({ content: new Uint8Array(PNG), contentType: "image/png" }),
+			true,
+		);
+		const output = {
+			text: "The image chart.png.",
+			images: [{ data: "iVBORw0KGgo=", mediaType: "image/png" }],
+		};
+
+		expect(tool.toModelOutput?.({ toolCallId: "call", input: { file: FILE }, output })).toEqual({
+			type: "content",
+			value: [
+				{ type: "text", text: "The image chart.png." },
+				{
+					type: "file",
+					data: { type: "data", data: "iVBORw0KGgo=" },
+					mediaType: "image/png",
+				},
+			],
+		});
 	});
 
 	it("refuses, so the model can try again, while the blob store can't be reached", async () => {