feat: support image content in tool responses

apps/desktop/src/bun/remote/remote-runtime-client.ts

@@ -323,7 +323,10 @@ export class RemoteRuntimeClient implements RuntimeClient {
     return this._rpc<McpCallToolResponse>("mcp.callTool", input);
   }
   builtInCallTool(input: { name: string; arguments: Record<string, unknown> }) {
-    return this._rpc<{ contentText: string }>("builtinTools.call", input);
+    return this._rpc<Awaited<ReturnType<RuntimeClient["builtInCallTool"]>>>(
+      "builtinTools.call",
+      input
+    );
   }
   setSearchSettings(settings: SearchSettings) {
     return this._rpc<SearchSettings>("search.set", { settings });

apps/desktop/src/client/built-in-tools.ts

@@ -1,4 +1,4 @@
-import type { BuiltinTool } from "@llm-space/core";
+import type { BuiltinTool, BuiltinToolCallResponse } from "@llm-space/core";
 
 import { electrobun } from "@/lib/electrobun";
 import type { RuntimeId } from "@/shared/runtime";
@@ -24,7 +24,7 @@ export async function callBuiltInTool(
     arguments: Record<string, unknown>;
   },
   runtimeId?: RuntimeId
-): Promise<{ contentText: string }> {
+): Promise<BuiltinToolCallResponse> {
   return _rpc().request.builtInCallTool({
     ...runtimeScope(runtimeId),
     ...input,

apps/desktop/src/client/tool-execution.ts

@@ -1,4 +1,8 @@
-import type { BuiltinTool, McpTool } from "@llm-space/core";
+import type {
+  BuiltinTool,
+  BuiltinToolCallResponse,
+  McpTool,
+} from "@llm-space/core";
 
 import { callBuiltInTool } from "@/client/built-in-tools";
 import { callMcpTool } from "@/client/mcp";
@@ -9,8 +13,7 @@ import type { RuntimeId } from "@/shared/runtime";
  * `isError` on the response; built-in tools signal failure by throwing, so a
  * successful built-in result is always `isError: false`.
  */
-export interface ToolCallResult {
-  contentText: string;
+export interface ToolCallResult extends BuiltinToolCallResponse {
   isError: boolean;
 }
 
@@ -34,13 +37,16 @@ export async function executeTool(
       runtimeId
     );
     return {
-      contentText: result.contentText,
+      content: result.content,
       isError: result.isError ?? false,
     };
   }
   const result = await callBuiltInTool(
     { name: tool.name, arguments: args },
     runtimeId
   );
-  return { contentText: result.contentText, isError: false };
+  return {
+    content: result.content,
+    isError: false,
+  };
 }

apps/desktop/src/shared/rpc.ts

@@ -2,6 +2,7 @@ import type {
   AgentEvent,
   AgentStreamRequest,
   BuiltinTool,
+  BuiltinToolCallResponse,
   CustomModel,
   FileNode,
   ModelConfig,
@@ -389,7 +390,7 @@ export interface DesktopRPCType {
           name: string;
           arguments: Record<string, unknown>;
         };
-        response: { contentText: string };
+        response: BuiltinToolCallResponse;
       };
       // The user's anonymous-analytics opt-out preference plus whether the
       // hard gates allow sending at all (see `shared/analytics.ts`).

apps/server/src/rpc.test.ts

@@ -73,8 +73,8 @@ function createRuntime(): RuntimeClient {
         },
         tools: [],
       }) as never,
-    mcpCallTool: () => Promise.resolve({ contentText: "" }),
-    builtInCallTool: () => Promise.resolve({ contentText: "" }),
+    mcpCallTool: () => Promise.resolve({ content: [] }),
+    builtInCallTool: () => Promise.resolve({ content: [] }),
     setSearchSettings: (settings) => settings,
     setNetworkSettings: (settings) => settings,
     detectSystemProxy: () => ({

docs/core-concepts.md

@@ -138,7 +138,7 @@ Messages are the conversation history of a Thread. LLM Space currently uses two
 User message content can be:
 
 - Text: `{ "type": "text", "text": "..." }`
-- Image data: `{ "type": "image_data", "mimeType": "image/png", "data": "..." }`
+- Image data: `{ "type": "image", "mimeType": "image/png", "data": "..." }`
 
 Assistant message content is mainly text, and may also include:
 

docs/core-concepts.zh-CN.md

@@ -138,7 +138,7 @@ Messages 是 Thread 的对话历史。LLM Space 当前主要使用两种消息
 用户消息的内容可以是：
 
 - 文本：`{ "type": "text", "text": "..." }`
-- 图片数据：`{ "type": "image_data", "mimeType": "image/png", "data": "..." }`
+- 图片数据：`{ "type": "image", "mimeType": "image/png", "data": "..." }`
 
 助手消息的内容主要是文本，也可以附带：
 

packages/core/src/client/converters.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ThreadContext } from "../types";
+
+import { convertToPiContext } from "./converters";
+
+describe("convertToPiContext", () => {
+  test("passes pi-compatible tool-result image content through", () => {
+    const context: ThreadContext = {
+      messages: [
+        {
+          id: "assistant-1",
+          role: "assistant",
+          content: [],
+          toolCalls: [
+            {
+              id: "tool-1",
+              input: { name: "read", arguments: { path: "/tmp/pixel.png" } },
+              output: {
+                content: [
+                  { type: "text", text: "[image file: pixel.png]" },
+                  {
+                    type: "image",
+                    data: "cG5nLWJ5dGVz",
+                    mimeType: "image/png",
+                  },
+                ],
+                isError: false,
+              },
+            },
+          ],
+        },
+      ],
+    };
+
+    expect(convertToPiContext(context).messages[1]).toMatchObject({
+      role: "toolResult",
+      toolCallId: "tool-1",
+      toolName: "read",
+      content: [
+        { type: "text", text: "[image file: pixel.png]" },
+        { type: "image", data: "cG5nLWJ5dGVz", mimeType: "image/png" },
+      ],
+      isError: false,
+    });
+  });
+});

packages/core/src/client/converters.ts

@@ -81,19 +81,7 @@ function _convertMessageContents(
   message: Message
 ): (pi.TextContent | pi.ImageContent | pi.ThinkingContent | pi.ToolCall)[] {
   if (message.role === "user") {
-    return message.content.map((content) => {
-      if (content.type === "text") {
-        return { ...content } satisfies pi.TextContent;
-      } else if (content.type === "image_data") {
-        return {
-          type: "image",
-          mimeType: content.mimeType,
-          data: content.data,
-        } satisfies pi.ImageContent;
-      } else {
-        throw new Error(`Unsupported content type: ${JSON.stringify(content)}`);
-      }
-    });
+    return message.content;
   } else if (message.role === "assistant") {
     const contents: (
       pi.TextContent | pi.ImageContent | pi.ThinkingContent | pi.ToolCall
@@ -107,7 +95,7 @@ function _convertMessageContents(
     for (const content of message.content) {
       if (content.type === "text") {
         contents.push({ ...content } satisfies pi.TextContent);
-      } else if (content.type === "image_data") {
+      } else if (content.type === "image") {
         // Assistant messages may not carry images; drop them on conversion.
         continue;
       } else {

packages/core/src/parsers/normalize-thread.ts

@@ -1,6 +1,6 @@
 import type {
   AssistantMessage,
-  ImageDataContent,
+  ImageContent,
   Message,
   ModelConfig,
   ModelProviderGroup,
@@ -29,7 +29,7 @@ import type { ThreadParseContext } from "./thread-parser";
  */
 interface ResolvedContent {
   text: TextContent[];
-  images: ImageDataContent[];
+  images: ImageContent[];
   thinking: string[];
   toolUses: RawToolUse[];
   toolResults: RawToolResult[];
@@ -323,7 +323,10 @@ function _resolveContent(content: unknown): ResolvedContent {
       }
 
       case "image": {
-        const image = _resolveAnthropicImage(b);
+        const image =
+          typeof b.mimeType === "string" && typeof b.data === "string"
+            ? _imageContent(b.mimeType, b.data)
+            : _resolveAnthropicImage(b);
         if (image) {
           result.images.push(image);
         }
@@ -338,17 +341,6 @@ function _resolveContent(content: unknown): ResolvedContent {
         break;
       }
 
-      case "image_data": {
-        // Already our internal shape.
-        if (typeof b.mimeType === "string" && typeof b.data === "string") {
-          const image = _imageData(b.mimeType, b.data);
-          if (image) {
-            result.images.push(image);
-          }
-        }
-        break;
-      }
-
       default:
         break;
     }
@@ -502,7 +494,7 @@ function _resolveToolSource(source: unknown): LegacyMcpToolSource | undefined {
 /** Anthropic image block: only base64 sources can be inlined. */
 function _resolveAnthropicImage(
   b: Record<string, unknown>
-): ImageDataContent | undefined {
+): ImageContent | undefined {
   const source = _asRecord(b.source);
   if (!source) {
     return undefined;
@@ -512,15 +504,15 @@ function _resolveAnthropicImage(
     typeof source.media_type === "string" &&
     typeof source.data === "string"
   ) {
-    return _imageData(source.media_type, source.data);
+    return _imageContent(source.media_type, source.data);
   }
   return undefined;
 }
 
 /** OpenAI image block: only `data:` base64 URLs can be inlined. */
 function _resolveOpenAiImage(
   b: Record<string, unknown>
-): ImageDataContent | undefined {
+): ImageContent | undefined {
   const imageUrl = _asRecord(b.image_url);
   const rawUrl = imageUrl?.url ?? b.image_url;
   const url = typeof rawUrl === "string" ? rawUrl : undefined;
@@ -531,18 +523,15 @@ function _resolveOpenAiImage(
   if (match?.[1] === undefined || match[2] === undefined) {
     return undefined;
   }
-  return _imageData(match[1], match[2]);
+  return _imageContent(match[1], match[2]);
 }
 
-/** Build an {@link ImageDataContent}, enforcing the `image/<subtype>` shape. */
-function _imageData(
+/** Build image content in pi's model-facing shape without another conversion. */
+function _imageContent(
   mimeType: string,
   data: string
-): ImageDataContent | undefined {
-  if (!/^image\/\w+$/.test(mimeType)) {
-    return undefined;
-  }
-  return { type: "image_data", mimeType, data };
+): ImageContent {
+  return { type: "image", mimeType, data };
 }
 
 /** Set a tool call's output, matching by id. Unmatched results are dropped. */

packages/core/src/server/storage/blob/image-blobs.ts

@@ -1,9 +1,9 @@
 import { createHash } from "node:crypto";
 
-import type { ImageDataContent, Thread } from "../../../types";
+import type { Thread } from "../../../types";
 
 /**
- * Sentinel prefix marking an `image_data.data` string as a reference into the
+ * Sentinel prefix marking an image `data` string as a reference into the
  * file's own blob table rather than inline base64. The colon guarantees it can
  * never collide with real base64 (whose alphabet excludes `:`), so the two are
  * unambiguous.
@@ -30,11 +30,17 @@ interface PackedThread extends Thread {
   blobs?: Record<string, string>;
 }
 
-function _isImageData(value: unknown): value is ImageDataContent {
+interface StoredImageContent {
+  type: "image";
+  data: string;
+}
+
+/** Match image content while packing or unpacking persisted threads. */
+function _isImageContent(value: unknown): value is StoredImageContent {
   return (
     !!value &&
     typeof value === "object" &&
-    (value as { type?: unknown }).type === "image_data" &&
+    (value as { type?: unknown }).type === "image" &&
     typeof (value as { data?: unknown }).data === "string"
   );
 }
@@ -51,7 +57,7 @@ function _parseRef(data: string): string | null {
 }
 
 /**
- * Return a copy of `value` with every `image_data.data` string mapped through
+ * Return a copy of `value` with every image `data` string mapped through
  * `fn`. Structurally shares (copy-on-write) any subtree that didn't change, so
  * an untouched thread is returned by reference.
  */
@@ -67,7 +73,7 @@ function _map(value: unknown, fn: (data: string) => string): unknown {
     return changed ? next : value;
   }
   if (value && typeof value === "object") {
-    if (_isImageData(value)) {
+    if (_isImageContent(value)) {
       const nextData = fn(value.data);
       return nextData === value.data ? value : { ...value, data: nextData };
     }

packages/core/src/server/storage/local/file-system.test.ts

@@ -116,12 +116,12 @@ describe("LocalFileSystem.write", () => {
             role: "user",
             content: [
               {
-                type: "image_data",
+                type: "image",
                 data: imageData,
                 mimeType: "image/png",
               },
               {
-                type: "image_data",
+                type: "image",
                 data: imageData,
                 mimeType: "image/png",
               },

packages/core/src/thread/run-history-utils.ts

@@ -5,6 +5,8 @@ import {
   type ThreadSnapshot,
 } from "../types";
 
+import { getToolResultText } from "./tool-call-status";
+
 /**
  * A short summary of a run's resulting thread, derived from its last message.
  */
@@ -19,7 +21,7 @@ export function summarizeRun(thread: ThreadSnapshot): string {
       .map((toolCall) => `${toolCall.input.name}()`)
       .join(", ");
   }
-  const imageCount = last.content.filter((c) => c.type === "image_data").length;
+  const imageCount = last.content.filter((c) => c.type === "image").length;
   if (imageCount > 0) {
     return `[${imageCount} image${imageCount > 1 ? "s" : ""}]`;
   }
@@ -62,7 +64,7 @@ export function runLastUserText(thread: ThreadSnapshot): string {
     return "No user message";
   }
   const text = getMessageText(message).trim();
-  const imageCount = message.content.filter((c) => c.type === "image_data")
+  const imageCount = message.content.filter((c) => c.type === "image")
     .length;
   if (text && imageCount > 0) {
     return `${text}\n[${imageCount} image${imageCount > 1 ? "s" : ""}]`;
@@ -93,10 +95,7 @@ function _toolResultText(message: AssistantMessage): string {
   }
   return message.toolCalls
     .map((toolCall) => {
-      const output = toolCall.output?.content
-        ?.map((content) => content.text)
-        .join("\n")
-        .trim();
+      const output = getToolResultText(toolCall.output?.content).trim();
       const args = JSON.stringify(toolCall.input.arguments);
       return output
         ? `${toolCall.input.name}(${args})\n${output}`