diff --git a/packages/ai/src/api/transform-messages.ts b/packages/ai/src/api/transform-messages.ts index ec7035a38..7ac3a946b 100644 --- a/packages/ai/src/api/transform-messages.ts +++ b/packages/ai/src/api/transform-messages.ts @@ -62,11 +62,25 @@ function replaceImagesWithPlaceholder(content: (TextContent | ImageContent)[], p return result; } +function normalizeToolResultMediaData(content: (TextContent | ImageContent)[]): (TextContent | ImageContent)[] { + let changed = false; + const normalized = content.map((block) => { + if (block.type !== "image") return block; + const prefix = `data:${block.mimeType};base64,`; + if (!block.data.startsWith(prefix)) return block; + changed = true; + return { ...block, data: block.data.slice(prefix.length) }; + }); + return changed ? normalized : content; +} + function downgradeUnsupportedImages(messages: Message[], model: Model): Message[] { const supportsImages = model.input.includes("image"); const supportsVideo = model.input.includes("video"); if (supportsImages && supportsVideo) { - return messages; + return messages.map((msg) => + msg.role === "toolResult" ? { ...msg, content: normalizeToolResultMediaData(msg.content) } : msg, + ); } return messages.map((msg) => { @@ -86,7 +100,7 @@ function downgradeUnsupportedImages(messages: Message[], model } if (msg.role === "toolResult") { - let content = msg.content; + let content = normalizeToolResultMediaData(msg.content); if (!supportsVideo) { content = replaceMediaWithPlaceholder(content, NO_VIDEO_TOOL_PLACEHOLDER, (b) => isVideoMimeType(b.mimeType), diff --git a/packages/ai/src/changes.md b/packages/ai/src/changes.md index fefe5b89b..fa55648e1 100644 --- a/packages/ai/src/changes.md +++ b/packages/ai/src/changes.md @@ -1,5 +1,24 @@ ## PR #1304 review fixes: shared auth-miss prefix, login merge, sentinel repair (2026-09-10) +## 2026-09-11 - Normalize canonical tool-result image data URLs before provider serialization + +### What changed + +- `api/transform-messages.ts`: canonical `data:${mimeType};base64,` prefixes on tool-result image blocks are stripped back to raw base64 at the shared message-normalization boundary. Raw base64 and mismatched/arbitrary data URLs remain unchanged. +- `test/issue-1260-tool-result-image-data-url.test.ts`: covers raw base64 and already-prefixed canonical image data through the Responses serializer and pins exactly one wire prefix. + +### Why + +- Issue #1260: `ImageContent.data` is an internal raw-base64 contract, but tool results can arrive already prefixed. Passing that value through made Responses adapters prepend a second data-URL prefix and emit an invalid `image_url`. + +### Why an extension could not handle it + +- This is the shared provider-normalization boundary used before protocol serializers. Fixing one provider or extension would leave the malformed internal representation visible to other adapters. + +### Expected merge conflict zones + +- LOW: `api/transform-messages.ts` around tool-result media normalization and `downgradeUnsupportedImages()`. + ### What changed - `packages/ai/src/auth/resolve.ts`: `PROVIDER_NOT_CONFIGURED_PREFIX` / `providerNotConfiguredMessage()` export the exact auth-miss wording every resolution site throws; `packages/ai/src/models.ts` re-exports both and throws through the helper. Consumers keying recovery decisions off that message (the coding-agent session layer and the credential-pool classifier) can never drift from the throw sites. diff --git a/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts b/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts new file mode 100644 index 000000000..f66799da2 --- /dev/null +++ b/packages/ai/test/issue-1260-tool-result-image-data-url.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { convertResponsesMessages } from "../src/providers/openai-responses-shared.ts"; +import type { Context, Model } from "../src/types.ts"; + +const model = { + id: "gpt-5.5", + name: "GPT-5.5", + provider: "openai-codex", + api: "openai-codex-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_000_000, + maxTokens: 128_000, +} as Model<"openai-codex-responses">; + +function zeroUsage() { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function convertToolImage(data: string) { + const context: Context = { + messages: [ + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_image|fc_image", + name: "image_tool", + arguments: {}, + }, + ], + api: "openai-codex-responses", + provider: "openai-codex", + model: "gpt-5.5", + usage: zeroUsage(), + stopReason: "toolUse", + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call_image|fc_image", + toolName: "image_tool", + content: [{ type: "image", data, mimeType: "image/png" }], + isError: false, + timestamp: 2, + }, + ], + tools: [], + }; + + return convertResponsesMessages(model, context, new Set(["openai-codex"])); +} + +describe("issue #1260 Responses tool-result image data URLs", () => { + it("adds exactly one data URL prefix to raw base64", () => { + expect(convertToolImage("QUJD")).toMatchObject([ + { type: "function_call", call_id: "call_image", name: "image_tool" }, + { + type: "function_call_output", + call_id: "call_image", + output: [{ type: "input_image", detail: "auto", image_url: "data:image/png;base64,QUJD" }], + }, + ]); + }); + + it("preserves an already-prefixed canonical image data URL", () => { + const dataUrl = "data:image/png;base64,QUJD"; + expect(convertToolImage(dataUrl)).toMatchObject([ + { type: "function_call", call_id: "call_image", name: "image_tool" }, + { + type: "function_call_output", + call_id: "call_image", + output: [{ type: "input_image", detail: "auto", image_url: dataUrl }], + }, + ]); + }); +});