From ec92a44dbc2628693b4a884c63e85a91a9a55456 Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Mon, 20 Jul 2026 10:39:05 +0700 Subject: [PATCH] fix(vision): forward MCP tool result images to vision-capable models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images nested inside LanguageModelToolResultPart.content (e.g. screenshots returned by chrome-devtools-mcp, playwright-mcp) were silently dropped by convertMessage(): partToText() returned "" for any image DataPart via its catch-all fallback. The result was an empty tool message sent to the model, so vision-capable models like Kimi K2.7 Code honestly reported they could not see the image — even though the same model worked fine when the image was pasted directly into chat. Root cause: partToText() handled TextPart / ToolCallPart / internal DataPart / string, but image DataParts nested inside tool results fell through to the catch-all return "". Top-level image attachments worked because they hit a separate handler in convertMessage() that explicitly checked part.mimeType.startsWith("image/"). Fix: - convertMessage(): walk part.content and partition each item into either text (via partToText) or an OpenAiContentPart image part (via dataPartToBase64). Emit a multimodal OpenAiContentPart[] on the tool message when any image is present; otherwise emit a plain string so text-only tool results stay byte-identical to the previous behavior. - AnthropicToolResultBlock.content type widened from string to string | AnthropicContentBlock[]. New anthropicToolResultContent() returns the string form when there are no images and the array form (text + image blocks) only when an image is present. - responsesInputItemsFromMessage() tool branch: new responsesToolOutput() helper that degrades images to a placeholder note (Responses API function_call_output.output is string-only). - googleContentsFromMessages() tool branch: new googleFunctionResponseContent() that emits parts:[{text},{inlineData}] when an image is present. - New MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000 constant. Oversized images (>1 MB raw bytes) are replaced with an actionable placeholder note so a single full-page MCP screenshot can't push the request payload past the upstream limit. Manual testing revealed agent screenshot loops producing 4.6 MB payloads that triggered upstream 400 "Upstream request failed". Drive-by log noise reduction (pre-existing, surfaced during testing): - provideLanguageModelChatInformation: replaced per-model log spam (22 lines per call x ~3 calls/sec) with one summary line per invocation. - fetchModels: replaced vscode.window.showWarningMessage on transient upstream 400/503 (common from OpenCode's shared gateway, especially on auto-registered AGENT_* variants the user may not actively use) with an Output-channel log. Bundled fallbackModels snapshot keeps the picker functional. Verification: - tsc -p ./ exit 0 (TypeScript strict, no errors) - node --test out/test/**/*.test.js — 107/107 pass, 0 regression - Manual test: chrome-devtools-mcp + OpenCode Go model successfully read and described the returned screenshot Docs: - docs/issues/34-20260720-mcp-tool-result-image-dropped.md (root-cause investigation + timeline + follow-up bugs uncovered during testing) - docs/features/12-20260720-mcp-tool-result-image-support.md (architecture, per-transport behavior, code locations, limitations) - CHANGELOG.md [Unreleased] section - docs/devlog.md session entry Fixes #77 --- CHANGELOG.md | 11 + docs/devlog.md | 25 +- ...-20260720-mcp-tool-result-image-support.md | 137 +++++++ ...-20260720-mcp-tool-result-image-dropped.md | 339 ++++++++++++++++++ package-lock.json | 6 +- src/extension.ts | 225 +++++++++++- 6 files changed, 718 insertions(+), 25 deletions(-) create mode 100644 docs/features/12-20260720-mcp-tool-result-image-support.md create mode 100644 docs/issues/34-20260720-mcp-tool-result-image-dropped.md diff --git a/CHANGELOG.md b/CHANGELOG.md index af171f9..d3f1e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to the **OpenCode Go BYOK Provider** extension are documented here. +## [Unreleased] + +### Added + +- **`[Vision]` Multimodal tool results — MCP screenshot forwarding (#77).** Images returned inside a `LanguageModelToolResultPart` (e.g. screenshots from `chrome-devtools-mcp`, `playwright-mcp`) are now forwarded to vision-capable models. Previously these images were silently dropped by the serialization layer and the model would report "I cannot see the image". Images are encoded as OpenAI-style `image_url` content parts and translated into the native multimodal shape for each transport: chat-completions (native array content), Anthropic messages (`tool_result.content: AnthropicContentBlock[]`), Google Gemini (`functionResponse.response.parts: [{inlineData}]`). Oversized images (>1 MB raw bytes) are replaced with an actionable placeholder note so a single full-page MCP screenshot can't push the request payload past the upstream limit. The Responses API cannot carry images in tool output and degrades to a placeholder note on that transport only. + +### Fixed + +- **`[Logging]` Model registration log spam during UI refresh.** VS Code refreshes model info on roughly a 300 ms cadence during chat UI activity; each call previously produced one log line per registered model (22+ lines per call). `provideLanguageModelChatInformation` now emits a single summary line per invocation (`Models registered: count=N provider=… first=… last=…`). Output channel is dramatically cleaner during testing. +- **`[Logging]` Transient model-list fetch failures no longer pop a modal warning.** OpenCode's shared gateway occasionally returns transient 400/503 responses that resolve on retry within seconds, and the previous behavior called `showWarningMessage` on every failure — including from auto-registered provider variants the user may not actively use (e.g. `OpenCode Zen (Agents)`). Failures now log to the Output channel only; the bundled `fallbackModels` snapshot keeps the picker functional. + ## [0.4.1] — 2026-07-15 ### Added diff --git a/docs/devlog.md b/docs/devlog.md index 058d6f7..034b5ff 100644 --- a/docs/devlog.md +++ b/docs/devlog.md @@ -1,5 +1,5 @@ # 🧠 OPENCODE COPILOT CHAT DEVLOG -**Branch:** `main` | **Updated:** 2026-07-15 Asia/Jakarta | **Current Phase:** v0.4.1 — Vision Proxy + Context Overflow Fix + Output Popup Fix ✅ Merged +**Branch:** `fix/issue-77-mcp-image-tool-result` | **Updated:** 2026-07-20 Asia/Jakarta | **Current Phase:** Issue #77 — MCP Tool Result Image Forwarding ✅ Fixed, pending PR --- @@ -7,11 +7,24 @@ | Field | Value | |-------|-------| -| **Last Session** | 2026-07-15 | -| **Worked On** | Reviewed PR #76 (@Wallacy, vision proxy for text-only models + context overflow safety margin + output pane focus steal fix). Two review rounds: first round flagged 1 blocker (context overflow fix missing from code, only in CHANGELOG) + 2 formatting issues (README table collapsed, CHANGELOG `[0.3.7]` heading removed) + 2 nice-to-haves (dummy CancellationToken, quota not documented). Wallacy force-pushed `8a0d813` addressing all blockers. Re-verified: 64-token margin in `modelLimits()`, README rows split, heading restored, real `token` wired to `proxyVision`. Merged via merge commit `d2fcbe4` (NOT squash, preserved 4 commits). Post-merge: wrote feature doc `docs/features/11-20260715-vision-proxy.md`, bumped version to 0.4.1, updated CHANGELOG, updated this devlog. | -| **Stopped At** | v0.4.1 docs + version bump complete. Pending `npm run compile` + `vsce package` + local VSIX install + commit. | -| **Next Action** | → Compile, package VSIX, install locally for smoke test. Then commit docs + version bump. Push only after user approval. | -| **Open Issues** | (1) VS Code API gap: thread ID → session cost. (2) Qwen image quota. (3) `qwen3.6-plus-free` tool-call loop. (4) #57/#58 agent model visibility. (5) Vision proxy quota not documented in README (minor, logged to Output channel). | +| **Last Session** | 2026-07-20 | +| **Worked On** | Investigated issue #77 (MCP tool result images dropped — vision-capable models couldn't see screenshots from `chrome-devtools-mcp`). Root cause: `convertMessage()` serialized `LanguageModelToolResultPart.content` via `partToText()` which silently dropped nested image `LanguageModelDataPart` (catch-all returned `""`). Pasted image attachments worked because they hit the separate top-level image handler. Fix: rewrote the tool-result branch to walk `part.content` and emit multimodal `OpenAiContentPart[]` when an image is present; updated all 4 transports (chat-completions native, Anthropic `tool_result.content: AnthropicContentBlock[]`, Google `functionResponse.parts: [{inlineData}]`, Responses API degraded to placeholder note). Two follow-up bugs uncovered during manual testing: (a) 4.6 MB payload → upstream 400 because MCP screenshot loops accumulate full-page PNGs; fixed with `MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000` size guard; (b) model registration log spam (22 lines × ~3 calls/sec) + transient model-list fetch failures popping modal warnings — both replaced with Output-channel logs. Wrote `docs/issues/34-*` + `docs/features/12-*` + updated CHANGELOG `[Unreleased]`. All tests pass (107/107), compile clean. | +| **Stopped At** | Ready to commit on `fix/issue-77-mcp-image-tool-result` and open PR. User will push + open PR after reviewing. | +| **Next Action** | → User reviews commit → push → open PR `fix/issue-77-mcp-image-tool-result` → merge with merge commit (NEVER squash) → closes #77 automatically via `Fixes #77` in PR body. | +| **Open Issues** | (1) VS Code API gap: thread ID → session cost. (2) Qwen image quota. (3) `qwen3.6-plus-free` tool-call loop. (4) #57/#58 agent model visibility. (5) Vision proxy quota not documented in README (minor). (6) `estimateTokenCount` under-counts base64 payloads — no history-level image trimming yet (tracked in issue doc #34 limitations). | + +--- + +## 🔬 Issue #77 — MCP Tool Result Image Forwarding — Session 2026-07-20 ✅ FIXED + +**Action:** Triaged issue #77 (reported by @yinhx3). Deep-dived VS Code Chat API to confirm `LanguageModelToolResultPart.content: unknown[]` may contain nested image `LanguageModelDataPart` (that's how MCP screenshot tools deliver images). Confirmed Kimi K2.7 is vision-capable via `VISION_CAPABLE_MODELS`. Designed and implemented a 4-transport fix with per-image size guard. Two pre-existing log-noise bugs fixed in the same session because they made manual testing very hard to follow. + +**Branch:** `fix/issue-77-mcp-image-tool-result` (created from `main` @ `55fb6ad`) + +**Compile:** `./node_modules/.bin/tsc -p ./` exit 0 +**Tests:** 107/107 pass, 0 regression (vision proxy, metadata, thinking, retry, usage, goUsageTracker, usageProfile) + +**Manual verification:** Chrome DevTools MCP + OpenCode Go model — model successfully read and described the screenshot. --- diff --git a/docs/features/12-20260720-mcp-tool-result-image-support.md b/docs/features/12-20260720-mcp-tool-result-image-support.md new file mode 100644 index 0000000..3051687 --- /dev/null +++ b/docs/features/12-20260720-mcp-tool-result-image-support.md @@ -0,0 +1,137 @@ +**Status:** 🟢 Active + +# Multimodal Tool Results — Images Returned by MCP Tools + +**Topic:** vision / tool-calling / streaming / provider / mcp +**Updated:** 2026-07-20 +**Tags:** #vision #tool-calling #streaming #provider #mcp +**Issues:** [#77](https://github.com/ltmoerdani/opencode-copilot-chat/issues/77) +**Released:** `Unreleased` (post-`0.4.1`) + +--- + +## Overview + +Extension-side support for forwarding images that arrive inside a +`LanguageModelToolResultPart` to vision-capable models. This is the shape +MCP tools such as `chrome-devtools-mcp` and `playwright-mcp` use to return +screenshots. Before this feature, those images were silently dropped at +serialization time and the model received an empty tool result. Now they +are encoded as OpenAI-style `image_url` content parts and translated into +the native multimodal format of each supported transport. + +--- + +## Architecture + +``` +MCP tool returns image + (chrome-devtools-mcp screenshot, etc.) + ↓ +VS Code delivers LanguageModelToolResultPart.content = [LanguageModelDataPart] + ↓ +convertMessage() walks part.content: + • TextPart / internal DataPart / string → joined into toolTextParts + • image DataPart → encoded as OpenAiContentPart + {type:"image_url", image_url:{url:"data:…"}} + (subject to MAX_TOOL_RESULT_IMAGE_BYTES = 1 MB) + ↓ +Tool message content: + • string if no images present (byte-identical to old behavior) + • OpenAiContentPart[] multimodal if ≥1 image present + ↓ +Per-transport builder converts to native shape: + • chat-completions: passes the array through as-is + • Anthropic messages: tool_result.content becomes string | AnthropicContentBlock[] + • Google Gemini: functionResponse.response gains parts:[{text},{inlineData}] + • Responses API: image replaced with placeholder note (API limit: string only) +``` + +--- + +## Configuration + +No settings, no toggle. If a model reports `imageInput: true` (i.e. it is in +`VISION_CAPABLE_MODELS` or `models.dev` says it supports vision), nested tool +result images are forwarded automatically. If the model is text-only, the +existing **vision proxy** (see +[`docs/features/11-20260715-vision-proxy.md`](11-20260715-vision-proxy.md)) +handles it. + +--- + +## Size Guard + +```ts +const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; // 1 MB raw bytes +``` + +Single images larger than 1 MB are replaced with an inline placeholder note so +the request payload stays bounded. Typical MCP screenshots are 50–300 KB; the +cap exists to prevent agent loops that re-capture full-page screenshots from +producing multi-MB payloads that get rejected upstream with +`400 Upstream request failed`. + +The placeholder text is: + +> `[Image attachment omitted: N bytes exceeds the 1000000-byte limit for tool +> results. Ask the tool to produce a smaller screenshot or save it to a file.]` + +--- + +## Per-Transport Behavior + +| Transport | Image in tool result | Notes | +|-----------|----------------------|-------| +| OpenAI chat-completions | ✅ Native — array content forwarded as-is | Used by Kimi, Mimo, GLM, DeepSeek, Grok on OpenCode Go | +| Anthropic messages | ✅ Native — `tool_result.content: AnthropicContentBlock[]` | Used by MiniMax M2.5/M2.7, Qwen3.5/3.6/3.7-max | +| Google Gemini | ✅ Native — `functionResponse.response.parts: [{inlineData}]` | Used by Gemini family (when available on OpenCode) | +| OpenAI Responses API | ❌ API limit — replaced with placeholder note | `function_call_output.output` is string-only | + +--- + +## Code Locations + +| Concern | Location | +|---------|----------| +| `convertMessage` tool-result image branch | `src/extension.ts` `convertMessage()` | +| `MAX_TOOL_RESULT_IMAGE_BYTES` constant | `src/extension.ts` (near `IMAGE_TOKEN_ESTIMATE`) | +| Anthropic tool result content | `src/extension.ts` `anthropicToolResultContent()` | +| Responses API tool output | `src/extension.ts` `responsesToolOutput()` | +| Google tool response content | `src/extension.ts` `googleFunctionResponseContent()` | +| Anthropic content-block type widening | `src/extension.ts` `AnthropicToolResultBlock` | + +--- + +## Verification + +- `tsc -p ./` — compile pass, no errors +- `node --test out/test/**/*.test.js` — 107/107 pass, no regression +- Manual test with `chrome-devtools-mcp` + Kimi K2.7 Code on OpenCode Go: + model successfully read and described the returned screenshot + +See [`docs/issues/34-20260720-mcp-tool-result-image-dropped.md`](../issues/34-20260720-mcp-tool-result-image-dropped.md) +for the full investigation, timeline, and follow-up bugs uncovered during +manual testing. + +--- + +## Limitations + +1. **Responses API has no image support in tool output.** Only `gpt-5-codex` + is affected today. Images are replaced with an actionable placeholder. +2. **No history-level image trimming.** The size guard bounds each image + individually but does not trim smaller images that collectively exceed + the model's context window. VS Code's own trimming is expected to handle + that, but our `estimateTokenCount` under-counts base64 payloads. +3. **Top-level image attachments still have no size cap.** Only tool-result + images are bounded. Consistent limit can be added in a follow-up if + users hit oversized pasted images. + +--- + +## Related Docs + +- [`docs/features/01-20260514-vision-image-input.md`](01-20260514-vision-image-input.md) — top-level image attachments +- [`docs/features/11-20260715-vision-proxy.md`](11-20260715-vision-proxy.md) — proxy for text-only models +- [`docs/issues/34-20260720-mcp-tool-result-image-dropped.md`](../issues/34-20260720-mcp-tool-result-image-dropped.md) — root-cause + fix writeup diff --git a/docs/issues/34-20260720-mcp-tool-result-image-dropped.md b/docs/issues/34-20260720-mcp-tool-result-image-dropped.md new file mode 100644 index 0000000..2082c5d --- /dev/null +++ b/docs/issues/34-20260720-mcp-tool-result-image-dropped.md @@ -0,0 +1,339 @@ +**Status:** ✅ Solved + +# MCP Tool Result Images Dropped — Vision-Capable Models Could Not See Screenshots + +**Topic:** vision / tool-calling / streaming / provider / vscode / mcp +**Updated:** 2026-07-20 +**Tags:** #vision #tool-calling #streaming #provider #vscode #mcp +**Issues:** [#77](https://github.com/ltmoerdani/opencode-copilot-chat/issues/77) + +--- + +## Overview + +When an MCP tool (e.g. `chrome-devtools-mcp`, `playwright-mcp`) returned an image +as part of its `LanguageModelToolResultPart`, vision-capable models connected via +this extension silently lost the image: the model would reply "I cannot see the +image" or ask the user to upload it again. The same MCP tool worked correctly with +Copilot's built-in models. Pasted image attachments also worked fine — only images +nested inside tool results were dropped. + +This documents the full root-cause investigation, the four-transport fix, two +follow-up bugs uncovered during manual testing (payload explosion and warning +spam), and the size-guard + log-noise mitigations that ship alongside the core +fix. + +--- + +## Problem Statement + +Reported in [#77](https://github.com/ltmoerdani/opencode-copilot-chat/issues/77) +by [@yinhx3](https://github.com/yinhx3): + +> When using the extension to connect a model, image responses from MCP tools +> (e.g., `chrome-devtools-mcp` screenshots) are not visible to the model. The +> same MCP tool output works correctly when using Copilot's built-in model. +> +> When the same image is uploaded directly through the chat input (for example, +> pasted as an attachment), the Kimi K2.7 Code model recognizes and describes +> it correctly. This confirms the model itself is capable of vision, and the +> problem is specifically with how MCP tool image results are passed to the +> model. + +### Observed behavior + +| Path | Worked? | +|------|---------| +| Paste/drag image into chat input (top-level `LanguageModelDataPart`) | ✅ Yes | +| MCP tool returns image (`LanguageModelDataPart` nested in `LanguageModelToolResultPart.content`) | ❌ No — model said "no image visible" | +| Built-in Copilot model + same MCP tool | ✅ Yes | + +### Confirmation that Kimi K2.7 is vision-capable + +`VISION_CAPABLE_MODELS` in `metadata.ts` (see +[`docs/features/01-20260514-vision-image-input.md`](../features/01-20260514-vision-image-input.md)) +lists `kimi-k2.6`, `kimi-k2.5` from MoonshotAI as multimodal. K2.7 Code inherits +that capability. The bug is in the extension, not the model. + +--- + +## Root Cause + +### VS Code API recap + +From the [VS Code API reference](https://code.visualstudio.com/api/references/vscode-api#LanguageModelToolResultPart): + +```ts +class LanguageModelToolResultPart { + constructor(callId: string, content: unknown[]); + readonly callId: string; + readonly content: unknown[]; +} +``` + +> `LanguageModelDataPart` — "Can be used in responses, chat messages, tool +> results, and other language model interactions." + +A `LanguageModelToolResultPart.content` is `unknown[]` and **may contain nested +`LanguageModelDataPart` instances** with image MIME types. This is exactly how +MCP tools like `chrome-devtools-mcp` deliver screenshots. + +### The broken serialization path + +In `src/extension.ts` `convertMessage()`, tool results were serialized as: + +```ts +if (part instanceof vscode.LanguageModelToolResultPart) { + toolResults.push({ + role: "tool", + tool_call_id: part.callId, + content: part.content.map(partToText).filter(Boolean).join("\n") + // ^^^^^^^^^^^^^^^^ + // partToText() only handles TextPart / ToolCallPart / + // internal DataPart / string. Image DataPart fell through to + // the catch-all `return ""` → silently dropped. + }); + continue; +} +``` + +And `partToText()`: + +```ts +function partToText(part: vscode.LanguageModelInputPart | unknown): string { + if (part instanceof vscode.LanguageModelTextPart) return part.value; + if (part instanceof vscode.LanguageModelToolResultPart) return /* recurse */; + if (part instanceof vscode.LanguageModelToolCallPart) return /* … */; + if (part instanceof vscode.LanguageModelDataPart && isInternalDataPart(part)) return ""; + if (typeof part === "string") return part; + return ""; // ← image DataPart in a tool result landed here → DROPPED +} +``` + +The extension sent `content: ""` to the provider, and the model honestly replied +that it had no image. + +### Why top-level paste worked + +Top-level image attachments arrive as a sibling of `LanguageModelToolResultPart` +inside `message.content`. They hit a separate handler in `convertMessage()` that +explicitly checked `part.mimeType.startsWith("image/")` and produced an +`OpenAiContentPart` of `type: "image_url"`. That code path was never reached for +images nested inside tool results. + +--- + +## Solution + +### 1. Serialize nested images in tool results (`convertMessage`) + +The tool-result branch was rewritten to walk `part.content` and partition each +item into either text (via `partToText`) or an `OpenAiContentPart` image part +(via `dataPartToBase64`). When at least one image is present, the resulting +`ApiMessage.content` becomes a multimodal array; otherwise it stays a plain +string so text-only tool results remain byte-for-byte identical to the previous +behavior. + +```ts +const toolTextParts: string[] = []; +const toolImageParts: OpenAiContentPart[] = []; +for (const resultPart of part.content) { + if (resultPart instanceof vscode.LanguageModelDataPart + && resultPart.mimeType.startsWith("image/") + && !isInternalDataPart(resultPart)) { + const base64 = dataPartToBase64(resultPart.data); + toolImageParts.push({ + type: "image_url", + image_url: { url: `data:${resultPart.mimeType};base64,${base64}` }, + }); + continue; + } + const text = partToText(resultPart); + if (text) toolTextParts.push(text); +} + +let toolContent: string | OpenAiContentPart[]; +if (toolImageParts.length > 0) { + const multimodal: OpenAiContentPart[] = []; + const joinedText = toolTextParts.join("\n"); + if (joinedText) multimodal.push({ type: "text", text: joinedText }); + multimodal.push(...toolImageParts); + toolContent = multimodal; +} else { + toolContent = toolTextParts.join("\n"); +} +``` + +### 2. Per-transport tool result handling + +Each transport builder was updated so the multimodal tool content survives into +the final request body: + +| Transport | Change | +|-----------|--------| +| **Anthropic messages** | `AnthropicToolResultBlock.content` type widened from `string` to `string \| AnthropicContentBlock[]`. New helper `anthropicToolResultContent()` returns the string form when there are no images (byte-identical to the old behavior) and the array form (text + image blocks) only when an image is present. | +| **OpenAI Responses API** | `function_call_output.output` is spec'd as a plain string and does not accept image content blocks. New helper `responsesToolOutput()` joins the text parts and appends a short note when an image was omitted. No image is forwarded on this transport. | +| **Google Gemini** | New helper `googleFunctionResponseContent()` emits the legacy `{ name, content }` shape for text-only tool results and extends it with `parts: [{text}, {inlineData}]` when an image is present. | +| **OpenAI chat-completions** | No builder change required — the multimodal `content` array produced by `convertMessage` flows straight into the request body. | + +### 3. Size guard for oversized images + +Manual testing (see "Follow-up bugs" below) revealed that MCP screenshot loops +can push a single request payload to 4.6 MB and trigger upstream `400 "Upstream +request failed"`. A hard cap was added in `convertMessage`: + +```ts +const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; // 1 MB raw bytes + +if (resultPart.data.byteLength > MAX_TOOL_RESULT_IMAGE_BYTES) { + toolTextParts.push( + `[Image attachment omitted: ${resultPart.data.byteLength} bytes exceeds the ` + + `${MAX_TOOL_RESULT_IMAGE_BYTES}-byte limit for tool results. Ask the tool ` + + `to produce a smaller screenshot or save it to a file.]` + ); + continue; +} +``` + +The model still knows an image was returned, the request payload stays bounded, +and the user gets an actionable hint. + +### 4. Log noise reduction (pre-existing, surfaced during testing) + +Two unrelated log-spam issues were addressed in the same session because they +made manual testing of the image fix very hard to follow: + +- `provideLanguageModelChatInformation` previously logged one line per model + per call (22 lines per call × ~3 calls/sec). Replaced with a single summary + log line: `Models registered: count=22 provider=opencodego first=… last=…`. +- `fetchModels` previously called `vscode.window.showWarningMessage` on every + transient upstream failure (400/503 from OpenCode's shared gateway are + common and resolve on retry within seconds). Replaced with an Output-channel + log line. The bundled `fallbackModels` snapshot keeps the picker functional. + +--- + +## Files Changed + +| File | Change | +|------|--------| +| `src/extension.ts` | `convertMessage()`: serialize nested image DataParts into multimodal `OpenAiContentPart[]` on tool messages; enforce `MAX_TOOL_RESULT_IMAGE_BYTES` cap. | +| `src/extension.ts` | `AnthropicToolResultBlock.content` type widened to `string \| AnthropicContentBlock[]`; new `anthropicToolResultContent()`. | +| `src/extension.ts` | `responsesInputItemsFromMessage()` tool branch: new `responsesToolOutput()` helper that degrades images to a placeholder note (Responses API does not support images in tool output). | +| `src/extension.ts` | `googleContentsFromMessages()` tool branch: new `googleFunctionResponseContent()` that emits `parts: [{text}, {inlineData}]` when an image is present. | +| `src/extension.ts` | `provideLanguageModelChatInformation()`: replaced per-model log spam with one summary line per invocation. | +| `src/extension.ts` | `fetchModels()`: replaced `showWarningMessage` with Output-channel log on transient upstream failures. | +| `src/extension.ts` | Added `MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000` constant. | + +**Total:** 1 file, +180 / −45 lines (approx). + +--- + +## Follow-up Bugs Uncovered During Manual Testing + +### A. 4.6 MB payload → upstream 400 + +After the core fix was in place, manual testing with `chrome-devtools-mcp` on +`mimo-v2.5` produced a series of `400 Bad Request` errors from the OpenCode Go +gateway. Inspecting the Output channel revealed: + +``` +[request] url=https://opencode.ai/zen/go/v1/chat/completions payloadBytes=4665383 +[http-error-body] {"error":{"message":"Error from provider (Console Go): Upstream + request failed","type":"invalid_request_error","code":"invalid_request_error"}} +``` + +Timeline (each turn re-sends the entire conversation history): + +``` +02:58:03 payloadBytes=1356 messages=1 +02:58:41 payloadBytes=144169 messages=3 (first screenshot) +02:59:17 payloadBytes=189950 messages=7 (loop screenshots accumulate) +03:00:07 payloadBytes=200065 messages=11 +03:00:28 payloadBytes=201561 messages=13 +03:00:42 payloadBytes=202412 messages=15 +03:00:55 payloadBytes=4665383 messages=17 ← BOOM (4.6 MB) +03:01:05 → 400 Upstream request failed +03:01:07 → retry → 400 +03:01:15 → retry → 400 +03:01:21 → retry → 400 +``` + +Cause: full-page MCP screenshots can be 1–2 MB each. Once several are embedded +in history as base64 data URIs (~1.33× the byte size), the payload crosses the +upstream limit and every subsequent retry in the same agent loop fails. The +`MAX_TOOL_RESULT_IMAGE_BYTES` size guard above is the direct mitigation. + +A longer-term follow-up would be to trim old images from conversation history +when the estimated token count exceeds the model's context window, but VS Code +Copilot Chat is supposed to perform that trimming based on +`advertisedMaxInputTokens` and the extension's local estimator currently +under-counts base64 image data. That's tracked separately. + +### B. "Model registered" log spam + warning popups + +Investigation of the "loading forever" UX complaint surfaced that: + +- VS Code refreshes model info on roughly a 300 ms cadence during UI activity. +- Each call produced 22 log lines (one per registered model). +- A single transient 400 from the shared OpenCode gateway on the auto-registered + `opencodezen-agent` provider (the user was on `opencodego`) triggered a modal + `showWarningMessage` popup, even though the bundled `fallbackModels` snapshot + kept everything functional. + +Both were noise rather than correctness bugs, but they obscured the real signal +during testing and made the extension feel broken. Fixed as described in +section 4 above. + +--- + +## Verification + +```bash +# 1. TypeScript strict compile +./node_modules/.bin/tsc -p ./ +# Result: exit 0, no errors + +# 2. Full unit test suite +node --test out/test/**/*.test.js +# Result: 107/107 pass, 0 fail, 0 regression on vision proxy / metadata / +# thinking / retry / usage / goUsageTracker / usageProfile tests +``` + +### Manual test (performed by @ltmoerdani) + +1. Set up `chrome-devtools-mcp` in `~/Library/Application Support/Code/User/mcp.json`. +2. Launched Extension Development Host (`F5` → "Run Extension with Copilot"). +3. Picked an OpenCode Go model in Copilot Chat. +4. Prompt: `Take a screenshot of https://example.com using chrome-devtools, + then describe what you see.` +5. ✅ The model read and described the screenshot. +6. No `400 Upstream request failed` for normal-sized screenshots. + +--- + +## Limitations & Follow-ups + +1. **Responses API cannot carry images in tool output.** The OpenAI Responses + API's `function_call_output.output` field is spec'd as a string. Images in + tool results are degraded to a placeholder note on that transport. The only + Responses-API model in the catalog today is `gpt-5-codex`, so the impact is + limited. A future improvement would be to surface this limitation to the + user via the model's tooltip. +2. **No history-level image trimming.** When multiple screenshots accumulate + across turns, the size guard trims each oversized image but does not trim + smaller images that collectively exceed the context window. VS Code's own + trimming should handle that, but its decisions rely on our + `estimateTokenCount`, which under-counts base64 payloads. Worth revisiting + if users hit context overflow with many small screenshots. +3. **Mimo v2.5 is slow on multimodal prompts.** Time-to-first-byte of 10–20 s + is normal for this model on payloads >100 KB. For faster verification of + vision fixes, prefer `qwen3.6-plus`, `glm-5.2`, or `kimi-k2.7-code`. + +--- + +## Related Docs + +- [`docs/features/01-20260514-vision-image-input.md`](../features/01-20260514-vision-image-input.md) — native vision support (the foundation this builds on) +- [`docs/features/11-20260715-vision-proxy.md`](../features/11-20260715-vision-proxy.md) — vision proxy for text-only models (separate feature, not required for this fix) +- [`docs/issues/08-20260520-vision-image-request-fixes.md`](08-20260520-vision-image-request-fixes.md) — earlier vision fixes (stack overflow, Qwen budget) diff --git a/package-lock.json b/package-lock.json index 81213ee..06de1ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opencode-copilot-chat", - "version": "0.3.6", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opencode-copilot-chat", - "version": "0.3.6", + "version": "0.4.1", "license": "MIT", "devDependencies": { "@types/node": "^26.1.0", @@ -15,7 +15,7 @@ "typescript": "^6.0.3" }, "engines": { - "vscode": "^1.120.0" + "vscode": "^1.125.0" } }, "node_modules/@azu/format-text": { diff --git a/src/extension.ts b/src/extension.ts index b471771..e9373d2 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -437,6 +437,16 @@ const MESSAGE_NAME_TOKEN_OVERHEAD = 1; const TOOL_CALL_TOKEN_OVERHEAD = 10; const TOOL_RESULT_TOKEN_OVERHEAD = 6; const IMAGE_TOKEN_ESTIMATE = 1024; +/** + * Hard upper limit (in bytes of raw image data) for a single image embedded + * in a tool result. MCP screenshots from chrome-devtools-mcp / playwright-mcp + * are typically 50–300 KB; anything above 1 MB is almost always an oversized + * raw capture that bloats the request payload (each image becomes a base64 + * data URI ≈ 1.33× its byte size) and triggers upstream 400 "Upstream request + * failed" rejections from OpenCode Go. Larger images are replaced with a + * placeholder text part so the model still knows an image was returned. + */ +const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; type CopilotCompatibleCapabilities = vscode.LanguageModelChatCapabilities & { supportsToolCalling: boolean; @@ -511,7 +521,11 @@ interface AnthropicToolUseBlock { interface AnthropicToolResultBlock { type: "tool_result"; tool_use_id: string; - content: string; + // Anthropic tool_result.content may be either a plain string or a list of + // content blocks (text + image) per the Messages API spec. We support the + // array form so MCP tool results that include images (e.g. screenshots) are + // forwarded to vision-capable Anthropic models instead of being dropped. + content: string | AnthropicContentBlock[]; cache_control?: AnthropicCacheControl; } @@ -1701,7 +1715,15 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider { + // CONTRACT: VS Code calls provideLanguageModelChatInformation frequently + // (every ~300ms during UI refresh). Per-model logging produces thousands + // of log lines per minute and obscures real signal. We accumulate a + // single summary line per invocation instead of one line per model. + let registeredCount = 0; + let firstModelId = ""; + let lastModelId = ""; + + const results = models.flatMap((modelId) => { const metadata = this.resolveModelMetadata(modelId, metadataSnapshot); const routing = resolveModelRouting(modelId, this.definition); const effectiveModelId = toEffectiveModelId(modelId, this.definition.vendor); @@ -1764,17 +1786,33 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider 0) { + this.log( + `Models registered: count=${registeredCount} provider=${this.definition.vendor}` + + ` first=${firstModelId} last=${lastModelId}` + + (this.definition.isAgentVariant ? " (agents)" : "") + ); + } + + return results; } async provideLanguageModelChatResponse( @@ -2066,8 +2104,19 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider { cache_control?: AnthropicCacheControl }, +): string | AnthropicContentBlock[] { + if (typeof content === "string") { + return content; + } + + if (!Array.isArray(content)) { + return ""; + } + + const hasImage = content.some((part) => part.type === "image_url" && part.image_url?.url); + if (!hasImage) { + return joinedTextContent(content, "\n"); + } + + return anthropicUserBlocks(content, nextCacheControl); +} + function anthropicAssistantBlocks( message: ApiMessage, nextCacheControl: () => { cache_control?: AnthropicCacheControl }, @@ -2446,12 +2521,18 @@ function responsesInputItemsFromMessage(message: ApiMessage): Array part.type === "image_url" && part.image_url?.url); + if (!hasImage) { + return text || ""; + } + + return [text, "[Image attachment omitted — Responses API does not support images in tool output]"] + .filter(Boolean) + .join("\n\n"); +} + function joinedTextContent( content: ApiMessage["content"], separator = "", @@ -2571,14 +2674,11 @@ function googleContentsFromMessages(messages: ApiMessage[]): Array> } { + if (typeof content === "string") { + return { name, content }; + } + + if (!Array.isArray(content)) { + return { name, content: JSON.stringify(content ?? "") }; + } + + const text = joinedTextContent(content, "\n"); + const hasImage = content.some((part) => part.type === "image_url" && part.image_url?.url); + if (!hasImage) { + return { name, content: text }; + } + + const parts: Array> = []; + if (text) { + parts.push({ text }); + } + for (const part of content) { + if (part.type === "image_url" && part.image_url?.url) { + const inlineData = dataUrlToInlineData(part.image_url.url); + if (inlineData) { + parts.push({ inlineData }); + } + } + } + + return { name, content: text, parts }; +} + function parseToolInput(value: string): object { if (!value.trim()) { return {}; @@ -2861,10 +3001,63 @@ function convertMessage( } if (part instanceof vscode.LanguageModelToolResultPart) { + // CONTRACT: A LanguageModelToolResultPart.content is unknown[] and may + // contain nested LanguageModelDataPart instances with image MIME types. + // This happens when MCP tools (e.g. chrome-devtools-mcp screenshots) + // return images. Previously we only ran partToText() which silently + // dropped image DataParts (returned "" via the catch-all fallback), + // so vision-capable models saw an empty tool result. We now serialize + // nested images into OpenAiContentPart image_url parts and emit a + // multimodal array on the tool message when any image is present. + // + // SIZE GUARD: Images larger than MAX_TOOL_RESULT_IMAGE_BYTES are + // replaced with a placeholder text part. This prevents a single + // oversized MCP screenshot from producing multi-MB payloads that + // trigger upstream 400 errors when the conversation history grows. + // Fallback for any non-text, non-image DataPart stays as plain text. + const toolTextParts: string[] = []; + const toolImageParts: OpenAiContentPart[] = []; + for (const resultPart of part.content) { + if (resultPart instanceof vscode.LanguageModelDataPart + && resultPart.mimeType.startsWith("image/") + && !isInternalDataPart(resultPart)) { + if (resultPart.data.byteLength > MAX_TOOL_RESULT_IMAGE_BYTES) { + toolTextParts.push( + `[Image attachment omitted: ${resultPart.data.byteLength} bytes exceeds the ${MAX_TOOL_RESULT_IMAGE_BYTES}-byte limit for tool results. Ask the tool to produce a smaller screenshot or save it to a file.]` + ); + continue; + } + const base64 = dataPartToBase64(resultPart.data); + toolImageParts.push({ + type: "image_url", + image_url: { url: `data:${resultPart.mimeType};base64,${base64}` }, + }); + continue; + } + + const text = partToText(resultPart); + if (text) { + toolTextParts.push(text); + } + } + + let toolContent: string | OpenAiContentPart[]; + if (toolImageParts.length > 0) { + const multimodal: OpenAiContentPart[] = []; + const joinedText = toolTextParts.join("\n"); + if (joinedText) { + multimodal.push({ type: "text", text: joinedText }); + } + multimodal.push(...toolImageParts); + toolContent = multimodal; + } else { + toolContent = toolTextParts.join("\n"); + } + toolResults.push({ role: "tool", tool_call_id: part.callId, - content: part.content.map(partToText).filter(Boolean).join("\n") + content: toolContent, }); continue; }