diff --git a/devlog/_plan/260905_external_image_roundtrip/000_plan.md b/devlog/_plan/260905_external_image_roundtrip/000_plan.md index 8c3e09a673..fd540fbeec 100644 --- a/devlog/_plan/260905_external_image_roundtrip/000_plan.md +++ b/devlog/_plan/260905_external_image_roundtrip/000_plan.md @@ -9,12 +9,13 @@ release, image synthesis, or unrelated adapter refactor. - Verifier: standalone converter/parser/adapter body inspection, TypeScript, exact-head GitHub CI. ALL local test suites are forbidden by the user, including focused suites. -- Stop: reviewed two-layer stack merged bottom-up to dev with green CI and ancestry. +- Stop: reviewed image-repair stack merged bottom-up to dev with green CI and ancestry. - Memory: this unit and the session-bound goalplan/ledger. - Outcomes: DONE only with proof; external dependencies may be BLOCKED/NEEDS_HUMAN; unsafe expansion is UNSAFE. No implementation-success claim from docs-only work. - Scope: this managed checkout, read-only Aside official docs, GitHub stack/CI/admin - merge. Maximum four concurrent agents; reassess after 90 minutes; no token cap set. + merge. The user's follow-up permits unlimited useful parallel agents (subject to + actual tool capacity); reassess after 90 minutes; no token cap set. - Escalation: reclaim a lane after two distinct failed agents; any delegated writes must be planned with disjoint paths before B. No production credentials in artifacts. @@ -45,8 +46,13 @@ serialization. Do not add a generic image helper or patch correct Responses code 1. wp0: docs-only roadmap and independent audit (this cycle). 2. wp1 / 010: preserve Chat image detail and structured tool output; lower PR to dev. -3. wp2 / 020: cross-protocol wire regressions and public contract; child PR to lower - branch, then CI/review/admin-merge bottom-up, retarget child and verify again. +3. wp2 / 020: cross-protocol wire regressions and public contract; child PR to lower. +4. User-expanded wp3 / 030: file references and explicit unsupported computer-output + boundary. 003 records full format coverage and rejected hypotheses. +5. wp4 / 040: orphan tool image carriers on Anthropic and Command Code. +6. wp5 / 050: active external Cursor tool screenshot attachments. +7. wp6 / 060: all-format finding disposition, CI/review/admin-merge bottom-up, retarget + children and verify exact heads/ancestry. Original completion criteria are unchanged. Existing placement is reused: src/chat/, tests/responses/, public reference/proxy-formats, structure/04_transports-and-sidecars.md. No new package, runtime module, or config. @@ -55,6 +61,12 @@ converter and protects the integrated contract independently of unit-level asser ## Continuity +wp1 outcome: commit `1f1daa368` implements 010 with ten converter regression cases; +draft PR #3586 targets dev. Independent patch reviewer inspected both changed files +and returned PASS. Standalone request JSON changed from image/detail missing (exit 1) +to both retained (exit 0); node TypeScript and privacy scan passed. Suites are CI-only, +not claimed green yet. wp2 inherits this verified converter and adds wire/HTTP evidence. + Roadmap audit: independent gpt-6-astra high reviewer returned GO-WITH-FIXES, two medium findings. Both folded: exact no-suite typecheck/push commands and actual Claude converter export. Direct node tsc exits 0. Standalone reproduction at diff --git a/devlog/_plan/260905_external_image_roundtrip/003_all_format_audit.md b/devlog/_plan/260905_external_image_roundtrip/003_all_format_audit.md new file mode 100644 index 0000000000..48d4c8cf39 --- /dev/null +++ b/devlog/_plan/260905_external_image_roundtrip/003_all_format_audit.md @@ -0,0 +1,47 @@ +# All-image-format audit, 2026-09-05 + +The user expanded the review from the reported OpenAI paths to every image format. +Two independent gpt-6-astra high reviewers inspected ingress and outgoing format +families. These are source-verified gaps, not claims that the first patch introduced +regressions. No unsupported provider-specific file resolver is being invented. + +| Surface | Supported transport / current boundary | Disposition | +| --- | --- | --- | +| Chat user image_url object/data/HTTPS/detail | Existing URLs preserved; detail fixed in 010 | Covered by lower and wire PRs | +| Chat tool image arrays/string URLs | OpenCodex extension, not standard upstream tool-role support | Fixed in 010; native Chat keeps its contract | +| Responses message input_image | URL/data; file_id native or text marker translated | Existing supported behavior | +| Responses function/custom output file_id | Raw native retains reference; translated parser drops it | 030: use existing file marker convention | +| Responses computer_screenshot output | Raw native retains item; translated parser ignores screenshot | 030: explicit translated400, native unchanged | +| Claude user/tool base64 and URL | Dedicated mapper and nested tool outputs | Covered by wire matrix | +| Claude source:file | Native reference valid; translated mapper drops it | 030: explicit translated error, no cross-provider resolution | +| Responses/Azure | Native raw inputs and repairable orphan images retained | No additional loss found | +| Chat/Mimo | User-image carrier after pending tool batch | No additional loss found | +| Anthropic | Paired image result works; orphan baseline JSON-inlines image data | 040: native image sibling with provenance; baseline standalone exit1 confirmed | +| Command Code | Paired image carrier works; orphan baseline skips it | 040: reuse wireImagePart on orphan carrier; baseline standalone exit1 confirmed | +| Google/Vertex/Antigravity | Data -> inline_data, tool image siblings | Remote-URL marker remains existing limitation | +| Kiro | Data images on user carrier; orphan pairing rejected | Remote URL remains existing limitation | +| Ollama native | Data/raw base64 images; unsupported URL/pairing rejected | Existing explicit contract | +| Cursor native MCP | Image bytes carried with tool result | Preserve unchanged | +| Cursor external wire | Active user images work; trailing tool images not prepared | 050: active trailing run only, existing count/byte limits | + +Key owners: src/responses/parser.ts:304, :732; src/claude/inbound.ts:134; +src/adapters/anthropic.ts:637, :753, :775; src/adapters/command-code.ts:110; +src/adapters/cursor/images.ts:647; cursor/live-transport.ts:621; +cursor/protobuf-request.ts:1383. Ordinary user images on OpenAI were already retained. + +The live 10100 process is version2.43.0 from the maintainer's main checkout, not this +worktree. A safe configuration inspection found no text-only declaration for native +OpenAI. No model request or personal request inspection was done, so the reported +specific OCR failure remains unattributed. Do not infer loaded commit from version. + +## Hypotheses and negative controls + +- H1 adapter cannot carry images: falsified by paired/native image branches. +- H2 image-bearing representation is dropped on a branch: source evidence above; + confirm each modified owner with a standalone body/encoder probe before editing. +- H3 capability policy intentionally omits images: true for documented URL/history + limits and text-only sidecars, excluded from universal vision-support claims. + +No new remote fetching, uploads, auth, provider metadata, tool execution, historical +image recall, or file-handle resolution. Full audit means every row has a disposition, +not that every upstream supports every representation. All new tests execute in CI only. diff --git a/devlog/_plan/260905_external_image_roundtrip/020_wire_contract.md b/devlog/_plan/260905_external_image_roundtrip/020_wire_contract.md index 7baf039379..4e9ae1c581 100644 --- a/devlog/_plan/260905_external_image_roundtrip/020_wire_contract.md +++ b/devlog/_plan/260905_external_image_roundtrip/020_wire_contract.md @@ -2,6 +2,17 @@ Depends on wp1 and its corrected Chat converter. One full PABCD cycle. +Delegated B lanes (user reconfirmed unlimited useful parallelism): worker A exclusively +edits tests/responses/openai-responses-passthrough.test.ts; worker B exclusively adds +the HTTP regression in tests/responses/chat-completions-endpoint.test.ts. Main owns +public docs, structure, devlog and git/CI. C reviewers are read-only and independent. +All lanes prohibit local test suites, services, config/auth and git/FSM mutation. + +User steering during B: all image representations must be audited before merge. This +cycle now publishes the wire-contract child; the original exact-head CI/merge/ancestry +criterion is unchanged and moves to appended wp3 after the expanded audit. No criterion +is dropped or marked met early. Only existing 020 implementation runs in this B. + ## MODIFY tests/responses/openai-responses-passthrough.test.ts Import real chatCompletionsToResponsesBody, anthropicToResponsesBody, @@ -38,6 +49,10 @@ existing Chat-to-Responses HTTP regression (line 2834). POST a user image with h detail and a paired tool screenshot to mock/grok-4.5; consume the stream and assert one captured /responses body with unchanged ordered image parts. This is real HTTP route proof in CI, not real-model OCR or canonical account authentication. +The manual HTTP probe showed that data-only mock Responses frames don't satisfy the +native event-name terminal observer. Add matching `event: response.output_text.delta` +and `event: response.completed` fields to mockDualWireUpstream's existing frames; +preserve all body assertions and require `[DONE]` on the new HTTP cases. ## MODIFY structure/04_transports-and-sidecars.md diff --git a/devlog/_plan/260905_external_image_roundtrip/030_image_input_forms.md b/devlog/_plan/260905_external_image_roundtrip/030_image_input_forms.md new file mode 100644 index 0000000000..f7944c680c --- /dev/null +++ b/devlog/_plan/260905_external_image_roundtrip/030_image_input_forms.md @@ -0,0 +1,86 @@ +# Accepted image representations without silent loss + +Depends on 020; wp3. C3 parsing, with C4 care for the explicit translated-file error. +Same resource/credential bounds as000; writes only paths below, no live credentials, +network fetching, remote upload, auth changes or local suites. Stop on unresolvable +native-vs-routed ambiguity, not by weakening native preservation. + +## MODIFY src/responses/parser.ts + +In outputToToolResultContent, input_image must follow the already-owned precedence: + +```diff +- else if (raw.type === "input_image" && typeof raw.image_url === "string") { ... } ++ else if (raw.type === "input_image") { ++ const imageUrl = nonEmptyString(raw.image_url); ++ const fileId = nonEmptyString(raw.file_id); ++ if (imageUrl) { /* existing image push and normalized detail; hasImage=true */ } ++ else if (fileId) parts.push({ type: "text", text: `[image: ${fileId}]` }); ++ } +``` + +Do not lower computer screenshots in this shared parser. Independent audit found +that parser-only lowering shifts vision-caption alignment and breaks native/raw +consistency. No new computer execution or observation-message semantics are added. + +## MODIFY src/server/responses/core.ts + +Immediately after existing isPassthrough determination (before vision planning), inspect +raw input items. A non-passthrough adapter receiving computer_call_output returns fixed +400 invalid_request_error: `computer_call_output requires a Responses passthrough +route; send screenshots as user input_image content on translated routes.` +No payload, source URL or call ID in the error. Passthrough stays unchanged, including +native/keyed Responses and routed compaction using a passthrough adapter. This avoids +shared-parser rejection of valid native traffic and preserves vision-caption alignment. +No helper/export/import is needed; use the existing raw body and formatted error owner. + +## MODIFY src/claude/inbound.ts + +In imageBlockToInputImage, after validating source object and before base64/URL cases: + +```diff ++ if (source.type === "file") throw new AnthropicRequestError( ++ "File-backed images require native Anthropic passthrough; use base64 or URL images on translated routes."); +``` + +The existing HTTP boundary catches AnthropicRequestError as400. Native Anthropic +passthrough never calls this converter. No file id, URL or payload echoed in errors. + +## MODIFY existing tests + +- tests/responses/responses-parser.test.ts: function/custom file-only output marker; + URL wins over file_id; malformed/empty refs never become images; original raw item and + caller object unchanged. No new computer tool declaration or toolCall emitted. +- tests/claude-integration/claude-inbound.test.ts: user/tool source:file throws the + existing error; base64/URL still preserve. Native negative control stays in existing + claude-native-passthrough.test.ts; add endpoint error coverage at its existing seam + only if the reviewer finds class-to400 mapping not covered. +- tests/responses/responses-compaction-routing.test.ts: beside the existing unpaired + output boundary, non-passthrough computer output returns400 with zero upstream fetch; + native/keyed Responses preserves exact raw screenshot and reaches its controlled + upstream; ordinary image message still works. Request includes another ordinary image + to prove there is no partial vision work or caption misassociation before rejection. +- docs-site/src/content/docs/reference/proxy-formats.md: file handles remain provider + scoped; native reference forwarding vs translated marker/error; hosted computer + outputs require Responses passthrough, screenshots can use ordinary input_image. + +Verifier: one standalone direct parser/converter probe before/after (no test runner), +node TypeScript, static test bundling, privacy scan, exact-head CI. Independent security +review confirms no native rejection, payload logging, new fetch, or execution authority. +Reuse existing modules; defer broad splits in large files to avoid unrelated churn. +Publish third stacked PR against codex/external-image-wire-contract; no merge yet. + +## C-review corrections + +Accepted consumer mismatch: output parser's nonempty-URL predicate must match raw vision +caption indexing. MODIFY src/vision/index.ts syncRawBodyImageDescriptions to skip empty +URLs for both message/tool fields, preserve existing file marker when available, and +never consume a later image's caption. Remove the now-unneeded private boolean argument. +MODIFY tests/vision/vision-cache.test.ts with function/custom arrays containing empty +URLs before two real images; actual describeImagesInPlace must preserve caption order. +Standalone .tmp/vision-caption-alignment-probe.ts demonstrated the misalignment (exit1). + +Accepted coverage gap: core guard test must activate vision in its ordinary-image +control. Explicit routed vision fixture, controlled description dependency, and no live +account resolution; control describes once, computer-output request describes zero. +These repairs preserve the original030scope and do not implement040or050. diff --git a/devlog/_plan/260905_external_image_roundtrip/040_orphan_image_carriers.md b/devlog/_plan/260905_external_image_roundtrip/040_orphan_image_carriers.md new file mode 100644 index 0000000000..6cce813447 --- /dev/null +++ b/devlog/_plan/260905_external_image_roundtrip/040_orphan_image_carriers.md @@ -0,0 +1,40 @@ +# Orphan image carriers + +Depends on030 observation/fallback contract; wp4. C3, same bounds as000. + +## MODIFY src/adapters/anthropic.ts + +Search found orphanToolResultText and toAnthropicContentPart as owners. Add one local +orphanToolResultContent helper returning string|unknown[]: image-free content returns +existing orphanToolResultText exactly; an +image-bearing array becomes an annotation text block followed by existing +toAnthropicContentPart mappings with empty text filtered as in toAnthropicToolResult. +Never JSON-stringify image bytes. Use helper at both sites: + +```diff +- orphanBlocks.push({ type: "text", text: orphanToolResultText(tr) }); ++ const orphan = orphanToolResultContent(tr); ++ orphanBlocks.push(...(typeof orphan === "string" ? [{ type: "text", text: orphan }] : orphan)); +- messages.push({ role: "user", content: orphanToolResultText(msg) }); ++ messages.push({ role: "user", content: orphanToolResultContent(msg) }); +``` + +Declare orphanBlocks unknown[] to match the existing content mapper's unknown return; +do not add a cast/export. Keep valid tool_result blocks before orphan siblings. + +## MODIFY src/adapters/command-code.ts + +Hoist existing image extraction and wireImagePart mapping before paired/orphan split. +Append mapped images to orphan user carrier after its provenance text; leave +closePendingCalls before the carrier. Reuse mapped images in paired result buffer. +No new shared utility or change to parallel-result ordering. + +## MODIFY tests/adapters/adapter-usage.test.ts and tests/providers/command-code-provider.test.ts + +Extend existing orphan/paired-image tests: standalone, duplicate, unmatched adjacent, +user barrier, outstanding other call; data+HTTPS; mixed/empty text; native image blocks +and no base64 in text; no fabricated tool pairing; existing exact text-only behavior. + +Main standalone adapter body probe must fail before and pass after. Workers have +disjoint adapter+test paths, no suites/services/git writes. Main owns verification, +docs note, fourth stacked PR and CI. No merge before050/060 acceptance. diff --git a/devlog/_plan/260905_external_image_roundtrip/050_cursor_tool_images.md b/devlog/_plan/260905_external_image_roundtrip/050_cursor_tool_images.md new file mode 100644 index 0000000000..77af8bee4c --- /dev/null +++ b/devlog/_plan/260905_external_image_roundtrip/050_cursor_tool_images.md @@ -0,0 +1,88 @@ +# Active Cursor external-tool screenshot attachments + +Depends on040 tool provenance/adjacency contract; wp5. C3 plus explicit size-boundary +review. Same resource bounds as000. No new remote fetching, credentials, protobuf +schema, historical-image recall, or native Composer/MCP behavior. + +## MODIFY src/adapters/cursor/images.ts + +Extend prepareCursorRawMessages with a default-off trailing-tool-image option. Only +when opted in and the final message is toolResult, find the contiguous trailing result +run and use existing prepareCursorContentParts on each in order. Apply MAX_CURSOR_IMAGES +to the aggregate run, before decoding; preserve earlier history and existing abort, +data-only normalization, compression and omission behavior. Return collected prepared +images in existing PreparedCursorRawMessages. Existing default/user/developer paths +and cursorVisionPrepareStartIndex callers stay unchanged. + +Extend existing ResolvedCursorImage with optional `sourceLabel?: string` metadata. +For each prepared image in an opted-in trailing result, copy the prepared image and +attach a bounded label identifying the trailing-result ordinal and prepared-image +ordinal, plus JSON-escaped tool name/call id truncated to128 characters each. The +ordinals disambiguate even truncated labels; no image bytes or result text enter labels. +Maximum12 labels, so action provenance stays bounded independently of history length. +User/developer and native MCP paths produce no sourceLabel. No new cross-module type. + +## MODIFY src/adapters/cursor/live-transport.ts + +```diff +- const preparedRaw = await prepareCursorRawMessages(request.rawMessages, signal); ++ const externalToolImages = isCursorExternalWireModel(request.modelId) ++ && request.rawMessages?.at(-1)?.role === "toolResult"; ++ const preparedRaw = await prepareCursorRawMessages(request.rawMessages, signal, { trailingToolImages: externalToolImages }); +- const selectedImages = await resolveActiveCursorImages(...); ++ const selectedImages = externalToolImages ? preparedRaw.images : await resolveActiveCursorImages(...); +``` + +Reuse isCursorExternalWireModel from its actual discovery owner. Do not use +cursorNeedsExternalToolContinuation, which includes native Composer2.5. Existing +protobuf buildPreparedCursorRunRequest already sends external continuation via +userMessageAction and selectedContext images. +Field chain: raw tool image -> prepared normalized part + ResolvedCursorImage -> existing +CursorRunRequest.selectedImages -> UserMessageAction.selectedContext -> blob/KV bytes. + +## MODIFY src/adapters/cursor/protobuf-request.ts and types.ts + +The A reviewer found that root pruning can remove source text while selectedImages +survive. Resolve this with active-action provenance, not post-prune reconstruction. +After computing existing actionText, for external tool continuations with source-labeled +selectedImages append a clearly marked client-supplied screenshot-source list in +attachment order. Each line includes attachment index and the bounded sourceLabel. +Use the augmented text only in UserMessageAction, never system instructions or native +MCP. It survives root pruning and checkpoint fallback because action text is outside +the prunable root. Existing echo-retry continuation text remains the prefix; append +provenance to it too. Other actions remain byte-equivalent. Update selectedImages +documentation in types.ts; no new CursorRunRequest field is required. +Truncate identifiers before JSON escaping, and use the same augmented actionText for +wire serialization and existing input-token estimation. Test escaped controls, long +identifiers, and an invalid earlier image omitted before a later valid image. +sourceLabel serialization: ResolvedCursorImage metadata -> active user action text; +buildSelectedImages ignores metadata and emits existing bytes/schema; no persisted +deserializer or separate consumer. Search all ResolvedCursorImage consumers before B. + +## MODIFY existing tests + +- tests/providers/cursor/cursor-images.test.ts: opted-in data images across parallel + trailing run (including final text-only result), aggregate count cap, invalid marker, + abort, detail, immutable source. Preserve text-only/default and stale-new-user cases. +- tests/providers/cursor/cursor-live-transport.test.ts: captureOpen actual encoded request + proves selectedContext bytes for external full replay/checkpoint continuation; Composer + negative control. Under pruning pressure use two distinct screenshot results, prove + root pruning actually occurred, then assert ordered source labels remain in the active + action alongside both attachment bytes. Cover checkpoint fallback and echo retry. + Existing fixture's valid PNG. +- tests/providers/cursor/cursor-tool-result-image.test.ts: correct stale blanket noVision + comment only; native MCP cases stay unchanged. +- public proxy-formats.md + transport SoT: active external tool data images use the + existing12-image aggregate limit; historical images and remote-URL policy unchanged. + +One worker owns images/test and sourceLabel contract; a second owns live transport, +protobuf action/types and live-transport tests only after the contract is agreed. +All preparation remains bounded. Main does standalone encode proof, static +type/bundle checks and CI; fresh independent review challenges the new activation path. +Publish fifth layer, preserving source/metadata association under history pruning. + +Review-size decision: publish this one050cycle as two dependency-ordered PR layers: +preparation API/sourceLabel plus its focused tests, then actual transport/protobuf +activation plus live-wire tests/public contract. Each layer has its own tests and CI; +the combined diff exceeds500lines largely due boundary/pruning regression coverage. +No acceptance is deferred beyond the full050cycle; both layers remain held for060. diff --git a/devlog/_plan/260905_external_image_roundtrip/060_delivery.md b/devlog/_plan/260905_external_image_roundtrip/060_delivery.md new file mode 100644 index 0000000000..8dce5cc4f0 --- /dev/null +++ b/devlog/_plan/260905_external_image_roundtrip/060_delivery.md @@ -0,0 +1,47 @@ +# Full-format closeout + +Depends on030/040/050; wp6. No new runtime features. Main owns all external writes. + +## Diagnosed CI prerequisite (test-only, no production storage changes) + +Additional CI fixture correction:3591job101240765762 failed loopback startup with +EADDRINUSE40895. The log cannot identify the owner of that port. Eleven fixtures used +rawstartServer(0), bypassing the existing reserved-port allocator used by the rollback +test. Reuse findAvailablePort with reservedPort for those public listener draws via one +local helper. No startup retry, productionlistener/auth change, or assertion removal. +Keep no-loopback/explicit-port/intentional-bind-failure tests unchanged. This removes a +reachable fixture self-collision; it does not claim everyexternal bindrace is solved. + +PR3589 job101236091166 on7783355f9 fails the late-async-spill ordering test before +its overwrite assertions: outer fallback uses real Date.now despite frozen inner +clocks, so its80ms reserve can expire during real cleanup. Images aren't in this +isolated batch. Do not blindly retry or weaken budgets/assertions. + +MODIFY only tests/responses/responses-state.test.ts: import existing spyOn and +awaitResponseSpillPublicationTailForTests; after `await started` in this single +late-completion test, capture Date.now and spy it to return that captured value. +Keep all assertions and40ms real drain timer. In finally restore spy FIRST, then +release the blocked writer and await the existing publication-tail barrier. +Do not freeze timers or other deadline/exhaustion tests. Existing superseded flag, +file-identity and replay assertions prove ordering independently of clock progression. +Publish correction on layer2, cascade all own higher branches with explicit leases, +and re-run exact-head CI; no new production clock hook/export. Independent reviewer +must verify scope and teardown. The earlier failed CI is the red evidence. + +MODIFY003 audit table with each exact final disposition, test names and CI links; +MODIFY000 continuity with exact commit/PR/reviewer proof. Archive unit _plan -> _fin +only when it describes a public outcome. Tests/code may not be weakened for green CI. + +Before each merge: refresh exact head, base, full status rollup, reviewer comments, +worktree identity and source ancestry. Resolve actual failures; never assume flakes. +Document user-authorized admin approval bypass. Merge bottom-up, prefer merge commits, +retain parent branches, retarget child to dev only after parent is public. Verify CI +against the exact child head and current base; restack with lease if necessary. Fetch +origin/dev and prove every merge SHA ancestor. No release, deployment or10100 restart. + +Local suites remain prohibited. Inspect and stop actual local Bun suite processes as +authorized, not SSH commands merely mentioning a remote suite, dev servers or the proxy. +Success: c-all fully accounted + unchanged c2 CI/review/merge/ancestry criterion met. +Report remaining native file/remote URL/history limitations honestly, separate from +fixed silent losses. Report original ordinary-image OCR mismatch unproven if no new +evidence establishes its cause. Do not equate model tokens or a mock reply with OCR. diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index 39251da8ba..adf9dc11e9 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -60,6 +60,13 @@ Responses shapes. Other Responses destinations preserve them. The same canonical boundary removes nested client-only `prompt_cache_breakpoint` markers and drops `item_reference` entries only on `store: false` continuations; tool call/result pairing is unchanged. +Image file IDs are provider-scoped references, not portable image bytes. Responses passthrough +retains them; translating adapters receive an `[image: file_id]` text marker for file-only image +parts in messages or function/custom tool outputs. Use an image URL or base64 data URL when the +translated model needs to see the image. Hosted `computer_call_output` items require a Responses +passthrough route; translated routes return HTTP 400 instead of silently dropping the screenshot. +For a screenshot observation without hosted computer-tool semantics, use a user `input_image`. + ### JSON and SSE output With `stream: true`, the response is `text/event-stream`. The bridge emits Responses events such as @@ -236,6 +243,16 @@ These endpoints speak the Anthropic Messages dialect used by Claude Code and com Most requests are translated to Responses, routed normally, then translated back to Anthropic JSON or Anthropic SSE. +Base64 and URL image sources are translated in user messages and nested tool results. File-backed +images (`source.type: "file"`) require native Anthropic passthrough; translated routes return a +fixed HTTP 400 error asking for base64 or URL input. OpenCodex does not resolve another provider's +file storage or upload the referenced image on the caller's behalf. + +When replay history contains an image-bearing tool result without its adjacent call, the +Anthropic and Command Code adapters retain the image in a provenance-labeled user carrier rather +than embedding its bytes in prompt text. They do not invent a successful tool call. Results for +valid pending calls still precede these carriers, preserving the upstream pairing contract. + Native Anthropic passthrough is eligible only when all of these are true: - native passthrough has not been disabled in Claude Code configuration; diff --git a/src/adapters/anthropic.ts b/src/adapters/anthropic.ts index e8e80dab1c..6eea4764a1 100644 --- a/src/adapters/anthropic.ts +++ b/src/adapters/anthropic.ts @@ -642,6 +642,19 @@ function orphanToolResultText(msg: OcxToolResultMessage): string { return `[tool_result without adjacent tool_use: ${label}]\n${content}`; } +function orphanToolResultContent(msg: OcxToolResultMessage): string | unknown[] { + if (typeof msg.content === "string" || !msg.content.some(p => p.type === "image")) { + return orphanToolResultText(msg); + } + const label = msg.toolName ? `${msg.toolName} (${msg.toolCallId})` : msg.toolCallId; + return [ + { type: "text", text: `[tool_result without adjacent tool_use: ${label}]` }, + ...msg.content + .map(toAnthropicContentPart) + .filter(p => !((p as { type?: string }).type === "text" && !(p as { text?: string }).text)), + ]; +} + /** * AgentRouter answers 400 `content-blocked` when the first user message is not in English * (#2074), while the same request in English returns 200. The gateway is inspecting the opening @@ -737,7 +750,7 @@ function messagesToAnthropicFormat( if (toolUseIds.length > 0) { const requiredIds = new Set(toolUseIds); const resultBlocks: Record[] = []; - const orphanBlocks: Record[] = []; + const orphanBlocks: unknown[] = []; const seen = new Set(); let j = i + 1; while (j < parsed.context.messages.length && parsed.context.messages[j].role === "toolResult") { @@ -750,7 +763,8 @@ function messagesToAnthropicFormat( resultBlocks.push(toAnthropicToolResult(tr, wireResultId)); seen.add(wireResultId); } else { - orphanBlocks.push({ type: "text", text: orphanToolResultText(tr) }); + const orphan = orphanToolResultContent(tr); + orphanBlocks.push(...(typeof orphan === "string" ? [{ type: "text", text: orphan }] : orphan)); } j++; } @@ -771,8 +785,8 @@ function messagesToAnthropicFormat( } case "toolResult": { // A standalone Anthropic tool_result is invalid unless it immediately follows an - // assistant tool_use. Preserve the information as text instead of sending a 400-prone block. - messages.push({ role: "user", content: orphanToolResultText(msg as OcxToolResultMessage) }); + // assistant tool_use. Preserve text and images as user content without fabricating a pairing. + messages.push({ role: "user", content: orphanToolResultContent(msg as OcxToolResultMessage) }); break; } } diff --git a/src/adapters/command-code.ts b/src/adapters/command-code.ts index df6843ca20..a9b429bc99 100644 --- a/src/adapters/command-code.ts +++ b/src/adapters/command-code.ts @@ -59,7 +59,7 @@ function wireImagePart(imageUrl: string): Record { * (#1383). This builder keeps the pairing invariant: * * - a `toolResult` that matches a declared assistant call emits the native `tool-result`; - * - a `toolResult` with no matching declared call degrades to a text carrier so the model + * - a `toolResult` with no matching declared call degrades to a user carrier so the model * still sees the outcome without a 400-prone standalone `tool` message; * - every declared assistant call that never received a result gets an explicit error * `tool-result`, so the upstream never sees an unpaired call. @@ -108,6 +108,9 @@ function wireMessages(messages: OcxMessage[]): Array> { continue; } if (message.role === "toolResult") { + const images = typeof message.content === "string" ? [] : message.content + .filter(part => part.type === "image") + .map(part => wireImagePart((part as { imageUrl: string }).imageUrl)); const callIndex = pendingCalls.findIndex(call => call.id === message.toolCallId); const paired = callIndex >= 0; if (paired) pendingCalls.splice(callIndex, 1); @@ -116,11 +119,11 @@ function wireMessages(messages: OcxMessage[]): Array> { // message lands, or their synthesized results would follow the orphan carrier. closePendingCalls(); // The upstream rejects a standalone tool message whose call was never declared by an - // assistant turn. Preserve the outcome as text so the model can still act on it. + // assistant turn. Preserve the outcome and images so the model can still act on it. const label = message.toolName ? `${message.toolName} (${message.toolCallId})` : message.toolCallId; const text = toolResultText(message.content); // The orphan result cannot ride a `tool` message; carry it in a user message instead. - out.push({ role: "user", content: [{ type: "text", text: `[tool result without adjacent tool call: ${label}]\n${text}` }] }); + out.push({ role: "user", content: [{ type: "text", text: `[tool result without adjacent tool call: ${label}]\n${text}` }, ...images] }); continue; } out.push({ role: "tool", content: [{ @@ -132,9 +135,8 @@ function wireMessages(messages: OcxMessage[]): Array> { // The proprietary wire's tool-result output is text-only; image parts returned by a // tool (e.g. Codex view_image) cannot live inside it. Carry them in a follow-up user // message using the same image encoding as the user branch so the bytes reach the model. - const images = typeof message.content === "string" ? [] : message.content.filter(part => part.type === "image"); if (images.length > 0) { - pendingImageCarriers.push({ role: "user", content: images.map(part => wireImagePart((part as { imageUrl: string }).imageUrl)) }); + pendingImageCarriers.push({ role: "user", content: images }); } continue; } diff --git a/src/adapters/cursor/images.ts b/src/adapters/cursor/images.ts index 84bb303f04..84a7995a53 100644 --- a/src/adapters/cursor/images.ts +++ b/src/adapters/cursor/images.ts @@ -84,6 +84,8 @@ export interface ResolvedCursorImage { uuid: string; /** Codex/OpenAI image detail hint; affects JPEG soft-cap tier. */ detail?: string; + /** Bounded client-supplied provenance for opted-in trailing tool-result images only. */ + sourceLabel?: string; } export type PrepareCursorImageOutcome = @@ -546,7 +548,7 @@ export function buildSelectedContext( /** * Resolve data: images for the active user/developer turn onto SelectedImage. - * Tool-result image promotion is intentionally out of scope in this slice. + * Opted-in tool-result runs use prepareCursorRawMessages directly instead. */ export async function resolveActiveCursorImages( messages: readonly OcxMessage[] | undefined, @@ -645,7 +647,7 @@ async function prepareCursorContentParts( * Historical messages before this index are left untouched (no decode). */ export function cursorVisionPrepareStartIndex(messages: readonly OcxMessage[]): number { - // Tool-result image preparation is out of scope in this slice. + // Default window excludes tool results; their preparation requires explicit opt-in. if (messages.at(-1)?.role === "toolResult") return messages.length; for (let i = messages.length - 1; i >= 0; i--) { const role = messages[i]?.role; @@ -658,7 +660,8 @@ export function cursorVisionPrepareStartIndex(messages: readonly OcxMessage[]): * Rewrite image data URLs in the active vision window (last user/developer turn) through * the JPEG soft-cap path before protobuf encode. Historical messages are left by * reference. Undecodable images become {@link CURSOR_VISION_IMAGE_OMITTED} text so - * image-only turns stay userMessageAction. + * image-only turns stay userMessageAction. Opted-in trailing tool results use the + * same preparation path, with an aggregate image cap and ready-image source labels. */ export interface PreparedCursorRawMessages { messages: readonly OcxMessage[] | undefined; @@ -668,10 +671,26 @@ export interface PreparedCursorRawMessages { export async function prepareCursorRawMessages( messages: readonly OcxMessage[] | undefined, signal?: AbortSignal, + options?: { trailingToolImages?: boolean }, ): Promise { if (!messages?.length) return { messages, images: [] }; throwIfImagePhaseAborted(signal); - const prepareFrom = cursorVisionPrepareStartIndex(messages); + const trailingToolImages = options?.trailingToolImages === true && messages.at(-1)?.role === "toolResult"; + let prepareFrom = cursorVisionPrepareStartIndex(messages); + if (trailingToolImages) { + let imageCount = 0; + // Count the entire contiguous run before any image URL is decoded or normalized. + while (prepareFrom > 0) { + throwIfImagePhaseAborted(signal); + const message = messages[prepareFrom - 1]!; + if (message.role !== "toolResult") break; + prepareFrom--; + imageCount += extractCursorImageParts(message.content).length; + if (imageCount > MAX_CURSOR_IMAGES) { + throw new CursorImageError(`Too many images in one request (max ${MAX_CURSOR_IMAGES}).`); + } + } + } const active = messages[prepareFrom]; if ( active @@ -688,10 +707,21 @@ export async function prepareCursorRawMessages( const message = messages[i]!; if ( i >= prepareFrom - && (message.role === "user" || message.role === "developer") + && (message.role === "user" || message.role === "developer" + || (trailingToolImages && message.role === "toolResult")) ) { const prepared = await prepareCursorContentParts(message.content, signal); - images.push(...prepared.images); + if (trailingToolImages && message.role === "toolResult") { + images.push(...prepared.images.map((image, index) => ({ + ...image, + sourceLabel: `tool result ${i - prepareFrom + 1}, image ${index + 1}: ${JSON.stringify({ + tool: message.toolName.slice(0, 128), + call_id: message.toolCallId.slice(0, 128), + })}`, + }))); + } else { + images.push(...prepared.images); + } if (prepared.content !== message.content) { changed = true; out.push({ ...message, content: prepared.content } as OcxMessage); diff --git a/src/claude/inbound.ts b/src/claude/inbound.ts index 90c4652cb1..de8f474341 100644 --- a/src/claude/inbound.ts +++ b/src/claude/inbound.ts @@ -134,6 +134,11 @@ function systemToInstructions(system: unknown): string | undefined { function imageBlockToInputImage(block: Rec): Rec | null { const source = block.source; if (!isRec(source)) return null; + if (source.type === "file") { + throw new AnthropicRequestError( + "File-backed images require native Anthropic passthrough; use base64 or URL images on translated routes.", + ); + } if (source.type === "base64" && typeof source.data === "string") { const media = typeof source.media_type === "string" ? source.media_type : "image/png"; return { type: "input_image", image_url: `data:${media};base64,${source.data}` }; diff --git a/src/responses/parser.ts b/src/responses/parser.ts index f26539945a..6aab9f6028 100644 --- a/src/responses/parser.ts +++ b/src/responses/parser.ts @@ -301,9 +301,15 @@ function outputToToolResultContent(output: string | unknown[] | undefined): stri if (typeof raw.text === "string") parts.push({ type: "text", text: raw.text }); } else if (raw.type === "refusal" && typeof raw.refusal === "string") { parts.push({ type: "text", text: `[refusal: ${raw.refusal}]` }); - } else if (raw.type === "input_image" && typeof raw.image_url === "string") { - parts.push({ type: "image", imageUrl: raw.image_url, ...(typeof raw.detail === "string" ? { detail: normalizeImageDetail(raw.detail) } : {}) }); - hasImage = true; + } else if (raw.type === "input_image") { + const imageUrl = nonEmptyString(raw.image_url); + const fileId = nonEmptyString(raw.file_id); + if (imageUrl) { + parts.push({ type: "image", imageUrl, ...(typeof raw.detail === "string" ? { detail: normalizeImageDetail(raw.detail) } : {}) }); + hasImage = true; + } else if (fileId) { + parts.push({ type: "text", text: `[image: ${fileId}]` }); + } } else if (raw.type === "encrypted_content") { // codex-rs FunctionCallOutputContentItem::EncryptedContent — opaque to routed models. parts.push({ type: "text", text: "[encrypted content omitted]" }); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index dafa660ba6..de5d2a2b02 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -3732,6 +3732,17 @@ async function handleResponsesInner( } const isPassthrough = "passthrough" in adapter && !!adapter.passthrough; + const rawInput = (parsed._rawBody as { input?: unknown }).input; + if (!isPassthrough && Array.isArray(rawInput) && rawInput.some( + item => item !== null && typeof item === "object" && item.type === "computer_call_output", + )) { + return formatErrorResponse( + 400, + "invalid_request_error", + "computer_call_output requires a Responses passthrough route; send screenshots as user input_image content on translated routes.", + ); + } + if (adapter.name === "kiro" && parsed.previousResponseId && !parsed._previousResponseInputExpanded) { return formatErrorResponse( 400, diff --git a/src/vision/index.ts b/src/vision/index.ts index 3f85258624..6f1a7392a9 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -406,11 +406,11 @@ function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: re if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return; let nextDescription = 0; - const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => { + const rewriteImages = (value: unknown): unknown => { if (Array.isArray(value)) { let changed = false; const rewritten = value.map(entry => { - const next = rewriteImages(entry, nonEmptyImageUrlsOnly); + const next = rewriteImages(entry); if (next !== entry) changed = true; return next; }); @@ -418,8 +418,10 @@ function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: re } if (!isPlainRecord(value)) return value; if (value.type === "input_image" && typeof value.image_url === "string") { - if (nonEmptyImageUrlsOnly && value.image_url.length === 0) { - return { type: "input_text", text: IMAGE_OMITTED_TEXT }; + // Both message and tool-output parsers exclude empty URLs from caption jobs. + if (value.image_url.length === 0) { + const fileId = typeof value.file_id === "string" && value.file_id.length > 0 ? value.file_id : undefined; + return { type: "input_text", text: fileId ? `[image: ${fileId}]` : IMAGE_OMITTED_TEXT }; } const description = descriptions[nextDescription++]; return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT }; @@ -442,7 +444,7 @@ function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: re ? "output" : undefined; if (!field) return item; - const rewritten = rewriteImages(item[field], isMessageContent); + const rewritten = rewriteImages(item[field]); if (rewritten === item[field]) return item; changed = true; return { ...item, [field]: rewritten }; diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index 2e9b9c047d..018ac91947 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -1444,7 +1444,7 @@ surface is listed here so a maintainer can find the owner without grepping: | Adapter execution support | `src/adapters/run-turn-queue.ts`, `src/adapters/tool-catalog-nudge.ts`, `src/adapters/identity.ts`, `src/adapters/image.ts`, `src/adapters/upstream-http-error.ts` | Shared machinery: turn ordering, tool-catalog nudging, client fingerprinting, image conversion, upstream error normalization. | | Cursor (beyond the sections above) | `src/adapters/cursor/live-transport.ts`, `src/adapters/cursor/http1-bidi.ts`, `src/adapters/cursor/live-models.ts`, `src/adapters/cursor/transport-retry.ts`, `src/adapters/cursor/mcp-manager.ts`, `src/adapters/cursor/thread-continuity.ts`, `src/adapters/cursor/checkpoint-store.ts` | Thread continuity is the point: a retry must not start a new Cursor thread, and a validated checkpoint must not rebuild the full root history. HTTP/2 remains the default; an explicit `http1.1`/`h1` pin maps the bidi run onto Cursor's `RunSSE` receive stream plus sequenced `BidiAppend` sends, and applies to live discovery too. | | Claude Messages | `src/server/claude-messages.ts` | Routed translation, a native Anthropic passthrough branch, and `count_tokens`. | -| Chat Completions inbound | `src/server/chat-completions.ts`, `src/chat/` | Inbound translation onto the same routing pipeline. | +| Chat Completions inbound | `src/server/chat-completions.ts`, `src/chat/` | Inbound translation onto the same routing pipeline. The content mapper preserves image URLs and supported detail, including screenshot-bearing tool results; target adapters own image placement on their wire. Image-free tool results stay strings. | | Hosted search relay | `src/server/search.ts` | Direct relay; distinct from the web-search sidecar loop below. | | Image/video generation loop | `src/images/loop.ts`, `src/images/plan.ts`, `src/images/fulfill.ts`, `src/images/xai-client.ts`, `src/images/xai-video-client.ts`, `src/images/artifacts.ts` | A provider-returned image URL is downloaded into a local artifact once, then served locally; warnings stay URL-free because provider CDN URLs may embed credentials. | | GitHub Copilot | `src/providers/xai-transport.ts` (`resolveProviderTransport`), `src/providers/github-copilot-transport.ts` | `resolveProviderTransport` selects the Copilot transport when the routed provider name is `github-copilot`; the Copilot module then resolves its headers and base URL, and the registry seeds the provider row and model fallback. | diff --git a/tests/adapters/adapter-usage.test.ts b/tests/adapters/adapter-usage.test.ts index 5912a65315..7967f2a796 100644 --- a/tests/adapters/adapter-usage.test.ts +++ b/tests/adapters/adapter-usage.test.ts @@ -4,6 +4,7 @@ import { createAnthropicAdapter as createAnthropicAdapterProduction } from "../. import { createGoogleAdapter as createGoogleAdapterProduction } from "../../src/adapters/google"; import { createOpenAIChatAdapter as createOpenAIChatAdapterProduction } from "../../src/adapters/openai-chat"; import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import type { OcxAssistantMessage, OcxContentPart, OcxToolResultMessage } from "../../src/types"; const createAnthropicAdapter = (...args: Parameters) => withTestTranslatorBudget(createAnthropicAdapterProduction(...args)); @@ -712,6 +713,85 @@ describe("anthropic tool result history repair", () => { }); }); + describe("orphan image carriers", () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + const httpsUrl = "https://example.test/image.png"; + const call: OcxAssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "view_image", arguments: {} }], + model: "claude-sonnet", + timestamp: 0, + }; + const paired: OcxToolResultMessage = { + role: "toolResult", toolCallId: "call_1", toolName: "view_image", + content: "first", isError: false, timestamp: 0, + }; + const validResult = { type: "tool_result", tool_use_id: "call_1", content: "first" }; + const missingResult = { + type: "tool_result", tool_use_id: "call_1", + content: "[missing tool_result for this tool_use in history]", is_error: true, + }; + + for (const source of [ + { name: "data", imageUrl: `data:image/png;base64,${png}`, wire: { type: "base64", media_type: "image/png", data: png } }, + { name: "HTTPS", imageUrl: httpsUrl, wire: { type: "url", url: httpsUrl } }, + ]) { + for (const mixed of [false, true]) { + const image: OcxContentPart = { type: "image", imageUrl: source.imageUrl }; + const content: OcxContentPart[] = mixed + ? [{ type: "text", text: "" }, { type: "text", text: "before" }, image, { type: "text", text: "" }, { type: "text", text: "after" }] + : [image]; + const orphan: OcxToolResultMessage = { ...paired, toolCallId: "orphan_call", content }; + const expectedParts = mixed + ? [{ type: "text", text: "before" }, { type: "image", source: source.wire }, { type: "text", text: "after" }] + : [{ type: "image", source: source.wire }]; + + for (const scenario of [ + { name: "standalone", history: [orphan], carrierIndex: 0, resultPrefix: [], orphanId: "orphan_call", pairedResults: [] }, + { name: "duplicate adjacent", history: [call, paired, { ...orphan, toolCallId: "call_1" }], carrierIndex: 1, resultPrefix: [validResult], orphanId: "call_1", pairedResults: [validResult] }, + // Orphan arrives BEFORE the valid result: tool_result blocks must still lead. + { name: "unmatched adjacent", history: [call, orphan, paired], carrierIndex: 1, resultPrefix: [validResult], orphanId: "orphan_call", pairedResults: [validResult] }, + { name: "outstanding other call", history: [call, orphan], carrierIndex: 1, resultPrefix: [missingResult], orphanId: "orphan_call", pairedResults: [missingResult] }, + { name: "user barrier", history: [call, { role: "user", content: "barrier", timestamp: 0 }, { ...orphan, toolCallId: "call_1" }], carrierIndex: 3, resultPrefix: [], orphanId: "call_1", pairedResults: [missingResult] }, + ]) { + test(`${scenario.name} preserves ${source.name} ${mixed ? "mixed/empty text" : "image-only"} content without pairing it`, async () => { + const body = await replay(scenario.history); + expect(body.messages).toHaveLength(scenario.carrierIndex + 1); + const carrier = body.messages[scenario.carrierIndex]; + expect(carrier.role).toBe("user"); + expect(carrier.content).toMatchObject([ + ...scenario.resultPrefix, + { type: "text", text: `[tool_result without adjacent tool_use: view_image (${scenario.orphanId})]` }, + ...expectedParts, + ]); + const blocks = body.messages.flatMap(message => + Array.isArray(message.content) ? message.content as Record[] : []); + const results = blocks.filter(block => block.type === "tool_result"); + expect(results).toHaveLength(scenario.pairedResults.length); + expect(results).toMatchObject(scenario.pairedResults); + const uses = blocks.filter(block => block.type === "tool_use"); + expect(uses.map(block => block.id)).toEqual(scenario.name === "standalone" ? [] : ["call_1"]); + const text = blocks.filter(block => block.type === "text").map(block => block.text); + expect(text).not.toContain(""); + expect(JSON.stringify(text)).not.toContain(png); + expect(JSON.stringify(text)).not.toContain(source.imageUrl); + if (scenario.name === "user barrier") { + expect(body.messages[2]).toMatchObject({ role: "user", content: [{ type: "text", text: "barrier" }] }); + } + }); + } + } + } + + test("image-free arrays keep the exact legacy orphan text", async () => { + const body = await replay([{ ...paired, content: [{ type: "text", text: "" }, { type: "text", text: "plain" }] }]); + expect(body.messages).toMatchObject([{ + role: "user", + content: [{ type: "text", text: '[tool_result without adjacent tool_use: view_image (call_1)]\n[{"type":"text","text":""},{"type":"text","text":"plain"}]' }], + }]); + }); + }); + test("maps non-string tool result content through Anthropic content blocks", async () => { const adapter = createAnthropicAdapter({ ...provider, adapter: "anthropic" }); const request = await adapter.buildRequest({ diff --git a/tests/claude-integration/claude-inbound.test.ts b/tests/claude-integration/claude-inbound.test.ts index b1d537055d..9943f14ede 100644 --- a/tests/claude-integration/claude-inbound.test.ts +++ b/tests/claude-integration/claude-inbound.test.ts @@ -91,6 +91,98 @@ describe("claude inbound translation", () => { expect(tail[1]).toEqual({ type: "input_image", image_url: "data:image/png;base64,aWc=" }); }); + for (const carrier of ["user", "tool_result"] as const) { + test(`${carrier} file-backed images throw the fixed AnthropicRequestError without file IDs`, () => { + const image = { type: "image", source: { type: "file", file_id: "file_private_image_030" } }; + const request = { + model: "m", max_tokens: 10, + messages: carrier === "user" + ? [{ role: "user", content: [image] }] + : [ + { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Read", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: [image] }] }, + ], + }; + let error: unknown; + try { + anthropicToResponsesBody(request); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(AnthropicRequestError); + expect(error).toHaveProperty( + "message", + "File-backed images require native Anthropic passthrough; use base64 or URL images on translated routes.", + ); + expect(String(error)).not.toContain(image.source.file_id); + }); + + test(`${carrier} base64 and URL images preserve their translated content`, () => { + const content = [ + { type: "image", source: { type: "base64", media_type: "image/png", data: "aWc=" } }, + { type: "image", source: { type: "url", url: "https://example.com/image.png" } }, + ]; + const body = anthropicToResponsesBody({ + model: "m", max_tokens: 10, + messages: carrier === "user" + ? [{ role: "user", content }] + : [ + { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Read", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content }] }, + ], + }); + const images = [ + { type: "input_image", image_url: "data:image/png;base64,aWc=" }, + { type: "input_image", image_url: "https://example.com/image.png" }, + ]; + expect(body.input).toEqual(carrier === "user" + ? [{ type: "message", role: "user", content: images }] + : [ + { type: "function_call", call_id: "t1", name: "Read", arguments: "{}" }, + { type: "function_call_output", call_id: "t1", output: images }, + ]); + }); + + test(`${carrier} file-backed documents retain attachment markers without rejection`, () => { + const content = [ + { type: "document", source: { type: "file", file_id: "file_document_030" }, title: "report.pdf" }, + ]; + const body = anthropicToResponsesBody({ + model: "m", max_tokens: 10, + messages: carrier === "user" + ? [{ role: "user", content }] + : [ + { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Read", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content }] }, + ], + }); + const marker = [{ type: "input_text", text: "[document: report.pdf]" }]; + expect(body.input).toEqual(carrier === "user" + ? [{ type: "message", role: "user", content: marker }] + : [ + { type: "function_call", call_id: "t1", name: "Read", arguments: "{}" }, + { type: "function_call_output", call_id: "t1", output: marker }, + ]); + }); + } + + test("file-backed image-shaped tool arguments remain opaque JSON without rejection", () => { + const body = anthropicToResponsesBody({ + model: "m", max_tokens: 10, + messages: [{ + role: "assistant", + content: [{ + type: "tool_use", id: "t1", name: "Read", + input: { image: { type: "image", source: { type: "file", file_id: "file_argument_030" } } }, + }], + }], + }); + expect(body.input).toEqual([{ + type: "function_call", call_id: "t1", name: "Read", + arguments: '{"image":{"type":"image","source":{"type":"file","file_id":"file_argument_030"}}}', + }]); + }); + test("thinking variants", () => { const base = { model: "m", max_tokens: 10, messages: [{ role: "user", content: "hi" }] }; expect((anthropicToResponsesBody({ ...base, thinking: { type: "adaptive" } }) as any).reasoning).toEqual({ summary: "auto" }); diff --git a/tests/providers/command-code-provider.test.ts b/tests/providers/command-code-provider.test.ts index e4bb3c4d90..3e369a9c6b 100644 --- a/tests/providers/command-code-provider.test.ts +++ b/tests/providers/command-code-provider.test.ts @@ -352,6 +352,144 @@ describe("Command Code provider", () => { }); }); + test.each(["", "before\n"])("preserves standalone orphan data and HTTPS images with text %j", async (text) => { + const image = "data:image/png;base64,QUJDRA=="; + const remote = "https://example.com/screenshot.JPEG?size=2#preview"; + const built = await builtRequest({ + ...parsed(), + context: { + ...parsed().context, + messages: [{ + role: "toolResult", toolCallId: "call_orphan", toolName: "view_image", + content: [ + { type: "text", text }, + { type: "image", imageUrl: image }, + { type: "text", text: "" }, + { type: "image", imageUrl: remote }, + { type: "text", text }, + ], + isError: false, timestamp: 1, + }], + }, + }); + const wire = JSON.parse(built.body).params.messages; + expect(wire).toEqual([{ + role: "user", + content: [ + { type: "text", text: `[tool result without adjacent tool call: view_image (call_orphan)]\n${text}[image][image]${text}` }, + { type: "image", image, mediaType: "image/png" }, + { type: "image", image: remote, mediaType: "image/jpeg" }, + ], + }]); + expect(wire[0].content[0].text).not.toContain("QUJDRA=="); + expect(wire[0].content[0].text).not.toContain(remote); + }); + + test.each(["duplicate", "user barrier"])("preserves orphan images after a %s without repairing the pairing", async (scenario) => { + const image = "data:image/png;base64,QUJDRA=="; + const remote = "https://example.com/late.webp"; + const request = parsed(); + request.context.messages = [{ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "view_image", arguments: {} }], + timestamp: 1, + }]; + if (scenario === "duplicate") { + request.context.messages.push({ + role: "toolResult", toolCallId: "call_1", toolName: "view_image", + content: [{ type: "text", text: "first" }, { type: "image", imageUrl: image }], + isError: false, timestamp: 2, + }); + } else { + request.context.messages.push({ role: "user", content: "continue", timestamp: 2 }); + } + request.context.messages.push({ + role: "toolResult", toolCallId: "call_1", toolName: "view_image", + content: [{ type: "text", text: "late:" }, { type: "image", imageUrl: remote }], + isError: false, timestamp: 3, + }); + const built = await builtRequest(request); + const wire = JSON.parse(built.body).params.messages; + expect(wire).toEqual([ + { role: "assistant", content: [{ type: "tool-call", toolCallId: "call_1", toolName: "view_image", input: {} }] }, + { role: "tool", content: [{ + type: "tool-result", toolCallId: "call_1", toolName: "view_image", + output: scenario === "duplicate" + ? { type: "text", value: "first[image]" } + : { type: "error-text", value: "[ocx] no tool result was recorded for this tool call; execution status unknown." }, + }] }, + scenario === "duplicate" + ? { role: "user", content: [{ type: "image", image, mediaType: "image/png" }] } + : { role: "user", content: [{ type: "text", text: "continue" }] }, + { role: "user", content: [ + { type: "text", text: "[tool result without adjacent tool call: view_image (call_1)]\nlate:[image]" }, + { type: "image", image: remote, mediaType: "image/webp" }, + ] }, + ]); + }); + + test("closes outstanding calls before buffered and unmatched orphan image carriers", async () => { + const image = "data:image/png;base64,QUJDRA=="; + const remote = "https://example.com/orphan.jpg"; + const built = await builtRequest({ + ...parsed(), + context: { + ...parsed().context, + messages: [ + { + role: "assistant", + content: [ + { type: "toolCall", id: "call_1", name: "view_image", arguments: {} }, + { type: "toolCall", id: "call_2", name: "lookup", arguments: {} }, + ], + timestamp: 1, + }, + { + role: "toolResult", toolCallId: "call_1", toolName: "view_image", + content: [{ type: "image", imageUrl: image }], isError: false, timestamp: 2, + }, + { + role: "toolResult", toolCallId: "call_orphan", toolName: "view_image", + content: [{ type: "text", text: "unmatched:" }, { type: "image", imageUrl: remote }], + isError: true, timestamp: 3, + }, + ], + }, + }); + expect(JSON.parse(built.body).params.messages).toEqual([ + { role: "assistant", content: [ + { type: "tool-call", toolCallId: "call_1", toolName: "view_image", input: {} }, + { type: "tool-call", toolCallId: "call_2", toolName: "lookup", input: {} }, + ] }, + { role: "tool", content: [{ type: "tool-result", toolCallId: "call_1", toolName: "view_image", output: { type: "text", value: "[image]" } }] }, + { role: "tool", content: [{ + type: "tool-result", toolCallId: "call_2", toolName: "lookup", + output: { type: "error-text", value: "[ocx] no tool result was recorded for this tool call; execution status unknown." }, + }] }, + { role: "user", content: [{ type: "image", image, mediaType: "image/png" }] }, + { role: "user", content: [ + { type: "text", text: "[tool result without adjacent tool call: view_image (call_orphan)]\nunmatched:[image]" }, + { type: "image", image: remote, mediaType: "image/jpeg" }, + ] }, + ]); + }); + + test.each(["", " outcome\n"])("preserves exact image-free orphan text %j for strings and arrays", async (text) => { + for (const content of [text, [{ type: "text" as const, text }, { type: "text" as const, text: "" }]]) { + const built = await builtRequest({ + ...parsed(), + context: { + ...parsed().context, + messages: [{ role: "toolResult", toolCallId: "call_orphan", toolName: "lookup", content, isError: false, timestamp: 1 }], + }, + }); + expect(JSON.parse(built.body).params.messages).toEqual([{ + role: "user", + content: [{ type: "text", text: `[tool result without adjacent tool call: lookup (call_orphan)]\n${text}` }], + }]); + } + }); + test("keeps the generate config to bounded workspace and git metadata", async () => { const built = await builtRequest(parsed()); const body = JSON.parse(built.body); diff --git a/tests/providers/cursor/cursor-images.test.ts b/tests/providers/cursor/cursor-images.test.ts index 515254af26..42d3d1ea2f 100644 --- a/tests/providers/cursor/cursor-images.test.ts +++ b/tests/providers/cursor/cursor-images.test.ts @@ -1,6 +1,7 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { createHash } from "node:crypto"; import { create, fromBinary } from "@bufbuild/protobuf"; +import type { OcxMessage, OcxToolResultMessage } from "../../../src/types"; import { CursorImageError, CURSOR_VISION_IMAGE_OMITTED, @@ -48,6 +49,223 @@ async function oversizedDecodablePng(): Promise { return new Uint8Array(await new Bun.Image(src).resize(2400, 2400).png().bytes()); } +function toolImageResult( + content: OcxToolResultMessage["content"], + toolCallId = "call_view", + toolName = "view_image", +): OcxToolResultMessage { + return { role: "toolResult", toolCallId, toolName, content, isError: false, timestamp: 1 }; +} + +describe("Cursor opted-in trailing tool image preparation", () => { + test("prepares all results in attachment order even when the final result is text-only", async () => { + const raw = [ + toolImageResult([ + { type: "text", text: "first screenshots" }, + { type: "image", imageUrl: PNG_DATA_URL, detail: "high" }, + { type: "image", imageUrl: PNG_DATA_URL, detail: "auto" }, + ], "call_a"), + toolImageResult("no screenshot", "call_b"), + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL, detail: "original" }], "call_c"), + toolImageResult("all done", "call_d"), + ]; + const prepared = await prepareCursorRawMessages(raw, undefined, { trailingToolImages: true }); + expect(prepared.images.map(image => image.sourceLabel)).toEqual([ + 'tool result 1, image 1: {"tool":"view_image","call_id":"call_a"}', + 'tool result 1, image 2: {"tool":"view_image","call_id":"call_a"}', + 'tool result 3, image 1: {"tool":"view_image","call_id":"call_c"}', + ]); + expect(prepared.images.map(image => image.detail)).toEqual(["high", "auto", "original"]); + for (const image of prepared.images) { + expect(image.mimeType).toBe("image/jpeg"); + expect(image.data.slice(0, 2)).toEqual(new Uint8Array([0xff, 0xd8])); + } + const normalizedParts = prepared.messages?.flatMap(message => + typeof message.content === "string" ? [] : message.content.filter(part => part.type === "image")); + expect(normalizedParts?.map(part => part.imageUrl)).toEqual( + prepared.images.map(image => `data:image/jpeg;base64,${Buffer.from(image.data).toString("base64")}`), + ); + expect(prepared.messages?.[1]).toBe(raw[1]); + expect(prepared.messages?.[3]).toBe(raw[3]); + }); + + test("omits invalid and remote images without gaps in ready-image ordinals", async () => { + const prepared = await prepareCursorRawMessages([ + toolImageResult([{ type: "image", imageUrl: "data:image/png;base64,!!!!" }], "call_bad"), + toolImageResult([ + { type: "image", imageUrl: "data:image/png;base64,!!!!" }, + { type: "image", imageUrl: PNG_DATA_URL }, + { type: "image", imageUrl: "https://example.com/remote.png" }, + { type: "image", imageUrl: PNG_DATA_URL }, + ], "call_good"), + ], undefined, { trailingToolImages: true }); + expect(prepared.images.map(image => image.sourceLabel)).toEqual([ + 'tool result 2, image 1: {"tool":"view_image","call_id":"call_good"}', + 'tool result 2, image 2: {"tool":"view_image","call_id":"call_good"}', + ]); + expect(prepared.messages?.[0]?.content).toEqual([{ type: "text", text: CURSOR_VISION_IMAGE_OMITTED }]); + const content = prepared.messages?.[1]?.content; + expect(Array.isArray(content)).toBe(true); + if (!Array.isArray(content)) throw new Error("expected parts"); + expect(content.map(part => part.type)).toEqual(["text", "image", "text", "image"]); + expect(content[0]).toEqual({ type: "text", text: CURSOR_VISION_IMAGE_OMITTED }); + expect(content[2]).toEqual({ type: "text", text: CURSOR_VISION_IMAGE_OMITTED }); + }); + + test("JSON-escapes quotes and controls in bounded source labels", async () => { + const prepared = await prepareCursorRawMessages([ + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }], 'call"\\\n\r\t\u0000', 'view"\\\n\u001b'), + ], undefined, { trailingToolImages: true }); + expect(prepared.images.map(image => image.sourceLabel)).toEqual([ + 'tool result 1, image 1: {"tool":"view\\"\\\\\\n\\u001b","call_id":"call\\"\\\\\\n\\r\\t\\u0000"}', + ]); + expect(prepared.images[0]?.sourceLabel).not.toMatch(/[\u0000-\u001f]/); + }); + + test("truncates identifiers before escaping and keeps colliding labels distinct by ordinal", async () => { + const name = '"'.repeat(128); + const id = "\\".repeat(128); + const prepared = await prepareCursorRawMessages([ + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }], `${id}first`, `${name}first`), + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }], `${id}second`, `${name}second`), + ], undefined, { trailingToolImages: true }); + expect(prepared.images).toHaveLength(2); + for (const [index, image] of prepared.images.entries()) { + const prefix = `tool result ${index + 1}, image 1: `; + expect(image.sourceLabel?.startsWith(prefix)).toBe(true); + expect(JSON.parse(image.sourceLabel!.slice(prefix.length))).toEqual({ tool: name, call_id: id }); + expect(image.sourceLabel!.length).toBeLessThan(600); + expect(image.sourceLabel).not.toContain("first"); + expect(image.sourceLabel).not.toContain("second"); + } + expect(prepared.images[0]?.sourceLabel).not.toBe(prepared.images[1]?.sourceLabel); + }); + + test("rejects aggregate counts over 12 across results before image decode", async () => { + const decode = spyOn(Bun.Image.prototype, "metadata"); + try { + await expect(prepareCursorRawMessages([ + toolImageResult(Array.from({ length: 6 }, () => ({ type: "image", imageUrl: PNG_DATA_URL }))), + toolImageResult(Array.from({ length: 7 }, () => ({ type: "image", imageUrl: "data:image/png;base64,!!!!" }))), + ], undefined, { trailingToolImages: true })).rejects.toMatchObject({ + name: "CursorImageError", + message: "Too many images in one request (max 12).", + }); + expect(decode).not.toHaveBeenCalled(); + } finally { + decode.mockRestore(); + } + }); + + test("accepts exactly 12 images across results", async () => { + const prepared = await prepareCursorRawMessages([ + toolImageResult(Array.from({ length: 6 }, () => ({ type: "image", imageUrl: PNG_DATA_URL })), "call_a"), + toolImageResult(Array.from({ length: 6 }, () => ({ type: "image", imageUrl: PNG_DATA_URL })), "call_b"), + ], undefined, { trailingToolImages: true }); + expect(prepared.images).toHaveLength(12); + expect(prepared.images[11]?.sourceLabel).toBe('tool result 2, image 6: {"tool":"view_image","call_id":"call_b"}'); + }); + + test("leaves history before the contiguous run untouched and never mutates input", async () => { + const raw: OcxMessage[] = [ + { role: "user", content: [{ type: "image", imageUrl: PNG_DATA_URL }], timestamp: 1 }, + toolImageResult(Array.from({ length: 13 }, () => ({ type: "image", imageUrl: PNG_DATA_URL })), "old"), + { role: "assistant", content: [{ type: "text", text: "next tool call" }], timestamp: 2 }, + toolImageResult([{ type: "text", text: "active" }, { type: "image", imageUrl: PNG_DATA_URL }], "new"), + ]; + const before = structuredClone(raw); + for (const message of raw) { + if (Array.isArray(message.content)) { + for (const part of message.content) Object.freeze(part); + Object.freeze(message.content); + } + Object.freeze(message); + } + Object.freeze(raw); + const prepared = await prepareCursorRawMessages(raw, undefined, { trailingToolImages: true }); + expect(raw).toEqual(before); + expect(prepared.messages).not.toBe(raw); + for (let index = 0; index < 3; index++) expect(prepared.messages?.[index]).toBe(raw[index]); + expect(prepared.messages?.[3]).not.toBe(raw[3]); + expect(prepared.images.map(image => image.sourceLabel)).toEqual([ + 'tool result 1, image 1: {"tool":"view_image","call_id":"new"}', + ]); + }); + + test("default, empty options and explicit false preserve trailing tool images unchanged", async () => { + const raw = [toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }])]; + for (const options of [undefined, {}, { trailingToolImages: false }]) { + const prepared = await prepareCursorRawMessages(raw, undefined, options); + expect(prepared.messages).toBe(raw); + expect(prepared.images).toEqual([]); + } + expect(cursorVisionPrepareStartIndex(raw)).toBe(raw.length); + expect(await resolveActiveCursorImages(raw)).toEqual([]); + }); + + test("text-only tool runs preserve identity", async () => { + const raw = [toolImageResult("done"), toolImageResult([{ type: "text", text: "also done" }])]; + const prepared = await prepareCursorRawMessages(raw, undefined, { trailingToolImages: true }); + expect(prepared.messages).toBe(raw); + expect(prepared.images).toEqual([]); + }); + + test("later user/developer turns do not revive stale tool images or receive source labels", async () => { + for (const role of ["user", "developer"] as const) { + const stale = toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }]); + const raw: OcxMessage[] = [stale, { role, content: "new question", timestamp: 2 }]; + const textOnly = await prepareCursorRawMessages(raw, undefined, { trailingToolImages: true }); + expect(textOnly.messages).toBe(raw); + expect(textOnly.images).toEqual([]); + const ordinary = await prepareCursorRawMessages([ + stale, { role, content: [{ type: "image", imageUrl: PNG_DATA_URL }], timestamp: 2 }, + ], undefined, { trailingToolImages: true }); + expect(ordinary.messages?.[0]).toBe(stale); + expect(ordinary.images).toHaveLength(1); + expect(ordinary.images[0]).not.toHaveProperty("sourceLabel"); + } + }); + + test("retains detail-dependent JPEG soft caps in opted-in results", async () => { + const pngPath = new URL("../../helpers/cursor-grumpy-fixture.png", import.meta.url); + const imageUrl = `data:image/png;base64,${Buffer.from(await Bun.file(pngPath).arrayBuffer()).toString("base64")}`; + const prepared = await prepareCursorRawMessages([ + toolImageResult([{ type: "image", imageUrl, detail: "auto" }]), + toolImageResult([{ type: "image", imageUrl, detail: "original" }]), + ], undefined, { trailingToolImages: true }); + expect(prepared.images).toHaveLength(2); + expect(prepared.images[0]!.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES); + expect(prepared.images[1]!.data.byteLength).toBeLessThanOrEqual(CURSOR_VISION_SOFT_MAX_BYTES_HIGH); + expect(prepared.images[1]!.data.byteLength).toBeGreaterThan(prepared.images[0]!.data.byteLength); + }); + + test("propagates pre-abort and abort during normalization without preparing later results", async () => { + const raw = [ + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }], "call_a"), + toolImageResult([{ type: "image", imageUrl: PNG_DATA_URL }], "call_b"), + ]; + const before = structuredClone(raw); + const preAborted = new AbortController(); + preAborted.abort(); + await expect(prepareCursorRawMessages(raw, preAborted.signal, { trailingToolImages: true })) + .rejects.toMatchObject({ name: "AbortError" }); + + const controller = new AbortController(); + const reason = new Error("stop image preparation"); + const decode = spyOn(Bun.Image.prototype, "metadata").mockImplementation(() => { + controller.abort(reason); + return Promise.reject(reason); + }); + try { + await expect(prepareCursorRawMessages(raw, controller.signal, { trailingToolImages: true })).rejects.toBe(reason); + expect(decode).toHaveBeenCalledTimes(1); + expect(raw).toEqual(before); + } finally { + decode.mockRestore(); + } + }); +}); + describe("Cursor image resolver", () => { test("rejects more than MAX_CURSOR_IMAGES in one request", async () => { const urls = Array.from({ length: MAX_CURSOR_IMAGES + 1 }, () => PNG_DATA_URL); @@ -102,6 +320,7 @@ describe("Cursor image resolver", () => { expect(resolved[0]!.data[0]).toBe(0xff); expect(resolved[0]!.data[1]).toBe(0xd8); expect(resolved[0]?.uuid.length).toBeGreaterThan(0); + expect(resolved[0]).not.toHaveProperty("sourceLabel"); }); test("soft-omits malformed and non-image data URLs", async () => { @@ -428,6 +647,7 @@ describe("Cursor image resolver", () => { expect(selectedImages).toHaveLength(1); expect(selectedImages[0]).toBe(prepared.images[0]); expect(selectedImages[0]?.data).toBe(prepared.images[0]?.data); + expect(selectedImages[0]).not.toHaveProperty("sourceLabel"); }); test("image-only remote soft-omit yields userMessageAction with omission text", async () => { diff --git a/tests/responses/chat-completions-endpoint.test.ts b/tests/responses/chat-completions-endpoint.test.ts index 4a4a5a5a12..b622027a97 100644 --- a/tests/responses/chat-completions-endpoint.test.ts +++ b/tests/responses/chat-completions-endpoint.test.ts @@ -122,8 +122,8 @@ function mockDualWireUpstream() { if (url.pathname.endsWith("/responses")) { const frames = [ - `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "ok" })}\n\n`, - `data: ${JSON.stringify({ + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: "ok" })}\n\n`, + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { id: "resp_1", @@ -2934,6 +2934,94 @@ test("inbound chat-completions honors the override when stripping sampling (#404 } }); +test.each([ + { model: "grok-4.5", pathname: "/v1/responses" }, + { model: "gemini-3-pro", pathname: "/v1/chat/completions" }, +])("inbound chat images and paired screenshots survive the $model wire", async ({ model, pathname }) => { + const png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + const screenshot = "https://example.com/tool-screenshot.png"; + const { server: upstream, captured } = mockDualWireUpstream(); + let server: ReturnType | undefined; + try { + saveConfig(dualWireConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); + server = startServer(0); + const response = await fetch(new URL("/v1/chat/completions", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: `mock/${model}`, + stream: true, + // Force the Chat sibling through Responses translation, not native Chat passthrough. + store: true, + messages: [ + { role: "user", content: [ + { type: "text", text: "Inspect this image." }, + { type: "image_url", image_url: { url: png, detail: "high" } }, + { type: "text", text: "Compare it with the screenshot." }, + ] }, + { role: "assistant", content: null, tool_calls: [ + { id: "call_screenshot", type: "function", function: { name: "screenshot", arguments: "{}" } }, + ] }, + { role: "tool", tool_call_id: "call_screenshot", content: [ + { type: "text", text: "Before screenshot." }, + { type: "image_url", image_url: { url: screenshot, detail: "low" } }, + { type: "text", text: "After screenshot." }, + { type: "image_url", image_url: { url: png, detail: "auto" } }, + ] }, + ], + }), + }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("[DONE]"); + + expect(captured).toHaveLength(1); + expect(captured[0]!.pathname).toBe(pathname); + expect(captured[0]!.body.model).toBe(model); + if (pathname === "/v1/responses") { + expect(captured[0]!.body.input).toEqual([ + { type: "message", role: "user", content: [ + { type: "input_text", text: "Inspect this image." }, + { type: "input_image", image_url: png, detail: "high" }, + { type: "input_text", text: "Compare it with the screenshot." }, + ] }, + { type: "function_call", call_id: "call_screenshot", name: "screenshot", arguments: "{}" }, + { type: "function_call_output", call_id: "call_screenshot", output: [ + { type: "input_text", text: "Before screenshot." }, + { type: "input_image", image_url: screenshot, detail: "low" }, + { type: "input_text", text: "After screenshot." }, + { type: "input_image", image_url: png, detail: "auto" }, + ] }, + ]); + } else { + const messages = captured[0]!.body.messages as Array>; + const conversation = messages.filter(message => message.role !== "system"); + expect(conversation.map(message => message.role)).toEqual(["user", "assistant", "tool", "user"]); + expect(conversation[0]).toEqual({ role: "user", content: [ + { type: "text", text: "Inspect this image." }, + { type: "image_url", image_url: { url: png, detail: "high" } }, + { type: "text", text: "Compare it with the screenshot." }, + ] }); + expect(conversation[1]!.tool_calls).toEqual([ + { id: "call_screenshot", type: "function", function: { name: "screenshot", arguments: "{}" } }, + ]); + expect(conversation[2]).toEqual({ + role: "tool", tool_call_id: "call_screenshot", content: "Before screenshot.After screenshot.", + }); + expect(conversation[3]).toEqual({ role: "user", content: [ + { type: "text", text: "[ocx] image output from the preceding tool result(s):" }, + { type: "image_url", image_url: { url: screenshot, detail: "low" } }, + { type: "image_url", image_url: { url: png, detail: "auto" } }, + ] }); + } + } finally { + try { + await server?.stop(true); + } finally { + await upstream.stop(true); + } + } +}); + test("/v1/chat/completions non-OK upstream preserves top-level structured cyber_policy type", async () => { const secret = `blocked by upstream policy Authorization: ${["Bear", "er"].join("")} chathttpsecret123456`; const safeMessage = "blocked by upstream policy Authorization: Bearer [REDACTED]"; diff --git a/tests/responses/openai-responses-passthrough.test.ts b/tests/responses/openai-responses-passthrough.test.ts index 8f9648e951..578a0f1906 100644 --- a/tests/responses/openai-responses-passthrough.test.ts +++ b/tests/responses/openai-responses-passthrough.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from "bun:test"; +import { createOpenAIChatAdapter } from "../../src/adapters/openai-chat"; import { createResponsesPassthroughAdapter as createResponsesPassthroughAdapterProduction } from "../../src/adapters/openai-responses"; import { openaiResponsesUrl } from "../../src/adapters/openai-responses-url"; +import { chatCompletionsToResponsesBody } from "../../src/chat/inbound"; +import { anthropicToResponsesBody } from "../../src/claude/inbound"; +import { parseRequest } from "../../src/responses/parser"; import { enrichProviderFromRegistry, providerConfigSeed } from "../../src/providers/derive"; import { getProviderRegistryEntry } from "../../src/providers/registry"; import { XAI_GROK_CLI_BASE_URL } from "../../src/providers/xai-transport"; @@ -25,6 +29,120 @@ const provider = { authMode: "forward" as const, }; +describe("external image wire matrix", () => { + // Same decodable 1x1 PNG as anthropic-image-normalize.test.ts; no fetch is needed. + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + const dataUrl = `data:image/png;base64,${png}`; + const httpsUrl = "https://images.example/second.png"; + const keyed = { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1", authMode: "key" as const, apiKey: "test-key" }; + const ingresses = [ + { + name: "Chat", convert: chatCompletionsToResponsesBody, + images: [ + { type: "image_url", image_url: { url: dataUrl, detail: "high" } }, + { type: "image_url", image_url: { url: httpsUrl, detail: "low" } }, + ], + call: { role: "assistant", tool_calls: [{ id: "call_image", type: "function", function: { name: "screenshot", arguments: "{}" } }] }, + responsesImages: [ + { type: "input_image", image_url: dataUrl, detail: "high" }, + { type: "input_image", image_url: httpsUrl, detail: "low" }, + ], + chatImages: [ + { type: "image_url", image_url: { url: dataUrl, detail: "high" } }, + { type: "image_url", image_url: { url: httpsUrl, detail: "low" } }, + ], + }, + { + name: "Claude", convert: anthropicToResponsesBody, + images: [ + { type: "image", source: { type: "base64", media_type: "image/png", data: png } }, + { type: "image", source: { type: "url", url: httpsUrl } }, + ], + call: { role: "assistant", content: [{ type: "tool_use", id: "call_image", name: "screenshot", input: {} }] }, + responsesImages: [ + { type: "input_image", image_url: dataUrl }, + { type: "input_image", image_url: httpsUrl }, + ], + chatImages: [ + { type: "image_url", image_url: { url: dataUrl } }, + { type: "image_url", image_url: { url: httpsUrl } }, + ], + }, + ]; + + for (const ingress of ingresses) { + for (const placement of ["user", "tool", "image-only tool"]) { + for (const target of ["API-key Responses", "ChatGPT forward", "Chat"]) { + test(`${ingress.name} ${placement} images -> ${target}`, async () => { + const isTool = placement !== "user"; + const imageOnly = placement === "image-only tool"; + const content = [...(imageOnly ? [] : [{ type: "text", text: "screenshot" }]), ...structuredClone(ingress.images)]; + const result = ingress.name === "Chat" + ? { role: "tool", tool_call_id: "call_image", content } + : { role: "user", content: [{ type: "tool_result", tool_use_id: "call_image", content }] }; + const raw = { + model: "test-model", stream: true, + messages: isTool + ? [structuredClone(ingress.call), result, { role: "user", content: "continue" }] + : [{ role: "user", content }], + }; + const original = structuredClone(raw); + const translated = ingress.convert(raw); + const translatedBefore = structuredClone(translated); + const parsed = parseRequest(translated); + const adapter = target === "Chat" + ? withTestTranslatorBudget(createOpenAIChatAdapter({ ...keyed, adapter: "openai-chat" })) + : createResponsesPassthroughAdapter(target === "ChatGPT forward" ? provider : keyed); + const request = await adapter.buildRequest(parsed, { headers: new Headers() }); + const body = JSON.parse(request.body) as { model: string; input?: unknown[]; messages?: unknown[] }; + expect(request.url).toBe(target === "ChatGPT forward" + ? "https://chatgpt.com/backend-api/codex/responses" + : target === "Chat" ? "https://api.openai.com/v1/chat/completions" : "https://api.openai.com/v1/responses"); + expect(body.model).toBe("test-model"); + // Expected payloads are hand-authored, never taken from translator/parser output. + if (target === "Chat") { + expect(body.messages).toEqual(isTool ? [ + { role: "assistant", content: "", tool_calls: [{ id: "call_image", type: "function", function: { name: "screenshot", arguments: "{}" } }] }, + { role: "tool", tool_call_id: "call_image", content: imageOnly ? "[image][image]" : "screenshot" }, + { role: "user", content: [{ type: "text", text: "[ocx] image output from the preceding tool result(s):" }, ...ingress.chatImages] }, + { role: "user", content: "continue" }, + ] : [{ role: "user", content: [{ type: "text", text: "screenshot" }, ...ingress.chatImages] }]); + } else { + const expectedContent = [...(imageOnly ? [] : [{ type: "input_text", text: "screenshot" }]), ...ingress.responsesImages]; + expect(body.input).toEqual(isTool ? [ + { type: "function_call", call_id: "call_image", name: "screenshot", arguments: "{}" }, + { type: "function_call_output", call_id: "call_image", output: expectedContent }, + { type: "message", role: "user", content: [{ type: "input_text", text: "continue" }] }, + ] : [{ type: "message", role: "user", content: expectedContent }]); + } + expect(raw).toEqual(original); + expect(translated).toEqual(translatedBefore); + }); + } + } + + test(`${ingress.name} orphan image-only output survives canonical forward repair`, async () => { + const content = structuredClone(ingress.images); + const raw = { model: "test-model", messages: [ingress.name === "Chat" + ? { role: "tool", tool_call_id: "call_orphan", content } + : { role: "user", content: [{ type: "tool_result", tool_use_id: "call_orphan", content }] }], + }; + const original = structuredClone(raw); + const translated = { ...ingress.convert(raw), previous_response_id: "resp_missing" }; + const translatedBefore = structuredClone(translated); + const request = await createResponsesPassthroughAdapter(provider).buildRequest(parseRequest(translated)); + const body = JSON.parse(request.body) as { previous_response_id?: string; input: unknown[] }; + expect(body.previous_response_id).toBeUndefined(); + expect(body.input).toEqual([{ + type: "message", role: "user", + content: [{ type: "input_text", text: "[tool output for call_orphan]" }, ...ingress.responsesImages], + }]); + expect(raw).toEqual(original); + expect(translated).toEqual(translatedBefore); + }); + } +}); + test("noncanonical forward providers cannot receive caller or runtime credentials", () => { const userInfoUrl = new URL("https://chatgpt.com/backend-api/codex"); userInfoUrl.username = "user"; diff --git a/tests/responses/responses-compaction-routing.test.ts b/tests/responses/responses-compaction-routing.test.ts index d9e8aa537c..e6f46ab3c7 100644 --- a/tests/responses/responses-compaction-routing.test.ts +++ b/tests/responses/responses-compaction-routing.test.ts @@ -10,6 +10,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; import * as adapterResolveModule from "../../src/server/adapter-resolve"; +import * as visionModule from "../../src/vision"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { CODEX_QUOTA_PROBE_INTERVAL_MS, @@ -1535,6 +1536,101 @@ test("a no-eligible policy compact request persists the evaluation trace", async * passthrough / routed compaction build from _rawBody, never reading context.messages — they * already degrade an unpaired output to "[tool output for unknown call]" on their own. */ +describe("computer screenshot output translation boundary", () => { + const screenshot = { + type: "computer_call_output", call_id: "call_screen", + output: { type: "computer_screenshot", image_url: "https://example.com/screen.png" }, + }; + const ordinaryImage = { + type: "message", role: "user", + content: [{ type: "input_image", image_url: "https://example.com/ordinary.png" }], + }; + + test("rejects before an otherwise active vision description", async () => { + const config = keyProviderConfig({ adapter: "openai-chat", noVisionModels: ["model"] }); + config.visionSidecar = { enabled: true, backend: "routed", model: "vision/seeing" }; + config.providers.vision = { adapter: "openai-chat", baseUrl: "https://vision.example/v1", apiKey: "test-key" }; + // Routed vision needs no live OpenAI account for this controlled description dependency. + const resolveAuth = spyOn(visionModule, "shouldResolveOpenAiVisionSidecar").mockReturnValue(false); + const describe = spyOn(visionModule, "describeImagesInPlace").mockImplementation(async () => {}); + let fetches = 0; + globalThis.fetch = (async () => { + fetches++; + return jsonResponse({ id: "chat_vision_control", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }); + }) as typeof fetch; + try { + const control = await handleResponses(compactionRequest({ + model: "gw/model", stream: false, input: [ordinaryImage], + }), config, { model: "", provider: "" }); + expect(control.status).toBe(200); + await control.text(); + expect(describe).toHaveBeenCalledTimes(1); + expect(fetches).toBe(1); + describe.mockClear(); + fetches = 0; + const rejected = await handleResponses(compactionRequest({ + model: "gw/model", stream: false, input: [screenshot, ordinaryImage], + }), config, { model: "", provider: "" }); + expect(rejected.status).toBe(400); + await rejected.text(); + expect(describe).not.toHaveBeenCalled(); + expect(fetches).toBe(0); + } finally { + describe.mockRestore(); + resolveAuth.mockRestore(); + } + }); + + test("rejects translated computer outputs before upstream or vision work", async () => { + let fetches = 0; + globalThis.fetch = (async () => { + fetches++; + throw new Error("unsupported computer output must not reach upstream"); + }) as typeof fetch; + const res = await handleResponses(compactionRequest({ + model: "gw/model", stream: false, input: [screenshot, ordinaryImage], + }), keyProviderConfig({ adapter: "openai-chat" }), { model: "", provider: "" }); + expect(res.status).toBe(400); + const json = await res.json() as { error: { type: string; message: string } }; + expect(json.error.type).toBe("invalid_request_error"); + expect(json.error.message).toBe("computer_call_output requires a Responses passthrough route; send screenshots as user input_image content on translated routes."); + expect(JSON.stringify(json)).not.toContain("example.com"); + expect(fetches).toBe(0); + }); + + test("keeps the exact screenshot output on a Responses passthrough route", async () => { + const captured: Array> = []; + globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { + captured.push(JSON.parse(String(init?.body))); + return jsonResponse(completedPayload("ok")); + }) as typeof fetch; + const res = await handleResponses(compactionRequest({ + model: "gw/model", stream: false, input: [screenshot, ordinaryImage], + }), keyProviderConfig(), { model: "", provider: "" }); + expect(res.status).toBe(200); + await res.text(); + expect(captured).toHaveLength(1); + expect(captured[0]!.input).toEqual([screenshot, ordinaryImage]); + }); + + test("ordinary user image input remains accepted by translated routes", async () => { + const captured: Array> = []; + globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { + captured.push(JSON.parse(String(init?.body))); + return jsonResponse({ id: "chat_probe", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }); + }) as typeof fetch; + const res = await handleResponses(compactionRequest({ + model: "gw/model", stream: false, input: [ordinaryImage], + }), keyProviderConfig({ adapter: "openai-chat" }), { model: "", provider: "" }); + expect(res.status).toBe(200); + await res.text(); + expect(captured).toHaveLength(1); + expect(captured[0]!.messages).toEqual([{ role: "user", content: [ + { type: "image_url", image_url: { url: "https://example.com/ordinary.png" } }, + ] }]); + }); +}); + describe("unpaired tool result boundary (#3259)", () => { function unpairedBody(item: Record): Record { return { diff --git a/tests/responses/responses-parser.test.ts b/tests/responses/responses-parser.test.ts index 1d0c46a05c..a15b104b6c 100644 --- a/tests/responses/responses-parser.test.ts +++ b/tests/responses/responses-parser.test.ts @@ -638,6 +638,86 @@ describe("codex-rs compat surface (260707)", () => { ]); }); + for (const outputType of ["function_call_output", "custom_tool_call_output"]) { + test.each([ + { + name: "file-only image becomes a text marker", + output: [{ type: "input_image", file_id: "file-only" }], + expected: "[image: file-only]", + }, + { + name: "empty URL falls back to file_id", + output: [{ type: "input_image", image_url: "", file_id: "file-fallback" }], + expected: "[image: file-fallback]", + }, + { + name: "non-string URLs fall back to usable file_ids", + output: [null, false, 42, {}, []].map(image_url => ({ type: "input_image", image_url, file_id: "file-valid" })), + expected: "[image: file-valid]".repeat(5), + }, + { + name: "nonempty URL wins over file_id and normalizes original detail", + output: [{ type: "input_image", image_url: "https://example.com/winner.png", file_id: "file-loser", detail: "original" }], + expected: [{ type: "image", imageUrl: "https://example.com/winner.png", detail: "high" }], + }, + { + name: "valid URL survives a malformed file_id", + output: [{ type: "input_image", image_url: "data:image/png;base64,aGVsbG8=", file_id: 42 }], + expected: [{ type: "image", imageUrl: "data:image/png;base64,aGVsbG8=" }], + }, + { + name: "empty arrays stay empty text", + output: [], + expected: "", + }, + { + name: "malformed blocks and unusable image references are omitted", + output: [ + null, false, 42, "", [], {}, + { type: "input_image" }, + { type: "input_image", image_url: "", file_id: "" }, + ...[null, false, 42, {}, []].flatMap(value => [ + { type: "input_image", image_url: value, file_id: "" }, + { type: "input_image", image_url: "", file_id: value }, + ]), + ], + expected: "", + }, + ])(`${outputType}: $name`, ({ output, expected }) => { + const parsed = parseRequest({ ...base, input: [{ type: outputType, call_id: "image-call", output }] }); + const result = parsed.context.messages.find(m => m.role === "toolResult"); + expect(result?.content).toEqual(expected); + }); + + test(`${outputType}: mixed image output preserves order, caller input and raw body`, () => { + const output = Object.freeze([ + { type: "input_text", text: "before" }, + { type: "input_image", image_url: "", file_id: "file-marker" }, + { type: "input_image", image_url: "https://example.com/kept.png", file_id: "file-ignored", detail: "original" }, + { type: "input_image", image_url: "", file_id: "" }, + { type: "input_text", text: "after" }, + ].map(block => Object.freeze(block))); + const item = Object.freeze({ type: outputType, call_id: "image-call", output }); + const body = Object.freeze({ ...base, input: Object.freeze([item]) }); + const before = JSON.stringify(body); + const parsed = parseRequest(body); + expect(parsed.context.messages).toHaveLength(1); + const result = parsed.context.messages[0]; + expect(result.role).toBe("toolResult"); + expect(result.content).toEqual([ + { type: "text", text: "before" }, + { type: "text", text: "[image: file-marker]" }, + { type: "image", imageUrl: "https://example.com/kept.png", detail: "high" }, + { type: "text", text: "after" }, + ]); + expect(parsed._rawBody).toBe(body); + expect(body.input[0]).toBe(item); + expect(item.output).toBe(output); + expect(JSON.stringify(body)).toBe(before); + expect(JSON.stringify(parsed._rawBody)).toBe(before); + }); + } + test("context_compaction with ocx1 payload replays the stored summary", () => { const summary = "previous work summary"; const encrypted = "ocx1:" + Buffer.from(summary, "utf-8").toString("base64"); diff --git a/tests/responses/responses-state.test.ts b/tests/responses/responses-state.test.ts index e4ae716a41..831a8df29c 100644 --- a/tests/responses/responses-state.test.ts +++ b/tests/responses/responses-state.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { BULK_DURABLE_IO_BUDGET_MS } from "../helpers/test-budget"; import { findDeadPid } from "../helpers/dead-pid"; import { @@ -30,6 +30,7 @@ import { evictOldestResponseContinuationForBudget, expandPreviousResponseInput, flushResponseState, + awaitResponseSpillPublicationTailForTests, markBodyNonPersistable, previousResponseConversationId, previousResponseProviderState, @@ -1513,13 +1514,19 @@ describe("Responses previous_response_id state", () => { let fallbackFile: string | undefined; let abandonedTempCount = -1; + // This checks late-write ordering, not elapsed time in the outer fallback reserve. + // Keep the real drain timer, but exclude unrelated cleanup wall time from that reserve. + const frozenNow = Date.now(); + const nowSpy = spyOn(Date, "now").mockReturnValue(frozenNow); try { await flushResponseState(); fallbackFile = spillFileNames(home)[0]; expect(fallbackFile).toBeDefined(); abandonedTempCount = spillTempNames(home).length; } finally { + nowSpy.mockRestore(); release(); + await awaitResponseSpillPublicationTailForTests(); } await hardened; await new Promise(resolve => setTimeout(resolve, 0)); diff --git a/tests/vision/vision-cache.test.ts b/tests/vision/vision-cache.test.ts index dce18586f0..7ceb3f2338 100644 --- a/tests/vision/vision-cache.test.ts +++ b/tests/vision/vision-cache.test.ts @@ -138,6 +138,41 @@ describe("vision description cache and per-turn cap", () => { expect(resolveMaxDescriptionsPerTurn(Number.NaN)).toBe(8); }); + test.each(["function_call_output", "custom_tool_call_output"])("%s empty URLs cannot consume another image's caption", async type => { + const request = parseRequest({ + model: "routed/blind", + input: [{ type, call_id: "call_images", output: [ + { type: "input_image", image_url: "", file_id: "file-marker" }, + { type: "input_image", image_url: "" }, + { type: "input_image", image_url: DATA_B }, + { type: "input_image", image_url: DATA_C }, + ] }], + }); + const seen: string[] = []; + globalThis.fetch = (async (_url: unknown, init?: RequestInit) => { + const caption = imageCaption(JSON.parse(String(init?.body))); + seen.push(caption); + return openaiSse(caption); + }) as typeof fetch; + await describeImagesInPlace(request, plan(), new Headers({ authorization: "Bearer test" })); + expect(seen).toEqual(["caption-b", "caption-c"]); + const raw = request._rawBody as { input: Array<{ output: Array<{ type: string; text: string }> }> }; + const output = raw.input[0]!.output; + expect(output[0]).toEqual({ type: "input_text", text: "[image: file-marker]" }); + expect(output[1]!.text).toContain("image omitted"); + expect(output[2]!.text).toContain("caption-b"); + expect(output[3]!.text).toContain("caption-c"); + expect(output[0]!.text).not.toContain("caption-"); + expect(output[1]!.text).not.toContain("caption-"); + const result = request.context.messages[0]!; + expect(result.role).toBe("toolResult"); + expect(result.content).toEqual([ + { type: "text", text: "[image: file-marker]" }, + { type: "text", text: output[2]!.text }, + { type: "text", text: output[3]!.text }, + ]); + }); + test("normalizes vision timeoutMs to the runtime bounds", () => { expect(resolveVisionTimeoutMs(undefined)).toBe(DEFAULT_VISION_TIMEOUT_MS); expect(resolveVisionTimeoutMs(12_000)).toBe(12_000);