From 4522b40b89c7e059ee240ee6605241b56d5a653c Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 13 Sep 2026 18:19:09 -0400 Subject: [PATCH] fix(compaction): retry image-rejected summaries without images --- packages/coding-agent/CHANGELOG.md | 2 + .../extensions/builtin/compaction/changes.md | 20 ++ .../builtin/compaction/speculative-summary.ts | 30 +- .../builtin/compaction/speculative.ts | 26 ++ .../summarization-image-fallback.test.ts | 277 ++++++++++++++++++ 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 packages/coding-agent/test/compaction/summarization-image-fallback.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 948d69b08..0f2bdf2bc 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,6 +10,8 @@ ### Fixed +- Retry compaction summaries rejected for image-format errors once without images, preserving the original transcript and normal multimodal requests. + ### Removed ## [2026.9.13-2] - 2026-09-13 diff --git a/packages/coding-agent/src/core/extensions/builtin/compaction/changes.md b/packages/coding-agent/src/core/extensions/builtin/compaction/changes.md index b932e2e49..21ad209d9 100644 --- a/packages/coding-agent/src/core/extensions/builtin/compaction/changes.md +++ b/packages/coding-agent/src/core/extensions/builtin/compaction/changes.md @@ -1,5 +1,25 @@ # changes.md — builtin compaction policy +## Retry local summaries without images after a provider image-format rejection (2026-09-13) + +### What changed + +- Speculative/local summary generation keeps native images on its normal first request. When that image-bearing request returns the narrowly recognized provider image-format/base64 rejection, compaction makes one text-only retry with image blocks replaced by non-empty placeholders. +- The retry projection runs after the context transformations in `prepareProviderRequest`, replacing their image blocks while preserving text and tool structure. Payload/header hooks remain active, and source messages and persisted session entries are not mutated. +- Other errors and image-free requests retain their existing retry behavior; a failed fallback propagates without another image retry, and aborts still stand down. + +### Why + +- A provider rejected a local compaction request at `input[*].output[*].image_url` even though every persisted PNG and base64 payload validated. Retrying only that failed summary without image transport recovers across downstream serialization causes without sanitizing stored data or degrading rich-image summarization by default. + +### Why an extension could not handle it + +- The builtin compaction extension owns the local summary request, its bounded retries, and provider error-stop handling. An external hook can transform a request but cannot conditionally replay this private summarization attempt. + +### Expected merge conflict zones + +- LOW: `speculative-summary.ts` request projection and `speculative.ts` retry classification. + ## Deterministic resume slice for an over-window restored context (2026-09-10) ### What changed diff --git a/packages/coding-agent/src/core/extensions/builtin/compaction/speculative-summary.ts b/packages/coding-agent/src/core/extensions/builtin/compaction/speculative-summary.ts index a69d284bd..d70373902 100644 --- a/packages/coding-agent/src/core/extensions/builtin/compaction/speculative-summary.ts +++ b/packages/coding-agent/src/core/extensions/builtin/compaction/speculative-summary.ts @@ -17,7 +17,7 @@ import { DEFAULT_SUMMARIZATION_IDLE_TIMEOUT_MS, summarizationMaxDurationMs, } from "../../../compaction/stream-watchdog.ts"; -import { convertToLlm } from "../../../messages.ts"; +import { convertToLlm, dedupeConsecutivePlaceholder } from "../../../messages.ts"; import type { buildPrompt } from "./prompts.ts"; import { repairOrphanedToolResults } from "./repair-tool-pairs.ts"; import type { SpeculativeCompactionContext, SpeculativeCompactionSnapshot } from "./speculative.ts"; @@ -25,8 +25,29 @@ import { normalizeSummarizationTurnOrder } from "./summarization-turn-order.ts"; const SUMMARY_TOKEN_HEADROOM = 32_768; const SUMMARY_CONTEXT_WINDOW_RESERVE_RATIO = 0.5; +const COMPACTION_IMAGE_RETRY_PLACEHOLDER = + "[Image omitted from this compaction retry after the provider rejected image input.]"; type CompactionProgressCallback = (delta: string) => void; +function omitImagesForCompactionRetry(messages: Message[]): Message[] { + return messages.map((message) => { + if ( + (message.role !== "user" && message.role !== "toolResult" && message.role !== "configurationUpdate") || + !Array.isArray(message.content) + ) { + return message; + } + if (!message.content.some((block) => block.type === "image")) return message; + const content = message.content.map((block) => + block.type === "image" ? { type: "text" as const, text: COMPACTION_IMAGE_RETRY_PLACEHOLDER } : block, + ); + return { + ...message, + content: dedupeConsecutivePlaceholder(content, COMPACTION_IMAGE_RETRY_PLACEHOLDER), + }; + }); +} + function summaryMaxTokens(model: Model, contextWindow: number): number { const headroom = model.maxTokens > 0 ? Math.min(SUMMARY_TOKEN_HEADROOM, model.maxTokens) : SUMMARY_TOKEN_HEADROOM; if (contextWindow > 0) { @@ -99,6 +120,8 @@ export async function generateSummaryMessage(options: { /** Resolved per-attempt duration budget; falls back to the size-adaptive default. */ maxDurationMs?: number; messages: AgentMessage[]; + /** Replace images after request hooks on the one provider-rejection fallback attempt. */ + omitImages?: boolean; onProgress?: CompactionProgressCallback; prompt: ReturnType; signal?: AbortSignal; @@ -136,10 +159,13 @@ export async function generateSummaryMessage(options: { options.maxDurationMs ?? summarizationMaxDurationMs(requestMessages.reduce((total, message) => total + estimateTokens(message), 0)); const providerRequest = await options.context.prepareProviderRequest?.(requestMessages); + const convertedMessages = convertToLlm(providerRequest?.messages ?? requestMessages); const requestContext = { systemPrompt: options.snapshot.systemPrompt ?? options.prompt.system, messages: repairOrphanedToolResults( - normalizeSummarizationTurnOrder(convertToLlm(providerRequest?.messages ?? requestMessages)), + normalizeSummarizationTurnOrder( + options.omitImages ? omitImagesForCompactionRetry(convertedMessages) : convertedMessages, + ), ), ...(options.snapshot.tools && options.snapshot.tools.length > 0 ? { tools: options.snapshot.tools } : {}), }; diff --git a/packages/coding-agent/src/core/extensions/builtin/compaction/speculative.ts b/packages/coding-agent/src/core/extensions/builtin/compaction/speculative.ts index 168005031..ca8d36be6 100644 --- a/packages/coding-agent/src/core/extensions/builtin/compaction/speculative.ts +++ b/packages/coding-agent/src/core/extensions/builtin/compaction/speculative.ts @@ -135,6 +135,24 @@ export class SummaryRequestError extends Error { } const UPSTREAM_STREAM_TRUNCATED_PATTERN = /(?:^|[^A-Za-z0-9_])upstream_stream_truncated(?:[^A-Za-z0-9_]|$)/; +const ESTABLISHED_IMAGE_FORMAT_REJECTION_PATTERN = + /unsupported image format|unsupported media type for base64 image|invalid data url for image/i; +const OPENAI_INVALID_IMAGE_DATA_URL_PATTERN = + /Invalid 'input\[\d+\]\.(?:content|output)\[\d+\]\.image_url'\.[\s\S]*Expected a base64-encoded data URL[\s\S]*invalid base64/i; + +function isImageFormatRejection(error: unknown): boolean { + return ( + error instanceof SummaryRequestError && + (ESTABLISHED_IMAGE_FORMAT_REJECTION_PATTERN.test(error.message) || + OPENAI_INVALID_IMAGE_DATA_URL_PATTERN.test(error.message)) + ); +} + +function hasSummarizationImages(messages: AgentMessage[]): boolean { + return convertToLlm(messages).some( + (message) => Array.isArray(message.content) && message.content.some((block) => block.type === "image"), + ); +} /** * Only failures with no cheaper recovery earn another billed request. @@ -277,6 +295,7 @@ export async function runExtensionCompaction( let overflowAttempts = 0; const summarizationToolsOffered = (requestSnapshot.tools?.length ?? 0) > 0; let toolUseRetrySpent = false; + let imageFallbackSpent = false; while (true) { if (signal?.aborted) return undefined; @@ -309,6 +328,7 @@ export async function runExtensionCompaction( forbidToolCalls: toolUseRetrySpent, maxDurationMs: attemptBudgetMs, messages: currentMessages, + omitImages: imageFallbackSpent, onProgress, prompt, signal, @@ -335,6 +355,8 @@ export async function runExtensionCompaction( return attempt; }, (error) => + !isImageFormatRejection(error) && + !imageFallbackSpent && retryEligible && allowSummarizationRetry(Date.now() - retryStartedMs, attemptBudgetMs) && isRetryableSummaryAttempt(error), @@ -343,6 +365,10 @@ export async function runExtensionCompaction( ); } catch (error) { if (signal?.aborted) return undefined; + if (!imageFallbackSpent && hasSummarizationImages(currentMessages) && isImageFormatRejection(error)) { + imageFallbackSpent = true; + continue; + } throw error; } if (!response) return undefined; diff --git a/packages/coding-agent/test/compaction/summarization-image-fallback.test.ts b/packages/coding-agent/test/compaction/summarization-image-fallback.test.ts new file mode 100644 index 000000000..af84842a3 --- /dev/null +++ b/packages/coding-agent/test/compaction/summarization-image-fallback.test.ts @@ -0,0 +1,277 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import { fauxAssistantMessage, fauxToolCall, type Message, registerFauxProvider } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { AuthStorage } from "../../src/core/auth-storage.ts"; +import { createFileOps, DEFAULT_COMPACTION_SETTINGS } from "../../src/core/compaction/index.ts"; +import { + runExtensionCompaction, + type SpeculativeCompactionContext, + type SpeculativeCompactionSnapshot, +} from "../../src/core/extensions/builtin/compaction/speculative.ts"; +import { ModelRegistry } from "../../src/core/model-registry.ts"; +import { SessionManager } from "../../src/core/session-manager.ts"; + +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; +const OPENAI_INVALID_IMAGE_ERROR = + "Invalid 'input[5].output[3].image_url'. Expected a base64-encoded data URL with an image MIME type (e.g. 'data:image/png;base64,aGVsbG8='), but got an invalid base64-encoded value."; +const registrations: Array<{ unregister(): void }> = []; + +function containsImage(messages: readonly AgentMessage[]): boolean { + return messages.some( + (message) => + "content" in message && + Array.isArray(message.content) && + message.content.some((block) => block.type === "image"), + ); +} + +function contentText(message: Message | undefined): string[] { + if (!message || !Array.isArray(message.content)) return []; + return message.content.flatMap((block) => (block.type === "text" ? [block.text] : [])); +} + +function richHistory(model: SpeculativeCompactionSnapshot["model"]): Message[] { + return [ + { + role: "user", + content: [ + { type: "text", text: "user text before image" }, + { type: "image", mimeType: "image/png", data: TINY_PNG_BASE64 }, + { type: "text", text: "user text after image" }, + ], + timestamp: 1, + }, + { + ...fauxAssistantMessage( + [ + { type: "text", text: "assistant tool preface" }, + fauxToolCall("read", { path: "mixed.png" }, { id: "call-mixed" }), + fauxToolCall("read", { path: "only.png" }, { id: "call-image-only" }), + ], + { timestamp: 2 }, + ), + api: model.api, + provider: model.provider, + model: model.id, + }, + { + role: "toolResult", + toolCallId: "call-mixed", + toolName: "read", + content: [ + { type: "text", text: "tool text before image" }, + { type: "image", mimeType: "image/png", data: TINY_PNG_BASE64 }, + { type: "text", text: "tool text after image" }, + ], + isError: false, + timestamp: 3, + }, + { + role: "toolResult", + toolCallId: "call-image-only", + toolName: "read", + content: [{ type: "image", mimeType: "image/png", data: TINY_PNG_BASE64 }], + isError: false, + timestamp: 4, + }, + { + role: "user", + content: [{ type: "image", mimeType: "image/png", data: TINY_PNG_BASE64 }], + timestamp: 5, + }, + ]; +} + +function createHarness(options: { rich?: boolean } = {}) { + const registration = registerFauxProvider({ + models: [{ id: "image-summary", contextWindow: 200_000, input: ["text", "image"] }], + }); + registrations.push(registration); + const model = registration.getModel(); + const messages = + options.rich === false + ? ([ + { role: "user", content: [{ type: "text", text: "text-only history" }], timestamp: 1 }, + ] satisfies Message[]) + : richHistory(model); + const sessionManager = SessionManager.inMemory(); + for (const message of messages) sessionManager.appendMessage(message); + const preparedRequests: AgentMessage[][] = []; + const authStorage = AuthStorage.inMemory(); + authStorage.setRuntimeApiKey(model.provider, "x"); + const modelRegistry = ModelRegistry.inMemory(authStorage); + modelRegistry.registerProvider(model.provider, { + api: registration.api, + baseUrl: model.baseUrl, + models: registration.models, + }); + const context = { + model, + modelRegistry, + sessionManager, + getContextUsage: () => ({ tokens: 0, percent: 0, contextWindow: 200_000 }), + getMessageRevision: () => 1, + prepareProviderRequest: async (requestMessages: AgentMessage[]) => { + preparedRequests.push(structuredClone(requestMessages)); + return { + messages: requestMessages, + transformHeaders: async (headers: Record) => ({ ...headers, "x-summary-hook": "kept" }), + transformPayload: async (payload: unknown) => payload, + }; + }, + applyCompaction: async () => ({ applied: true as const, reason: "ok" as const }), + } satisfies SpeculativeCompactionContext; + const snapshot = { + generation: 1, + expectedRevision: 1, + model, + contextWindow: 200_000, + preparation: { + firstKeptEntryId: "keep", + messagesToSummarize: messages, + turnPrefixMessages: [], + isSplitTurn: false, + tokensBefore: 10_000, + fileOps: createFileOps(), + settings: { ...DEFAULT_COMPACTION_SETTINGS }, + }, + promptVariant: "default", + origin: "blocking", + } satisfies SpeculativeCompactionSnapshot; + return { context, messages, preparedRequests, registration, snapshot }; +} + +afterEach(() => { + for (const registration of registrations.splice(0)) registration.unregister(); +}); + +describe("local compaction image-format fallback", () => { + it("retries a rejected rich request once without images while preserving text and tool identity", async () => { + const harness = createHarness(); + const sourceBefore = structuredClone(harness.messages); + const persistedBefore = structuredClone(harness.context.sessionManager.getBranch()); + harness.registration.setResponses([ + (context) => { + expect(containsImage(context.messages)).toBe(true); + return fauxAssistantMessage("", { stopReason: "error", errorMessage: OPENAI_INVALID_IMAGE_ERROR }); + }, + (context) => { + expect(containsImage(context.messages)).toBe(false); + return fauxAssistantMessage("summary recovered without image transport"); + }, + ]); + + await expect(runExtensionCompaction(harness.context, harness.snapshot)).resolves.toMatchObject({ + summary: "summary recovered without image transport", + }); + + const calls = harness.registration.getCallLog(); + expect(calls).toHaveLength(2); + expect(containsImage(calls[0]?.context.messages ?? [])).toBe(true); + expect(containsImage(calls[1]?.context.messages ?? [])).toBe(false); + expect(harness.preparedRequests).toHaveLength(2); + expect(containsImage(harness.preparedRequests[1] ?? [])).toBe(true); + expect(calls[1]?.options?.headers).toMatchObject({ "x-summary-hook": "kept" }); + + const retried = calls[1]?.context.messages ?? []; + expect(retried.flatMap((message) => contentText(message))).toEqual( + expect.arrayContaining([ + "user text before image", + "user text after image", + "assistant tool preface", + "tool text before image", + "tool text after image", + ]), + ); + const assistant = retried.find((message) => message.role === "assistant"); + expect(assistant?.content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "toolCall", id: "call-mixed", name: "read" }), + expect.objectContaining({ type: "toolCall", id: "call-image-only", name: "read" }), + ]), + ); + for (const toolCallId of ["call-mixed", "call-image-only"]) { + const result = retried.find((message) => message.role === "toolResult" && message.toolCallId === toolCallId); + expect(result).toMatchObject({ role: "toolResult", toolCallId, toolName: "read", isError: false }); + expect(contentText(result)).not.toHaveLength(0); + } + const imageOnlyToolResult = retried.find( + (message) => message.role === "toolResult" && message.toolCallId === "call-image-only", + ); + expect(contentText(imageOnlyToolResult)).toHaveLength(1); + const imageOnlyUser = retried.find((message) => message.role === "user" && message.timestamp === 5); + expect(contentText(imageOnlyUser)).toHaveLength(1); + expect(harness.messages).toEqual(sourceBefore); + expect(harness.context.sessionManager.getBranch()).toEqual(persistedBefore); + }); + + it.each([ + "Unsupported image format: image/bmp", + "Unsupported media type for base64 image: image/tiff", + "Invalid data URL for image content", + ])("recognizes established provider image-format wording: %s", async (errorMessage) => { + const harness = createHarness(); + harness.registration.setResponses([ + fauxAssistantMessage("", { stopReason: "error", errorMessage }), + fauxAssistantMessage("recovered summary"), + ]); + + await expect(runExtensionCompaction(harness.context, harness.snapshot)).resolves.toMatchObject({ + summary: "recovered summary", + }); + expect(harness.registration.getCallLog()).toHaveLength(2); + }); + + it.each(["request blocked by provider policy", "invalid base64 in an unrelated metadata field"])( + "does not retry an image-bearing request for an ordinary error: %s", + async (errorMessage) => { + const harness = createHarness(); + harness.registration.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage })]); + + await expect(runExtensionCompaction(harness.context, harness.snapshot)).rejects.toThrow(errorMessage); + expect(harness.registration.getCallLog()).toHaveLength(1); + }, + ); + + it("does not retry an image-format error when the request contains no images", async () => { + const harness = createHarness({ rich: false }); + harness.registration.setResponses([ + fauxAssistantMessage("", { stopReason: "error", errorMessage: OPENAI_INVALID_IMAGE_ERROR }), + ]); + + await expect(runExtensionCompaction(harness.context, harness.snapshot)).rejects.toThrow( + OPENAI_INVALID_IMAGE_ERROR, + ); + expect(harness.registration.getCallLog()).toHaveLength(1); + }); + + it("propagates a failed fallback without spending another image retry", async () => { + const harness = createHarness(); + harness.registration.setResponses([ + fauxAssistantMessage("", { stopReason: "error", errorMessage: OPENAI_INVALID_IMAGE_ERROR }), + fauxAssistantMessage("", { stopReason: "error", errorMessage: "fallback request failed" }), + ]); + + await expect(runExtensionCompaction(harness.context, harness.snapshot)).rejects.toThrow( + "fallback request failed", + ); + expect(harness.registration.getCallLog()).toHaveLength(2); + }); + + it("stands down when the caller aborts during the rejected rich attempt", async () => { + const harness = createHarness(); + const controller = new AbortController(); + harness.registration.setResponses([ + () => { + controller.abort("cancelled by caller"); + return fauxAssistantMessage("", { stopReason: "error", errorMessage: OPENAI_INVALID_IMAGE_ERROR }); + }, + ]); + + await expect( + runExtensionCompaction(harness.context, harness.snapshot, controller.signal), + ).resolves.toBeUndefined(); + expect(harness.registration.getCallLog()).toHaveLength(1); + }); +});