fix(compaction): retry image-rejected summaries without images

packages/coding-agent/CHANGELOG.md

@@ -10,6 +10,8 @@
 
 ### Fixed
 
+- Retry compaction summaries rejected for image-format errors once without images, preserving the original transcript and normal multimodal requests.
+
 ### Removed
 
 ## [2026.9.13-2] - 2026-09-13

packages/coding-agent/src/core/extensions/builtin/compaction/changes.md

@@ -1,5 +1,25 @@
 # changes.md — builtin compaction policy
 
+## Retry local summaries without images after a provider image-format rejection (2026-09-13)
+
+### What changed
+
+- Speculative/local summary generation keeps native images on its normal first request. When that image-bearing request returns the narrowly recognized provider image-format/base64 rejection, compaction makes one text-only retry with image blocks replaced by non-empty placeholders.
+- The retry projection runs after the context transformations in `prepareProviderRequest`, replacing their image blocks while preserving text and tool structure. Payload/header hooks remain active, and source messages and persisted session entries are not mutated.
+- Other errors and image-free requests retain their existing retry behavior; a failed fallback propagates without another image retry, and aborts still stand down.
+
+### Why
+
+- A provider rejected a local compaction request at `input[*].output[*].image_url` even though every persisted PNG and base64 payload validated. Retrying only that failed summary without image transport recovers across downstream serialization causes without sanitizing stored data or degrading rich-image summarization by default.
+
+### Why an extension could not handle it
+
+- The builtin compaction extension owns the local summary request, its bounded retries, and provider error-stop handling. An external hook can transform a request but cannot conditionally replay this private summarization attempt.
+
+### Expected merge conflict zones
+
+- LOW: `speculative-summary.ts` request projection and `speculative.ts` retry classification.
+
 ## Deterministic resume slice for an over-window restored context (2026-09-10)
 
 ### What changed

packages/coding-agent/src/core/extensions/builtin/compaction/speculative-summary.ts

@@ -17,16 +17,37 @@ import {
 	DEFAULT_SUMMARIZATION_IDLE_TIMEOUT_MS,
 	summarizationMaxDurationMs,
 } from "../../../compaction/stream-watchdog.ts";
-import { convertToLlm } from "../../../messages.ts";
+import { convertToLlm, dedupeConsecutivePlaceholder } from "../../../messages.ts";
 import type { buildPrompt } from "./prompts.ts";
 import { repairOrphanedToolResults } from "./repair-tool-pairs.ts";
 import type { SpeculativeCompactionContext, SpeculativeCompactionSnapshot } from "./speculative.ts";
 import { normalizeSummarizationTurnOrder } from "./summarization-turn-order.ts";
 
 const SUMMARY_TOKEN_HEADROOM = 32_768;
 const SUMMARY_CONTEXT_WINDOW_RESERVE_RATIO = 0.5;
+const COMPACTION_IMAGE_RETRY_PLACEHOLDER =
+	"[Image omitted from this compaction retry after the provider rejected image input.]";
 type CompactionProgressCallback = (delta: string) => void;
 
+function omitImagesForCompactionRetry(messages: Message[]): Message[] {
+	return messages.map((message) => {
+		if (
+			(message.role !== "user" && message.role !== "toolResult" && message.role !== "configurationUpdate") ||
+			!Array.isArray(message.content)
+		) {
+			return message;
+		}
+		if (!message.content.some((block) => block.type === "image")) return message;
+		const content = message.content.map((block) =>
+			block.type === "image" ? { type: "text" as const, text: COMPACTION_IMAGE_RETRY_PLACEHOLDER } : block,
+		);
+		return {
+			...message,
+			content: dedupeConsecutivePlaceholder(content, COMPACTION_IMAGE_RETRY_PLACEHOLDER),
+		};
+	});
+}
+
 function summaryMaxTokens(model: Model<any>, contextWindow: number): number {
 	const headroom = model.maxTokens > 0 ? Math.min(SUMMARY_TOKEN_HEADROOM, model.maxTokens) : SUMMARY_TOKEN_HEADROOM;
 	if (contextWindow > 0) {
@@ -99,6 +120,8 @@ export async function generateSummaryMessage(options: {
 	/** Resolved per-attempt duration budget; falls back to the size-adaptive default. */
 	maxDurationMs?: number;
 	messages: AgentMessage[];
+	/** Replace images after request hooks on the one provider-rejection fallback attempt. */
+	omitImages?: boolean;
 	onProgress?: CompactionProgressCallback;
 	prompt: ReturnType<typeof buildPrompt>;
 	signal?: AbortSignal;
@@ -136,10 +159,13 @@ export async function generateSummaryMessage(options: {
 			options.maxDurationMs ??
 			summarizationMaxDurationMs(requestMessages.reduce((total, message) => total + estimateTokens(message), 0));
 		const providerRequest = await options.context.prepareProviderRequest?.(requestMessages);
+		const convertedMessages = convertToLlm(providerRequest?.messages ?? requestMessages);
 		const requestContext = {
 			systemPrompt: options.snapshot.systemPrompt ?? options.prompt.system,
 			messages: repairOrphanedToolResults(
-				normalizeSummarizationTurnOrder(convertToLlm(providerRequest?.messages ?? requestMessages)),
+				normalizeSummarizationTurnOrder(
+					options.omitImages ? omitImagesForCompactionRetry(convertedMessages) : convertedMessages,
+				),
 			),
 			...(options.snapshot.tools && options.snapshot.tools.length > 0 ? { tools: options.snapshot.tools } : {}),
 		};

packages/coding-agent/src/core/extensions/builtin/compaction/speculative.ts

@@ -135,6 +135,24 @@ export class SummaryRequestError extends Error {
 }
 
 const UPSTREAM_STREAM_TRUNCATED_PATTERN = /(?:^|[^A-Za-z0-9_])upstream_stream_truncated(?:[^A-Za-z0-9_]|$)/;
+const ESTABLISHED_IMAGE_FORMAT_REJECTION_PATTERN =
+	/unsupported image format|unsupported media type for base64 image|invalid data url for image/i;
+const OPENAI_INVALID_IMAGE_DATA_URL_PATTERN =
+	/Invalid 'input\[\d+\]\.(?:content|output)\[\d+\]\.image_url'\.[\s\S]*Expected a base64-encoded data URL[\s\S]*invalid base64/i;
+
+function isImageFormatRejection(error: unknown): boolean {
+	return (
+		error instanceof SummaryRequestError &&
+		(ESTABLISHED_IMAGE_FORMAT_REJECTION_PATTERN.test(error.message) ||
+			OPENAI_INVALID_IMAGE_DATA_URL_PATTERN.test(error.message))
+	);
+}
+
+function hasSummarizationImages(messages: AgentMessage[]): boolean {
+	return convertToLlm(messages).some(
+		(message) => Array.isArray(message.content) && message.content.some((block) => block.type === "image"),
+	);
+}
 
 /**
  * Only failures with no cheaper recovery earn another billed request.
@@ -277,6 +295,7 @@ export async function runExtensionCompaction(
 	let overflowAttempts = 0;
 	const summarizationToolsOffered = (requestSnapshot.tools?.length ?? 0) > 0;
 	let toolUseRetrySpent = false;
+	let imageFallbackSpent = false;
 
 	while (true) {
 		if (signal?.aborted) return undefined;
@@ -309,6 +328,7 @@ export async function runExtensionCompaction(
 						forbidToolCalls: toolUseRetrySpent,
 						maxDurationMs: attemptBudgetMs,
 						messages: currentMessages,
+						omitImages: imageFallbackSpent,
 						onProgress,
 						prompt,
 						signal,
@@ -335,6 +355,8 @@ export async function runExtensionCompaction(
 					return attempt;
 				},
 				(error) =>
+					!isImageFormatRejection(error) &&
+					!imageFallbackSpent &&
 					retryEligible &&
 					allowSummarizationRetry(Date.now() - retryStartedMs, attemptBudgetMs) &&
 					isRetryableSummaryAttempt(error),
@@ -343,6 +365,10 @@ export async function runExtensionCompaction(
 			);
 		} catch (error) {
 			if (signal?.aborted) return undefined;
+			if (!imageFallbackSpent && hasSummarizationImages(currentMessages) && isImageFormatRejection(error)) {
+				imageFallbackSpent = true;
+				continue;
+			}
 			throw error;
 		}
 		if (!response) return undefined;