perf(ai): streaming requests stop pinning a full clone of their wire payload

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 ### Added
 
+- `VEYYON_DEBUG_STARTUP=1` writes one line per phase of a prompt submission (compaction check, plan arm, context build, memory context), so a slow submit names the phase that spent the time.
 - `read` takes `depth` and `limit` arguments for directory listings, and a read of the session working directory root with neither now returns a concise top-level listing with per-subdirectory entry counts instead of the recursive tree.
 - A tool result that carries an image now states whether the picture reached the screen, so a model reading a file describes what it shows instead of reporting that it displayed it.
 - A picture the block gives up on after the fact, because the session's image budget demoted it or a Kitty session could not convert it, is stated to the model as undrawn instead of being reported as displayed.
@@ -39,6 +40,8 @@
 - Daemon completion parsing and eval-store serialization errors use shared type guards; behavior is unchanged.
 - Superseded and useless tool results are now pruned as a batch whose combined size pays for the prompt-cache rewrite it forces, instead of only when a single result sits within 8,000 tokens of the end of the conversation.
 - The Anthropic provider reads its endpoint, credential placement, rejected betas and retry policy from the catalog's wire-capability table instead of comparing provider ids at seventeen call sites.
+- A streaming request no longer pins a parsed clone of its wire payload for the life of the stream: every provider's diagnostic dump retains only the exact sent bytes and materializes a body when a 400/413 dump is built.
+- The OpenAI-family, pi-native and Codex request builders serialize the request body once instead of deep-cloning the request graph, which took attempt preparation on a 32MiB context from 82ms to 9ms.
 - A message that names a dead socket reads the same everywhere: `namesDeadSocket` in `@veyyon/ai/error/flags` is the one list of errnos and phrases, and `ENETUNREACH`, `EHOSTUNREACH` and `EAI_AGAIN` now count as transient transport failures like the rest of them.
 - `MNEMOPI_NO_EMBEDDINGS=0`, `false`, `no` or `off` now leaves embeddings on everywhere instead of disabling them on the API path.
 - Every `MNEMOPI_*` value is read by `config.ts` alone; the local-model, extraction and embedding modules ask it instead of parsing the variable again.

packages/ai/CHANGELOG.md

@@ -10,6 +10,8 @@
 ### Changed
 
 - The Anthropic provider reads its endpoint, credential placement, rejected betas and retry policy from the catalog's wire-capability table instead of comparing provider ids at seventeen call sites.
+- A streaming request no longer pins a parsed clone of its wire payload for the life of the stream: every provider's diagnostic dump retains only the exact sent bytes and materializes a body when a 400/413 dump is built.
+- The OpenAI-family, pi-native and Codex request builders serialize the request body once instead of deep-cloning the request graph, which took attempt preparation on a 32MiB context from 82ms to 9ms.
 - A message that names a dead socket reads the same everywhere: `namesDeadSocket` in `@veyyon/ai/error/flags` is the one list of errnos and phrases, and `ENETUNREACH`, `EHOSTUNREACH` and `EAI_AGAIN` now count as transient transport failures like the rest of them.
 
 ### Fixed

packages/ai/src/providers/amazon-bedrock.ts

@@ -42,7 +42,7 @@ import {
 	kStreamingPartialJson,
 } from "../utils/block-symbols";
 import { AssistantMessageEventStream } from "../utils/event-stream";
-import type { RawHttpRequestDump } from "../utils/http-inspector";
+import { materializeDumpBody, type RawHttpRequestDump } from "../utils/http-inspector";
 import { armPreResponseTimeout, getStreamFirstEventTimeoutMs } from "../utils/idle-iterator";
 import { fetchProviderWithRetry } from "../utils/provider-fetch";
 import { notifyProviderResponse } from "../utils/provider-response";
@@ -307,6 +307,8 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream"> = (
 
 		const blocks = output.content as Block[];
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let wireBodyJson: string | undefined;
 		const region = resolveBedrockRegion(model.id, options);
 
 		try {
@@ -404,9 +406,11 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream"> = (
 					model: model.id,
 					method: "POST",
 					url,
-					body: commandInput,
 				};
-				const body = new TextEncoder().encode(JSON.stringify(commandInput));
+				// Retain the exact sent BYTES, not the parsed object: a dump body is
+				// read only on the 400/413 path.
+				wireBodyJson = JSON.stringify(commandInput);
+				const body = new TextEncoder().encode(wireBodyJson);
 
 				if (bearerToken) {
 					return {
@@ -590,7 +594,11 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream"> = (
 					diagnostics = `\n[thinking-diag] ${JSON.stringify(thinkingBlocks)}`;
 				}
 			}
-			const result = await AIError.finalize(error, { api: model.api, signal: options.signal, rawRequestDump });
+			const result = await AIError.finalize(error, {
+				api: model.api,
+				signal: options.signal,
+				rawRequestDump: materializeDumpBody(rawRequestDump, wireBodyJson),
+			});
 			output.stopReason = result.stopReason;
 			output.errorStatus = result.status;
 			output.errorId = result.id;

packages/ai/src/providers/anthropic.ts

@@ -75,7 +75,7 @@ import { withEmptyCompletionRetry } from "../utils/empty-completion-retry";
 import { AssistantMessageEventStream } from "../utils/event-stream";
 import { isPreResponseStall, openStallLadderBudget } from "../utils/first-event-budget";
 import { isFoundryEnabled } from "../utils/foundry";
-import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
+import { finalizeErrorMessage, materializeDumpBody, type RawHttpRequestDump } from "../utils/http-inspector";
 import { getStreamFirstEventTimeoutMs, getStreamIdleTimeoutMs, iterateWithIdleTimeout } from "../utils/idle-iterator";
 import { notifyProviderResponse } from "../utils/provider-response";
 import { COMBINATOR_KEYS, NO_STRICT, toolWireSchema } from "../utils/schema";
@@ -1879,6 +1879,8 @@ const streamAnthropicOnce = (
 			timestamp: Date.now(),
 		};
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let anthropicWireBodyJson: string | undefined;
 		let activeAbortTracker = createAbortSourceTracker(options?.signal);
 
 		const onSseEvent = options?.onSseEvent;
@@ -2035,14 +2037,17 @@ const streamAnthropicOnce = (
 					nextParams = replacementPayload as typeof nextParams;
 				}
 				nextParams = toWellFormedDeep(nextParams) as typeof nextParams;
+				// Retain the exact sent BYTES, not the parsed object: a dump body is
+				// read only on the 400/413 path, and holding the graph here pinned a
+				// full context-sized object for the whole stream.
 				rawRequestDump = {
 					provider: model.provider,
 					api: output.api,
 					model: model.id,
 					method: "POST",
 					url: `${baseUrl}/v1/messages${isOAuthToken ? "?beta=true" : ""}`,
-					body: nextParams,
 				};
+				anthropicWireBodyJson = JSON.stringify(nextParams);
 				return nextParams;
 			};
 			let params = await prepareParams();
@@ -2698,7 +2703,10 @@ const streamAnthropicOnce = (
 						// success (consumers treat its presence as failure).
 						logger.warn("anthropic: strict tools rejected, retrying without strict tools", {
 							model: model.id,
-							error: await finalizeErrorMessage(streamFailure, rawRequestDump),
+							error: await finalizeErrorMessage(
+								streamFailure,
+								materializeDumpBody(rawRequestDump, anthropicWireBodyJson),
+							),
 						});
 						if (providerSessionState) {
 							providerSessionState.strictToolsDisabled = true;
@@ -2843,10 +2851,11 @@ const streamAnthropicOnce = (
 				api: model.api,
 				provider: model.provider,
 				abortTracker: activeAbortTracker,
-				rawRequestDump,
+				rawRequestDump: materializeDumpBody(rawRequestDump, anthropicWireBodyJson),
 			});
 			output.stopReason = result.stopReason;
 			output.errorStatus = result.status;
+
 			output.errorId = result.id;
 			output.errorMessage = maybeAddReplayUnsignedThinkingHint(model, result.message);
 			output.duration = performance.now() - startTime;

packages/ai/src/providers/azure-openai-responses.ts

@@ -16,6 +16,7 @@ import { createAbortSourceTracker } from "../utils/abort";
 import { withEmptyCompletionRetry } from "../utils/empty-completion-retry";
 import { AssistantMessageEventStream } from "../utils/event-stream";
 import type { RawHttpRequestDump } from "../utils/http-inspector";
+import { materializeDumpBody } from "../utils/http-inspector";
 import {
 	getOpenAIStreamFirstEventTimeoutMs,
 	getOpenAIStreamIdleTimeoutMs,
@@ -100,6 +101,8 @@ const streamAzureOpenAIResponsesOnce = (
 			model.id,
 		);
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let wireBodyJson: string | undefined;
 		const abortTracker = createAbortSourceTracker(options?.signal);
 		const firstEventTimeoutAbortError = new AIError.StreamTimeoutError(
 			AZURE_OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE,
@@ -129,7 +132,6 @@ const streamAzureOpenAIResponsesOnce = (
 				model: model.id,
 				method: "POST",
 				url,
-				body: params,
 			};
 			let activeRequestParams = params;
 			let activeReasoningEffortFallbackKey = createOpenAIReasoningEffortFallbackKey(
@@ -138,17 +140,29 @@ const streamAzureOpenAIResponsesOnce = (
 				typeof params.model === "string" ? params.model : model.id,
 			);
 			const prepareRequest = async (): Promise<RequestInit> => {
-				const attemptParams = structuredClone(params);
-				const replacementPayload = await options?.onPayload?.(attemptParams, model);
-				const wireParams = replacementPayload !== undefined ? (replacementPayload as typeof params) : attemptParams;
+				// Serialize once; the hook gets an isolated parse of exactly those
+				// bytes, and when no extension handles the event the wire reuses
+				// `bodyJson` instead of re-serializing (structuredClone + stringify
+				// measured 82ms on a 32MiB context where serialize-once costs 9ms).
+				const bodyJson = JSON.stringify(params);
+				let wireParams = params;
+				if (options?.onPayload) {
+					const attemptParams = JSON.parse(bodyJson) as typeof params;
+					const replacementPayload = await options.onPayload(attemptParams, model);
+					wireParams =
+						replacementPayload !== undefined && replacementPayload !== attemptParams
+							? (replacementPayload as typeof params)
+							: attemptParams;
+				}
 				activeRequestParams = wireParams;
 				activeReasoningEffortFallbackKey = createOpenAIReasoningEffortFallbackKey(
 					"azure-responses",
 					url,
 					typeof wireParams.model === "string" ? wireParams.model : model.id,
 				);
-				if (rawRequestDump) rawRequestDump.body = wireParams;
-				return { body: JSON.stringify(wireParams) };
+				const body = wireParams === params ? bodyJson : JSON.stringify(wireParams);
+				wireBodyJson = body;
+				return { body };
 			};
 			const attemptedReasoningEffortFallbacks = new Set<string>();
 			let openaiHandle: OpenAIStreamHandle<ResponseStreamEvent>;
@@ -191,8 +205,9 @@ const streamAzureOpenAIResponsesOnce = (
 					const retryMarker = `${activeReasoningEffortFallbackKey}:${String(reasoningEffortFallback)}`;
 					if (attemptedReasoningEffortFallbacks.has(retryMarker)) throw error;
 					attemptedReasoningEffortFallbacks.add(retryMarker);
+					// The fallback-applied params reach `wireBodyJson` when the retried
+					// attempt's prepareRequest serializes them; no eager copy needed.
 					applyOpenAIReasoningEffortFallback(params, reasoningEffortFallback);
-					rawRequestDump.body = params;
 				} finally {
 					clearTimeout(requestTimeout);
 				}
@@ -249,7 +264,11 @@ const streamAzureOpenAIResponsesOnce = (
 			stream.push({ type: "done", reason: output.stopReason, message: output });
 			stream.end();
 		} catch (error) {
-			const result = await AIError.finalize(error, { api: model.api, abortTracker, rawRequestDump });
+			const result = await AIError.finalize(error, {
+				api: model.api,
+				abortTracker,
+				rawRequestDump: materializeDumpBody(rawRequestDump, wireBodyJson),
+			});
 			output.stopReason = result.stopReason;
 			output.errorStatus = result.status;
 			output.errorId = result.id;

packages/ai/src/providers/google-gemini-cli.ts

@@ -38,7 +38,8 @@ import type {
 import { normalizeSystemPrompts } from "../utils";
 import { AssistantMessageEventStream } from "../utils/event-stream";
 import { extractGoogleValidationUrl, formatGoogleValidationRequiredMessage } from "../utils/google-validation";
-import type { RawHttpRequestDump } from "../utils/http-inspector";
+import { materializeDumpBody, type RawHttpRequestDump } from "../utils/http-inspector";
+
 import { armPreResponseTimeout, getStreamFirstEventTimeoutMs } from "../utils/idle-iterator";
 import { fetchProviderWithRetry } from "../utils/provider-fetch";
 // Refresh is the sole responsibility of AuthStorage (broker-aware, single-flighted);
@@ -538,6 +539,8 @@ export const streamGoogleGeminiCli: StreamFunction<"google-gemini-cli"> = (
 			timestamp: Date.now(),
 		};
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let wireBodyJson: string | undefined;
 
 		try {
 			const apiKeyRaw = options?.apiKey;
@@ -630,9 +633,9 @@ export const streamGoogleGeminiCli: StreamFunction<"google-gemini-cli"> = (
 				api: output.api,
 				model: model.id,
 				method: "POST",
-				body: requestBody,
 				headers: requestHeaders,
 			};
+			wireBodyJson = requestBodyJson;
 
 			// Direct callers that skip `register-builtins` (which installs the
 			// iterator-level watchdog) need a pre-response timer alongside
@@ -1080,7 +1083,11 @@ export const streamGoogleGeminiCli: StreamFunction<"google-gemini-cli"> = (
 			stream.push({ type: "done", reason: output.stopReason, message: output });
 			stream.end();
 		} catch (error) {
-			const result = await AIError.finalize(error, { api: model.api, signal: options?.signal, rawRequestDump });
+			const result = await AIError.finalize(error, {
+				api: model.api,
+				signal: options?.signal,
+				rawRequestDump: materializeDumpBody(rawRequestDump, wireBodyJson),
+			});
 			output.stopReason = result.stopReason;
 			output.errorStatus = result.status;
 			output.errorId = result.id;

packages/ai/src/providers/google-shared.ts

@@ -32,7 +32,7 @@ import type {
 import { shouldSendServiceTier } from "../types";
 import { normalizeSystemPrompts } from "../utils";
 import { AssistantMessageEventStream } from "../utils/event-stream";
-import type { RawHttpRequestDump } from "../utils/http-inspector";
+import { materializeDumpBody, type RawHttpRequestDump } from "../utils/http-inspector";
 import { notifyProviderResponse } from "../utils/provider-response";
 import { normalizeSchemaForCCA, normalizeSchemaForGoogle, toolWireSchema } from "../utils/schema";
 import type {
@@ -1076,6 +1076,8 @@ export function streamGoogleGenAI<T extends "google-generative-ai" | "google-ver
 			timestamp: Date.now(),
 		};
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let wireBodyJson: string | undefined;
 
 		try {
 			const plan = await prepare();
@@ -1098,11 +1100,14 @@ export function streamGoogleGenAI<T extends "google-generative-ai" | "google-ver
 				model: model.id,
 				method: "POST",
 				url: plan.url,
-				body: params,
 				headers: plan.headers,
 			};
 
+			// Retain the exact sent BYTES, not the parsed object: a dump body is read
+			// only on the 400/413 path, and holding the graph here pinned a full
+			// context-sized object for the whole stream.
 			const bodyJson = JSON.stringify(paramsToWireBody(params));
+			wireBodyJson = bodyJson;
 			const fetchImpl = plan.fetch ?? options?.fetch ?? (globalThis.fetch.bind(globalThis) as FetchImpl);
 			const openStreamAt = async (requestUrl: string): Promise<ReadableStream<Uint8Array>> => {
 				const response = await fetchImpl(requestUrl, {
@@ -1189,7 +1194,11 @@ export function streamGoogleGenAI<T extends "google-generative-ai" | "google-ver
 			stream.push({ type: "done", reason: output.stopReason as "length" | "stop" | "toolUse", message: output });
 			stream.end();
 		} catch (error) {
-			const result = await AIError.finalize(error, { api: model.api, signal: options?.signal, rawRequestDump });
+			const result = await AIError.finalize(error, {
+				api: model.api,
+				signal: options?.signal,
+				rawRequestDump: materializeDumpBody(rawRequestDump, wireBodyJson),
+			});
 			output.stopReason = result.stopReason;
 			output.errorStatus = result.status;
 			output.errorId = result.id;

packages/ai/src/providers/ollama.ts

@@ -27,6 +27,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream";
 import {
 	type CapturedHttpErrorResponse,
 	captureHttpErrorResponse,
+	materializeDumpBody,
 	type RawHttpRequestDump,
 } from "../utils/http-inspector";
 import {
@@ -434,6 +435,8 @@ const streamOllamaOnce = (
 		let sawDone = false;
 		const output = createEmptyOutput(model);
 		let rawRequestDump: RawHttpRequestDump | undefined;
+		/** Exact bytes of the last sent request body; materialized into a dump only on the 400/413 path. */
+		let wireBodyJson: string | undefined;
 		let capturedErrorResponse: CapturedHttpErrorResponse | undefined;
 		let activeThinkingIndex: number | undefined;
 		let activeTextIndex: number | undefined;
@@ -552,8 +555,8 @@ const streamOllamaOnce = (
 				model: model.id,
 				method: "POST",
 				url: `${baseUrl}/api/chat`,
-				body,
 			};
+			wireBodyJson = JSON.stringify(body);
 			// Direct callers that bypass `register-builtins` (which installs
 			// the iterator-level watchdog) need a pre-response timer alongside
 			// `timeout: false`; otherwise an Ollama server that accepts the
@@ -738,8 +741,7 @@ const streamOllamaOnce = (
 			const result = await AIError.finalize(error, {
 				api: model.api,
 				provider: model.provider,
-				signal: options.signal,
-				rawRequestDump,
+				rawRequestDump: materializeDumpBody(rawRequestDump, wireBodyJson),
 				capturedErrorResponse,
 			});
 			output.stopReason = result.stopReason;