From 7b7c8e75f003d3c708699df0973f401a52f59c15 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:31:43 -0700 Subject: [PATCH 1/2] bench(coding-agent): committed instrument for session-load memory Measures heap after forced GC per phase (module baseline, entries loaded, context volume) plus the process high-water RSS while loading a synthetic multi-MB session through the real SessionManager pipeline, so memory claims about session retention have one reproducible instrument instead of a throwaway script per investigation. --- CHANGELOG.md | 1 + packages/coding-agent/CHANGELOG.md | 2 + .../bench/session-memory.bench.ts | 120 ++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 packages/coding-agent/bench/session-memory.bench.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 50190f6ea5..291b86790d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Added +- `bench/session-memory.bench.ts` measures what loading a large session actually holds: synthetic transcript in, real SessionManager load, heap after a forced GC per phase plus the process high-water RSS. Memory claims about session retention now have one committed instrument instead of an ad-hoc script per investigation. - A tool result that carries an image now states whether the picture reached the screen, so a model reading a file describes what it shows instead of reporting that it displayed it. - A picture the block gives up on after the fact, because the session's image budget demoted it or a Kitty session could not convert it, is stated to the model as undrawn instead of being reported as displayed. - `read` accepts a semicolon-delimited list of internal resources (`skill://demo/one.md;skill://demo/two.md`), the same list form `grep` and `glob` take, and returns one section per entry. diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index bf178493eb..409b56b421 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,6 +4,8 @@ ### Added +- `bench/session-memory.bench.ts` measures what loading a large session actually holds: synthetic transcript in, real SessionManager load, heap after a forced GC per phase plus the process high-water RSS. Memory claims about session retention now have one committed instrument instead of an ad-hoc script per investigation. + - A tool result that carries an image now states whether the picture reached the screen, so a model reading a file describes what it shows instead of reporting that it displayed it. - A picture the block gives up on after the fact, because the session's image budget demoted it or a Kitty session could not convert it, is stated to the model as undrawn instead of being reported as displayed. - `read` accepts a semicolon-delimited list of internal resources (`skill://demo/one.md;skill://demo/two.md`), the same list form `grep` and `glob` take, and returns one section per entry. diff --git a/packages/coding-agent/bench/session-memory.bench.ts b/packages/coding-agent/bench/session-memory.bench.ts new file mode 100644 index 0000000000..36c261d841 --- /dev/null +++ b/packages/coding-agent/bench/session-memory.bench.ts @@ -0,0 +1,120 @@ +/** + * Benchmark: resident memory of loading a large session through the real + * pipeline. + * + * Measures, per phase, the heap AFTER a forced synchronous GC plus the + * process high-water RSS, so steady-state retention and transient peaks are + * reported separately: + * + * 1. module baseline — heap before any session work + * 2. entries loaded — retention of the parsed entry graph + * 3. context build — buildSessionContext over the loaded branch + * + * The synthetic transcript mirrors real sessions: alternating user / + * assistant turns with multi-KB text blocks and periodic large tool outputs. + * + * Run: bun packages/coding-agent/bench/session-memory.bench.ts + * SESSION_MB=80 bun ... to size the synthetic transcript. + */ + +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionManager } from "../src/session/session-manager"; + +const TARGET_MB = Number(process.env.SESSION_MB ?? "32"); +const TOOL_OUTPUT_EVERY = 10; +const TEXT_KB = 4; + +function makeEntry(i: number, big: string): string { + const timestamp = new Date(Date.UTC(2026, 0, 1) + i * 1000).toISOString(); + const base = { type: "message", id: `e${i}`, parentId: i === 0 ? null : `e${i - 1}`, timestamp }; + if (i % TOOL_OUTPUT_EVERY === TOOL_OUTPUT_EVERY - 1) { + return JSON.stringify({ + ...base, + message: { role: "toolResult", toolCallId: `tc${i}`, toolName: "bash", output: big }, + }); + } + if (i % 2 === 0) { + return JSON.stringify({ + ...base, + message: { role: "user", content: [{ type: "text", text: `Task step ${i}: ${big.slice(0, 400)}` }] }, + }); + } + return JSON.stringify({ + ...base, + message: { + role: "assistant", + content: [{ type: "text", text: `Analysis ${i}: ${big}` }], + api: "openai-completions", + provider: "openai", + model: "gpt-test", + usage: { input: 1000, output: 2000, cacheRead: 0, cacheWrite: 0, totalTokens: 3000, cost: { total: 0.01 } }, + stopReason: "stop", + }, + }); +} + +function writeSyntheticSession(dir: string): { file: string; bytes: number; count: number } { + const file = path.join(dir, "memory-bench.jsonl"); + const big = "x".repeat(TEXT_KB * 1024); + let written = 0; + let count = 0; + const fd = fs.openSync(file, "w"); + fs.writeSync( + fd, + `${JSON.stringify({ type: "session", version: 3, id: "membench0000000000000000000000", timestamp: new Date().toISOString(), cwd: dir })}\n`, + ); + while (written < TARGET_MB * 1024 * 1024) { + const line = `${makeEntry(count++, big)}\n`; + fs.writeSync(fd, line); + written += Buffer.byteLength(line); + } + fs.closeSync(fd); + return { file, bytes: written, count }; +} + +/** High-water RSS in MiB. Linux reads the kernel's own VmHWM accounting; + * other platforms fall back to the current RSS sampling. */ +function peakRssMiB(): number { + try { + const status = fs.readFileSync("/proc/self/status", "utf8"); + const m = /VmHWM:\s+(\d+) kB/.exec(status); + if (m) return Number(m[1]) / 1024; + } catch { + // not Linux + } + return process.memoryUsage().rss / 1048576; +} + +function gcAndReport(label: string): void { + Bun.gc(true); + const heap = process.memoryUsage().heapUsed / 1048576; + const rss = process.memoryUsage().rss / 1048576; + console.log(`${label.padEnd(24)} heap ${heap.toFixed(0).padStart(5)} MiB rss ${rss.toFixed(0).padStart(5)} MiB`); +} + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "session-memory-")); +const { file, bytes, count } = writeSyntheticSession(dir); +console.log(`synthetic session: ${(bytes / 1048576).toFixed(1)} MiB, ${count} entries (${TARGET_MB} MiB target)`); +gcAndReport("module baseline"); + +const t0 = performance.now(); +const sm = await SessionManager.open(file, undefined, undefined, { suppressBreadcrumb: true }); +const loadMs = performance.now() - t0; +gcAndReport(`entries loaded (${loadMs.toFixed(0)}ms)`); + +const entries = sm.getEntries(); +let contextChars = 0; +for (const entry of entries) { + if (entry.type === "message") { + const content = entry.message.content; + if (typeof content === "string") contextChars += content.length; + else if (Array.isArray(content)) + for (const block of content) if (block.type === "text") contextChars += block.text.length; + } +} +console.log(`context text volume ${((contextChars * 2) / 1048576).toFixed(1)} MiB (UTF-16)`); +console.log(`peak RSS ${peakRssMiB().toFixed(0)} MiB`); + +fs.rmSync(dir, { recursive: true, force: true }); From b5aaef8aa47d9ff279f58cbe043dd839589691a2 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:09:20 -0700 Subject: [PATCH 2/2] perf(bench): measure the context build the header already promised The header named three phases and the script had two. Phase 3 was a character count over the loaded entries, which measures the transcript, not the pipeline, and buildSessionContext -- the pass that turns retained entries into the message array a turn carries -- was never called. It is called now, through SessionManager.buildSessionContext(), so the phase runs against the live entry array, leaf and id index rather than a re-derived copy. Each phase also prints high-water RSS beside heap and current RSS. VmHWM is monotonic, so the rise between two phases is that phase's transient peak; the header claimed peaks were reported per phase and one line at the end reported the whole run. The synthetic transcript moves from the system temp dir to the repo's gitignored .scratch root. SESSION_MB=500 is half a gigabyte, and where /tmp is a tmpfs that is half a gigabyte of RAM charged against the measurement being taken. Refs #903 --- CHANGELOG.md | 2 +- packages/coding-agent/CHANGELOG.md | 2 +- .../bench/session-memory.bench.ts | 46 +++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37ffc5139a..4ec4683883 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Added -- `bench/session-memory.bench.ts` measures what loading a large session actually holds: synthetic transcript in, real SessionManager load, heap after a forced GC per phase plus the process high-water RSS. Memory claims about session retention now have one committed instrument instead of an ad-hoc script per investigation. +- `bench/session-memory.bench.ts` reports heap after a forced GC, current RSS and high-water RSS at each of three phases (module baseline, `SessionManager.open`, `buildSessionContext`) over a synthetic transcript sized by `SESSION_MB`. - `VEYYON_DEBUG_STARTUP=1` writes one line per phase of a prompt submission (compaction check, plan arm, context build, memory context), so a slow submit names the phase that spent the time. - `read` takes `depth` and `limit` arguments for directory listings, and a read of the session working directory root with neither now returns a concise top-level listing with per-subdirectory entry counts instead of the recursive tree. - A tool result that carries an image now states whether the picture reached the screen, so a model reading a file describes what it shows instead of reporting that it displayed it. diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9e3612e005..156431d0bf 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -7,7 +7,7 @@ - Classified runner output (cargo, bun, Go, ctest, dotnet, clippy, golangci-lint, Gradle lint, pytest, and tsc/eslint-family) now opens with a result-contract header: `[clean] ` or `[errors]` / `[errors N] `. The header is the verdict and the body contains retained diagnostics. ### Added -- `bench/session-memory.bench.ts` measures what loading a large session actually holds: synthetic transcript in, real SessionManager load, heap after a forced GC per phase plus the process high-water RSS. Memory claims about session retention now have one committed instrument instead of an ad-hoc script per investigation. +- `bench/session-memory.bench.ts` reports heap after a forced GC, current RSS and high-water RSS at each of three phases (module baseline, `SessionManager.open`, `buildSessionContext`) over a synthetic transcript sized by `SESSION_MB`. - `VEYYON_DEBUG_STARTUP=1` writes one line per phase of a prompt submission (compaction check, plan arm, context build, memory context), so a slow submit names the phase that spent the time. - `read` takes `depth` and `limit` arguments for directory listings, and a read of the session working directory root with neither now returns a concise top-level listing with per-subdirectory entry counts instead of the recursive tree. diff --git a/packages/coding-agent/bench/session-memory.bench.ts b/packages/coding-agent/bench/session-memory.bench.ts index 36c261d841..4e780a97c3 100644 --- a/packages/coding-agent/bench/session-memory.bench.ts +++ b/packages/coding-agent/bench/session-memory.bench.ts @@ -2,11 +2,12 @@ * Benchmark: resident memory of loading a large session through the real * pipeline. * - * Measures, per phase, the heap AFTER a forced synchronous GC plus the - * process high-water RSS, so steady-state retention and transient peaks are - * reported separately: + * Reports three numbers per phase: the heap AFTER a forced synchronous GC, + * which is steady-state retention; the current RSS; and the process high-water + * RSS, which is monotonic, so the rise between two phases is the transient peak + * that phase reached. * - * 1. module baseline — heap before any session work + * 1. module baseline — before any session work * 2. entries loaded — retention of the parsed entry graph * 3. context build — buildSessionContext over the loaded branch * @@ -18,7 +19,6 @@ */ import * as fs from "node:fs"; -import * as os from "node:os"; import * as path from "node:path"; import { SessionManager } from "../src/session/session-manager"; @@ -88,13 +88,26 @@ function peakRssMiB(): number { } function gcAndReport(label: string): void { + // Bun.gc(true) has no portable equivalent: node's global.gc exists only under + // --expose-gc, which this script cannot set for its own process. Without a + // forced collection the heap number is whatever the collector last felt like + // doing, which is not retention. Bun.gc(true); const heap = process.memoryUsage().heapUsed / 1048576; const rss = process.memoryUsage().rss / 1048576; - console.log(`${label.padEnd(24)} heap ${heap.toFixed(0).padStart(5)} MiB rss ${rss.toFixed(0).padStart(5)} MiB`); + const peak = peakRssMiB(); + console.log( + `${label.padEnd(28)} heap ${heap.toFixed(0).padStart(5)} MiB rss ${rss.toFixed(0).padStart(5)} MiB peak ${peak.toFixed(0).padStart(5)} MiB`, + ); } -const dir = fs.mkdtempSync(path.join(os.tmpdir(), "session-memory-")); +// The transcript is written under the repo's gitignored scratch root rather than +// the system temp dir: at SESSION_MB=500 this is half a gigabyte, and on a host +// whose /tmp is a tmpfs that is half a gigabyte of RAM charged against the +// measurement the bench exists to take. +const scratchRoot = path.join(import.meta.dirname, "..", "..", "..", ".scratch"); +fs.mkdirSync(scratchRoot, { recursive: true }); +const dir = fs.mkdtempSync(path.join(scratchRoot, "session-memory-")); const { file, bytes, count } = writeSyntheticSession(dir); console.log(`synthetic session: ${(bytes / 1048576).toFixed(1)} MiB, ${count} entries (${TARGET_MB} MiB target)`); gcAndReport("module baseline"); @@ -104,17 +117,12 @@ const sm = await SessionManager.open(file, undefined, undefined, { suppressBread const loadMs = performance.now() - t0; gcAndReport(`entries loaded (${loadMs.toFixed(0)}ms)`); -const entries = sm.getEntries(); -let contextChars = 0; -for (const entry of entries) { - if (entry.type === "message") { - const content = entry.message.content; - if (typeof content === "string") contextChars += content.length; - else if (Array.isArray(content)) - for (const block of content) if (block.type === "text") contextChars += block.text.length; - } -} -console.log(`context text volume ${((contextChars * 2) / 1048576).toFixed(1)} MiB (UTF-16)`); -console.log(`peak RSS ${peakRssMiB().toFixed(0)} MiB`); +const t1 = performance.now(); +// The manager's own accessor, not the free function: it passes the live entry +// array, leaf and id index, which is the shape the production build sees. +const context = sm.buildSessionContext(); +const buildMs = performance.now() - t1; +gcAndReport(`context build (${buildMs.toFixed(0)}ms)`); +console.log(`context messages ${context.messages.length}`); fs.rmSync(dir, { recursive: true, force: true });