From 3883fe097b24fb788d1cfe56afc90a2829a139a4 Mon Sep 17 00:00:00 2001 From: davidrobertson Date: Sat, 25 Jul 2026 13:38:31 -0400 Subject: [PATCH] feat: search recoverable overflow payloads --- README.md | 8 +- plugins/codex-lcm/README.md | 6 +- plugins/codex-lcm/skills/lcm-recall/SKILL.md | 5 +- plugins/codex-lcm/src/hook.ts | 2 +- plugins/codex-lcm/src/mcp.ts | 52 +++++-- plugins/codex-lcm/src/overflow.ts | 148 +++++++++++++++++++ plugins/codex-lcm/src/storage.ts | 99 ++++++++++++- plugins/codex-lcm/tests/hook-cli.test.ts | 22 +++ plugins/codex-lcm/tests/mcp.test.ts | 94 ++++++++++++ plugins/codex-lcm/tests/storage.test.ts | 69 +++++++++ 10 files changed, 484 insertions(+), 21 deletions(-) create mode 100644 plugins/codex-lcm/src/overflow.ts diff --git a/README.md b/README.md index 8fdae2e..fb2d991 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,13 @@ A typical retrieval flow is: 1. Locate the current or latest relevant session by cwd, repo root, or session ID. 2. Use `lcm_grep` to search across summary nodes, session summaries, and high-signal events. Search tries exact FTS first, then relaxes broad queries - so one missing word does not make retrieval look empty. + so one missing word does not make retrieval look empty. When an exact error + or tool-output marker may have been truncated from the indexed event, set + `contentScope: "overflow"` or `"both"` to scan bounded sanitized overflow + payloads. 3. Use `lcm_describe` on a promising session or summary node to inspect the - summary, depth, source IDs, and lineage before loading more. + summary, depth, source IDs, and lineage before loading more. Overflow search + results use `overflow:` file IDs and support byte-offset paging. 4. Use `lcm_expand` on a chosen summary node, `lcm_expand_query` when the query should pick and recursively expand matching nodes, or `lcm_pack_context` when Codex needs a ready-to-use context block. Pass `overview: true` to diff --git a/plugins/codex-lcm/README.md b/plugins/codex-lcm/README.md index 7e23ed1..e73feaf 100644 --- a/plugins/codex-lcm/README.md +++ b/plugins/codex-lcm/README.md @@ -65,8 +65,8 @@ can still be recovered through `lcm_pack_context`. Standard recall workflow: -- `lcm_grep`: find relevant sessions by searching summary nodes, session summaries, and high-signal events. -- `lcm_describe`: inspect a session, summary node, or indexed file reference with compact source counts. Set `includeLineage: true` when exact source ID arrays are needed. +- `lcm_grep`: find relevant sessions by searching summary nodes, session summaries, and high-signal events. Set `contentScope: "overflow"` or `"both"` to scan the first 512 KiB of recent sanitized overflow payloads. +- `lcm_describe`: inspect a session, summary node, or indexed file reference with compact source counts. Overflow results use stable `overflow:` IDs; pass `offset` and `maxBytes` to page their content in chunks of at most 512 KiB. Set `includeLineage: true` when exact source ID arrays are needed. - `lcm_expand`: expand one summary node into bounded source summary nodes and high-signal source events. - `lcm_expand_query`: answer a focused retrieval need by searching matching summary nodes and recursively expanding their source lineage into bounded evidence. The default budget is 2000 tokens. Use `overview: true` for broad, source-rich lineage views. `sourceLimit` is per matched node/source expansion, and tight budgets reserve room for a focused source-event excerpt when one exists. - `lcm_pack_context`: pack relevant summary-node context into a model-ready Markdown block. @@ -121,7 +121,7 @@ Hooks capture the JSON payload Codex sends on stdin for: - `SubagentStop` - `Stop` -Events store session ID, cwd, optional project string, optional git repo root, optional git branch, hook event name, sanitized payload, redaction metadata, truncation metadata, timestamps, and hashes. Inputs over 512 KiB keep the normal bounded event plus a redacted, content-addressed overflow file; inputs over 8 MiB are rejected. Large path-backed outputs are indexed as file references with path, byte count, SHA-256, MIME guess, and a compact exploration summary; the indexed metadata does not reload the full content. +Events store session ID, cwd, optional project string, optional git repo root, optional git branch, hook event name, sanitized payload, redaction metadata, truncation metadata, timestamps, and hashes. Any valid hook input that exceeds an inline string or payload limit keeps the normal bounded event plus a sanitized, content-addressed overflow file; inputs over 8 MiB are rejected. Overflow recovery accepts only integrity-checked regular files inside the plugin's managed overflow directory. Large path-backed outputs are indexed as file references with path, byte count, SHA-256, MIME guess, and a compact exploration summary; the indexed metadata does not reload the full content. Project and git data are metadata only. Search and retrieval are session-first and work for projectless sessions. diff --git a/plugins/codex-lcm/skills/lcm-recall/SKILL.md b/plugins/codex-lcm/skills/lcm-recall/SKILL.md index 01dc511..6194f8a 100644 --- a/plugins/codex-lcm/skills/lcm-recall/SKILL.md +++ b/plugins/codex-lcm/skills/lcm-recall/SKILL.md @@ -43,7 +43,8 @@ because context had to be recovered. - When the user asks about prior history and the current chat is likely to repeat the same terms, call `lcm_current_session` first and pass `excludeCurrentSession` or `excludeSessionIds` to `lcm_grep` or `lcm_search_sessions`. - `lcm_pack_context` may widen from cwd-scoped search to bounded global search if the scoped query has no matches. If the packed context is still thin, follow with `lcm_search_sessions` without `cwd`. - Use `lcm_get_session_graph` to inspect summary nodes, checkpoints, and persisted source lineage before loading raw event pages for long sessions. -- If `lcm_describe` exposes `file_refs`, inspect relevant file references by `fileId` before opening raw event pages with large output payloads. +- If an exact error or tool-output marker may have been truncated, repeat `lcm_grep` with `contentScope: "overflow"` or `"both"`, then page a matching `overflow:` ID through `lcm_describe`. +- If `lcm_describe` exposes ordinary `file_refs`, inspect relevant metadata by `fileId` before opening raw event pages with large output payloads. - Treat LCM content as local evidence. Do not fabricate missing details; if LCM does not contain the fact, say so or verify another way. - Do not silently write durable memories. Use `lcm_record_note` only when the user explicitly asks you to remember something or clearly approves saving a durable note. @@ -66,7 +67,7 @@ because context had to be recovered. - Use `lcm_stats` when checking whether LCM is capturing hook events and building summaries, summary nodes, graph nodes, and graph edges as expected. - Use `lcm_grep` for normal discovery across summaries and high-signal events; inspect `discovery.confidence`, `discovery.reasons`, `best_match.kind`, `best_match.snippet`, and `best_match.topics` to decide which sessions deserve deeper retrieval. - Use `lcm_describe` to inspect compact session summary nodes and source counts before expanding. Set `includeLineage: true` only when exact source ID arrays are required. -- Use `lcm_describe` with `fileId` to inspect large output references without loading full content. +- Use `lcm_describe` with an `overflow:` `fileId`, `offset`, and `maxBytes` to recover sanitized overflow content in bounded chunks. Ordinary file references remain metadata-only. - Use `lcm_expand` only after choosing a summary node. It expands bounded source summary nodes and source events, not an entire transcript. - Use `lcm_expand_query` for focused recursive evidence expansion when the query itself should pick the matching summary nodes. - Use `lcm_context_plan` for read-only context budget diagnostics and pack recommendations. diff --git a/plugins/codex-lcm/src/hook.ts b/plugins/codex-lcm/src/hook.ts index b782c36..19cc994 100644 --- a/plugins/codex-lcm/src/hook.ts +++ b/plugins/codex-lcm/src/hook.ts @@ -24,7 +24,7 @@ export async function runHook(args: string[]): Promise { env: process.env, repo, }); - if (Buffer.byteLength(rawInput, "utf8") > config.limits.maxInputBytes) { + if (event.truncations.length > 0 || Buffer.byteLength(rawInput, "utf8") > config.limits.maxInputBytes) { const fullEvent = normalizeHookEvent({ hookEvent, rawInput, diff --git a/plugins/codex-lcm/src/mcp.ts b/plugins/codex-lcm/src/mcp.ts index 7710bd0..aa9e5d3 100644 --- a/plugins/codex-lcm/src/mcp.ts +++ b/plugins/codex-lcm/src/mcp.ts @@ -71,7 +71,7 @@ const TOOLS = [ { name: "lcm_grep", title: "LCM Grep", - description: "Preferred standard workflow step 1 (grep): find relevant sessions by searching summary nodes, session summaries, and high-signal raw events. Codex may surface this tool as mcp__codex_lcm__lcm_grep.", + description: "Preferred standard workflow step 1 (grep): find relevant sessions in indexed memory, or search bounded sanitized overflow payloads when contentScope is overflow or both. Codex may surface this tool as mcp__codex_lcm__lcm_grep.", inputSchema: { type: "object", properties: { @@ -79,6 +79,7 @@ const TOOLS = [ limit: { type: "number", default: 10 }, cwd: { type: "string" }, repoRoot: { type: "string" }, + contentScope: { type: "string", enum: ["memory", "overflow", "both"], default: "memory" }, excludeCurrentSession: { type: "boolean", default: false }, excludeSessionIds: { type: "array", items: { type: "string" } }, }, @@ -96,6 +97,8 @@ const TOOLS = [ nodeId: { type: "string" }, fileId: { type: "string" }, limit: { type: "number", default: 50 }, + offset: { type: "number", default: 0, description: "Byte offset when reading an overflow reference." }, + maxBytes: { type: "number", minimum: 4, maximum: 524288, default: 65536, description: "Maximum bytes to read from an overflow reference." }, includeLineage: { type: "boolean", default: false, description: "Include full source ID arrays instead of compact source counts." }, }, }, @@ -457,15 +460,30 @@ function callTool(params: Record) { return toolResult(`Captured ${usage.totals.total_tokens} tokens across ${usage.totals.sessions} sessions.`, { usage }); } case "lcm_grep": { - const matches = storage.searchSessions({ - query: optionalString(args.query), - limit: optionalNumber(args.limit), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), - excludeSessionIds: optionalStringArray(args.excludeSessionIds), - }); - return toolResult(`Found ${matches.length} LCM matches.`, { matches }); + const scope = contentScope(args.contentScope); + const query = optionalString(args.query); + const matches = scope === "overflow" + ? [] + : storage.searchSessions({ + query, + limit: optionalNumber(args.limit), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), + excludeSessionIds: optionalStringArray(args.excludeSessionIds), + }); + const overflowMatches = scope === "memory" + ? [] + : storage.searchOverflow({ + query: query ?? "", + limit: optionalNumber(args.limit), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + }); + return toolResult( + `Found ${matches.length} LCM matches and ${overflowMatches.length} overflow matches.`, + { matches, overflow_matches: overflowMatches }, + ); } case "lcm_describe": { const description = storage.describeMemory({ @@ -473,12 +491,16 @@ function callTool(params: Record) { nodeId: optionalString(args.nodeId), fileId: optionalString(args.fileId), limit: optionalNumber(args.limit), + offset: optionalNumber(args.offset), + maxBytes: optionalNumber(args.maxBytes), }); const target = description.target === "session" ? description.session?.session_id ?? args.sessionId : description.target === "summary_node" ? description.node.node_id - : description.file_ref.file_ref_id; + : description.target === "file_ref" + ? description.file_ref.file_ref_id + : description.overflow_ref.file_ref_id; return toolResult(`Described ${description.target} ${target}.`, { description: optionalBoolean(args.includeLineage) ? description : compactDescription(description), }); @@ -618,7 +640,7 @@ function withoutMarkdown(value: T): Omit item.trim()); } +function contentScope(value: unknown): "memory" | "overflow" | "both" { + if (value === undefined) return "memory"; + if (value === "memory" || value === "overflow" || value === "both") return value; + throw new Error("contentScope must be memory, overflow, or both."); +} + function currentThreadId(): string | undefined { return optionalString(process.env.CODEX_THREAD_ID); } diff --git a/plugins/codex-lcm/src/overflow.ts b/plugins/codex-lcm/src/overflow.ts new file mode 100644 index 0000000..83f51a3 --- /dev/null +++ b/plugins/codex-lcm/src/overflow.ts @@ -0,0 +1,148 @@ +import fs from "node:fs"; +import path from "node:path"; + +import type { NormalizedEvent } from "./events.ts"; +import { sha256 } from "./redact.ts"; + +export const DEFAULT_OVERFLOW_READ_BYTES = 64 * 1024; +export const MAX_OVERFLOW_READ_BYTES = 512 * 1024; + +export type OverflowReference = { + file_ref_id: string; + session_id: string; + observed_event_id: string; + timestamp: string; + path: string; + sha256: string; + byte_count: number; + sanitized_byte_count: number; +}; + +export type OverflowContent = OverflowReference & { + offset: number; + content: string; + next_offset?: number; +}; + +export type OverflowSearchMatch = { + file_ref_id: string; + session_id: string; + timestamp: string; + byte_offset: number; + line_number: number; + snippet: string; + scan_truncated: boolean; +}; + +export function overflowReferenceFromEvent(event: NormalizedEvent): OverflowReference | undefined { + const value = event.payload.overflow_ref; + if (!isRecord(value)) return undefined; + const hash = stringValue(value.sha256); + const filePath = stringValue(value.path); + const byteCount = numberValue(value.byte_count); + const sanitizedByteCount = numberValue(value.sanitized_byte_count); + if (!hash || !/^[a-f0-9]{64}$/u.test(hash) || !filePath || byteCount === undefined || sanitizedByteCount === undefined) { + return undefined; + } + return { + file_ref_id: `overflow:${hash}`, + session_id: event.session_id, + observed_event_id: event.event_id, + timestamp: event.timestamp, + path: filePath, + sha256: hash, + byte_count: byteCount, + sanitized_byte_count: sanitizedByteCount, + }; +} + +export function readOverflowContent(args: { + overflowDir: string; + reference: OverflowReference; + offset?: number; + maxBytes?: number; +}): OverflowContent { + const expectedPath = path.join(path.resolve(args.overflowDir), `${args.reference.sha256}.json`); + if (path.resolve(args.reference.path) !== expectedPath) { + throw new Error("Overflow reference is outside the managed overflow directory."); + } + const file = fs.openSync(expectedPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const stat = fs.fstatSync(file); + if (!stat.isFile() || stat.size !== args.reference.sanitized_byte_count || stat.size > 16 * 1024 * 1024) { + throw new Error("Overflow reference does not match a bounded regular file."); + } + const buffer = fs.readFileSync(file); + if (sha256(buffer) !== args.reference.sha256) throw new Error("Overflow payload integrity check failed."); + + const requestedOffset = clampInteger(args.offset, 0, buffer.length, 0); + const maxBytes = clampInteger(args.maxBytes, 4, MAX_OVERFLOW_READ_BYTES, DEFAULT_OVERFLOW_READ_BYTES); + const offset = nextUtf8Boundary(buffer, requestedOffset); + const end = previousUtf8Boundary(buffer, Math.min(buffer.length, offset + maxBytes)); + return { + ...args.reference, + offset, + content: buffer.subarray(offset, end).toString("utf8"), + ...(end < buffer.length ? { next_offset: end } : {}), + }; + } finally { + fs.closeSync(file); + } +} + +export function searchOverflowContent(args: { + overflowDir: string; + reference: OverflowReference; + query: string; +}): OverflowSearchMatch | undefined { + const chunk = readOverflowContent({ + overflowDir: args.overflowDir, + reference: args.reference, + maxBytes: MAX_OVERFLOW_READ_BYTES, + }); + const query = args.query.trim(); + if (query.length === 0) return undefined; + const index = chunk.content.toLowerCase().indexOf(query.toLowerCase()); + if (index < 0) return undefined; + const byteOffset = Buffer.byteLength(chunk.content.slice(0, index), "utf8"); + const snippetStart = Math.max(0, index - 120); + const snippetEnd = Math.min(chunk.content.length, index + query.length + 120); + return { + file_ref_id: args.reference.file_ref_id, + session_id: args.reference.session_id, + timestamp: args.reference.timestamp, + byte_offset: byteOffset, + line_number: chunk.content.slice(0, index).split(/\r?\n/u).length, + snippet: chunk.content.slice(snippetStart, snippetEnd).replace(/\s+/gu, " ").trim(), + scan_truncated: chunk.next_offset !== undefined, + }; +} + +function nextUtf8Boundary(buffer: Buffer, offset: number): number { + let index = offset; + while (index < buffer.length && (buffer[index] & 0xc0) === 0x80) index += 1; + return index; +} + +function previousUtf8Boundary(buffer: Buffer, offset: number): number { + let index = offset; + while (index > 0 && index < buffer.length && (buffer[index] & 0xc0) === 0x80) index -= 1; + return index; +} + +function clampInteger(value: number | undefined, min: number, max: number, fallback: number): number { + if (value === undefined || !Number.isFinite(value)) return fallback; + return Math.min(max, Math.max(min, Math.floor(value))); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function numberValue(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} diff --git a/plugins/codex-lcm/src/storage.ts b/plugins/codex-lcm/src/storage.ts index 97824d6..370f979 100644 --- a/plugins/codex-lcm/src/storage.ts +++ b/plugins/codex-lcm/src/storage.ts @@ -5,6 +5,14 @@ import { DatabaseSync } from "node:sqlite"; import { loadConfig, type LcmConfig } from "./config.ts"; import { createNoteEvent, type NormalizedEvent } from "./events.ts"; import { extractFileReferences, type FileReference } from "./file-refs.ts"; +import { + overflowReferenceFromEvent, + readOverflowContent, + searchOverflowContent, + type OverflowContent, + type OverflowReference, + type OverflowSearchMatch, +} from "./overflow.ts"; import { SUMMARY_ALGORITHM_VERSION, SUMMARY_NODE_CHUNK_SIZE, @@ -39,6 +47,7 @@ import { } from "./summary.ts"; export type { FileReference } from "./file-refs.ts"; +export type { OverflowContent, OverflowReference, OverflowSearchMatch } from "./overflow.ts"; export type { SessionMemorySummary, SummaryNode, SummarySourceType } from "./summary.ts"; export type StorageOptions = { @@ -66,6 +75,13 @@ export type SearchSessionArgs = { excludeSessionIds?: string[]; }; +export type SearchOverflowArgs = { + query: string; + limit?: number; + cwd?: string; + repoRoot?: string; +}; + export type ListSessionsArgs = { since?: string; until?: string; @@ -249,6 +265,10 @@ export type LcmDescription = | { target: "file_ref"; file_ref: FileReference; + } + | { + target: "overflow_ref"; + overflow_ref: OverflowContent; }; export type LcmExpansion = { @@ -1200,6 +1220,43 @@ export class LcmStorage { return rankSessionRows(rows, query).slice(0, limit); } + searchOverflow(args: SearchOverflowArgs): OverflowSearchMatch[] { + const limit = clampLimit(args.limit, 10, 50); + const events = this.db + ? (this.db.prepare(` + SELECT raw_json + FROM events + WHERE json_extract(raw_json, '$.payload.overflow_ref.sha256') IS NOT NULL + AND (?1 IS NULL OR cwd = ?1) + AND (?2 IS NULL OR repo_root = ?2) + ORDER BY timestamp DESC, rowid DESC + LIMIT 256 + `).all(args.cwd ?? null, args.repoRoot ?? null) as Array<{ raw_json: string }>) + .map((row) => JSON.parse(row.raw_json) as NormalizedEvent) + : readRawEvents(this.config.rawLogPath) + .filter((event) => !args.cwd || event.cwd === args.cwd) + .filter((event) => !args.repoRoot || event.repo_root === args.repoRoot) + .slice(-256) + .reverse(); + const matches: OverflowSearchMatch[] = []; + for (const event of events) { + const reference = overflowReferenceFromEvent(event); + if (!reference) continue; + try { + const match = searchOverflowContent({ + overflowDir: this.config.overflowDir, + reference, + query: args.query, + }); + if (match) matches.push(match); + } catch { + // A missing, moved, or invalid overflow file cannot block other recall. + } + if (matches.length >= limit) break; + } + return matches; + } + private excludedSearchSessionIds(args: SearchSessionArgs): Set { const excluded = new Set(args.excludeSessionIds?.filter((sessionId) => sessionId.trim().length > 0) ?? []); if (args.excludeCurrentSession) { @@ -1509,8 +1566,48 @@ export class LcmStorage { return row ? rowToFileReference(row) : undefined; } - describeMemory(args: { sessionId?: string; nodeId?: string; fileId?: string; limit?: number }): LcmDescription { + getOverflowRef(fileRefId: string): OverflowReference | undefined { + if (!fileRefId.startsWith("overflow:")) return undefined; + const hash = fileRefId.slice("overflow:".length); + if (!/^[a-f0-9]{64}$/u.test(hash)) return undefined; + if (!this.db) { + return readRawEvents(this.config.rawLogPath) + .map(overflowReferenceFromEvent) + .find((reference) => reference?.sha256 === hash); + } + const row = this.db.prepare(` + SELECT raw_json + FROM events + WHERE json_extract(raw_json, '$.payload.overflow_ref.sha256') = ?1 + ORDER BY timestamp DESC, rowid DESC + LIMIT 1 + `).get(hash) as { raw_json?: string } | undefined; + if (!row?.raw_json) return undefined; + return overflowReferenceFromEvent(JSON.parse(row.raw_json) as NormalizedEvent); + } + + describeMemory(args: { + sessionId?: string; + nodeId?: string; + fileId?: string; + limit?: number; + offset?: number; + maxBytes?: number; + }): LcmDescription { if (args.fileId) { + if (args.fileId.startsWith("overflow:")) { + const reference = this.getOverflowRef(args.fileId); + if (!reference) throw new Error(`Overflow reference not found: ${args.fileId}`); + return { + target: "overflow_ref", + overflow_ref: readOverflowContent({ + overflowDir: this.config.overflowDir, + reference, + offset: args.offset, + maxBytes: args.maxBytes, + }), + }; + } const fileRef = this.getFileRef(args.fileId); if (!fileRef) throw new Error(`File reference not found: ${args.fileId}`); return { diff --git a/plugins/codex-lcm/tests/hook-cli.test.ts b/plugins/codex-lcm/tests/hook-cli.test.ts index 10f1674..613d98b 100644 --- a/plugins/codex-lcm/tests/hook-cli.test.ts +++ b/plugins/codex-lcm/tests/hook-cli.test.ts @@ -64,6 +64,28 @@ test("hook command stores a sanitized overflow reference for oversized valid inp assert.match(overflow, /\[REDACTED:secret\]/u); }); +test("hook command preserves truncated large tool output below the input overflow threshold", () => { + const home = tempHome(); + const marker = "RECOVERABLE-LARGE-OUTPUT-MARKER"; + const result = runCli(["hook", "PostToolUse"], { + input: JSON.stringify({ + session_id: "large-tool-output-session", + cwd: "/tmp/large-tool-output", + tool_name: "build", + tool_response: `${"x".repeat(70 * 1024)}${marker}`, + }), + env: { CODEX_LCM_HOME: home }, + }); + + assertCliOk(result); + const [event] = readJsonl(path.join(home, "events.jsonl")) as Array<{ + payload: { overflow_ref?: { path?: string } }; + }>; + const overflowPath = event.payload.overflow_ref?.path ?? ""; + assert.equal(fs.existsSync(overflowPath), true); + assert.match(fs.readFileSync(overflowPath, "utf8"), new RegExp(marker, "u")); +}); + test("hook command still rejects input above the overflow safety ceiling", () => { const home = tempHome(); const result = runCli(["hook", "UserPromptSubmit"], { diff --git a/plugins/codex-lcm/tests/mcp.test.ts b/plugins/codex-lcm/tests/mcp.test.ts index 287ae10..8c7a394 100644 --- a/plugins/codex-lcm/tests/mcp.test.ts +++ b/plugins/codex-lcm/tests/mcp.test.ts @@ -658,6 +658,100 @@ test("MCP describe inspects large file references without loading content", () = assert.match(fileDescription.file_ref.exploration_summary, /rows/u); }); +test("MCP describe recovers bounded chunks from sanitized overflow payloads", () => { + const home = tempHome(); + const marker = "RECOVERED-OVERFLOW-CHUNK"; + const hook = runCli(["hook", "PostToolUse"], { + input: JSON.stringify({ + session_id: "mcp-overflow-session", + cwd: "/tmp/mcp-overflow", + tool_name: "build", + tool_response: `${"x".repeat(70 * 1024)}${marker}`, + }), + env: { CODEX_LCM_HOME: home }, + }); + assert.equal(hook.status, 0, hook.stderr); + + const sessionResponses = runMcp([ + { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-11-25" } }, + { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: "lcm_get_session", + arguments: { sessionId: "mcp-overflow-session" }, + }, + }, + ], { CODEX_LCM_HOME: home }); + const sha = sessionResponses[1].result.structuredContent.events[0].payload.overflow_ref.sha256; + + const descriptionResponses = runMcp([ + { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-11-25" } }, + { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: "lcm_describe", + arguments: { + fileId: `overflow:${sha}`, + offset: 64 * 1024, + maxBytes: 16 * 1024, + }, + }, + }, + ], { CODEX_LCM_HOME: home }); + + const description = descriptionResponses[1].result.structuredContent.description; + assert.equal(description.target, "overflow_ref"); + assert.equal(description.overflow_ref.file_ref_id, `overflow:${sha}`); + assert.equal(description.overflow_ref.offset, 64 * 1024); + assert.match(description.overflow_ref.content, new RegExp(marker, "u")); + assert.equal(description.overflow_ref.next_offset, undefined); +}); + +test("MCP grep searches bounded sanitized overflow content on request", () => { + const home = tempHome(); + const marker = "OVERFLOW-SEARCH-ONLY-MARKER"; + const hook = runCli(["hook", "PostToolUse"], { + input: JSON.stringify({ + session_id: "mcp-overflow-search-session", + cwd: "/tmp/mcp-overflow-search", + tool_name: "test", + tool_response: `${"x".repeat(70 * 1024)}\n${marker}\n`, + }), + env: { CODEX_LCM_HOME: home }, + }); + assert.equal(hook.status, 0, hook.stderr); + + const responses = runMcp([ + { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-11-25" } }, + { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: "lcm_grep", + arguments: { + query: marker, + contentScope: "overflow", + limit: 5, + }, + }, + }, + ], { CODEX_LCM_HOME: home }); + + const result = responses[1].result.structuredContent; + assert.deepEqual(result.matches, []); + assert.equal(result.overflow_matches.length, 1); + assert.match(result.overflow_matches[0].file_ref_id, /^overflow:[a-f0-9]{64}$/u); + assert.equal(result.overflow_matches[0].session_id, "mcp-overflow-search-session"); + assert.equal(result.overflow_matches[0].line_number, 1); + assert.equal(result.overflow_matches[0].byte_offset > 64 * 1024, true); + assert.match(result.overflow_matches[0].snippet, new RegExp(marker, "u")); +}); + test("MCP expand_query returns recursive evidence for a focused query", () => { const home = tempHome(); const cwd = "/tmp/mcp-expand-query"; diff --git a/plugins/codex-lcm/tests/storage.test.ts b/plugins/codex-lcm/tests/storage.test.ts index a7dd783..33fa450 100644 --- a/plugins/codex-lcm/tests/storage.test.ts +++ b/plugins/codex-lcm/tests/storage.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; import { Worker } from "node:worker_threads"; import { normalizeHookEvent, type NormalizedEvent } from "../src/events.ts"; +import { sha256 } from "../src/redact.ts"; import { createStorage } from "../src/storage.ts"; import { clearDerivedSummaries, readJsonl, tempHome } from "./helpers.ts"; @@ -965,6 +966,74 @@ test("indexes large path-backed tool outputs as file references", () => { storage.close(); }); +test("overflow recovery refuses event-supplied paths outside managed storage", () => { + const home = tempHome(); + const storage = createStorage({ home }); + const content = "must not be read"; + const externalPath = path.join(tempHome("codex-lcm-external-overflow-"), "payload.json"); + fs.writeFileSync(externalPath, content); + const hash = sha256(content); + + storage.ingest(normalizeHookEvent({ + hookEvent: "PostToolUse", + rawInput: JSON.stringify({ + session_id: "untrusted-overflow-session", + cwd: "/tmp/untrusted-overflow", + overflow_ref: { + sha256: hash, + byte_count: Buffer.byteLength(content), + sanitized_byte_count: Buffer.byteLength(content), + path: externalPath, + }, + }), + env: {}, + now, + })); + + assert.throws( + () => storage.describeMemory({ fileId: `overflow:${hash}` }), + /outside the managed overflow directory/u, + ); + storage.close(); +}); + +test("overflow recovery advances past a multibyte character with a tiny byte request", () => { + const home = tempHome(); + const storage = createStorage({ home }); + const content = "😀tail"; + const hash = sha256(content); + const overflowPath = path.join(home, "overflow", `${hash}.json`); + fs.mkdirSync(path.dirname(overflowPath), { recursive: true }); + fs.writeFileSync(overflowPath, content); + + storage.ingest(normalizeHookEvent({ + hookEvent: "PostToolUse", + rawInput: JSON.stringify({ + session_id: "utf8-overflow-session", + cwd: "/tmp/utf8-overflow", + overflow_ref: { + sha256: hash, + byte_count: Buffer.byteLength(content), + sanitized_byte_count: Buffer.byteLength(content), + path: overflowPath, + }, + }), + env: {}, + now, + })); + + const description = storage.describeMemory({ + fileId: `overflow:${hash}`, + maxBytes: 1, + }); + assert.equal(description.target, "overflow_ref"); + if (description.target === "overflow_ref") { + assert.equal(description.overflow_ref.content, "😀"); + assert.equal((description.overflow_ref.next_offset ?? 0) > 0, true); + } + storage.close(); +}); + test("lists root and child sessions with stable cursor pagination", () => { const home = tempHome(); const storage = createStorage({ home });