feat: search recoverable overflow payloads

README.md

@@ -44,9 +44,13 @@ A typical retrieval flow is:
 1. Locate the current or latest relevant session by cwd, repo root, or session ID.
 2. Use `lcm_grep` to search across summary nodes, session summaries, and
    high-signal events. Search tries exact FTS first, then relaxes broad queries
-   so one missing word does not make retrieval look empty.
+   so one missing word does not make retrieval look empty. When an exact error
+   or tool-output marker may have been truncated from the indexed event, set
+   `contentScope: "overflow"` or `"both"` to scan bounded sanitized overflow
+   payloads.
 3. Use `lcm_describe` on a promising session or summary node to inspect the
-   summary, depth, source IDs, and lineage before loading more.
+   summary, depth, source IDs, and lineage before loading more. Overflow search
+   results use `overflow:<sha256>` file IDs and support byte-offset paging.
 4. Use `lcm_expand` on a chosen summary node, `lcm_expand_query` when the query
    should pick and recursively expand matching nodes, or `lcm_pack_context` when
    Codex needs a ready-to-use context block. Pass `overview: true` to

plugins/codex-lcm/README.md

@@ -65,8 +65,8 @@ can still be recovered through `lcm_pack_context`.
 
 Standard recall workflow:
 
-- `lcm_grep`: find relevant sessions by searching summary nodes, session summaries, and high-signal events.
-- `lcm_describe`: inspect a session, summary node, or indexed file reference with compact source counts. Set `includeLineage: true` when exact source ID arrays are needed.
+- `lcm_grep`: find relevant sessions by searching summary nodes, session summaries, and high-signal events. Set `contentScope: "overflow"` or `"both"` to scan the first 512 KiB of recent sanitized overflow payloads.
+- `lcm_describe`: inspect a session, summary node, or indexed file reference with compact source counts. Overflow results use stable `overflow:<sha256>` IDs; pass `offset` and `maxBytes` to page their content in chunks of at most 512 KiB. Set `includeLineage: true` when exact source ID arrays are needed.
 - `lcm_expand`: expand one summary node into bounded source summary nodes and high-signal source events.
 - `lcm_expand_query`: answer a focused retrieval need by searching matching summary nodes and recursively expanding their source lineage into bounded evidence. The default budget is 2000 tokens. Use `overview: true` for broad, source-rich lineage views. `sourceLimit` is per matched node/source expansion, and tight budgets reserve room for a focused source-event excerpt when one exists.
 - `lcm_pack_context`: pack relevant summary-node context into a model-ready Markdown block.
@@ -121,7 +121,7 @@ Hooks capture the JSON payload Codex sends on stdin for:
 - `SubagentStop`
 - `Stop`
 
-Events store session ID, cwd, optional project string, optional git repo root, optional git branch, hook event name, sanitized payload, redaction metadata, truncation metadata, timestamps, and hashes. Inputs over 512 KiB keep the normal bounded event plus a redacted, content-addressed overflow file; inputs over 8 MiB are rejected. Large path-backed outputs are indexed as file references with path, byte count, SHA-256, MIME guess, and a compact exploration summary; the indexed metadata does not reload the full content.
+Events store session ID, cwd, optional project string, optional git repo root, optional git branch, hook event name, sanitized payload, redaction metadata, truncation metadata, timestamps, and hashes. Any valid hook input that exceeds an inline string or payload limit keeps the normal bounded event plus a sanitized, content-addressed overflow file; inputs over 8 MiB are rejected. Overflow recovery accepts only integrity-checked regular files inside the plugin's managed overflow directory. Large path-backed outputs are indexed as file references with path, byte count, SHA-256, MIME guess, and a compact exploration summary; the indexed metadata does not reload the full content.
 
 Project and git data are metadata only. Search and retrieval are session-first and work for projectless sessions.
 

plugins/codex-lcm/skills/lcm-recall/SKILL.md

@@ -43,7 +43,8 @@ because context had to be recovered.
 - When the user asks about prior history and the current chat is likely to repeat the same terms, call `lcm_current_session` first and pass `excludeCurrentSession` or `excludeSessionIds` to `lcm_grep` or `lcm_search_sessions`.
 - `lcm_pack_context` may widen from cwd-scoped search to bounded global search if the scoped query has no matches. If the packed context is still thin, follow with `lcm_search_sessions` without `cwd`.
 - Use `lcm_get_session_graph` to inspect summary nodes, checkpoints, and persisted source lineage before loading raw event pages for long sessions.
-- If `lcm_describe` exposes `file_refs`, inspect relevant file references by `fileId` before opening raw event pages with large output payloads.
+- If an exact error or tool-output marker may have been truncated, repeat `lcm_grep` with `contentScope: "overflow"` or `"both"`, then page a matching `overflow:<sha256>` ID through `lcm_describe`.
+- If `lcm_describe` exposes ordinary `file_refs`, inspect relevant metadata by `fileId` before opening raw event pages with large output payloads.
 - Treat LCM content as local evidence. Do not fabricate missing details; if LCM does not contain the fact, say so or verify another way.
 - Do not silently write durable memories. Use `lcm_record_note` only when the user explicitly asks you to remember something or clearly approves saving a durable note.
 
@@ -66,7 +67,7 @@ because context had to be recovered.
 - Use `lcm_stats` when checking whether LCM is capturing hook events and building summaries, summary nodes, graph nodes, and graph edges as expected.
 - Use `lcm_grep` for normal discovery across summaries and high-signal events; inspect `discovery.confidence`, `discovery.reasons`, `best_match.kind`, `best_match.snippet`, and `best_match.topics` to decide which sessions deserve deeper retrieval.
 - Use `lcm_describe` to inspect compact session summary nodes and source counts before expanding. Set `includeLineage: true` only when exact source ID arrays are required.
-- Use `lcm_describe` with `fileId` to inspect large output references without loading full content.
+- Use `lcm_describe` with an `overflow:<sha256>` `fileId`, `offset`, and `maxBytes` to recover sanitized overflow content in bounded chunks. Ordinary file references remain metadata-only.
 - Use `lcm_expand` only after choosing a summary node. It expands bounded source summary nodes and source events, not an entire transcript.
 - Use `lcm_expand_query` for focused recursive evidence expansion when the query itself should pick the matching summary nodes.
 - Use `lcm_context_plan` for read-only context budget diagnostics and pack recommendations.

plugins/codex-lcm/src/hook.ts

@@ -24,7 +24,7 @@ export async function runHook(args: string[]): Promise<void> {
     env: process.env,
     repo,
   });
-  if (Buffer.byteLength(rawInput, "utf8") > config.limits.maxInputBytes) {
+  if (event.truncations.length > 0 || Buffer.byteLength(rawInput, "utf8") > config.limits.maxInputBytes) {
     const fullEvent = normalizeHookEvent({
       hookEvent,
       rawInput,

plugins/codex-lcm/src/mcp.ts

@@ -71,14 +71,15 @@ const TOOLS = [
   {
     name: "lcm_grep",
     title: "LCM Grep",
-    description: "Preferred standard workflow step 1 (grep): find relevant sessions by searching summary nodes, session summaries, and high-signal raw events. Codex may surface this tool as mcp__codex_lcm__lcm_grep.",
+    description: "Preferred standard workflow step 1 (grep): find relevant sessions in indexed memory, or search bounded sanitized overflow payloads when contentScope is overflow or both. Codex may surface this tool as mcp__codex_lcm__lcm_grep.",
     inputSchema: {
       type: "object",
       properties: {
         query: { type: "string" },
         limit: { type: "number", default: 10 },
         cwd: { type: "string" },
         repoRoot: { type: "string" },
+        contentScope: { type: "string", enum: ["memory", "overflow", "both"], default: "memory" },
         excludeCurrentSession: { type: "boolean", default: false },
         excludeSessionIds: { type: "array", items: { type: "string" } },
       },
@@ -96,6 +97,8 @@ const TOOLS = [
         nodeId: { type: "string" },
         fileId: { type: "string" },
         limit: { type: "number", default: 50 },
+        offset: { type: "number", default: 0, description: "Byte offset when reading an overflow reference." },
+        maxBytes: { type: "number", minimum: 4, maximum: 524288, default: 65536, description: "Maximum bytes to read from an overflow reference." },
         includeLineage: { type: "boolean", default: false, description: "Include full source ID arrays instead of compact source counts." },
       },
     },
@@ -457,28 +460,47 @@ function callTool(params: Record<string, unknown>) {
         return toolResult(`Captured ${usage.totals.total_tokens} tokens across ${usage.totals.sessions} sessions.`, { usage });
       }
       case "lcm_grep": {
-        const matches = storage.searchSessions({
-          query: optionalString(args.query),
-          limit: optionalNumber(args.limit),
-          cwd: optionalString(args.cwd),
-          repoRoot: optionalString(args.repoRoot),
-          excludeCurrentSession: optionalBoolean(args.excludeCurrentSession),
-          excludeSessionIds: optionalStringArray(args.excludeSessionIds),
-        });
-        return toolResult(`Found ${matches.length} LCM matches.`, { matches });
+        const scope = contentScope(args.contentScope);
+        const query = optionalString(args.query);
+        const matches = scope === "overflow"
+          ? []
+          : storage.searchSessions({
+            query,
+            limit: optionalNumber(args.limit),
+            cwd: optionalString(args.cwd),
+            repoRoot: optionalString(args.repoRoot),
+            excludeCurrentSession: optionalBoolean(args.excludeCurrentSession),
+            excludeSessionIds: optionalStringArray(args.excludeSessionIds),
+          });
+        const overflowMatches = scope === "memory"
+          ? []
+          : storage.searchOverflow({
+            query: query ?? "",
+            limit: optionalNumber(args.limit),
+            cwd: optionalString(args.cwd),
+            repoRoot: optionalString(args.repoRoot),
+          });
+        return toolResult(
+          `Found ${matches.length} LCM matches and ${overflowMatches.length} overflow matches.`,
+          { matches, overflow_matches: overflowMatches },
+        );
       }
       case "lcm_describe": {
         const description = storage.describeMemory({
           sessionId: optionalString(args.sessionId),
           nodeId: optionalString(args.nodeId),
           fileId: optionalString(args.fileId),
           limit: optionalNumber(args.limit),
+          offset: optionalNumber(args.offset),
+          maxBytes: optionalNumber(args.maxBytes),
         });
         const target = description.target === "session"
           ? description.session?.session_id ?? args.sessionId
           : description.target === "summary_node"
             ? description.node.node_id
-            : description.file_ref.file_ref_id;
+            : description.target === "file_ref"
+              ? description.file_ref.file_ref_id
+              : description.overflow_ref.file_ref_id;
         return toolResult(`Described ${description.target} ${target}.`, {
           description: optionalBoolean(args.includeLineage) ? description : compactDescription(description),
         });
@@ -618,7 +640,7 @@ function withoutMarkdown<T extends { markdown: string }>(value: T): Omit<T, "mar
 }
 
 function compactDescription(description: LcmDescription): unknown {
-  if (description.target === "file_ref") return description;
+  if (description.target === "file_ref" || description.target === "overflow_ref") return description;
   if (description.target === "summary_node") {
     return {
       ...description,
@@ -678,6 +700,12 @@ function optionalStringArray(value: unknown): string[] | undefined {
   return value.map((item) => item.trim());
 }
 
+function contentScope(value: unknown): "memory" | "overflow" | "both" {
+  if (value === undefined) return "memory";
+  if (value === "memory" || value === "overflow" || value === "both") return value;
+  throw new Error("contentScope must be memory, overflow, or both.");
+}
+
 function currentThreadId(): string | undefined {
   return optionalString(process.env.CODEX_THREAD_ID);
 }

plugins/codex-lcm/src/overflow.ts

@@ -0,0 +1,148 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import type { NormalizedEvent } from "./events.ts";
+import { sha256 } from "./redact.ts";
+
+export const DEFAULT_OVERFLOW_READ_BYTES = 64 * 1024;
+export const MAX_OVERFLOW_READ_BYTES = 512 * 1024;
+
+export type OverflowReference = {
+  file_ref_id: string;
+  session_id: string;
+  observed_event_id: string;
+  timestamp: string;
+  path: string;
+  sha256: string;
+  byte_count: number;
+  sanitized_byte_count: number;
+};
+
+export type OverflowContent = OverflowReference & {
+  offset: number;
+  content: string;
+  next_offset?: number;
+};
+
+export type OverflowSearchMatch = {
+  file_ref_id: string;
+  session_id: string;
+  timestamp: string;
+  byte_offset: number;
+  line_number: number;
+  snippet: string;
+  scan_truncated: boolean;
+};
+
+export function overflowReferenceFromEvent(event: NormalizedEvent): OverflowReference | undefined {
+  const value = event.payload.overflow_ref;
+  if (!isRecord(value)) return undefined;
+  const hash = stringValue(value.sha256);
+  const filePath = stringValue(value.path);
+  const byteCount = numberValue(value.byte_count);
+  const sanitizedByteCount = numberValue(value.sanitized_byte_count);
+  if (!hash || !/^[a-f0-9]{64}$/u.test(hash) || !filePath || byteCount === undefined || sanitizedByteCount === undefined) {
+    return undefined;
+  }
+  return {
+    file_ref_id: `overflow:${hash}`,
+    session_id: event.session_id,
+    observed_event_id: event.event_id,
+    timestamp: event.timestamp,
+    path: filePath,
+    sha256: hash,
+    byte_count: byteCount,
+    sanitized_byte_count: sanitizedByteCount,
+  };
+}
+
+export function readOverflowContent(args: {
+  overflowDir: string;
+  reference: OverflowReference;
+  offset?: number;
+  maxBytes?: number;
+}): OverflowContent {
+  const expectedPath = path.join(path.resolve(args.overflowDir), `${args.reference.sha256}.json`);
+  if (path.resolve(args.reference.path) !== expectedPath) {
+    throw new Error("Overflow reference is outside the managed overflow directory.");
+  }
+  const file = fs.openSync(expectedPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+  try {
+    const stat = fs.fstatSync(file);
+    if (!stat.isFile() || stat.size !== args.reference.sanitized_byte_count || stat.size > 16 * 1024 * 1024) {
+      throw new Error("Overflow reference does not match a bounded regular file.");
+    }
+    const buffer = fs.readFileSync(file);
+    if (sha256(buffer) !== args.reference.sha256) throw new Error("Overflow payload integrity check failed.");
+
+    const requestedOffset = clampInteger(args.offset, 0, buffer.length, 0);
+    const maxBytes = clampInteger(args.maxBytes, 4, MAX_OVERFLOW_READ_BYTES, DEFAULT_OVERFLOW_READ_BYTES);
+    const offset = nextUtf8Boundary(buffer, requestedOffset);
+    const end = previousUtf8Boundary(buffer, Math.min(buffer.length, offset + maxBytes));
+    return {
+      ...args.reference,
+      offset,
+      content: buffer.subarray(offset, end).toString("utf8"),
+      ...(end < buffer.length ? { next_offset: end } : {}),
+    };
+  } finally {
+    fs.closeSync(file);
+  }
+}
+
+export function searchOverflowContent(args: {
+  overflowDir: string;
+  reference: OverflowReference;
+  query: string;
+}): OverflowSearchMatch | undefined {
+  const chunk = readOverflowContent({
+    overflowDir: args.overflowDir,
+    reference: args.reference,
+    maxBytes: MAX_OVERFLOW_READ_BYTES,
+  });
+  const query = args.query.trim();
+  if (query.length === 0) return undefined;
+  const index = chunk.content.toLowerCase().indexOf(query.toLowerCase());
+  if (index < 0) return undefined;
+  const byteOffset = Buffer.byteLength(chunk.content.slice(0, index), "utf8");
+  const snippetStart = Math.max(0, index - 120);
+  const snippetEnd = Math.min(chunk.content.length, index + query.length + 120);
+  return {
+    file_ref_id: args.reference.file_ref_id,
+    session_id: args.reference.session_id,
+    timestamp: args.reference.timestamp,
+    byte_offset: byteOffset,
+    line_number: chunk.content.slice(0, index).split(/\r?\n/u).length,
+    snippet: chunk.content.slice(snippetStart, snippetEnd).replace(/\s+/gu, " ").trim(),
+    scan_truncated: chunk.next_offset !== undefined,
+  };
+}
+
+function nextUtf8Boundary(buffer: Buffer, offset: number): number {
+  let index = offset;
+  while (index < buffer.length && (buffer[index] & 0xc0) === 0x80) index += 1;
+  return index;
+}
+
+function previousUtf8Boundary(buffer: Buffer, offset: number): number {
+  let index = offset;
+  while (index > 0 && index < buffer.length && (buffer[index] & 0xc0) === 0x80) index -= 1;
+  return index;
+}
+
+function clampInteger(value: number | undefined, min: number, max: number, fallback: number): number {
+  if (value === undefined || !Number.isFinite(value)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(value)));
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringValue(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function numberValue(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+}