diff --git a/plugins/codex-lcm/skills/lcm-recall/SKILL.md b/plugins/codex-lcm/skills/lcm-recall/SKILL.md index 39d3525..4868834 100644 --- a/plugins/codex-lcm/skills/lcm-recall/SKILL.md +++ b/plugins/codex-lcm/skills/lcm-recall/SKILL.md @@ -1,6 +1,6 @@ --- name: lcm-recall -description: Use when a task may depend on prior Codex session context, long-running work, compaction recovery, projectless memory, or a user asks what happened before. +description: Recover or resume Codex work after compaction, interruption, or handoff with lcm_pack_context; also use for prior session context, long-running work, projectless memory, or questions about what happened before. --- # LCM Recall diff --git a/plugins/codex-lcm/src/event-codec.ts b/plugins/codex-lcm/src/event-codec.ts new file mode 100644 index 0000000..faa3dba --- /dev/null +++ b/plugins/codex-lcm/src/event-codec.ts @@ -0,0 +1,61 @@ +import type { NormalizedEvent } from "./events.ts"; + +export class PersistedEventError extends Error { + constructor() { + super("Persisted event does not match schema version 1."); + this.name = "PersistedEventError"; + } +} + +export function decodePersistedEvent(serialized: string): NormalizedEvent { + const event = parsePersistedEvent(serialized); + if (!event) throw new PersistedEventError(); + return event; +} + +export function parsePersistedEvent(serialized: string): NormalizedEvent | undefined { + let value: unknown; + try { + value = JSON.parse(serialized); + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } + return isNormalizedEvent(value) ? value : undefined; +} + +function isNormalizedEvent(value: unknown): value is NormalizedEvent { + if (!isRecord(value)) return false; + return value.schema_version === 1 + && isString(value.event_id) + && isString(value.timestamp) + && isString(value.hook_event) + && isString(value.session_id) + && isString(value.cwd) + && isOptionalString(value.project) + && isOptionalString(value.repo_root) + && isOptionalString(value.git_branch) + && isOptionalString(value.tool_name) + && isRecord(value.payload) + && Array.isArray(value.redactions) + && Array.isArray(value.truncations) + && isString(value.raw_input_sha256) + && isNonNegativeNumber(value.original_bytes) + && isNonNegativeNumber(value.sanitized_bytes); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +function isOptionalString(value: unknown): value is string | undefined { + return value === undefined || isString(value); +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} diff --git a/plugins/codex-lcm/src/mcp-catalog.ts b/plugins/codex-lcm/src/mcp-catalog.ts new file mode 100644 index 0000000..a7b8ccb --- /dev/null +++ b/plugins/codex-lcm/src/mcp-catalog.ts @@ -0,0 +1,266 @@ +// allow: SIZE_OK — declarative MCP tool schema table. +export const TOOLS = [ + { + name: "lcm_health", + title: "LCM Health", + description: "Report Codex LCM storage and index health.", + inputSchema: { type: "object", properties: {} }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_stats", + title: "LCM Stats", + description: "Report aggregate LCM index shape, summary depths, graph counts, and freshness without raw transcript text.", + inputSchema: { type: "object", properties: {} }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_list_sessions", + title: "LCM List Sessions", + description: "List sessions across projects with time, root/child, pagination, and optional compact summary filters.", + inputSchema: { + type: "object", + properties: { + since: { type: "string" }, + until: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + parentSessionId: { type: "string" }, + rootsOnly: { type: "boolean", default: false }, + includeSummaries: { type: "boolean", default: false, description: "Include compact titles, overviews, prompts, outcomes, and topics in this one response." }, + limit: { type: "number", default: 50 }, + cursor: { type: "string" }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_usage", + title: "LCM Usage", + description: "Aggregate captured Codex token usage across filtered sessions. With rootsOnly, selected roots include all descendant sessions.", + inputSchema: { + type: "object", + properties: { + since: { type: "string" }, + until: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + parentSessionId: { type: "string" }, + rootsOnly: { type: "boolean", default: false }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_grep", + title: "LCM Grep", + description: "Preferred standard workflow step 1 (grep): find relevant sessions in indexed memory, or search bounded sanitized overflow payloads when contentScope is overflow or both. Codex may surface this tool as mcp__codex_lcm__lcm_grep.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + limit: { type: "number", default: 10 }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + contentScope: { type: "string", enum: ["memory", "overflow", "both"], default: "memory" }, + excludeCurrentSession: { type: "boolean", default: false }, + excludeSessionIds: { type: "array", items: { type: "string" } }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_describe", + title: "LCM Describe", + description: "Preferred standard workflow step 2 (describe): inspect a session, summary node, or file reference, including summary-node depth and source lineage metadata. Codex may surface this tool as mcp__codex_lcm__lcm_describe.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + nodeId: { type: "string" }, + fileId: { type: "string" }, + limit: { type: "number", default: 50 }, + offset: { type: "number", default: 0, description: "Byte offset when reading an overflow reference." }, + maxBytes: { type: "number", minimum: 4, maximum: 524288, default: 65536, description: "Maximum bytes to read from an overflow reference." }, + includeLineage: { type: "boolean", default: false, description: "Include full source ID arrays instead of compact source counts." }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_expand", + title: "LCM Expand", + description: "Preferred standard workflow step 3 (expand): expand one summary node into bounded source summary nodes and high-signal source events. Codex may surface this tool as mcp__codex_lcm__lcm_expand.", + inputSchema: { + type: "object", + properties: { + nodeId: { type: "string" }, + query: { type: "string" }, + limit: { type: "number", default: 4 }, + }, + required: ["nodeId"], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_expand_query", + title: "LCM Expand Query", + description: "Find matching summary nodes and recursively expand their source lineage into bounded evidence for a focused query.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + sessionIds: { type: "array", items: { type: "string" } }, + budgetTokens: { type: "number", default: 2000 }, + limit: { type: "number", default: 4 }, + sourceLimit: { type: "number", default: 6, description: "Maximum source events or source summary nodes considered per matched node." }, + overview: { type: "boolean", default: false, description: "Prefer higher-depth, source-rich summary nodes for broad overview queries." }, + }, + required: ["query"], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_context_plan", + title: "LCM Context Plan", + description: "Estimate recent-session token pressure and report compaction proximity. Returns diagnostics only; Codex owns compaction.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + modelContextWindow: { type: "number", default: 128000 }, + autoCompactTokenLimit: { type: "number", default: 96000 }, + recentEventLimit: { type: "number", default: 80 }, + canControlCompaction: { + type: "boolean", + const: false, + default: false, + description: "Always false; this tool reports context pressure but cannot own Codex compaction.", + }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_current_session", + title: "LCM Current Session", + description: "Find the current or latest known session by session ID, cwd, or repo root.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_search_sessions", + title: "LCM Search Sessions", + description: "Search across Codex sessions with SQLite FTS.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + limit: { type: "number", default: 10 }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + excludeCurrentSession: { type: "boolean", default: false }, + excludeSessionIds: { type: "array", items: { type: "string" } }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_get_session", + title: "LCM Get Session", + description: "Retrieve sanitized raw events for a session, optionally paged for long sessions.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + limit: { type: "number" }, + cursor: { type: "string" }, + }, + required: ["sessionId"], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_get_session_summary", + title: "LCM Get Session Summary", + description: "Retrieve the deterministic extractive summary for a session, including topics and source event pointers.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + }, + required: ["sessionId"], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_get_session_graph", + title: "LCM Get Session Graph", + description: "Retrieve a bounded DAG slice for a session, including session, turn, event, checkpoint, and summary nodes.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + limit: { type: "number", default: 200 }, + }, + required: ["sessionId"], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_get_recent_context", + title: "LCM Get Recent Context", + description: "Retrieve recent events for a session or latest cwd-matching session.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + cwd: { type: "string" }, + repoRoot: { type: "string" }, + limit: { type: "number", default: 20 }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_pack_context", + title: "LCM Pack Context", + description: "Recover and resume prior Codex work after compaction, interruption, or handoff. Returns model-ready, token-budgeted Markdown from relevant session summaries and source evidence.", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + sessionIds: { type: "array", items: { type: "string" } }, + budgetTokens: { type: "number", default: 1200 }, + cwd: { type: "string" }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "lcm_record_note", + title: "LCM Record Note", + description: "Record a user-authored note as a first-class LCM event.", + inputSchema: { + type: "object", + properties: { + sessionId: { type: "string" }, + cwd: { type: "string" }, + text: { type: "string" }, + }, + required: ["sessionId", "cwd", "text"], + }, + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, + }, +]; diff --git a/plugins/codex-lcm/src/mcp-result.ts b/plugins/codex-lcm/src/mcp-result.ts new file mode 100644 index 0000000..f59ac07 --- /dev/null +++ b/plugins/codex-lcm/src/mcp-result.ts @@ -0,0 +1,48 @@ +import type { LcmDescription, SummaryNode } from "./storage.ts"; + +export function toolResult(text: string, structuredContent: unknown) { + return { + content: [{ type: "text", text }], + structuredContent, + }; +} + +export function withoutMarkdown(value: T): Omit { + const { markdown: _markdown, ...rest } = value; + return rest; +} + +export function compactDescription(description: LcmDescription): unknown { + if (description.target === "file_ref" || description.target === "overflow_ref") return description; + if (description.target === "summary_node") { + return { + ...description, + node: compactSummaryNode(description.node), + source_nodes: description.source_nodes.map(compactSummaryNode), + }; + } + return { + ...description, + summary: description.summary ? compactSessionSummary(description.summary) : undefined, + summary_nodes: description.summary_nodes.map(compactSummaryNode), + }; +} + +function compactSummaryNode(node: SummaryNode): Omit & { + readonly source_count: number; + readonly source_event_count: number; +} { + const { source_ids: sourceIds, source_event_ids: sourceEventIds, ...rest } = node; + return { + ...rest, + source_count: sourceIds.length, + source_event_count: sourceEventIds.length, + }; +} + +function compactSessionSummary( + summary: T, +): Omit & { readonly source_event_count: number } { + const { source_event_ids: sourceEventIds, ...rest } = summary; + return { ...rest, source_event_count: sourceEventIds.length }; +} diff --git a/plugins/codex-lcm/src/mcp-tools.ts b/plugins/codex-lcm/src/mcp-tools.ts new file mode 100644 index 0000000..82cba3e --- /dev/null +++ b/plugins/codex-lcm/src/mcp-tools.ts @@ -0,0 +1,234 @@ +import { compactDescription, toolResult, withoutMarkdown } from "./mcp-result.ts"; +import { createStorage } from "./storage.ts"; + +export function callTool(params: Record) { + const name = stringArg(params.name, "name"); + const args = isRecord(params.arguments) ? params.arguments : {}; + const storage = createStorage({ readOnly: name !== "lcm_record_note" }); + try { + switch (name) { + case "lcm_health": { + const health = storage.health(); + return toolResult(`Codex LCM has ${health.event_count} events across ${health.session_count} sessions.`, { health }); + } + case "lcm_stats": { + const stats = storage.stats(); + return toolResult( + `Codex LCM has ${stats.event_count} events, ${stats.summary_node_count ?? 0} summary nodes, and ${stats.graph_node_count ?? 0} graph nodes.`, + { stats }, + ); + } + case "lcm_list_sessions": { + const page = storage.listSessions({ + since: optionalString(args.since), + until: optionalString(args.until), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + parentSessionId: optionalString(args.parentSessionId), + rootsOnly: optionalBoolean(args.rootsOnly), + includeSummaries: optionalBoolean(args.includeSummaries), + limit: optionalNumber(args.limit), + cursor: optionalString(args.cursor), + }); + return toolResult(`Loaded ${page.sessions.length} sessions.`, { page }); + } + case "lcm_usage": { + const usage = storage.usage({ + since: optionalString(args.since), + until: optionalString(args.until), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + parentSessionId: optionalString(args.parentSessionId), + rootsOnly: optionalBoolean(args.rootsOnly), + }); + return toolResult(`Captured ${usage.totals.total_tokens} tokens across ${usage.totals.sessions} sessions.`, { usage }); + } + case "lcm_grep": { + const scope = contentScope(args.contentScope); + const query = optionalString(args.query); + const matches = scope === "overflow" + ? [] + : storage.searchSessions({ + query, + limit: optionalNumber(args.limit), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), + excludeSessionIds: optionalStringArray(args.excludeSessionIds), + }); + const overflowMatches = scope === "memory" + ? [] + : storage.searchOverflow({ + query: query ?? "", + limit: optionalNumber(args.limit), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + }); + return toolResult( + `Found ${matches.length} LCM matches and ${overflowMatches.length} overflow matches.`, + { matches, overflow_matches: overflowMatches }, + ); + } + case "lcm_describe": { + const description = storage.describeMemory({ + sessionId: optionalString(args.sessionId), + nodeId: optionalString(args.nodeId), + fileId: optionalString(args.fileId), + limit: optionalNumber(args.limit), + offset: optionalNumber(args.offset), + maxBytes: optionalNumber(args.maxBytes), + }); + const target = description.target === "session" + ? description.session?.session_id ?? args.sessionId + : description.target === "summary_node" + ? description.node.node_id + : description.target === "file_ref" + ? description.file_ref.file_ref_id + : description.overflow_ref.file_ref_id; + return toolResult(`Described ${description.target} ${target}.`, { + description: optionalBoolean(args.includeLineage) ? description : compactDescription(description), + }); + } + case "lcm_expand": { + const expansion = storage.expandMemory({ + nodeId: stringArg(args.nodeId, "nodeId"), + query: optionalString(args.query), + limit: optionalNumber(args.limit), + }); + return toolResult(expansion.markdown, { expansion: withoutMarkdown(expansion) }); + } + case "lcm_expand_query": { + const expansion = storage.expandQuery({ + query: stringArg(args.query, "query"), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + sessionIds: optionalStringArray(args.sessionIds), + budgetTokens: optionalNumber(args.budgetTokens), + limit: optionalNumber(args.limit), + sourceLimit: optionalNumber(args.sourceLimit), + overview: optionalBoolean(args.overview), + }); + return toolResult(expansion.markdown, { expansion: withoutMarkdown(expansion) }); + } + case "lcm_context_plan": { + const plan = storage.getContextPlan({ + sessionId: optionalString(args.sessionId), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + modelContextWindow: optionalNumber(args.modelContextWindow), + autoCompactTokenLimit: optionalNumber(args.autoCompactTokenLimit), + recentEventLimit: optionalNumber(args.recentEventLimit), + }); + return toolResult(`Context plan state: ${plan.state}. ${plan.recommendation}`, { plan }); + } + case "lcm_current_session": { + const session = storage.getCurrentSession({ + sessionId: optionalString(args.sessionId), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + }); + return toolResult(session ? `Current session: ${session.session_id}` : "No matching session found.", { session }); + } + case "lcm_search_sessions": { + const matches = storage.searchSessions({ + query: optionalString(args.query), + limit: optionalNumber(args.limit), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), + excludeSessionIds: optionalStringArray(args.excludeSessionIds), + }); + return toolResult(`Found ${matches.length} matching sessions.`, { matches }); + } + case "lcm_get_session": { + const session = storage.getSession(stringArg(args.sessionId, "sessionId"), { + limit: optionalNumber(args.limit), + cursor: optionalString(args.cursor), + }); + return toolResult(`Loaded ${session.events.length} events.`, session); + } + case "lcm_get_session_summary": { + const summary = storage.getSessionMemorySummary(stringArg(args.sessionId, "sessionId")); + return toolResult(summary ? `Loaded summary for ${summary.session_id}.` : "No summary found.", { summary }); + } + case "lcm_get_session_graph": { + const graph = storage.getSessionGraph(stringArg(args.sessionId, "sessionId"), { + limit: optionalNumber(args.limit), + }); + return toolResult(`Loaded graph with ${graph.nodes.length} nodes and ${graph.edges.length} edges.`, graph); + } + case "lcm_get_recent_context": { + const context = storage.getRecentContext({ + sessionId: optionalString(args.sessionId), + cwd: optionalString(args.cwd), + repoRoot: optionalString(args.repoRoot), + limit: optionalNumber(args.limit), + }); + return toolResult(`Loaded ${context.events.length} recent events.`, context); + } + case "lcm_pack_context": { + const packed = storage.packContext({ + query: optionalString(args.query), + sessionIds: optionalStringArray(args.sessionIds), + currentThreadId: currentThreadId(), + budgetTokens: optionalNumber(args.budgetTokens), + cwd: optionalString(args.cwd), + }); + return toolResult("Packed context is in structuredContent.markdown.", packed); + } + case "lcm_record_note": { + const event = storage.recordNote({ + sessionId: stringArg(args.sessionId, "sessionId"), + cwd: stringArg(args.cwd, "cwd"), + text: stringArg(args.text, "text"), + }); + return toolResult(`Recorded note for ${event.session_id}.`, { event }); + } + default: + throw new Error(`Unknown tool: ${name}`); + } + } finally { + storage.close(); + } +} + +function stringArg(value: unknown, name: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${name} must be a non-empty string.`); + } + return value.trim(); +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function optionalNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function optionalBoolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +function optionalStringArray(value: unknown): string[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim().length === 0)) { + throw new Error("value must be an array of non-empty strings."); + } + return value.map((item) => item.trim()); +} + +function contentScope(value: unknown): "memory" | "overflow" | "both" { + if (value === undefined) return "memory"; + if (value === "memory" || value === "overflow" || value === "both") return value; + throw new Error("contentScope must be memory, overflow, or both."); +} + +function currentThreadId(): string | undefined { + return optionalString(process.env.CODEX_THREAD_ID); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/plugins/codex-lcm/src/mcp.ts b/plugins/codex-lcm/src/mcp.ts index 3aaef2a..fb9836c 100644 --- a/plugins/codex-lcm/src/mcp.ts +++ b/plugins/codex-lcm/src/mcp.ts @@ -1,5 +1,6 @@ import { DEFAULT_LIMITS } from "./config.ts"; -import { createStorage, type LcmDescription, type SummaryNode } from "./storage.ts"; +import { TOOLS } from "./mcp-catalog.ts"; +import { callTool } from "./mcp-tools.ts"; type JsonRpcRequest = { readonly jsonrpc: "2.0"; @@ -16,272 +17,6 @@ const MAX_MESSAGE_BYTES = DEFAULT_LIMITS.maxInputBytes; let responseFraming: "line" | "header" = "line"; -const TOOLS = [ - { - name: "lcm_health", - title: "LCM Health", - description: "Report Codex LCM storage and index health.", - inputSchema: { type: "object", properties: {} }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_stats", - title: "LCM Stats", - description: "Report aggregate LCM index shape, summary depths, graph counts, and freshness without raw transcript text.", - inputSchema: { type: "object", properties: {} }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_list_sessions", - title: "LCM List Sessions", - description: "List sessions across projects with time, root/child, pagination, and optional compact summary filters.", - inputSchema: { - type: "object", - properties: { - since: { type: "string" }, - until: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - parentSessionId: { type: "string" }, - rootsOnly: { type: "boolean", default: false }, - includeSummaries: { type: "boolean", default: false, description: "Include compact titles, overviews, prompts, outcomes, and topics in this one response." }, - limit: { type: "number", default: 50 }, - cursor: { type: "string" }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_usage", - title: "LCM Usage", - description: "Aggregate captured Codex token usage across filtered sessions. With rootsOnly, selected roots include all descendant sessions.", - inputSchema: { - type: "object", - properties: { - since: { type: "string" }, - until: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - parentSessionId: { type: "string" }, - rootsOnly: { type: "boolean", default: false }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_grep", - title: "LCM Grep", - description: "Preferred standard workflow step 1 (grep): find relevant sessions in indexed memory, or search bounded sanitized overflow payloads when contentScope is overflow or both. Codex may surface this tool as mcp__codex_lcm__lcm_grep.", - inputSchema: { - type: "object", - properties: { - query: { type: "string" }, - limit: { type: "number", default: 10 }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - contentScope: { type: "string", enum: ["memory", "overflow", "both"], default: "memory" }, - excludeCurrentSession: { type: "boolean", default: false }, - excludeSessionIds: { type: "array", items: { type: "string" } }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_describe", - title: "LCM Describe", - description: "Preferred standard workflow step 2 (describe): inspect a session, summary node, or file reference, including summary-node depth and source lineage metadata. Codex may surface this tool as mcp__codex_lcm__lcm_describe.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - nodeId: { type: "string" }, - fileId: { type: "string" }, - limit: { type: "number", default: 50 }, - offset: { type: "number", default: 0, description: "Byte offset when reading an overflow reference." }, - maxBytes: { type: "number", minimum: 4, maximum: 524288, default: 65536, description: "Maximum bytes to read from an overflow reference." }, - includeLineage: { type: "boolean", default: false, description: "Include full source ID arrays instead of compact source counts." }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_expand", - title: "LCM Expand", - description: "Preferred standard workflow step 3 (expand): expand one summary node into bounded source summary nodes and high-signal source events. Codex may surface this tool as mcp__codex_lcm__lcm_expand.", - inputSchema: { - type: "object", - properties: { - nodeId: { type: "string" }, - query: { type: "string" }, - limit: { type: "number", default: 4 }, - }, - required: ["nodeId"], - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_expand_query", - title: "LCM Expand Query", - description: "Find matching summary nodes and recursively expand their source lineage into bounded evidence for a focused query.", - inputSchema: { - type: "object", - properties: { - query: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - sessionIds: { type: "array", items: { type: "string" } }, - budgetTokens: { type: "number", default: 2000 }, - limit: { type: "number", default: 4 }, - sourceLimit: { type: "number", default: 6, description: "Maximum source events or source summary nodes considered per matched node." }, - overview: { type: "boolean", default: false, description: "Prefer higher-depth, source-rich summary nodes for broad overview queries." }, - }, - required: ["query"], - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_context_plan", - title: "LCM Context Plan", - description: "Estimate recent-session token pressure and recommend whether to pack LCM context. This observes pressure only; Codex owns compaction.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - modelContextWindow: { type: "number", default: 128000 }, - autoCompactTokenLimit: { type: "number", default: 96000 }, - recentEventLimit: { type: "number", default: 80 }, - canControlCompaction: { - type: "boolean", - const: false, - default: false, - description: "Always false; this tool reports context pressure but cannot own Codex compaction.", - }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_current_session", - title: "LCM Current Session", - description: "Find the current or latest known session by session ID, cwd, or repo root.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_search_sessions", - title: "LCM Search Sessions", - description: "Search across Codex sessions with SQLite FTS.", - inputSchema: { - type: "object", - properties: { - query: { type: "string" }, - limit: { type: "number", default: 10 }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - excludeCurrentSession: { type: "boolean", default: false }, - excludeSessionIds: { type: "array", items: { type: "string" } }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_get_session", - title: "LCM Get Session", - description: "Retrieve sanitized raw events for a session, optionally paged for long sessions.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - limit: { type: "number" }, - cursor: { type: "string" }, - }, - required: ["sessionId"], - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_get_session_summary", - title: "LCM Get Session Summary", - description: "Retrieve the deterministic extractive summary for a session, including topics and source event pointers.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - }, - required: ["sessionId"], - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_get_session_graph", - title: "LCM Get Session Graph", - description: "Retrieve a bounded DAG slice for a session, including session, turn, event, checkpoint, and summary nodes.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - limit: { type: "number", default: 200 }, - }, - required: ["sessionId"], - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_get_recent_context", - title: "LCM Get Recent Context", - description: "Retrieve recent events for a session or latest cwd-matching session.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - cwd: { type: "string" }, - repoRoot: { type: "string" }, - limit: { type: "number", default: 20 }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_pack_context", - title: "LCM Pack Context", - description: "Pack relevant summary nodes and bounded source lineage into a token-budgeted Markdown context block.", - inputSchema: { - type: "object", - properties: { - query: { type: "string" }, - sessionIds: { type: "array", items: { type: "string" } }, - budgetTokens: { type: "number", default: 1200 }, - cwd: { type: "string" }, - }, - }, - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, - }, - { - name: "lcm_record_note", - title: "LCM Record Note", - description: "Record a user-authored note as a first-class LCM event.", - inputSchema: { - type: "object", - properties: { - sessionId: { type: "string" }, - cwd: { type: "string" }, - text: { type: "string" }, - }, - required: ["sessionId", "cwd", "text"], - }, - annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, - }, -]; - export function startMcpServer(): void { let buffer: Buffer = Buffer.alloc(0); process.stdin.on("data", (chunk: Buffer | string) => { @@ -417,196 +152,6 @@ function handleMessage(message: JsonRpcRequest): void { if (id !== undefined) sendError(id, -32601, `Method not found: ${method ?? ""}`); } -function callTool(params: Record) { - const name = stringArg(params.name, "name"); - const args = isRecord(params.arguments) ? params.arguments : {}; - const storage = createStorage({ readOnly: name !== "lcm_record_note" }); - try { - switch (name) { - case "lcm_health": { - const health = storage.health(); - return toolResult(`Codex LCM has ${health.event_count} events across ${health.session_count} sessions.`, { health }); - } - case "lcm_stats": { - const stats = storage.stats(); - return toolResult( - `Codex LCM has ${stats.event_count} events, ${stats.summary_node_count ?? 0} summary nodes, and ${stats.graph_node_count ?? 0} graph nodes.`, - { stats }, - ); - } - case "lcm_list_sessions": { - const page = storage.listSessions({ - since: optionalString(args.since), - until: optionalString(args.until), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - parentSessionId: optionalString(args.parentSessionId), - rootsOnly: optionalBoolean(args.rootsOnly), - includeSummaries: optionalBoolean(args.includeSummaries), - limit: optionalNumber(args.limit), - cursor: optionalString(args.cursor), - }); - return toolResult(`Loaded ${page.sessions.length} sessions.`, { page }); - } - case "lcm_usage": { - const usage = storage.usage({ - since: optionalString(args.since), - until: optionalString(args.until), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - parentSessionId: optionalString(args.parentSessionId), - rootsOnly: optionalBoolean(args.rootsOnly), - }); - return toolResult(`Captured ${usage.totals.total_tokens} tokens across ${usage.totals.sessions} sessions.`, { usage }); - } - case "lcm_grep": { - const scope = contentScope(args.contentScope); - const query = optionalString(args.query); - const matches = scope === "overflow" - ? [] - : storage.searchSessions({ - query, - limit: optionalNumber(args.limit), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), - excludeSessionIds: optionalStringArray(args.excludeSessionIds), - }); - const overflowMatches = scope === "memory" - ? [] - : storage.searchOverflow({ - query: query ?? "", - limit: optionalNumber(args.limit), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - }); - return toolResult( - `Found ${matches.length} LCM matches and ${overflowMatches.length} overflow matches.`, - { matches, overflow_matches: overflowMatches }, - ); - } - case "lcm_describe": { - const description = storage.describeMemory({ - sessionId: optionalString(args.sessionId), - nodeId: optionalString(args.nodeId), - fileId: optionalString(args.fileId), - limit: optionalNumber(args.limit), - offset: optionalNumber(args.offset), - maxBytes: optionalNumber(args.maxBytes), - }); - const target = description.target === "session" - ? description.session?.session_id ?? args.sessionId - : description.target === "summary_node" - ? description.node.node_id - : description.target === "file_ref" - ? description.file_ref.file_ref_id - : description.overflow_ref.file_ref_id; - return toolResult(`Described ${description.target} ${target}.`, { - description: optionalBoolean(args.includeLineage) ? description : compactDescription(description), - }); - } - case "lcm_expand": { - const expansion = storage.expandMemory({ - nodeId: stringArg(args.nodeId, "nodeId"), - query: optionalString(args.query), - limit: optionalNumber(args.limit), - }); - return toolResult(expansion.markdown, { expansion: withoutMarkdown(expansion) }); - } - case "lcm_expand_query": { - const expansion = storage.expandQuery({ - query: stringArg(args.query, "query"), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - sessionIds: optionalStringArray(args.sessionIds), - budgetTokens: optionalNumber(args.budgetTokens), - limit: optionalNumber(args.limit), - sourceLimit: optionalNumber(args.sourceLimit), - overview: optionalBoolean(args.overview), - }); - return toolResult(expansion.markdown, { expansion: withoutMarkdown(expansion) }); - } - case "lcm_context_plan": { - const plan = storage.getContextPlan({ - sessionId: optionalString(args.sessionId), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - modelContextWindow: optionalNumber(args.modelContextWindow), - autoCompactTokenLimit: optionalNumber(args.autoCompactTokenLimit), - recentEventLimit: optionalNumber(args.recentEventLimit), - }); - return toolResult(`Context plan state: ${plan.state}. ${plan.recommendation}`, { plan }); - } - case "lcm_current_session": { - const session = storage.getCurrentSession({ - sessionId: optionalString(args.sessionId), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - }); - return toolResult(session ? `Current session: ${session.session_id}` : "No matching session found.", { session }); - } - case "lcm_search_sessions": { - const matches = storage.searchSessions({ - query: optionalString(args.query), - limit: optionalNumber(args.limit), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - excludeCurrentSession: optionalBoolean(args.excludeCurrentSession), - excludeSessionIds: optionalStringArray(args.excludeSessionIds), - }); - return toolResult(`Found ${matches.length} matching sessions.`, { matches }); - } - case "lcm_get_session": { - const session = storage.getSession(stringArg(args.sessionId, "sessionId"), { - limit: optionalNumber(args.limit), - cursor: optionalString(args.cursor), - }); - return toolResult(`Loaded ${session.events.length} events.`, session); - } - case "lcm_get_session_summary": { - const summary = storage.getSessionMemorySummary(stringArg(args.sessionId, "sessionId")); - return toolResult(summary ? `Loaded summary for ${summary.session_id}.` : "No summary found.", { summary }); - } - case "lcm_get_session_graph": { - const graph = storage.getSessionGraph(stringArg(args.sessionId, "sessionId"), { - limit: optionalNumber(args.limit), - }); - return toolResult(`Loaded graph with ${graph.nodes.length} nodes and ${graph.edges.length} edges.`, graph); - } - case "lcm_get_recent_context": { - const context = storage.getRecentContext({ - sessionId: optionalString(args.sessionId), - cwd: optionalString(args.cwd), - repoRoot: optionalString(args.repoRoot), - limit: optionalNumber(args.limit), - }); - return toolResult(`Loaded ${context.events.length} recent events.`, context); - } - case "lcm_pack_context": { - const packed = storage.packContext({ - query: optionalString(args.query), - sessionIds: optionalStringArray(args.sessionIds), - currentThreadId: currentThreadId(), - budgetTokens: optionalNumber(args.budgetTokens), - cwd: optionalString(args.cwd), - }); - return toolResult("Packed context is in structuredContent.markdown.", packed); - } - case "lcm_record_note": { - const event = storage.recordNote({ - sessionId: stringArg(args.sessionId, "sessionId"), - cwd: stringArg(args.cwd, "cwd"), - text: stringArg(args.text, "text"), - }); - return toolResult(`Recorded note for ${event.session_id}.`, { event }); - } - default: - throw new Error(`Unknown tool: ${name}`); - } - } finally { - storage.close(); - } -} function send(message: unknown): void { const body = JSON.stringify(message); @@ -627,89 +172,6 @@ function sendError(id: JsonRpcRequest["id"], code: number, message: string): voi send({ jsonrpc: "2.0", id, error: { code, message } }); } -function toolResult(text: string, structuredContent: unknown) { - return { - content: [{ type: "text", text }], - structuredContent, - }; -} - -function withoutMarkdown(value: T): Omit { - const { markdown: _markdown, ...rest } = value; - return rest; -} - -function compactDescription(description: LcmDescription): unknown { - if (description.target === "file_ref" || description.target === "overflow_ref") return description; - if (description.target === "summary_node") { - return { - ...description, - node: compactSummaryNode(description.node), - source_nodes: description.source_nodes.map(compactSummaryNode), - }; - } - const summary = description.summary; - return { - ...description, - summary: summary ? compactSessionSummary(summary) : undefined, - summary_nodes: description.summary_nodes.map(compactSummaryNode), - }; -} - -function compactSummaryNode(node: SummaryNode): Omit & { - source_count: number; - source_event_count: number; -} { - const { source_ids: sourceIds, source_event_ids: sourceEventIds, ...rest } = node; - return { - ...rest, - source_count: sourceIds.length, - source_event_count: sourceEventIds.length, - }; -} - -function compactSessionSummary(summary: T): Omit & { source_event_count: number } { - const { source_event_ids: sourceEventIds, ...rest } = summary; - return { ...rest, source_event_count: sourceEventIds.length }; -} - -function stringArg(value: unknown, name: string): string { - if (typeof value !== "string" || value.trim().length === 0) { - throw new Error(`${name} must be a non-empty string.`); - } - return value.trim(); -} - -function optionalString(value: unknown): string | undefined { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; -} - -function optionalNumber(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) ? value : undefined; -} - -function optionalBoolean(value: unknown): boolean | undefined { - return typeof value === "boolean" ? value : undefined; -} - -function optionalStringArray(value: unknown): string[] | undefined { - if (value === undefined) return undefined; - if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim().length === 0)) { - throw new Error("value must be an array of non-empty strings."); - } - return value.map((item) => item.trim()); -} - -function contentScope(value: unknown): "memory" | "overflow" | "both" { - if (value === undefined) return "memory"; - if (value === "memory" || value === "overflow" || value === "both") return value; - throw new Error("contentScope must be memory, overflow, or both."); -} - -function currentThreadId(): string | undefined { - return optionalString(process.env.CODEX_THREAD_ID); -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/plugins/codex-lcm/src/raw-log.ts b/plugins/codex-lcm/src/raw-log.ts new file mode 100644 index 0000000..aaeba19 --- /dev/null +++ b/plugins/codex-lcm/src/raw-log.ts @@ -0,0 +1,54 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { parsePersistedEvent } from "./event-codec.ts"; +import type { NormalizedEvent } from "./events.ts"; + +export type RawLogReadResult = { + readonly events: NormalizedEvent[]; + readonly malformedLineCount: number; +}; + +export type RawLogState = { + readonly size: number; + readonly mtimeMs: number; + readonly ctimeMs: number; +}; + +export function appendRawEvents(rawLogPath: string, events: readonly NormalizedEvent[]): void { + if (events.length === 0) return; + fs.mkdirSync(path.dirname(rawLogPath), { recursive: true, mode: 0o700 }); + fs.appendFileSync(rawLogPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`, { mode: 0o600 }); +} + +export function readRawLog(rawLogPath: string): RawLogReadResult { + if (!fs.existsSync(rawLogPath)) return { events: [], malformedLineCount: 0 }; + const events: NormalizedEvent[] = []; + let malformedLineCount = 0; + for (const line of fs.readFileSync(rawLogPath, "utf8").split(/\r?\n/u)) { + if (line.trim().length === 0) continue; + const event = parsePersistedEvent(line); + if (event) events.push(event); + else malformedLineCount += 1; + } + return { events, malformedLineCount }; +} + +export function readRawEvents(rawLogPath: string): NormalizedEvent[] { + return readRawLog(rawLogPath).events; +} + +export function readRawEventIds(rawLogPath: string): Set { + return new Set(readRawEvents(rawLogPath).map((event) => event.event_id)); +} + +export function rawLogStat(rawLogPath: string): fs.Stats | undefined { + return fs.existsSync(rawLogPath) ? fs.statSync(rawLogPath) : undefined; +} + +export function rawLogState(rawLogPath: string): RawLogState { + const stat = rawLogStat(rawLogPath); + return stat + ? { size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs } + : { size: 0, mtimeMs: 0, ctimeMs: 0 }; +} diff --git a/plugins/codex-lcm/src/redact-assignments.ts b/plugins/codex-lcm/src/redact-assignments.ts index 607d1d9..6d4aaf5 100644 --- a/plugins/codex-lcm/src/redact-assignments.ts +++ b/plugins/codex-lcm/src/redact-assignments.ts @@ -1,7 +1,8 @@ import type { RedactionRecord } from "./redact.ts"; const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gu; -const SECRET_ASSIGNMENT_RE = /(^|[\s{,;])((?:"[^"\n:=]+"|'[^'\n:=]+'|[A-Za-z_][A-Za-z0-9_.-]*)\s*[:=]\s*)(?:"([^"\n]*)"|'([^'\n]*)'|((?:Bearer|Basic|Digest|Token|OAuth)\s+[^\s,;}]+|[^\s,;{}]+))/giu; +const ASSIGNMENT_PREFIX_RE = /(? { redactions.push({ path, reason: "token-pattern" }); @@ -66,49 +73,109 @@ export function shouldRedactSecretKey(key: string, value: unknown): boolean { function redactSecretAssignmentLine(value: string, path: string, redactions: RedactionRecord[]): string { if (!value.includes(":") && !value.includes("=")) return value; - return value.replace( - SECRET_ASSIGNMENT_RE, - ( - match: string, - prefix: string, - assignmentPrefix: string, - doubleQuotedValue: string | undefined, - singleQuotedValue: string | undefined, - bareValue: string | undefined, - ) => redactSecretAssignmentMatch({ - match, - prefix, - assignmentPrefix, - value: doubleQuotedValue ?? singleQuotedValue ?? bareValue ?? "", - quote: doubleQuotedValue !== undefined ? "\"" : singleQuotedValue !== undefined ? "'" : "", - path, - redactions, - }), + let output = ""; + let copiedUntil = 0; + const quoteContext: QuoteContext = { plain: undefined, escaped: undefined, offset: 0 }; + ASSIGNMENT_PREFIX_RE.lastIndex = 0; + for (let match = ASSIGNMENT_PREFIX_RE.exec(value); match; match = ASSIGNMENT_PREFIX_RE.exec(value)) { + advanceQuoteContext(value, match.index, quoteContext); + const key = assignmentKey(match[0]); + if (!isSecretAssignmentKey(key)) continue; + + const wrapper = quoteContext.escaped + ? { quote: quoteContext.escaped, escaped: true } + : quoteContext.plain + ? { quote: quoteContext.plain, escaped: false } + : undefined; + const bounds = assignmentValueBounds(value, ASSIGNMENT_PREFIX_RE.lastIndex, wrapper); + const assignmentValue = value.slice(bounds.start, bounds.end); + if (assignmentValue.length === 0 || + isBenignTokenMetricAssignment(key, assignmentValue) || + assignmentValue.startsWith("[REDACTED:secret]")) { + continue; + } + + output += value.slice(copiedUntil, bounds.start); + output += /^Bearer\s+/u.test(assignmentValue) + ? assignmentValue.replace(/^Bearer\s+[^\s"']+/u, "Bearer [REDACTED:token]") + : "[REDACTED:secret]"; + copiedUntil = bounds.end; + ASSIGNMENT_PREFIX_RE.lastIndex = bounds.end; + redactions.push({ path, reason: "token-pattern" }); + } + return output + value.slice(copiedUntil); +} + +function assignmentValueBounds( + value: string, + start: number, + wrapper?: { readonly quote: string; readonly escaped: boolean }, +): { + readonly start: number; + readonly end: number; +} { + const firstNonWhitespace = value.slice(start).search(/\S/u); + if (firstNonWhitespace > 0 && + !ASSIGNMENT_AT_START_RE.test(value.slice(start + firstNonWhitespace))) { + start += firstNonWhitespace; + } + const openingQuote = value[start]; + if (openingQuote === "\"" || openingQuote === "'") { + return { start: start + 1, end: closingQuoteIndex(value, start + 1, openingQuote, false) }; + } + if (openingQuote === "\\" && (value[start + 1] === "\"" || value[start + 1] === "'")) { + return { start: start + 2, end: closingQuoteIndex(value, start + 2, value[start + 1], true) }; + } + + const end = bareValueEnd( + value, + start, + wrapper?.quote, + wrapper?.escaped, ); + return { start, end }; +} + +function advanceQuoteContext(value: string, end: number, context: QuoteContext): void { + for (let index = context.offset; index < end; index += 1) { + const quote = value[index]; + if (quote !== "\"" && quote !== "'") continue; + if (hasOddBackslashPrefix(value, index)) { + context.escaped = context.escaped === quote ? undefined : context.escaped ?? quote; + } else { + context.plain = context.plain === quote ? undefined : context.plain ?? quote; + } + } + context.offset = end; } -function redactSecretAssignmentMatch(args: { - readonly match: string; - readonly prefix: string; - readonly assignmentPrefix: string; - readonly value: string; - readonly quote: string; - readonly path: string; - readonly redactions: RedactionRecord[]; -}): string { - const key = assignmentKey(args.assignmentPrefix); - if (!isSecretAssignmentKey(key)) return args.match; - if (isBenignTokenMetricAssignment(key, args.value)) return args.match; - if (args.value.startsWith("[REDACTED:secret]")) return args.match; - - if (/^Bearer\s+[^\s"']+/u.test(args.value)) { - args.redactions.push({ path: args.path, reason: "token-pattern" }); - return `${args.prefix}${args.assignmentPrefix}${args.quote}${args.value.replace(/^Bearer\s+[^\s"']+/u, "Bearer [REDACTED:token]")}${args.quote}`; +function closingQuoteIndex(value: string, start: number, quote: string, escaped: boolean): number { + for (let index = start; index < value.length; index += 1) { + if (value[index] !== quote) continue; + if (escaped === hasOddBackslashPrefix(value, index)) { + return escaped ? index - 1 : index; + } } + return value.length; +} + +function bareValueEnd(value: string, start: number, wrapperQuote?: string, escapedWrapper = false): number { + const scheme = /^(?:Bearer|Basic|Digest|Token|OAuth)\s+/iu.exec(value.slice(start)); + const scanStart = start + (scheme?.[0].length ?? 0); + for (let index = scanStart; index < value.length; index += 1) { + if (wrapperQuote && value[index] === wrapperQuote && + escapedWrapper === hasOddBackslashPrefix(value, index)) { + return escapedWrapper ? index - 1 : index; + } + if (/[\s,;{}]/u.test(value[index])) return index; + } + return value.length; +} - if (args.value.length === 0) return args.match; - args.redactions.push({ path: args.path, reason: "token-pattern" }); - return `${args.prefix}${args.assignmentPrefix}${args.quote}[REDACTED:secret]${args.quote}`; +function hasOddBackslashPrefix(value: string, index: number): boolean { + let count = 0; + while (index > count && value[index - count - 1] === "\\") count += 1; + return count % 2 === 1; } function assignmentKey(assignmentPrefix: string): string { @@ -121,8 +188,8 @@ function assignmentKey(assignmentPrefix: string): string { function isBenignTokenMetricAssignment(key: string, value: string): boolean { if (!isTokenMetricKey(key)) return false; - return /^[+-]?(?:\d+(?:\.\d+)?|\.\d+)(?:[eE][+-]?\d+)?[.!?]?\s*$/u.test(value) || - /^(?:true|false|null)[.!?]?\s*$/iu.test(value); + return /^[+-]?(?:\d+(?:\.\d+)?|\.\d+)(?:[eE][+-]?\d+)?[.!?]?(?:\\?["'])*\s*$/u.test(value) || + /^(?:true|false|null)[.!?]?(?:\\?["'])*\s*$/iu.test(value); } function isTokenMetricKey(key: string): boolean { diff --git a/plugins/codex-lcm/src/storage-rows.ts b/plugins/codex-lcm/src/storage-rows.ts new file mode 100644 index 0000000..d61935f --- /dev/null +++ b/plugins/codex-lcm/src/storage-rows.ts @@ -0,0 +1,172 @@ +import type { FileReference } from "./file-refs.ts"; +import type { GraphEdge, GraphNode, SessionSummary } from "./storage.ts"; +import type { SessionMemorySummary, SummaryNode } from "./summary.ts"; + +export function rowToSessionSummary(row: unknown): SessionSummary { + const record = recordValue(row); + return { + session_id: String(record.session_id), + first_seen: String(record.first_seen), + last_seen: String(record.last_seen), + cwd: String(record.cwd), + ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), + ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), + event_count: Number(record.event_count), + ...(record.parent_session_id ? { parent_session_id: String(record.parent_session_id) } : {}), + ...(record.agent_role ? { agent_role: String(record.agent_role) } : {}), + ...(record.agent_nickname ? { agent_nickname: String(record.agent_nickname) } : {}), + ...(record.model ? { model: String(record.model) } : {}), + ...(record.reasoning_effort ? { reasoning_effort: String(record.reasoning_effort) } : {}), + ...optionalNumber(record, "total_input_tokens"), + ...optionalNumber(record, "cached_input_tokens"), + ...optionalNumber(record, "output_tokens"), + ...optionalNumber(record, "reasoning_output_tokens"), + ...optionalNumber(record, "total_tokens"), + ...optionalNumber(record, "match_count"), + ...(record.summary_title ? { + summary: { + updated_at: String(record.summary_updated_at), + title: String(record.summary_title), + overview: String(record.summary_overview), + topics: parseStringArray(record.summary_topics_json), + key_prompts: parseStringArray(record.summary_key_prompts_json), + outcomes: parseStringArray(record.summary_outcomes_json), + source_event_count: parseStringArray(record.summary_source_event_ids_json).length, + }, + } : {}), + }; +} + +export function rowToSessionMemorySummary(row: unknown): SessionMemorySummary { + const record = recordValue(row); + return { + session_id: String(record.session_id), + updated_at: String(record.updated_at), + cwd: String(record.cwd), + ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), + ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), + title: String(record.title), + overview: String(record.overview), + topics: parseStringArray(record.topics_json), + key_prompts: parseStringArray(record.key_prompts_json), + outcomes: parseStringArray(record.outcomes_json), + tools: parseStringArray(record.tools_json), + source_event_ids: parseStringArray(record.source_event_ids_json), + }; +} + +export function rowToSummaryNode(row: unknown): SummaryNode { + const record = recordValue(row); + return { + node_id: String(record.node_id), + session_id: String(record.session_id), + depth: Number(record.depth), + summary_text: String(record.summary_text), + token_count: Number(record.token_count), + source_token_count: Number(record.source_token_count), + source_type: String(record.source_type) === "nodes" ? "nodes" : "events", + source_ids: parseStringArray(record.source_ids_json), + source_event_ids: parseStringArray(record.source_event_ids_json), + earliest_at: String(record.earliest_at), + latest_at: String(record.latest_at), + created_at: String(record.created_at), + cwd: String(record.cwd), + ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), + ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), + topics: parseStringArray(record.topics_json), + }; +} + +export function rowToFileReference(row: unknown): FileReference { + const record = recordValue(row); + return { + file_ref_id: String(record.file_ref_id), + session_id: String(record.session_id), + observed_event_id: String(record.observed_event_id), + timestamp: String(record.timestamp), + path: String(record.path), + mime_type: String(record.mime_type), + byte_count: Number(record.byte_count), + sha256: String(record.sha256), + exploration_summary: String(record.exploration_summary), + metadata: parseMetadata(record.metadata_json), + }; +} + +export function rowToGraphNode(row: unknown): GraphNode { + const record = recordValue(row); + return { + node_id: String(record.node_id), + kind: graphNodeKind(record.kind), + session_id: String(record.session_id), + ...(record.event_id ? { event_id: String(record.event_id) } : {}), + ...(record.turn_id ? { turn_id: String(record.turn_id) } : {}), + timestamp: String(record.timestamp), + cwd: String(record.cwd), + ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), + ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), + label: String(record.label), + metadata: parseMetadata(record.metadata_json), + }; +} + +export function rowToGraphEdge(row: unknown): GraphEdge { + const record = recordValue(row); + return { + from_node_id: String(record.from_node_id), + to_node_id: String(record.to_node_id), + kind: String(record.kind), + session_id: String(record.session_id), + position: Number(record.position), + created_at: String(record.created_at), + metadata: parseMetadata(record.metadata_json), + }; +} + +function optionalNumber(record: Record, key: string): Record { + const value = record[key]; + return value === null || value === undefined ? {} : { [key]: Number(value) }; +} + +function graphNodeKind(value: unknown): GraphNode["kind"] { + switch (value) { + case "session": + case "turn": + case "event": + case "checkpoint": + case "summary": + return value; + default: + throw new TypeError(`Invalid graph node kind: ${String(value)}`); + } +} + +function parseMetadata(value: unknown): Record { + if (typeof value !== "string") return {}; + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) ? parsed : {}; + } catch (error) { + if (error instanceof SyntaxError) return {}; + throw error; + } +} + +export function parseStringArray(value: unknown): string[] { + if (typeof value !== "string") return []; + try { + const parsed: unknown = JSON.parse(value); + return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : []; + } catch (error) { + if (error instanceof SyntaxError) return []; + throw error; + } +} + +function recordValue(value: unknown): Record { + return isRecord(value) ? value : {}; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/plugins/codex-lcm/src/storage-schema.ts b/plugins/codex-lcm/src/storage-schema.ts new file mode 100644 index 0000000..2beb8b5 --- /dev/null +++ b/plugins/codex-lcm/src/storage-schema.ts @@ -0,0 +1,202 @@ +import type { DatabaseSync } from "node:sqlite"; + +import { SUMMARY_ALGORITHM_VERSION, SUMMARY_NODE_VERSION } from "./summary.ts"; + +export type SchemaInitialization = { + readonly backfillSessionMetadata: boolean; +}; + +export function initializeStorageSchema(db: DatabaseSync): SchemaInitialization { + db.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT PRIMARY KEY, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + cwd TEXT NOT NULL, + repo_root TEXT, + git_branch TEXT, + event_count INTEGER NOT NULL DEFAULT 0, + parent_session_id TEXT, + agent_role TEXT, + agent_nickname TEXT, + model TEXT, + reasoning_effort TEXT, + total_input_tokens INTEGER, + cached_input_tokens INTEGER, + output_tokens INTEGER, + reasoning_output_tokens INTEGER, + total_tokens INTEGER + ); + CREATE TABLE IF NOT EXISTS events ( + event_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + hook_event TEXT NOT NULL, + cwd TEXT NOT NULL, + repo_root TEXT, + git_branch TEXT, + turn_id TEXT, + tool_use_id TEXT, + text TEXT NOT NULL DEFAULT '', + raw_json TEXT NOT NULL + ); + CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5( + event_id UNINDEXED, + session_id, + cwd, + repo_root, + hook_event, + content + ); + CREATE TABLE IF NOT EXISTS graph_nodes ( + node_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + session_id TEXT NOT NULL, + event_id TEXT, + turn_id TEXT, + timestamp TEXT NOT NULL, + cwd TEXT NOT NULL, + repo_root TEXT, + git_branch TEXT, + label TEXT NOT NULL, + metadata_json TEXT NOT NULL, + UNIQUE(event_id) + ); + CREATE TABLE IF NOT EXISTS graph_edges ( + from_node_id TEXT NOT NULL, + to_node_id TEXT NOT NULL, + kind TEXT NOT NULL, + session_id TEXT NOT NULL, + position INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (from_node_id, to_node_id, kind), + CHECK (from_node_id <> to_node_id) + ); + CREATE TABLE IF NOT EXISTS session_summaries ( + session_id TEXT PRIMARY KEY, + summary_version INTEGER NOT NULL DEFAULT ${SUMMARY_ALGORITHM_VERSION}, + updated_at TEXT NOT NULL, + cwd TEXT NOT NULL, + repo_root TEXT, + git_branch TEXT, + title TEXT NOT NULL, + overview TEXT NOT NULL, + topics_json TEXT NOT NULL, + key_prompts_json TEXT NOT NULL, + outcomes_json TEXT NOT NULL, + tools_json TEXT NOT NULL, + source_event_ids_json TEXT NOT NULL, + summary_text TEXT NOT NULL + ); + CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_fts USING fts5( + session_id UNINDEXED, + cwd, + repo_root, + content + ); + CREATE TABLE IF NOT EXISTS summary_nodes ( + node_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + summary_version INTEGER NOT NULL DEFAULT ${SUMMARY_NODE_VERSION}, + depth INTEGER NOT NULL, + summary_text TEXT NOT NULL, + token_count INTEGER NOT NULL, + source_token_count INTEGER NOT NULL, + source_type TEXT NOT NULL, + source_ids_json TEXT NOT NULL, + source_event_ids_json TEXT NOT NULL, + earliest_at TEXT NOT NULL, + latest_at TEXT NOT NULL, + created_at TEXT NOT NULL, + cwd TEXT NOT NULL, + repo_root TEXT, + git_branch TEXT, + topics_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS file_refs ( + file_ref_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + observed_event_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + path TEXT NOT NULL, + mime_type TEXT NOT NULL, + byte_count INTEGER NOT NULL, + sha256 TEXT NOT NULL, + exploration_summary TEXT NOT NULL, + metadata_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS index_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE VIRTUAL TABLE IF NOT EXISTS summary_node_fts USING fts5( + node_id UNINDEXED, + session_id, + cwd, + repo_root, + depth, + content + ); + CREATE INDEX IF NOT EXISTS idx_graph_nodes_session_kind_time ON graph_nodes(session_id, kind, timestamp); + CREATE INDEX IF NOT EXISTS idx_graph_nodes_event ON graph_nodes(event_id); + CREATE INDEX IF NOT EXISTS idx_graph_edges_from ON graph_edges(from_node_id, kind); + CREATE INDEX IF NOT EXISTS idx_graph_edges_to ON graph_edges(to_node_id, kind); + CREATE INDEX IF NOT EXISTS idx_graph_edges_session ON graph_edges(session_id, kind, position); + CREATE INDEX IF NOT EXISTS idx_session_summaries_updated ON session_summaries(updated_at); + CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_depth_latest ON summary_nodes(session_id, depth, latest_at); + CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_latest ON summary_nodes(session_id, latest_at); + CREATE INDEX IF NOT EXISTS idx_file_refs_session_time ON file_refs(session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_file_refs_path ON file_refs(path); + `); + ensureColumn(db, "events", "turn_id", "TEXT"); + ensureColumn(db, "events", "tool_use_id", "TEXT"); + ensureColumn(db, "session_summaries", "summary_version", "INTEGER"); + const backfillSessionMetadata = [ + ensureColumn(db, "sessions", "parent_session_id", "TEXT"), + ensureColumn(db, "sessions", "agent_role", "TEXT"), + ensureColumn(db, "sessions", "agent_nickname", "TEXT"), + ensureColumn(db, "sessions", "model", "TEXT"), + ensureColumn(db, "sessions", "reasoning_effort", "TEXT"), + ensureColumn(db, "sessions", "total_input_tokens", "INTEGER"), + ensureColumn(db, "sessions", "cached_input_tokens", "INTEGER"), + ensureColumn(db, "sessions", "output_tokens", "INTEGER"), + ensureColumn(db, "sessions", "reasoning_output_tokens", "INTEGER"), + ensureColumn(db, "sessions", "total_tokens", "INTEGER"), + ].some(Boolean); + db.exec(` + CREATE INDEX IF NOT EXISTS idx_events_session_turn ON events(session_id, turn_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_events_tool_use ON events(session_id, tool_use_id, hook_event, timestamp); + CREATE INDEX IF NOT EXISTS idx_events_session_hook_time ON events(session_id, hook_event, timestamp); + CREATE INDEX IF NOT EXISTS idx_events_session_time ON events(session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id, last_seen); + CREATE INDEX IF NOT EXISTS idx_sessions_last_seen ON sessions(last_seen); + `); + return { backfillSessionMetadata }; +} + +function ensureColumn(db: DatabaseSync, table: string, column: string, type: string): boolean { + const tableName = sqlIdentifier(table); + const columnName = sqlIdentifier(column); + const columnType = sqlColumnType(type); + const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() + .map((row) => String(row.name)); + if (columns.includes(columnName)) return false; + db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${columnType}`); + return true; +} + +function sqlIdentifier(value: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(value)) { + throw new TypeError(`Invalid SQL identifier: ${value}`); + } + return value; +} + +function sqlColumnType(value: string): string { + if (value !== "TEXT" && value !== "INTEGER") { + throw new TypeError(`Invalid SQL column type: ${value}`); + } + return value; +} diff --git a/plugins/codex-lcm/src/storage.ts b/plugins/codex-lcm/src/storage.ts index d3d1730..6e8f575 100644 --- a/plugins/codex-lcm/src/storage.ts +++ b/plugins/codex-lcm/src/storage.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; -import path from "node:path"; import { DatabaseSync } from "node:sqlite"; import { loadConfig, type LcmConfig } from "./config.ts"; +import { decodePersistedEvent } from "./event-codec.ts"; import { createNoteEvent, type NormalizedEvent } from "./events.ts"; import { extractFileReferences, type FileReference } from "./file-refs.ts"; import { @@ -13,6 +13,25 @@ import { type OverflowReference, type OverflowSearchMatch, } from "./overflow.ts"; +import { + appendRawEvents, + rawLogState, + rawLogStat, + readRawEventIds, + readRawEvents, + readRawLog, + type RawLogState, +} from "./raw-log.ts"; +import { initializeStorageSchema } from "./storage-schema.ts"; +import { + parseStringArray, + rowToFileReference, + rowToGraphEdge, + rowToGraphNode, + rowToSessionMemorySummary, + rowToSessionSummary, + rowToSummaryNode, +} from "./storage-rows.ts"; import { SUMMARY_ALGORITHM_VERSION, SUMMARY_NODE_CHUNK_SIZE, @@ -365,12 +384,6 @@ type RawEventIdCache = { eventIds: Set; }; -type RawLogState = { - size: number; - mtimeMs: number; - ctimeMs: number; -}; - export class LcmStorage { readonly config: LcmConfig; private db?: DatabaseSync; @@ -501,8 +514,7 @@ export class LcmStorage { if (eventsToAppend.length > 0) { try { - fs.mkdirSync(path.dirname(this.config.rawLogPath), { recursive: true, mode: 0o700 }); - fs.appendFileSync(this.config.rawLogPath, `${eventsToAppend.map((event) => JSON.stringify(event)).join("\n")}\n`, { mode: 0o600 }); + appendRawEvents(this.config.rawLogPath, eventsToAppend); } catch (error) { if (this.db) { try { @@ -584,8 +596,7 @@ export class LcmStorage { } private rawLogStat(): fs.Stats | undefined { - if (!fs.existsSync(this.config.rawLogPath)) return undefined; - return fs.statSync(this.config.rawLogPath); + return rawLogStat(this.config.rawLogPath); } rebuildSessionMemorySummaries(sessionIds: Iterable): string[] { @@ -637,7 +648,7 @@ export class LcmStorage { WHERE hook_event IN ${SUMMARY_SOURCE_HOOKS} ORDER BY timestamp ASC, rowid ASC `).all() as Array<{ raw_json: string }>) - .map((row) => JSON.parse(row.raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent(row.raw_json)) .filter(isSearchIndexEvent) .filter((event) => !isCodexLcmToolEvent(event)); const summarySessionIds = new Set(searchableEvents @@ -947,10 +958,7 @@ export class LcmStorage { } private rawLogState(): RawLogState { - const stat = this.rawLogStat(); - return stat - ? { size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs } - : { size: 0, mtimeMs: 0, ctimeMs: 0 }; + return rawLogState(this.config.rawLogPath); } private rebuildTouchedSummarySessions(sessionIds: Iterable): string[] { @@ -1251,7 +1259,7 @@ export class LcmStorage { ORDER BY timestamp DESC, rowid DESC LIMIT 256 `).all(args.cwd ?? null, args.repoRoot ?? null) as Array<{ raw_json: string }>) - .map((row) => JSON.parse(row.raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent(row.raw_json)) : readRawEvents(this.config.rawLogPath) .filter((event) => !args.cwd || event.cwd === args.cwd) .filter((event) => !args.repoRoot || event.repo_root === args.repoRoot) @@ -1354,7 +1362,7 @@ export class LcmStorage { ORDER BY timestamp ASC, rowid ASC LIMIT ?2 OFFSET ?3 `).all(sessionId, limit, offset); - const events = rows.map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent); + const events = rows.map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)); const total = session?.event_count ?? events.length; return { session, @@ -1449,7 +1457,7 @@ export class LcmStorage { `).all(session.session_id, limit); return { session_id: session.session_id, - events: rows.map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent), + events: rows.map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)), }; } @@ -1602,7 +1610,7 @@ export class LcmStorage { LIMIT 1 `).get(hash) as { raw_json?: string } | undefined; if (!row?.raw_json) return undefined; - return overflowReferenceFromEvent(JSON.parse(row.raw_json) as NormalizedEvent); + return overflowReferenceFromEvent(decodePersistedEvent(row.raw_json)); } describeMemory(args: { @@ -1996,7 +2004,7 @@ export class LcmStorage { ORDER BY timestamp ASC, rowid ASC `).all(...selectedIds); return rows - .map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)) .filter(isSummarySourceEvent) .filter((event) => !isCodexLcmToolEvent(event)) .sort((a, b) => @@ -2019,7 +2027,7 @@ export class LcmStorage { WHERE event_id IN (${placeholders}) ORDER BY timestamp ASC, rowid ASC `).all(...summary.source_event_ids) as Array<{ raw_json: string }>) - .map((row) => JSON.parse(row.raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent(row.raw_json)) .filter(isSummarySourceEvent) .filter((event) => !isCodexLcmToolEvent(event)); const matching = events.filter((event) => matchesQueryText(eventSignalText(event), query)); @@ -2047,7 +2055,7 @@ export class LcmStorage { ) ORDER BY timestamp ASC, rowid ASC `).all(sessionId, limit); - return rows.map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent); + return rows.map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)); } private getContextPlanSummaryStats(sessionId: string): { summaryNodeCount: number; estimatedSummaryTokens: number } { @@ -2269,172 +2277,7 @@ export class LcmStorage { private initialize(): void { if (!this.db) return; - this.db.exec(` - PRAGMA journal_mode = WAL; - CREATE TABLE IF NOT EXISTS sessions ( - session_id TEXT PRIMARY KEY, - first_seen TEXT NOT NULL, - last_seen TEXT NOT NULL, - cwd TEXT NOT NULL, - repo_root TEXT, - git_branch TEXT, - event_count INTEGER NOT NULL DEFAULT 0, - parent_session_id TEXT, - agent_role TEXT, - agent_nickname TEXT, - model TEXT, - reasoning_effort TEXT, - total_input_tokens INTEGER, - cached_input_tokens INTEGER, - output_tokens INTEGER, - reasoning_output_tokens INTEGER, - total_tokens INTEGER - ); - CREATE TABLE IF NOT EXISTS events ( - event_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - hook_event TEXT NOT NULL, - cwd TEXT NOT NULL, - repo_root TEXT, - git_branch TEXT, - turn_id TEXT, - tool_use_id TEXT, - text TEXT NOT NULL DEFAULT '', - raw_json TEXT NOT NULL - ); - CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5( - event_id UNINDEXED, - session_id, - cwd, - repo_root, - hook_event, - content - ); - CREATE TABLE IF NOT EXISTS graph_nodes ( - node_id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - session_id TEXT NOT NULL, - event_id TEXT, - turn_id TEXT, - timestamp TEXT NOT NULL, - cwd TEXT NOT NULL, - repo_root TEXT, - git_branch TEXT, - label TEXT NOT NULL, - metadata_json TEXT NOT NULL, - UNIQUE(event_id) - ); - CREATE TABLE IF NOT EXISTS graph_edges ( - from_node_id TEXT NOT NULL, - to_node_id TEXT NOT NULL, - kind TEXT NOT NULL, - session_id TEXT NOT NULL, - position INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - metadata_json TEXT NOT NULL DEFAULT '{}', - PRIMARY KEY (from_node_id, to_node_id, kind), - CHECK (from_node_id <> to_node_id) - ); - CREATE TABLE IF NOT EXISTS session_summaries ( - session_id TEXT PRIMARY KEY, - summary_version INTEGER NOT NULL DEFAULT ${SUMMARY_ALGORITHM_VERSION}, - updated_at TEXT NOT NULL, - cwd TEXT NOT NULL, - repo_root TEXT, - git_branch TEXT, - title TEXT NOT NULL, - overview TEXT NOT NULL, - topics_json TEXT NOT NULL, - key_prompts_json TEXT NOT NULL, - outcomes_json TEXT NOT NULL, - tools_json TEXT NOT NULL, - source_event_ids_json TEXT NOT NULL, - summary_text TEXT NOT NULL - ); - CREATE VIRTUAL TABLE IF NOT EXISTS session_summary_fts USING fts5( - session_id UNINDEXED, - cwd, - repo_root, - content - ); - CREATE TABLE IF NOT EXISTS summary_nodes ( - node_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - summary_version INTEGER NOT NULL DEFAULT ${SUMMARY_NODE_VERSION}, - depth INTEGER NOT NULL, - summary_text TEXT NOT NULL, - token_count INTEGER NOT NULL, - source_token_count INTEGER NOT NULL, - source_type TEXT NOT NULL, - source_ids_json TEXT NOT NULL, - source_event_ids_json TEXT NOT NULL, - earliest_at TEXT NOT NULL, - latest_at TEXT NOT NULL, - created_at TEXT NOT NULL, - cwd TEXT NOT NULL, - repo_root TEXT, - git_branch TEXT, - topics_json TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS file_refs ( - file_ref_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - observed_event_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - path TEXT NOT NULL, - mime_type TEXT NOT NULL, - byte_count INTEGER NOT NULL, - sha256 TEXT NOT NULL, - exploration_summary TEXT NOT NULL, - metadata_json TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS index_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE VIRTUAL TABLE IF NOT EXISTS summary_node_fts USING fts5( - node_id UNINDEXED, - session_id, - cwd, - repo_root, - depth, - content - ); - CREATE INDEX IF NOT EXISTS idx_graph_nodes_session_kind_time ON graph_nodes(session_id, kind, timestamp); - CREATE INDEX IF NOT EXISTS idx_graph_nodes_event ON graph_nodes(event_id); - CREATE INDEX IF NOT EXISTS idx_graph_edges_from ON graph_edges(from_node_id, kind); - CREATE INDEX IF NOT EXISTS idx_graph_edges_to ON graph_edges(to_node_id, kind); - CREATE INDEX IF NOT EXISTS idx_graph_edges_session ON graph_edges(session_id, kind, position); - CREATE INDEX IF NOT EXISTS idx_session_summaries_updated ON session_summaries(updated_at); - CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_depth_latest ON summary_nodes(session_id, depth, latest_at); - CREATE INDEX IF NOT EXISTS idx_summary_nodes_session_latest ON summary_nodes(session_id, latest_at); - CREATE INDEX IF NOT EXISTS idx_file_refs_session_time ON file_refs(session_id, timestamp); - CREATE INDEX IF NOT EXISTS idx_file_refs_path ON file_refs(path); - `); - this.ensureColumn("events", "turn_id", "TEXT"); - this.ensureColumn("events", "tool_use_id", "TEXT"); - this.ensureColumn("session_summaries", "summary_version", "INTEGER"); - const backfillSessionMetadata = [ - this.ensureColumn("sessions", "parent_session_id", "TEXT"), - this.ensureColumn("sessions", "agent_role", "TEXT"), - this.ensureColumn("sessions", "agent_nickname", "TEXT"), - this.ensureColumn("sessions", "model", "TEXT"), - this.ensureColumn("sessions", "reasoning_effort", "TEXT"), - this.ensureColumn("sessions", "total_input_tokens", "INTEGER"), - this.ensureColumn("sessions", "cached_input_tokens", "INTEGER"), - this.ensureColumn("sessions", "output_tokens", "INTEGER"), - this.ensureColumn("sessions", "reasoning_output_tokens", "INTEGER"), - this.ensureColumn("sessions", "total_tokens", "INTEGER"), - ].some(Boolean); - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_events_session_turn ON events(session_id, turn_id, timestamp); - CREATE INDEX IF NOT EXISTS idx_events_tool_use ON events(session_id, tool_use_id, hook_event, timestamp); - CREATE INDEX IF NOT EXISTS idx_events_session_hook_time ON events(session_id, hook_event, timestamp); - CREATE INDEX IF NOT EXISTS idx_events_session_time ON events(session_id, timestamp); - CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id, last_seen); - CREATE INDEX IF NOT EXISTS idx_sessions_last_seen ON sessions(last_seen); - `); + const { backfillSessionMetadata } = initializeStorageSchema(this.db); if (backfillSessionMetadata) this.backfillExistingSessionMetadata(); } @@ -2769,20 +2612,6 @@ export class LcmStorage { return row !== undefined; } - private ensureColumn(table: string, column: string, type: string): boolean { - if (!this.db) return false; - const tableName = sqlIdentifier(table); - const columnName = sqlIdentifier(column); - const columnType = sqlColumnType(type); - const columns = this.db.prepare(`PRAGMA table_info(${tableName})`).all() - .map((row) => String((row as { name: string }).name)); - if (!columns.includes(columnName)) { - this.db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${columnType}`); - return true; - } - return false; - } - private backfillExistingSessionMetadata(): void { if (!this.db) return; const rows = this.db.prepare("SELECT raw_json FROM events WHERE hook_event = 'SessionStart'").all() as Array<{ raw_json: string }>; @@ -2790,7 +2619,7 @@ export class LcmStorage { UPDATE sessions SET parent_session_id = ?2, agent_role = ?3, agent_nickname = ?4 WHERE session_id = ?1 `); for (const row of rows) { - const event = JSON.parse(row.raw_json) as NormalizedEvent; + const event = decodePersistedEvent(row.raw_json); const metadata = extractSessionMetadata(event); update.run(event.session_id, metadata.parent_session_id ?? null, metadata.agent_role ?? null, metadata.agent_nickname ?? null); } @@ -2816,7 +2645,7 @@ export class LcmStorage { this.db.exec("BEGIN IMMEDIATE"); try { for (const row of rows) { - const event = JSON.parse(row.raw_json) as NormalizedEvent; + const event = decodePersistedEvent(row.raw_json); const parentId = extractSessionMetadata(event).parent_session_id; if (parentId && parentId !== event.session_id) update.run(event.session_id, parentId); } @@ -2852,7 +2681,7 @@ export class LcmStorage { this.db.exec("BEGIN IMMEDIATE"); try { for (const row of rows) { - const event = JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent; + const event = decodePersistedEvent((row as { raw_json: string }).raw_json); const metadata = extractEventMetadata(event); const count = (counts.get(event.session_id) ?? Number(this.db.prepare(` SELECT COUNT(*) AS count FROM graph_nodes WHERE session_id = ?1 AND kind = 'event' @@ -2897,7 +2726,7 @@ export class LcmStorage { this.db.exec("BEGIN IMMEDIATE"); try { for (const row of rows) { - const event = JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent; + const event = decodePersistedEvent((row as { raw_json: string }).raw_json); this.indexFileRefsForEvent(event); } this.db.prepare(` @@ -3180,7 +3009,7 @@ export class LcmStorage { ORDER BY timestamp ASC, rowid ASC `).all(sessionId); return rows - .map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)) .filter((event) => !isCodexLcmToolEvent(event)) .filter(isSummarySourceEvent); } @@ -3208,7 +3037,7 @@ export class LcmStorage { LIMIT ?2 `).all(sessionId, SUMMARY_RECENT_EVENT_LIMIT); const events = uniqueEvents([...earlySignals, ...latestSignals, ...recentEvents] - .map((row) => JSON.parse((row as { raw_json: string }).raw_json) as NormalizedEvent) + .map((row) => decodePersistedEvent((row as { raw_json: string }).raw_json)) .filter((event) => !isCodexLcmToolEvent(event)) .filter((event) => !isSummaryHook(event.hook_event) || isSummarySourceEvent(event))) .sort((a, b) => a.timestamp.localeCompare(b.timestamp) || a.event_id.localeCompare(b.event_id)); @@ -3220,104 +3049,6 @@ export function createStorage(options: StorageOptions = {}): LcmStorage { return new LcmStorage(options); } -function rowToSessionSummary(row: unknown): SessionSummary { - const record = row as Record; - return { - session_id: String(record.session_id), - first_seen: String(record.first_seen), - last_seen: String(record.last_seen), - cwd: String(record.cwd), - ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), - ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), - event_count: Number(record.event_count), - ...(record.parent_session_id ? { parent_session_id: String(record.parent_session_id) } : {}), - ...(record.agent_role ? { agent_role: String(record.agent_role) } : {}), - ...(record.agent_nickname ? { agent_nickname: String(record.agent_nickname) } : {}), - ...(record.model ? { model: String(record.model) } : {}), - ...(record.reasoning_effort ? { reasoning_effort: String(record.reasoning_effort) } : {}), - ...(record.total_input_tokens !== null && record.total_input_tokens !== undefined - ? { total_input_tokens: Number(record.total_input_tokens) } - : {}), - ...(record.cached_input_tokens !== null && record.cached_input_tokens !== undefined - ? { cached_input_tokens: Number(record.cached_input_tokens) } - : {}), - ...(record.output_tokens !== null && record.output_tokens !== undefined ? { output_tokens: Number(record.output_tokens) } : {}), - ...(record.reasoning_output_tokens !== null && record.reasoning_output_tokens !== undefined - ? { reasoning_output_tokens: Number(record.reasoning_output_tokens) } - : {}), - ...(record.total_tokens !== null && record.total_tokens !== undefined ? { total_tokens: Number(record.total_tokens) } : {}), - ...(record.match_count !== undefined ? { match_count: Number(record.match_count) } : {}), - ...(record.summary_title ? { - summary: { - updated_at: String(record.summary_updated_at), - title: String(record.summary_title), - overview: String(record.summary_overview), - topics: parseStringArray(record.summary_topics_json), - key_prompts: parseStringArray(record.summary_key_prompts_json), - outcomes: parseStringArray(record.summary_outcomes_json), - source_event_count: parseStringArray(record.summary_source_event_ids_json).length, - }, - } : {}), - }; -} - -function rowToSessionMemorySummary(row: unknown): SessionMemorySummary { - const record = row as Record; - return { - session_id: String(record.session_id), - updated_at: String(record.updated_at), - cwd: String(record.cwd), - ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), - ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), - title: String(record.title), - overview: String(record.overview), - topics: parseStringArray(record.topics_json), - key_prompts: parseStringArray(record.key_prompts_json), - outcomes: parseStringArray(record.outcomes_json), - tools: parseStringArray(record.tools_json), - source_event_ids: parseStringArray(record.source_event_ids_json), - }; -} - -function rowToSummaryNode(row: unknown): SummaryNode { - const record = row as Record; - const sourceType = String(record.source_type) === "nodes" ? "nodes" : "events"; - return { - node_id: String(record.node_id), - session_id: String(record.session_id), - depth: Number(record.depth), - summary_text: String(record.summary_text), - token_count: Number(record.token_count), - source_token_count: Number(record.source_token_count), - source_type: sourceType, - source_ids: parseStringArray(record.source_ids_json), - source_event_ids: parseStringArray(record.source_event_ids_json), - earliest_at: String(record.earliest_at), - latest_at: String(record.latest_at), - created_at: String(record.created_at), - cwd: String(record.cwd), - ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), - ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), - topics: parseStringArray(record.topics_json), - }; -} - -function rowToFileReference(row: unknown): FileReference { - const record = row as Record; - return { - file_ref_id: String(record.file_ref_id), - session_id: String(record.session_id), - observed_event_id: String(record.observed_event_id), - timestamp: String(record.timestamp), - path: String(record.path), - mime_type: String(record.mime_type), - byte_count: Number(record.byte_count), - sha256: String(record.sha256), - exploration_summary: String(record.exploration_summary), - metadata: parseMetadata(record.metadata_json), - }; -} - function summaryNodeToGraphNode(node: SummaryNode): GraphNode { return { node_id: node.node_id, @@ -3526,7 +3257,7 @@ function isSearchDiscoveryRow(row: unknown, query: string): boolean { if (searchMatchKind(record.match_kind) !== "event") return true; if (typeof record.match_text !== "string") return true; try { - return isSearchDiscoveryEvent(JSON.parse(record.match_text) as NormalizedEvent, query); + return isSearchDiscoveryEvent(decodePersistedEvent(record.match_text), query); } catch { return true; } @@ -3588,7 +3319,7 @@ function searchMatchText(kind: SessionSearchMatch["kind"], value: unknown): stri if (typeof value !== "string") return ""; if (kind !== "event") return value; try { - const event = JSON.parse(value) as NormalizedEvent; + const event = decodePersistedEvent(value); return eventSignalText(event) || `${event.hook_event}: ${JSON.stringify(event.payload)}`; } catch { return value; @@ -3632,56 +3363,6 @@ function compactWhitespace(text: string): string { return text.replace(/\s+/gu, " ").trim(); } -function rowToGraphNode(row: unknown): GraphNode { - const record = row as Record; - return { - node_id: String(record.node_id), - kind: String(record.kind) as GraphNode["kind"], - session_id: String(record.session_id), - ...(record.event_id ? { event_id: String(record.event_id) } : {}), - ...(record.turn_id ? { turn_id: String(record.turn_id) } : {}), - timestamp: String(record.timestamp), - cwd: String(record.cwd), - ...(record.repo_root ? { repo_root: String(record.repo_root) } : {}), - ...(record.git_branch ? { git_branch: String(record.git_branch) } : {}), - label: String(record.label), - metadata: parseMetadata(record.metadata_json), - }; -} - -function rowToGraphEdge(row: unknown): GraphEdge { - const record = row as Record; - return { - from_node_id: String(record.from_node_id), - to_node_id: String(record.to_node_id), - kind: String(record.kind), - session_id: String(record.session_id), - position: Number(record.position), - created_at: String(record.created_at), - metadata: parseMetadata(record.metadata_json), - }; -} - -function parseMetadata(value: unknown): Record { - if (typeof value !== "string") return {}; - try { - const parsed = JSON.parse(value) as unknown; - return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; - } catch { - return {}; - } -} - -function parseStringArray(value: unknown): string[] { - if (typeof value !== "string") return []; - try { - const parsed = JSON.parse(value) as unknown; - return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : []; - } catch { - return []; - } -} - function clampLimit(limit: number | undefined, fallback: number, max = 200): number { return Math.min(Math.max(Number(limit ?? fallback), 1), max); } @@ -3785,20 +3466,6 @@ function parseTimestamp(value: string | undefined, name: string): string | undef return timestamp.toISOString(); } -function sqlIdentifier(value: string): string { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(value)) { - throw new Error(`Invalid SQL identifier: ${value}`); - } - return value; -} - -function sqlColumnType(value: string): string { - if (!/^[A-Z][A-Z0-9_]*(?:\s+[A-Z][A-Z0-9_]*)*$/u.test(value)) { - throw new Error(`Invalid SQL column type: ${value}`); - } - return value; -} - function eventSearchText(event: NormalizedEvent): string { const metadata = extractEventMetadata(event); return [ @@ -3828,29 +3495,6 @@ function checkpointToMarkdown(node: GraphNode): string { ].join("\n"); } -function readRawLog(rawLogPath: string): { events: NormalizedEvent[]; malformedLineCount: number } { - if (!fs.existsSync(rawLogPath)) return { events: [], malformedLineCount: 0 }; - const events: NormalizedEvent[] = []; - let malformedLineCount = 0; - for (const line of fs.readFileSync(rawLogPath, "utf8").split(/\r?\n/u)) { - if (line.trim().length === 0) continue; - try { - events.push(JSON.parse(line) as NormalizedEvent); - } catch { - malformedLineCount += 1; - } - } - return { events, malformedLineCount }; -} - -function readRawEvents(rawLogPath: string): NormalizedEvent[] { - return readRawLog(rawLogPath).events; -} - -function readRawEventIds(rawLogPath: string): Set { - return new Set(readRawEvents(rawLogPath).map((event) => event.event_id)); -} - function countEventsByHook(events: NormalizedEvent[]): Record { const counts: Record = {}; for (const event of events) { diff --git a/plugins/codex-lcm/tests/event-codec.test.ts b/plugins/codex-lcm/tests/event-codec.test.ts new file mode 100644 index 0000000..216f84a --- /dev/null +++ b/plugins/codex-lcm/tests/event-codec.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { decodePersistedEvent, parsePersistedEvent, PersistedEventError } from "../src/event-codec.ts"; + +const serializedEvent = JSON.stringify({ + schema_version: 1, + event_id: "event-1", + timestamp: "2026-07-27T12:00:00.000Z", + hook_event: "UserPromptSubmit", + session_id: "session-1", + cwd: "/tmp/project", + payload: { prompt: "remember this" }, + redactions: [], + truncations: [], + raw_input_sha256: "abc123", + original_bytes: 13, + sanitized_bytes: 13, +}); + +test("parses a complete persisted event", () => { + const event = parsePersistedEvent(serializedEvent); + + assert.equal(event?.event_id, "event-1"); + assert.deepEqual(event?.payload, { prompt: "remember this" }); +}); + +test("rejects malformed or structurally invalid persisted events", () => { + assert.equal(parsePersistedEvent("{not-json"), undefined); + assert.equal(parsePersistedEvent(JSON.stringify({ event_id: "event-1" })), undefined); + assert.throws( + () => decodePersistedEvent(JSON.stringify({ event_id: "event-1" })), + PersistedEventError, + ); +}); diff --git a/plugins/codex-lcm/tests/redact-nested.test.ts b/plugins/codex-lcm/tests/redact-nested.test.ts new file mode 100644 index 0000000..42707c9 --- /dev/null +++ b/plugins/codex-lcm/tests/redact-nested.test.ts @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { sanitizeForStorage } from "../src/redact.ts"; + +test("redacts secret assignments nested inside JSON-like command strings", () => { + const result = sanitizeForStorage({ + command: '{"text":"CLI password=test_cli_password token_budget=4096"}', + }); + + assert.deepEqual(result.value, { + command: '{"text":"CLI password=[REDACTED:secret] token_budget=4096"}', + }); + assert.equal(result.redactions.length, 1); +}); + +test("redacts shell-quoted secret assignments", () => { + const result = sanitizeForStorage({ + command: "run --env 'password=test_cli_password' --env \"api_token=test_cli_token\"", + }); + + assert.deepEqual(result.value, { + command: "run --env 'password=[REDACTED:secret]' --env \"api_token=[REDACTED:secret]\"", + }); + assert.equal(result.redactions.length, 2); +}); + +test("redacts quotes inside unquoted secret values without leaking suffixes", () => { + const result = sanitizeForStorage({ + command: "password=abc'def api_token=abc\"def", + }); + + assert.deepEqual(result.value, { + command: "password=[REDACTED:secret] api_token=[REDACTED:secret]", + }); +}); + +test("preserves empty assignments and scans the following assignment", () => { + const result = sanitizeForStorage({ + command: "password= api_token='' token_budget=4096 password= actual_secret", + }); + + assert.deepEqual(result.value, { + command: "password= api_token='' token_budget=4096 password= [REDACTED:secret]", + }); +}); + +test("preserves closing quotes after an even number of backslashes", () => { + const cases = [ + { + input: String.raw`password="abc\\" token_budget=4096 api_token="actual_secret"`, + expected: 'password="[REDACTED:secret]" token_budget=4096 api_token="[REDACTED:secret]"', + }, + { + input: String.raw`password="abc\"def" token_budget=4096`, + expected: 'password="[REDACTED:secret]" token_budget=4096', + }, + { + input: String.raw`{"text":"password=\"abc\\\" token_budget=4096"}`, + expected: String.raw`{"text":"password=\"[REDACTED:secret]\" token_budget=4096"}`, + }, + ]; + + for (const testCase of cases) { + const result = sanitizeForStorage({ command: testCase.input }); + assert.deepEqual(result.value, { command: testCase.expected }); + } +}); + +test("redacts deeply nested assignments without overflowing the call stack", () => { + const result = sanitizeForStorage({ + command: `${"x=".repeat(1000)}password=deep_secret`, + }); + + assert.equal( + (result.value as { command: string }).command, + `${"x=".repeat(1000)}password=[REDACTED:secret]`, + ); +}); + +test("preserves escaped JSON structure and benign metrics while redacting nested secrets", () => { + const result = sanitizeForStorage({ + command: '{"command":"{\\"text\\":\\"password=deep_secret token_budget=4096\\"}"}', + }); + + assert.deepEqual(result.value, { + command: '{"command":"{\\"text\\":\\"password=[REDACTED:secret] token_budget=4096\\"}"}', + }); +}); + +test("preserves enclosing quotes for terminal nested assignments", () => { + const cases = [ + '{"text":"CLI password=actual_secret"}', + String.raw`{\"text\":\"CLI password=actual_secret\"}`, + ]; + + for (const command of cases) { + const result = sanitizeForStorage({ command }); + assert.equal( + (result.value as { command: string }).command, + command.replace("actual_secret", "[REDACTED:secret]"), + ); + } +}); + +test("scans many secret assignments in linear time", () => { + const command = "password=value ".repeat(25_000); + const startedAt = performance.now(); + const result = sanitizeForStorage( + { command }, + { maxStringBytes: 1024 * 1024, maxPayloadBytes: 2 * 1024 * 1024 }, + ); + + assert.equal(result.redactions.length, 25_000); + assert.ok( + performance.now() - startedAt < 1_000, + "expected 25,000 assignments to sanitize within one second", + ); +});