diff --git a/plugins/codex-lcm/README.md b/plugins/codex-lcm/README.md index e73feaf..932f52e 100644 --- a/plugins/codex-lcm/README.md +++ b/plugins/codex-lcm/README.md @@ -26,6 +26,7 @@ node bin/codex-lcm usage --since 2026-07-13T00:00:00Z node bin/codex-lcm cleanup node bin/codex-lcm context-plan node bin/codex-lcm benchmark long-context +node bin/codex-lcm benchmark retrieval-quality ``` Storage defaults to `~/.codex-lcm`. Override storage for hook and MCP operations with: @@ -42,6 +43,7 @@ node bin/codex-lcm import-codex-sessions --dry-run --json node bin/codex-lcm import-codex-sessions --json node bin/codex-lcm context-plan --session-id SESSION --json node bin/codex-lcm benchmark long-context --json +node bin/codex-lcm benchmark retrieval-quality --json node bin/codex-lcm cleanup --apply --json ``` @@ -218,6 +220,12 @@ retrieval check. It creates temporary storage, imports a long session with an ol needle event, and verifies `lcm_pack_context` recovers that source under a bounded budget. Pass `--home PATH` to keep the benchmark storage for inspection. +`node bin/codex-lcm benchmark retrieval-quality --json` runs a fixed, +dependency-free corpus through production session search. It reports Recall@1, +Recall@5, mean reciprocal rank, and per-case ranks across exact, cross-session, +temporal, and paraphrase queries. The paraphrase cases expose lexical retrieval +gaps instead of hiding them behind exact needles. + ## Privacy And Safety Before writing to disk, Codex LCM: diff --git a/plugins/codex-lcm/package.json b/plugins/codex-lcm/package.json index 354473a..cec1bce 100644 --- a/plugins/codex-lcm/package.json +++ b/plugins/codex-lcm/package.json @@ -12,6 +12,7 @@ "typecheck": "npm exec --yes --package=typescript@7.0.2 -- tsc --noEmit", "test": "node --no-warnings --test tests/*.test.ts", "benchmark:long-context": "node --no-warnings bin/codex-lcm benchmark long-context --json", + "benchmark:retrieval-quality": "node --no-warnings bin/codex-lcm benchmark retrieval-quality --json", "smoke": "node --no-warnings scripts/smoke-test.ts" }, "engines": { diff --git a/plugins/codex-lcm/src/benchmark.ts b/plugins/codex-lcm/src/benchmark.ts index c620a42..c326252 100644 --- a/plugins/codex-lcm/src/benchmark.ts +++ b/plugins/codex-lcm/src/benchmark.ts @@ -10,6 +10,37 @@ const BENCHMARK_SESSION_ID = "codex-lcm-benchmark-long-context"; const BENCHMARK_CWD = "/tmp/codex-lcm-benchmark"; const BENCHMARK_NEEDLE = "BENCHMARK-NEEDLE recursive evidence recovery source event"; const BENCHMARK_QUERY = "BENCHMARK-NEEDLE recursive evidence recovery"; +const RETRIEVAL_BENCHMARK_CWD = "/tmp/codex-lcm-retrieval-quality"; + +type RetrievalCategory = "exact" | "cross-session" | "temporal" | "paraphrase"; + +const RETRIEVAL_CORPUS = [ + ["retrieval-storage", "SQLite WAL concurrent readers were chosen for local session storage."], + ["retrieval-websocket", "WebSocket reconnect jitter prevents synchronized client retries."], + ["retrieval-pool", "The pool chemistry pH target is 7.4 after probe calibration."], + ["retrieval-overflow", "Overflow payload integrity uses a content hash before bounded recovery."], + ["retrieval-release-old", "The release signing certificate belongs to the legacy build account."], + ["retrieval-release-new", "The release signing certificate moved to the production build account."], + ["retrieval-migration-old", "Schema migration rollback used a manual database snapshot."], + ["retrieval-migration-new", "Schema migration rollback now uses the automated restore job."], + ["retrieval-backoff", "Failed requests retry with exponential delay and random spread."], +] as const; + +const RETRIEVAL_QUERIES: ReadonlyArray<{ + id: string; + category: RetrievalCategory; + query: string; + expectedSessionId: string; +}> = [ + { id: "exact-storage", category: "exact", query: "SQLite WAL concurrent readers", expectedSessionId: "retrieval-storage" }, + { id: "exact-websocket", category: "exact", query: "WebSocket reconnect jitter", expectedSessionId: "retrieval-websocket" }, + { id: "cross-pool", category: "cross-session", query: "pool chemistry pH target", expectedSessionId: "retrieval-pool" }, + { id: "cross-overflow", category: "cross-session", query: "overflow payload integrity hash", expectedSessionId: "retrieval-overflow" }, + { id: "temporal-release", category: "temporal", query: "release signing certificate", expectedSessionId: "retrieval-release-new" }, + { id: "temporal-migration", category: "temporal", query: "schema migration rollback", expectedSessionId: "retrieval-migration-new" }, + { id: "paraphrase-overflow", category: "paraphrase", query: "retain oversized command output", expectedSessionId: "retrieval-overflow" }, + { id: "paraphrase-backoff", category: "paraphrase", query: "randomized backoff for failed calls", expectedSessionId: "retrieval-backoff" }, +]; export type LongContextBenchmarkOptions = { events?: number; @@ -30,6 +61,30 @@ export type LongContextBenchmarkResult = { storage_home?: string; }; +export type RetrievalQualityMetrics = { + queries: number; + recall_at_1: number; + recall_at_5: number; + mean_reciprocal_rank: number; +}; + +export type RetrievalQualityBenchmarkResult = RetrievalQualityMetrics & { + name: "retrieval-quality"; + corpus_version: 1; + sessions: number; + by_category: Record; + cases: Array<{ + id: string; + category: RetrievalCategory; + query: string; + expected_session_id: string; + rank: number | null; + top_session_ids: string[]; + }>; + duration_ms: number; + storage_home?: string; +}; + export function runLongContextBenchmark(options: LongContextBenchmarkOptions = {}): LongContextBenchmarkResult { const eventCount = Math.max(16, Math.floor(options.events ?? 128)); const budgetTokens = Math.max(64, Math.floor(options.budgetTokens ?? 1200)); @@ -77,3 +132,71 @@ export function runLongContextBenchmark(options: LongContextBenchmarkOptions = { if (cleanup) fs.rmSync(home, { recursive: true, force: true }); } } + +export function runRetrievalQualityBenchmark(options: { home?: string } = {}): RetrievalQualityBenchmarkResult { + const home = options.home ?? fs.mkdtempSync(path.join(os.tmpdir(), "codex-lcm-retrieval-quality-")); + const cleanup = options.home === undefined; + const startedAt = performance.now(); + const storage = createStorage({ home }); + + try { + storage.ingestMany(RETRIEVAL_CORPUS.map(([sessionId, prompt], index) => normalizeHookEvent({ + hookEvent: "UserPromptSubmit", + rawInput: JSON.stringify({ + session_id: sessionId, + cwd: RETRIEVAL_BENCHMARK_CWD, + prompt, + }), + env: {}, + now: () => new Date(Date.UTC(2026, 6, 1, 12, index)), + }))); + const cases = RETRIEVAL_QUERIES.map((entry) => { + const topSessionIds = storage.searchSessions({ + query: entry.query, + cwd: RETRIEVAL_BENCHMARK_CWD, + limit: 5, + }).map((match) => match.session_id); + const index = topSessionIds.indexOf(entry.expectedSessionId); + return { + id: entry.id, + category: entry.category, + query: entry.query, + expected_session_id: entry.expectedSessionId, + rank: index < 0 ? null : index + 1, + top_session_ids: topSessionIds, + }; + }); + return { + name: "retrieval-quality", + corpus_version: 1, + sessions: RETRIEVAL_CORPUS.length, + ...retrievalMetrics(cases), + by_category: { + exact: retrievalMetrics(cases.filter((entry) => entry.category === "exact")), + "cross-session": retrievalMetrics(cases.filter((entry) => entry.category === "cross-session")), + temporal: retrievalMetrics(cases.filter((entry) => entry.category === "temporal")), + paraphrase: retrievalMetrics(cases.filter((entry) => entry.category === "paraphrase")), + }, + cases, + duration_ms: Math.round(performance.now() - startedAt), + ...(cleanup ? {} : { storage_home: home }), + }; + } finally { + storage.close(); + if (cleanup) fs.rmSync(home, { recursive: true, force: true }); + } +} + +function retrievalMetrics(cases: ReadonlyArray<{ rank: number | null }>): RetrievalQualityMetrics { + const queries = cases.length; + return { + queries, + recall_at_1: ratio(cases.filter((entry) => entry.rank === 1).length, queries), + recall_at_5: ratio(cases.filter((entry) => entry.rank !== null && entry.rank <= 5).length, queries), + mean_reciprocal_rank: ratio(cases.reduce((sum, entry) => sum + (entry.rank ? 1 / entry.rank : 0), 0), queries), + }; +} + +function ratio(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4)); +} diff --git a/plugins/codex-lcm/src/cli.ts b/plugins/codex-lcm/src/cli.ts index 11902f7..cc00326 100644 --- a/plugins/codex-lcm/src/cli.ts +++ b/plugins/codex-lcm/src/cli.ts @@ -1,5 +1,5 @@ import { loadConfig, pluginRoot } from "./config.ts"; -import { runLongContextBenchmark } from "./benchmark.ts"; +import { runLongContextBenchmark, runRetrievalQualityBenchmark } from "./benchmark.ts"; import { importCodexSessions } from "./codex-import.ts"; import { buildDoctorReport } from "./doctor.ts"; import { runHook } from "./hook.ts"; @@ -61,13 +61,21 @@ export async function main(argv: string[]): Promise { } if (command === "benchmark") { const benchmarkName = rest[0]; - if (benchmarkName !== "long-context") throw new Error("Usage: codex-lcm benchmark long-context [--events N] [--budget-tokens N] [--home PATH] [--json]"); - printObjectOrText(runLongContextBenchmark({ - events: numberOptionValue(rest, "--events"), - budgetTokens: numberOptionValue(rest, "--budget-tokens"), - home: optionValue(rest, "--home"), - })); - return; + if (benchmarkName === "long-context") { + printObjectOrText(runLongContextBenchmark({ + events: numberOptionValue(rest, "--events"), + budgetTokens: numberOptionValue(rest, "--budget-tokens"), + home: optionValue(rest, "--home"), + })); + return; + } + if (benchmarkName === "retrieval-quality") { + printObjectOrText(runRetrievalQualityBenchmark({ + home: optionValue(rest, "--home"), + })); + return; + } + throw new Error("Usage: codex-lcm benchmark long-context|retrieval-quality [options] [--json]"); } if (command === "health") { const storage = createStorage({ config: loadConfig(), readOnly: true }); @@ -166,6 +174,7 @@ Commands: codex-lcm usage [--since ISO] [--until ISO] [--cwd PATH] [--repo-root PATH] [--parent-session-id ID] [--roots-only] [--json] codex-lcm context-plan [--session-id ID] [--cwd PATH] [--repo-root PATH] [--model-context-window N] [--auto-compact-token-limit N] [--recent-event-limit N] [--json] codex-lcm benchmark long-context [--events N] [--budget-tokens N] [--home PATH] [--json] + codex-lcm benchmark retrieval-quality [--home PATH] [--json] codex-lcm import-codex-sessions [--from PATH] [--dry-run] [--progress] [--batch-size N] [--json] `); } diff --git a/plugins/codex-lcm/tests/benchmark.test.ts b/plugins/codex-lcm/tests/benchmark.test.ts index 2d636f6..d293742 100644 --- a/plugins/codex-lcm/tests/benchmark.test.ts +++ b/plugins/codex-lcm/tests/benchmark.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { runLongContextBenchmark } from "../src/benchmark.ts"; +import { runLongContextBenchmark, runRetrievalQualityBenchmark } from "../src/benchmark.ts"; import { assertCliOk, runCli, tempHome } from "./helpers.ts"; test("long-context benchmark recovers an old source event through packed context", () => { @@ -19,6 +19,22 @@ test("long-context benchmark recovers an old source event through packed context assert.equal(result.duration_ms >= 0, true); }); +test("retrieval-quality benchmark reports ranked results across labeled query categories", () => { + const result = runRetrievalQualityBenchmark(); + + assert.equal(result.name, "retrieval-quality"); + assert.equal(result.queries, 8); + assert.equal(result.cases.length, result.queries); + assert.deepEqual(Object.keys(result.by_category).sort(), ["cross-session", "exact", "paraphrase", "temporal"]); + assert.equal(result.recall_at_1 >= 0 && result.recall_at_1 <= 1, true); + assert.equal(result.recall_at_5 >= result.recall_at_1 && result.recall_at_5 <= 1, true); + assert.equal(result.mean_reciprocal_rank >= 0 && result.mean_reciprocal_rank <= 1, true); + assert.equal(result.by_category.exact.recall_at_5, 1); + assert.equal(result.by_category.paraphrase.recall_at_5 < 1, true); + assert.equal(result.cases.every((entry) => entry.rank === null || entry.rank > 0), true); + assert.equal(result.duration_ms >= 0, true); +}); + test("benchmark long-context command prints JSON results", () => { const result = runCli(["benchmark", "long-context", "--events", "64", "--budget-tokens", "800", "--json"], { timeout: 10_000, @@ -31,6 +47,18 @@ test("benchmark long-context command prints JSON results", () => { assert.equal(parsed.recovered, true); }); +test("benchmark retrieval-quality command prints ranked metrics", () => { + const result = runCli(["benchmark", "retrieval-quality", "--json"], { + timeout: 10_000, + }); + + assertCliOk(result); + const parsed = JSON.parse(result.stdout); + assert.equal(parsed.name, "retrieval-quality"); + assert.equal(parsed.queries, 8); + assert.equal(parsed.cases.length, 8); +}); + test("benchmark long-context command can keep caller-provided storage", () => { const home = tempHome("codex-lcm-benchmark-keep-"); const result = runCli(["benchmark", "long-context", "--events", "64", "--budget-tokens", "800", "--home", home, "--json"], {