feat: add retrieval quality benchmark

plugins/codex-lcm/README.md

@@ -26,6 +26,7 @@ node bin/codex-lcm usage --since 2026-07-13T00:00:00Z
 node bin/codex-lcm cleanup
 node bin/codex-lcm context-plan
 node bin/codex-lcm benchmark long-context
+node bin/codex-lcm benchmark retrieval-quality
 ```
 
 Storage defaults to `~/.codex-lcm`. Override storage for hook and MCP operations with:
@@ -42,6 +43,7 @@ node bin/codex-lcm import-codex-sessions --dry-run --json
 node bin/codex-lcm import-codex-sessions --json
 node bin/codex-lcm context-plan --session-id SESSION --json
 node bin/codex-lcm benchmark long-context --json
+node bin/codex-lcm benchmark retrieval-quality --json
 node bin/codex-lcm cleanup --apply --json
 ```
 
@@ -218,6 +220,12 @@ retrieval check. It creates temporary storage, imports a long session with an ol
 needle event, and verifies `lcm_pack_context` recovers that source under a
 bounded budget. Pass `--home PATH` to keep the benchmark storage for inspection.
 
+`node bin/codex-lcm benchmark retrieval-quality --json` runs a fixed,
+dependency-free corpus through production session search. It reports Recall@1,
+Recall@5, mean reciprocal rank, and per-case ranks across exact, cross-session,
+temporal, and paraphrase queries. The paraphrase cases expose lexical retrieval
+gaps instead of hiding them behind exact needles.
+
 ## Privacy And Safety
 
 Before writing to disk, Codex LCM:

plugins/codex-lcm/package.json

@@ -12,6 +12,7 @@
     "typecheck": "npm exec --yes --package=typescript@7.0.2 -- tsc --noEmit",
     "test": "node --no-warnings --test tests/*.test.ts",
     "benchmark:long-context": "node --no-warnings bin/codex-lcm benchmark long-context --json",
+    "benchmark:retrieval-quality": "node --no-warnings bin/codex-lcm benchmark retrieval-quality --json",
     "smoke": "node --no-warnings scripts/smoke-test.ts"
   },
   "engines": {

plugins/codex-lcm/src/benchmark.ts

@@ -10,6 +10,37 @@ const BENCHMARK_SESSION_ID = "codex-lcm-benchmark-long-context";
 const BENCHMARK_CWD = "/tmp/codex-lcm-benchmark";
 const BENCHMARK_NEEDLE = "BENCHMARK-NEEDLE recursive evidence recovery source event";
 const BENCHMARK_QUERY = "BENCHMARK-NEEDLE recursive evidence recovery";
+const RETRIEVAL_BENCHMARK_CWD = "/tmp/codex-lcm-retrieval-quality";
+
+type RetrievalCategory = "exact" | "cross-session" | "temporal" | "paraphrase";
+
+const RETRIEVAL_CORPUS = [
+  ["retrieval-storage", "SQLite WAL concurrent readers were chosen for local session storage."],
+  ["retrieval-websocket", "WebSocket reconnect jitter prevents synchronized client retries."],
+  ["retrieval-pool", "The pool chemistry pH target is 7.4 after probe calibration."],
+  ["retrieval-overflow", "Overflow payload integrity uses a content hash before bounded recovery."],
+  ["retrieval-release-old", "The release signing certificate belongs to the legacy build account."],
+  ["retrieval-release-new", "The release signing certificate moved to the production build account."],
+  ["retrieval-migration-old", "Schema migration rollback used a manual database snapshot."],
+  ["retrieval-migration-new", "Schema migration rollback now uses the automated restore job."],
+  ["retrieval-backoff", "Failed requests retry with exponential delay and random spread."],
+] as const;
+
+const RETRIEVAL_QUERIES: ReadonlyArray<{
+  id: string;
+  category: RetrievalCategory;
+  query: string;
+  expectedSessionId: string;
+}> = [
+  { id: "exact-storage", category: "exact", query: "SQLite WAL concurrent readers", expectedSessionId: "retrieval-storage" },
+  { id: "exact-websocket", category: "exact", query: "WebSocket reconnect jitter", expectedSessionId: "retrieval-websocket" },
+  { id: "cross-pool", category: "cross-session", query: "pool chemistry pH target", expectedSessionId: "retrieval-pool" },
+  { id: "cross-overflow", category: "cross-session", query: "overflow payload integrity hash", expectedSessionId: "retrieval-overflow" },
+  { id: "temporal-release", category: "temporal", query: "release signing certificate", expectedSessionId: "retrieval-release-new" },
+  { id: "temporal-migration", category: "temporal", query: "schema migration rollback", expectedSessionId: "retrieval-migration-new" },
+  { id: "paraphrase-overflow", category: "paraphrase", query: "retain oversized command output", expectedSessionId: "retrieval-overflow" },
+  { id: "paraphrase-backoff", category: "paraphrase", query: "randomized backoff for failed calls", expectedSessionId: "retrieval-backoff" },
+];
 
 export type LongContextBenchmarkOptions = {
   events?: number;
@@ -30,6 +61,30 @@ export type LongContextBenchmarkResult = {
   storage_home?: string;
 };
 
+export type RetrievalQualityMetrics = {
+  queries: number;
+  recall_at_1: number;
+  recall_at_5: number;
+  mean_reciprocal_rank: number;
+};
+
+export type RetrievalQualityBenchmarkResult = RetrievalQualityMetrics & {
+  name: "retrieval-quality";
+  corpus_version: 1;
+  sessions: number;
+  by_category: Record<RetrievalCategory, RetrievalQualityMetrics>;
+  cases: Array<{
+    id: string;
+    category: RetrievalCategory;
+    query: string;
+    expected_session_id: string;
+    rank: number | null;
+    top_session_ids: string[];
+  }>;
+  duration_ms: number;
+  storage_home?: string;
+};
+
 export function runLongContextBenchmark(options: LongContextBenchmarkOptions = {}): LongContextBenchmarkResult {
   const eventCount = Math.max(16, Math.floor(options.events ?? 128));
   const budgetTokens = Math.max(64, Math.floor(options.budgetTokens ?? 1200));
@@ -77,3 +132,71 @@ export function runLongContextBenchmark(options: LongContextBenchmarkOptions = {
     if (cleanup) fs.rmSync(home, { recursive: true, force: true });
   }
 }
+
+export function runRetrievalQualityBenchmark(options: { home?: string } = {}): RetrievalQualityBenchmarkResult {
+  const home = options.home ?? fs.mkdtempSync(path.join(os.tmpdir(), "codex-lcm-retrieval-quality-"));
+  const cleanup = options.home === undefined;
+  const startedAt = performance.now();
+  const storage = createStorage({ home });
+
+  try {
+    storage.ingestMany(RETRIEVAL_CORPUS.map(([sessionId, prompt], index) => normalizeHookEvent({
+      hookEvent: "UserPromptSubmit",
+      rawInput: JSON.stringify({
+        session_id: sessionId,
+        cwd: RETRIEVAL_BENCHMARK_CWD,
+        prompt,
+      }),
+      env: {},
+      now: () => new Date(Date.UTC(2026, 6, 1, 12, index)),
+    })));
+    const cases = RETRIEVAL_QUERIES.map((entry) => {
+      const topSessionIds = storage.searchSessions({
+        query: entry.query,
+        cwd: RETRIEVAL_BENCHMARK_CWD,
+        limit: 5,
+      }).map((match) => match.session_id);
+      const index = topSessionIds.indexOf(entry.expectedSessionId);
+      return {
+        id: entry.id,
+        category: entry.category,
+        query: entry.query,
+        expected_session_id: entry.expectedSessionId,
+        rank: index < 0 ? null : index + 1,
+        top_session_ids: topSessionIds,
+      };
+    });
+    return {
+      name: "retrieval-quality",
+      corpus_version: 1,
+      sessions: RETRIEVAL_CORPUS.length,
+      ...retrievalMetrics(cases),
+      by_category: {
+        exact: retrievalMetrics(cases.filter((entry) => entry.category === "exact")),
+        "cross-session": retrievalMetrics(cases.filter((entry) => entry.category === "cross-session")),
+        temporal: retrievalMetrics(cases.filter((entry) => entry.category === "temporal")),
+        paraphrase: retrievalMetrics(cases.filter((entry) => entry.category === "paraphrase")),
+      },
+      cases,
+      duration_ms: Math.round(performance.now() - startedAt),
+      ...(cleanup ? {} : { storage_home: home }),
+    };
+  } finally {
+    storage.close();
+    if (cleanup) fs.rmSync(home, { recursive: true, force: true });
+  }
+}
+
+function retrievalMetrics(cases: ReadonlyArray<{ rank: number | null }>): RetrievalQualityMetrics {
+  const queries = cases.length;
+  return {
+    queries,
+    recall_at_1: ratio(cases.filter((entry) => entry.rank === 1).length, queries),
+    recall_at_5: ratio(cases.filter((entry) => entry.rank !== null && entry.rank <= 5).length, queries),
+    mean_reciprocal_rank: ratio(cases.reduce((sum, entry) => sum + (entry.rank ? 1 / entry.rank : 0), 0), queries),
+  };
+}
+
+function ratio(numerator: number, denominator: number): number {
+  return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4));
+}

plugins/codex-lcm/src/cli.ts

@@ -1,5 +1,5 @@
 import { loadConfig, pluginRoot } from "./config.ts";
-import { runLongContextBenchmark } from "./benchmark.ts";
+import { runLongContextBenchmark, runRetrievalQualityBenchmark } from "./benchmark.ts";
 import { importCodexSessions } from "./codex-import.ts";
 import { buildDoctorReport } from "./doctor.ts";
 import { runHook } from "./hook.ts";
@@ -61,13 +61,21 @@ export async function main(argv: string[]): Promise<void> {
   }
   if (command === "benchmark") {
     const benchmarkName = rest[0];
-    if (benchmarkName !== "long-context") throw new Error("Usage: codex-lcm benchmark long-context [--events N] [--budget-tokens N] [--home PATH] [--json]");
-    printObjectOrText(runLongContextBenchmark({
-      events: numberOptionValue(rest, "--events"),
-      budgetTokens: numberOptionValue(rest, "--budget-tokens"),
-      home: optionValue(rest, "--home"),
-    }));
-    return;
+    if (benchmarkName === "long-context") {
+      printObjectOrText(runLongContextBenchmark({
+        events: numberOptionValue(rest, "--events"),
+        budgetTokens: numberOptionValue(rest, "--budget-tokens"),
+        home: optionValue(rest, "--home"),
+      }));
+      return;
+    }
+    if (benchmarkName === "retrieval-quality") {
+      printObjectOrText(runRetrievalQualityBenchmark({
+        home: optionValue(rest, "--home"),
+      }));
+      return;
+    }
+    throw new Error("Usage: codex-lcm benchmark long-context|retrieval-quality [options] [--json]");
   }
   if (command === "health") {
     const storage = createStorage({ config: loadConfig(), readOnly: true });
@@ -166,6 +174,7 @@ Commands:
   codex-lcm usage [--since ISO] [--until ISO] [--cwd PATH] [--repo-root PATH] [--parent-session-id ID] [--roots-only] [--json]
   codex-lcm context-plan [--session-id ID] [--cwd PATH] [--repo-root PATH] [--model-context-window N] [--auto-compact-token-limit N] [--recent-event-limit N] [--json]
   codex-lcm benchmark long-context [--events N] [--budget-tokens N] [--home PATH] [--json]
+  codex-lcm benchmark retrieval-quality [--home PATH] [--json]
   codex-lcm import-codex-sessions [--from PATH] [--dry-run] [--progress] [--batch-size N] [--json]
 `);
 }

plugins/codex-lcm/tests/benchmark.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { runLongContextBenchmark } from "../src/benchmark.ts";
+import { runLongContextBenchmark, runRetrievalQualityBenchmark } from "../src/benchmark.ts";
 import { assertCliOk, runCli, tempHome } from "./helpers.ts";
 
 test("long-context benchmark recovers an old source event through packed context", () => {
@@ -19,6 +19,22 @@ test("long-context benchmark recovers an old source event through packed context
   assert.equal(result.duration_ms >= 0, true);
 });
 
+test("retrieval-quality benchmark reports ranked results across labeled query categories", () => {
+  const result = runRetrievalQualityBenchmark();
+
+  assert.equal(result.name, "retrieval-quality");
+  assert.equal(result.queries, 8);
+  assert.equal(result.cases.length, result.queries);
+  assert.deepEqual(Object.keys(result.by_category).sort(), ["cross-session", "exact", "paraphrase", "temporal"]);
+  assert.equal(result.recall_at_1 >= 0 && result.recall_at_1 <= 1, true);
+  assert.equal(result.recall_at_5 >= result.recall_at_1 && result.recall_at_5 <= 1, true);
+  assert.equal(result.mean_reciprocal_rank >= 0 && result.mean_reciprocal_rank <= 1, true);
+  assert.equal(result.by_category.exact.recall_at_5, 1);
+  assert.equal(result.by_category.paraphrase.recall_at_5 < 1, true);
+  assert.equal(result.cases.every((entry) => entry.rank === null || entry.rank > 0), true);
+  assert.equal(result.duration_ms >= 0, true);
+});
+
 test("benchmark long-context command prints JSON results", () => {
   const result = runCli(["benchmark", "long-context", "--events", "64", "--budget-tokens", "800", "--json"], {
     timeout: 10_000,
@@ -31,6 +47,18 @@ test("benchmark long-context command prints JSON results", () => {
   assert.equal(parsed.recovered, true);
 });
 
+test("benchmark retrieval-quality command prints ranked metrics", () => {
+  const result = runCli(["benchmark", "retrieval-quality", "--json"], {
+    timeout: 10_000,
+  });
+
+  assertCliOk(result);
+  const parsed = JSON.parse(result.stdout);
+  assert.equal(parsed.name, "retrieval-quality");
+  assert.equal(parsed.queries, 8);
+  assert.equal(parsed.cases.length, 8);
+});
+
 test("benchmark long-context command can keep caller-provided storage", () => {
   const home = tempHome("codex-lcm-benchmark-keep-");
   const result = runCli(["benchmark", "long-context", "--events", "64", "--budget-tokens", "800", "--home", home, "--json"], {