From cb6e36c8cd75262853575149cd0f4b3a825b867f Mon Sep 17 00:00:00 2001 From: davidrobertson Date: Sun, 26 Jul 2026 16:06:21 -0400 Subject: [PATCH] feat: improve paraphrase recall coverage --- plugins/codex-lcm/README.md | 6 +- plugins/codex-lcm/skills/lcm-recall/SKILL.md | 21 ++++-- plugins/codex-lcm/src/benchmark.ts | 78 +++++++++++++++++--- plugins/codex-lcm/tests/benchmark.test.ts | 14 +++- 4 files changed, 95 insertions(+), 24 deletions(-) diff --git a/plugins/codex-lcm/README.md b/plugins/codex-lcm/README.md index 932f52e..29901a0 100644 --- a/plugins/codex-lcm/README.md +++ b/plugins/codex-lcm/README.md @@ -223,8 +223,10 @@ bounded budget. Pass `--home PATH` to keep the benchmark storage for inspection. `node bin/codex-lcm benchmark retrieval-quality --json` runs a fixed, dependency-free corpus through production session search. It reports Recall@1, Recall@5, mean reciprocal rank, and per-case ranks across exact, cross-session, -temporal, and paraphrase queries. The paraphrase cases expose lexical retrieval -gaps instead of hiding them behind exact needles. +temporal, and paraphrase queries. Corpus version 2 has 39 sessions and 38 +queries, including 32 paraphrase cases split evenly between development and +holdout sets. The paraphrase cases expose lexical retrieval gaps instead of +hiding them behind exact needles. ## Privacy And Safety diff --git a/plugins/codex-lcm/skills/lcm-recall/SKILL.md b/plugins/codex-lcm/skills/lcm-recall/SKILL.md index 6194f8a..39d3525 100644 --- a/plugins/codex-lcm/skills/lcm-recall/SKILL.md +++ b/plugins/codex-lcm/skills/lcm-recall/SKILL.md @@ -34,6 +34,11 @@ because context had to be recovered. - Keep Codex LCM MCP calls sequential. Do not fan out one call per session or run multiple LCM calls concurrently; each call opens an index reader and concurrent fan-out can multiply process and SQLite memory. - For date-range or multi-session reviews, call `lcm_list_sessions` once with `includeSummaries: true`, then follow its cursor sequentially if needed. Only describe or expand the small set of sessions whose compact summaries need source evidence. - The preferred standard workflow is `lcm_grep` -> `lcm_describe` -> `lcm_expand`: find candidates, inspect session or summary-node lineage, then expand a chosen summary node into bounded source evidence. +- If `lcm_grep` returns no useful candidate, retry sequentially with at most two + concrete reformulations while keeping the same scope filters. Replace likely + concept wording rather than merely rearranging terms, for example `oversized + command output` -> `overflow payload` -> `large tool result`. Stop after the + first useful candidate; do not fan out retries. - In Codex, `mcp__codex_lcm__lcm_grep` -> `mcp__codex_lcm__lcm_describe` -> `mcp__codex_lcm__lcm_expand` are those same standard tools. Agents must use the host-qualified form Codex shows when the bare names are absent rather than fall back to lower-level session APIs. - Use `lcm_expand_query` as the focused query-first alternative when you do not yet know which summary node to expand. It searches matching summary nodes and recursively expands source lineage into bounded evidence. It does not synthesize an answer. Use `overview: true` for broad lineage views, and remember `sourceLimit` is per matched node/source expansion. - Treat `lcm_pack_context` as the model-ready retrieval path. It searches summary nodes first and expands bounded source lineage, so it is usually better than loading raw events for broad recall. @@ -53,13 +58,15 @@ because context had to be recovered. 1. Identify the current `cwd` and repo root if available. Projectless sessions are valid; do not require git metadata. 2. Call `lcm_current_session` with the current `cwd` and repo root if known. 3. Call `lcm_grep` with a concrete query and `cwd` or `repoRoot` when scope is known. -4. For a promising hit, call `lcm_describe` with the session ID. If it exposes a relevant summary node, call `lcm_expand` with that node ID. -5. Call `lcm_expand_query` when a focused query needs deeper source-lineage evidence and manual node selection would add friction. -6. Call `lcm_context_plan` when a long-running session may be near a soft context limit. -7. Call `lcm_pack_context` when you need a model-ready context block instead of manually reading descriptions and expansions. -8. If the task may involve older work or another thread, repeat `lcm_grep` or `lcm_search_sessions` with broader scope. -9. For long sessions, prefer `lcm_describe`, `lcm_get_session_graph`, and paged `lcm_get_session` with `limit` and `cursor`; do not load the entire session unless the user explicitly asks for a full raw dump. -10. Use `lcm_get_recent_context` for the latest bounded tail of a known session. +4. If no useful candidate appears, retry `lcm_grep` with at most two concrete + reformulations before widening the scope. +5. For a promising hit, call `lcm_describe` with the session ID. If it exposes a relevant summary node, call `lcm_expand` with that node ID. +6. Call `lcm_expand_query` when a focused query needs deeper source-lineage evidence and manual node selection would add friction. +7. Call `lcm_context_plan` when a long-running session may be near a soft context limit. +8. Call `lcm_pack_context` when you need a model-ready context block instead of manually reading descriptions and expansions. +9. If the task may involve older work or another thread, repeat `lcm_grep` or `lcm_search_sessions` with broader scope. +10. For long sessions, prefer `lcm_describe`, `lcm_get_session_graph`, and paged `lcm_get_session` with `limit` and `cursor`; do not load the entire session unless the user explicitly asks for a full raw dump. +11. Use `lcm_get_recent_context` for the latest bounded tail of a known session. ## Tool Hints diff --git a/plugins/codex-lcm/src/benchmark.ts b/plugins/codex-lcm/src/benchmark.ts index c326252..7122901 100644 --- a/plugins/codex-lcm/src/benchmark.ts +++ b/plugins/codex-lcm/src/benchmark.ts @@ -13,8 +13,9 @@ const BENCHMARK_QUERY = "BENCHMARK-NEEDLE recursive evidence recovery"; const RETRIEVAL_BENCHMARK_CWD = "/tmp/codex-lcm-retrieval-quality"; type RetrievalCategory = "exact" | "cross-session" | "temporal" | "paraphrase"; +type RetrievalSplit = "development" | "holdout"; -const RETRIEVAL_CORPUS = [ +const RETRIEVAL_BASE_CORPUS: ReadonlyArray = [ ["retrieval-storage", "SQLite WAL concurrent readers were chosen for local session storage."], ["retrieval-websocket", "WebSocket reconnect jitter prevents synchronized client retries."], ["retrieval-pool", "The pool chemistry pH target is 7.4 after probe calibration."], @@ -24,22 +25,68 @@ const RETRIEVAL_CORPUS = [ ["retrieval-migration-old", "Schema migration rollback used a manual database snapshot."], ["retrieval-migration-new", "Schema migration rollback now uses the automated restore job."], ["retrieval-backoff", "Failed requests retry with exponential delay and random spread."], -] as const; +]; + +const RETRIEVAL_PARAPHRASE_FIXTURES: ReadonlyArray = [ + ["retrieval-cache", "The cache removes expired entries when resident memory passes the configured ceiling.", "clear old cached data after hitting the RAM limit", "development"], + ["retrieval-auth", "Authentication credentials rotate automatically without interrupting active sessions.", "replace login secrets without downtime", "holdout"], + ["retrieval-queue", "Workers move poison messages to a dead-letter queue after repeated delivery failures.", "isolate jobs that keep failing", "development"], + ["retrieval-thumbnails", "Image thumbnails are generated asynchronously after uploads complete.", "make preview pictures in the background", "holdout"], + ["retrieval-dns", "DNS failover sends traffic to the secondary region when the primary health check fails.", "route users to a backup region during an outage", "development"], + ["retrieval-audit", "Audit records are retained for seven years in immutable object storage.", "how long compliance history stays stored", "holdout"], + ["retrieval-flags", "Feature flags roll out to five percent of accounts before wider activation.", "release a change gradually to a small user cohort", "development"], + ["retrieval-rate-limit", "API rate limits use a token bucket so brief request bursts can pass.", "allow short traffic spikes while enforcing a quota", "holdout"], + ["retrieval-backup", "Database backups run nightly and a restore drill verifies them each week.", "check that nightly snapshots can actually recover data", "development"], + ["retrieval-email", "Email delivery retries soft bounces but stops immediately on permanent rejection.", "try temporary mail failures again without retrying hard failures", "holdout"], + ["retrieval-encryption", "Uploaded documents use envelope encryption with a distinct wrapped data key.", "protect each file with its own wrapped key", "development"], + ["retrieval-localization", "Missing translation keys fall back to the default English message.", "show the default language when localized text is absent", "holdout"], + ["retrieval-pagination", "List endpoints use opaque cursor pagination to stay stable while rows change.", "page through changing results without skips", "development"], + ["retrieval-session-expiry", "Idle browser sessions expire after thirty minutes without activity.", "sign out inactive users after half an hour", "holdout"], + ["retrieval-readiness", "The readiness probe fails while database migrations are still pending.", "keep an instance out of traffic until database upgrades finish", "development"], + ["retrieval-tracing", "Trace identifiers propagate through HTTP calls and queued background jobs.", "follow one request across services and queue workers", "holdout"], + ["retrieval-scheduler", "Recurring jobs use UTC schedules to avoid daylight-saving clock changes.", "prevent seasonal clock changes from shifting scheduled work", "development"], + ["retrieval-multipart", "Multipart uploads checksum each part before assembling the final object.", "detect damaged chunks before joining a large file", "holdout"], + ["retrieval-indexing", "Search indexing batches document writes before refreshing visible results.", "group content updates before making search results current", "development"], + ["retrieval-config", "The service reloads configuration on SIGHUP without restarting the process.", "apply new settings without a restart", "holdout"], + ["retrieval-locks", "Lease-based locks renew during long jobs and expire if their worker dies.", "stop two workers owning the same task during lengthy work", "development"], + ["retrieval-offline", "Offline edits enter an outbox and synchronize after network connectivity returns.", "sync changes made without a network after reconnecting", "holdout"], + ["retrieval-percentile", "Metrics aggregate request latency into percentile histograms.", "measure the slowest one percent of requests", "development"], + ["retrieval-payments", "Payment intents require an idempotency key before processing a charge.", "prevent duplicate billing when checkout retries", "holdout"], + ["retrieval-temp-files", "Orphaned temporary files are deleted after twenty-four hours.", "remove abandoned scratch data the next day", "development"], + ["retrieval-canary", "Canary deployments mirror one percent of production traffic to the candidate build.", "send a tiny live sample to the new version", "holdout"], + ["retrieval-password-reset", "Password reset links expire after one use or fifteen minutes.", "make an account recovery URL single-use and short-lived", "development"], + ["retrieval-compression", "HTTP responses use Brotli compression when the client advertises support.", "shrink browser payloads with br encoding", "holdout"], + ["retrieval-json-schema", "Unknown JSON fields are preserved so older clients can round-trip future properties.", "keep newer properties intact when old clients save data", "development"], + ["retrieval-connections", "The connection pool closes idle sockets before the database server timeout.", "discard unused database connections before the server does", "holdout"], +]; + +const RETRIEVAL_CORPUS: ReadonlyArray = [ + ...RETRIEVAL_BASE_CORPUS, + ...RETRIEVAL_PARAPHRASE_FIXTURES.map(([sessionId, prompt]) => [sessionId, prompt] as const), +]; const RETRIEVAL_QUERIES: ReadonlyArray<{ id: string; category: RetrievalCategory; + split: RetrievalSplit; query: string; expectedSessionId: string; }> = [ - { id: "exact-storage", category: "exact", query: "SQLite WAL concurrent readers", expectedSessionId: "retrieval-storage" }, - { id: "exact-websocket", category: "exact", query: "WebSocket reconnect jitter", expectedSessionId: "retrieval-websocket" }, - { id: "cross-pool", category: "cross-session", query: "pool chemistry pH target", expectedSessionId: "retrieval-pool" }, - { id: "cross-overflow", category: "cross-session", query: "overflow payload integrity hash", expectedSessionId: "retrieval-overflow" }, - { id: "temporal-release", category: "temporal", query: "release signing certificate", expectedSessionId: "retrieval-release-new" }, - { id: "temporal-migration", category: "temporal", query: "schema migration rollback", expectedSessionId: "retrieval-migration-new" }, - { id: "paraphrase-overflow", category: "paraphrase", query: "retain oversized command output", expectedSessionId: "retrieval-overflow" }, - { id: "paraphrase-backoff", category: "paraphrase", query: "randomized backoff for failed calls", expectedSessionId: "retrieval-backoff" }, + { id: "exact-storage", category: "exact", split: "development", query: "SQLite WAL concurrent readers", expectedSessionId: "retrieval-storage" }, + { id: "exact-websocket", category: "exact", split: "holdout", query: "WebSocket reconnect jitter", expectedSessionId: "retrieval-websocket" }, + { id: "cross-pool", category: "cross-session", split: "development", query: "pool chemistry pH target", expectedSessionId: "retrieval-pool" }, + { id: "cross-overflow", category: "cross-session", split: "holdout", query: "overflow payload integrity hash", expectedSessionId: "retrieval-overflow" }, + { id: "temporal-release", category: "temporal", split: "development", query: "release signing certificate", expectedSessionId: "retrieval-release-new" }, + { id: "temporal-migration", category: "temporal", split: "holdout", query: "schema migration rollback", expectedSessionId: "retrieval-migration-new" }, + { id: "paraphrase-overflow", category: "paraphrase", split: "development", query: "retain oversized command output", expectedSessionId: "retrieval-overflow" }, + { id: "paraphrase-backoff", category: "paraphrase", split: "holdout", query: "randomized backoff for failed calls", expectedSessionId: "retrieval-backoff" }, + ...RETRIEVAL_PARAPHRASE_FIXTURES.map(([sessionId, _prompt, query, split], index) => ({ + id: `paraphrase-${String(index + 1).padStart(2, "0")}`, + category: "paraphrase" as const, + split, + query, + expectedSessionId: sessionId, + })), ]; export type LongContextBenchmarkOptions = { @@ -70,12 +117,14 @@ export type RetrievalQualityMetrics = { export type RetrievalQualityBenchmarkResult = RetrievalQualityMetrics & { name: "retrieval-quality"; - corpus_version: 1; + corpus_version: 2; sessions: number; by_category: Record; + by_split: Record; cases: Array<{ id: string; category: RetrievalCategory; + split: RetrievalSplit; query: string; expected_session_id: string; rank: number | null; @@ -160,6 +209,7 @@ export function runRetrievalQualityBenchmark(options: { home?: string } = {}): R return { id: entry.id, category: entry.category, + split: entry.split, query: entry.query, expected_session_id: entry.expectedSessionId, rank: index < 0 ? null : index + 1, @@ -168,7 +218,7 @@ export function runRetrievalQualityBenchmark(options: { home?: string } = {}): R }); return { name: "retrieval-quality", - corpus_version: 1, + corpus_version: 2, sessions: RETRIEVAL_CORPUS.length, ...retrievalMetrics(cases), by_category: { @@ -177,6 +227,10 @@ export function runRetrievalQualityBenchmark(options: { home?: string } = {}): R temporal: retrievalMetrics(cases.filter((entry) => entry.category === "temporal")), paraphrase: retrievalMetrics(cases.filter((entry) => entry.category === "paraphrase")), }, + by_split: { + development: retrievalMetrics(cases.filter((entry) => entry.split === "development")), + holdout: retrievalMetrics(cases.filter((entry) => entry.split === "holdout")), + }, cases, duration_ms: Math.round(performance.now() - startedAt), ...(cleanup ? {} : { storage_home: home }), diff --git a/plugins/codex-lcm/tests/benchmark.test.ts b/plugins/codex-lcm/tests/benchmark.test.ts index d293742..041feda 100644 --- a/plugins/codex-lcm/tests/benchmark.test.ts +++ b/plugins/codex-lcm/tests/benchmark.test.ts @@ -23,14 +23,21 @@ test("retrieval-quality benchmark reports ranked results across labeled query ca const result = runRetrievalQualityBenchmark(); assert.equal(result.name, "retrieval-quality"); - assert.equal(result.queries, 8); + assert.equal(result.corpus_version, 2); + assert.equal(result.sessions, 39); + assert.equal(result.queries, 38); assert.equal(result.cases.length, result.queries); assert.deepEqual(Object.keys(result.by_category).sort(), ["cross-session", "exact", "paraphrase", "temporal"]); + assert.deepEqual(Object.keys(result.by_split).sort(), ["development", "holdout"]); + assert.equal(result.by_category.paraphrase.queries, 32); + assert.equal(result.by_split.development.queries, 19); + assert.equal(result.by_split.holdout.queries, 19); assert.equal(result.recall_at_1 >= 0 && result.recall_at_1 <= 1, true); assert.equal(result.recall_at_5 >= result.recall_at_1 && result.recall_at_5 <= 1, true); assert.equal(result.mean_reciprocal_rank >= 0 && result.mean_reciprocal_rank <= 1, true); assert.equal(result.by_category.exact.recall_at_5, 1); assert.equal(result.by_category.paraphrase.recall_at_5 < 1, true); + assert.equal(result.cases.every((entry) => entry.split === "development" || entry.split === "holdout"), true); assert.equal(result.cases.every((entry) => entry.rank === null || entry.rank > 0), true); assert.equal(result.duration_ms >= 0, true); }); @@ -55,8 +62,9 @@ test("benchmark retrieval-quality command prints ranked metrics", () => { assertCliOk(result); const parsed = JSON.parse(result.stdout); assert.equal(parsed.name, "retrieval-quality"); - assert.equal(parsed.queries, 8); - assert.equal(parsed.cases.length, 8); + assert.equal(parsed.corpus_version, 2); + assert.equal(parsed.queries, 38); + assert.equal(parsed.cases.length, 38); }); test("benchmark long-context command can keep caller-provided storage", () => {