Add anchor-drift quality eval (graded quality marker)

evals/cases/anchor-drift-quality/bootstrap-seed.proposal.json

@@ -0,0 +1,117 @@
+{
+  "title": "Seed memory for anchor drift quality eval",
+  "summary": "Baseline code_verified claims spanning a difficulty ladder. Ten are anchored to symbols the session patch changes in varied ways (value, behavior, signature, enum, rename, half-truth, removal) and should be repaired; four are anchored to stable code the patch leaves intact (some in files that change nearby) and should be left alone.",
+  "creates": {
+    "components": [
+      { "id": "component.retrieval", "name": "Graph context retrieval", "code_anchor": "libs/knowledge-graph/graph-context/config.ts" },
+      { "id": "component.proposal_validation", "name": "Proposal validation", "code_anchor": "libs/knowledge-graph/validate-proposal.ts" },
+      { "id": "component.install", "name": "Install platforms", "code_anchor": "libs/install/paths.ts" },
+      { "id": "component.session_hooks", "name": "Session hooks", "code_anchor": "libs/hooks/session-state.ts" },
+      { "id": "component.code_anchors", "name": "Code anchor resolver", "code_anchor": "libs/knowledge-graph/code-anchors/resolver.ts" },
+      { "id": "component.storage", "name": "SQLite storage", "code_anchor": "libs/storage/sqlite/db.ts" }
+    ],
+    "flows": [],
+    "claims": [
+      {
+        "id": "claim.minimum_selected_claims",
+        "kind": "fact",
+        "text": "The retrieval ranking config selects at least 3 claims for a context packet (minimumSelectedClaims = 3).",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/config.ts", "symbol": "rankingConfig" }]
+      },
+      {
+        "id": "claim.default_embedding_model",
+        "kind": "fact",
+        "text": "Graph context defaults to the local all-mpnet-base-v2 embedding model with 768 dimensions.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/config.ts", "symbol": "graphContextConfig" }]
+      },
+      {
+        "id": "claim.max_code_anchors",
+        "kind": "fact",
+        "text": "Proposal validation allows a claim at most 3 code anchors (maxCodeAnchorsPerClaim = 3).",
+        "truth": "code_verified", "intent": "intended", "about": ["component.proposal_validation"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/validate-proposal.ts", "symbol": "maxCodeAnchorsPerClaim" }]
+      },
+      {
+        "id": "claim.bm25_normalization",
+        "kind": "fact",
+        "text": "scoreBm25 min-max normalizes results by dividing every document score by the top score, so the highest-ranked document scores 1.0.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "scoreBm25" }]
+      },
+      {
+        "id": "claim.tokenize_stemming",
+        "kind": "fact",
+        "text": "Tokenization adds stemming variants for the suffixes -ing, -ed, -es, -s, and -ation.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "tokenVariants" }]
+      },
+      {
+        "id": "claim.selection_rule",
+        "kind": "fact",
+        "text": "selectRankedDocuments keeps a document if it is within the minimumSelected count OR its score is at least the selection threshold.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "selectRankedDocuments" }]
+      },
+      {
+        "id": "claim.install_platforms",
+        "kind": "fact",
+        "text": "Greplica supports 8 install platforms: codex, claude, copilot, cursor, opencode, openhands, factory-droid, and antigravity.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.install"],
+        "code_anchors": [{ "file": "libs/install/paths.ts", "symbol": "installPlatforms" }]
+      },
+      {
+        "id": "claim.round_score",
+        "kind": "fact",
+        "text": "roundScore rounds a ranking score to 3 decimal places.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "roundScore" }]
+      },
+      {
+        "id": "claim.session_update_thresholds",
+        "kind": "fact",
+        "text": "Background memory updates trigger after 7 Stop hooks or after 40 minutes of uncovered activity, with a 5-minute grace window.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.session_hooks"],
+        "code_anchors": [{ "file": "libs/hooks/session-state.ts", "symbol": "defaultTimeThresholdMinutes" }]
+      },
+      {
+        "id": "claim.cfamily_fallback",
+        "kind": "fact",
+        "text": "When a file has no tree-sitter grammar, fallbackSymbolForFile locates a symbol by a C-family text search over the source.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.code_anchors"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/code-anchors/resolver.ts", "symbol": "fallbackSymbolForFile" }]
+      },
+      {
+        "id": "claim.index_scores_helper",
+        "kind": "fact",
+        "text": "indexScores builds a lookup Map from a document id to its score entry.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "indexScores" }]
+      },
+      {
+        "id": "claim.split_camel_case",
+        "kind": "fact",
+        "text": "splitCamelCase splits an identifier on camelCase, PascalCase, and letter/digit boundaries.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"],
+        "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "splitCamelCase" }]
+      },
+      {
+        "id": "claim.default_database_path",
+        "kind": "fact",
+        "text": "The SQLite graph database lives at graph.db inside the Greplica home directory.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.storage"],
+        "code_anchors": [{ "file": "libs/storage/sqlite/db.ts", "symbol": "defaultDatabasePath" }]
+      },
+      {
+        "id": "claim.bundled_skills",
+        "kind": "fact",
+        "text": "Greplica bundles 3 skills: greplica-bootstrap, greplica-update-working-memory, and greplica-fast-session-bootstrap.",
+        "truth": "code_verified", "intent": "intended", "about": ["component.install"],
+        "code_anchors": [{ "file": "libs/install/paths.ts", "symbol": "skillNames" }]
+      }
+    ],
+    "sources": [],
+    "edges": []
+  }
+}

evals/cases/anchor-drift-quality/rubric.json

@@ -0,0 +1,52 @@
+{
+  "case_id": "anchor-drift-quality",
+  "base_commit": "ec947c7e4728c2e6ef84868cc9428d473e22326e",
+  "score": {
+    "pass_threshold": 65,
+    "dimension_weights": {
+      "detection": 0.25,
+      "correctness": 0.35,
+      "completeness": 0.15,
+      "preservation": 0.15,
+      "anchor_accuracy": 0.1,
+      "restraint": 1.0
+    },
+    "difficulty_weights": { "easy": 1, "medium": 2, "hard": 3, "trap": 1.5 }
+  },
+  "patch_summary": "One refactor-style patch to graph-context/config.ts, bm25.ts, rank.ts, validate-proposal.ts, install/paths.ts, and hooks/session-state.ts. Value changes: minimumSelectedClaims 3->5; embedding all-mpnet-base-v2/768 -> all-MiniLM-L6-v2/384; maxCodeAnchorsPerClaim 3->4; defaultTimeThresholdMinutes 40->30. Behavior: scoreBm25 no longer divides by the top score (returns raw bm25 scores); selectRankedDocuments now keeps a document only when it is within minimumSelected AND its score meets the threshold. Rename: tokenVariants -> stemVariants. Removal: indexScores deleted and inlined into rankContextDocuments. installPlatforms gains 'windsurf' (now 9 platforms).",
+  "cases": [
+    { "id": "claim.minimum_selected_claims", "category": "value", "difficulty": "easy", "action": "supersede",
+      "expected": "Replacement states the ranking config selects at least 5 claims (minimumSelectedClaims = 5).",
+      "dimensions": ["detection", "correctness"] },
+    { "id": "claim.default_embedding_model", "category": "value", "difficulty": "easy", "action": "supersede",
+      "expected": "Replacement states the default local embedding is all-MiniLM-L6-v2 with 384 dimensions (both the model name and the dimension).",
+      "dimensions": ["detection", "correctness", "completeness"] },
+    { "id": "claim.max_code_anchors", "category": "value", "difficulty": "easy", "action": "supersede",
+      "expected": "Replacement states a claim may have at most 4 code anchors (maxCodeAnchorsPerClaim = 4).",
+      "dimensions": ["detection", "correctness"] },
+    { "id": "claim.bm25_normalization", "category": "behavior", "difficulty": "medium", "action": "supersede",
+      "expected": "Replacement reflects that scoreBm25 no longer min-max normalizes; it returns the raw BM25 scores and the top document is no longer forced to 1.0.",
+      "dimensions": ["detection", "correctness"] },
+    { "id": "claim.tokenize_stemming", "category": "rename", "difficulty": "hard", "action": "supersede",
+      "expected": "The stemming-suffix fact stays accurate (behavior is unchanged) and the claim is re-anchored to the renamed symbol stemVariants. The value here is fixing the broken anchor, not rewriting the text.",
+      "expected_anchor_symbol": "stemVariants",
+      "dimensions": ["detection", "anchor_accuracy", "preservation"] },
+    { "id": "claim.selection_rule", "category": "behavior", "difficulty": "medium", "action": "supersede",
+      "expected": "Replacement reflects that selectRankedDocuments now keeps a document only when it is within the minimumSelected count AND its score meets the selection threshold (both conditions), not either one.",
+      "dimensions": ["detection", "correctness"] },
+    { "id": "claim.install_platforms", "category": "enum", "difficulty": "medium", "action": "supersede",
+      "expected": "Replacement lists 9 install platforms, adding windsurf to the previous 8.",
+      "dimensions": ["detection", "correctness", "completeness"] },
+    { "id": "claim.session_update_thresholds", "category": "half_wrong", "difficulty": "hard", "action": "supersede",
+      "expected": "Replacement corrects only the time threshold to 30 minutes while PRESERVING the 7 Stop hooks and the 5-minute grace window. Dropping or changing the still-correct 7 and 5 is a preservation failure.",
+      "dimensions": ["detection", "correctness", "completeness", "preservation"] },
+    { "id": "claim.index_scores_helper", "category": "removal", "difficulty": "hard", "action": "retire",
+      "expected": "indexScores was removed (its logic was inlined into rankContextDocuments). The claim should be retired/superseded and the agent must NOT keep asserting indexScores still exists or fabricate a replacement helper.",
+      "dimensions": ["detection", "correctness"] },
+    { "id": "claim.round_score", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] },
+    { "id": "claim.cfamily_fallback", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] },
+    { "id": "claim.split_camel_case", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] },
+    { "id": "claim.default_database_path", "category": "fresh", "difficulty": "easy", "action": "leave", "dimensions": ["restraint"] },
+    { "id": "claim.bundled_skills", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] }
+  ]
+}