diff --git a/evals/cases/anchor-drift-quality/bootstrap-seed.proposal.json b/evals/cases/anchor-drift-quality/bootstrap-seed.proposal.json new file mode 100644 index 0000000..98f9352 --- /dev/null +++ b/evals/cases/anchor-drift-quality/bootstrap-seed.proposal.json @@ -0,0 +1,117 @@ +{ + "title": "Seed memory for anchor drift quality eval", + "summary": "Baseline code_verified claims spanning a difficulty ladder. Ten are anchored to symbols the session patch changes in varied ways (value, behavior, signature, enum, rename, half-truth, removal) and should be repaired; four are anchored to stable code the patch leaves intact (some in files that change nearby) and should be left alone.", + "creates": { + "components": [ + { "id": "component.retrieval", "name": "Graph context retrieval", "code_anchor": "libs/knowledge-graph/graph-context/config.ts" }, + { "id": "component.proposal_validation", "name": "Proposal validation", "code_anchor": "libs/knowledge-graph/validate-proposal.ts" }, + { "id": "component.install", "name": "Install platforms", "code_anchor": "libs/install/paths.ts" }, + { "id": "component.session_hooks", "name": "Session hooks", "code_anchor": "libs/hooks/session-state.ts" }, + { "id": "component.code_anchors", "name": "Code anchor resolver", "code_anchor": "libs/knowledge-graph/code-anchors/resolver.ts" }, + { "id": "component.storage", "name": "SQLite storage", "code_anchor": "libs/storage/sqlite/db.ts" } + ], + "flows": [], + "claims": [ + { + "id": "claim.minimum_selected_claims", + "kind": "fact", + "text": "The retrieval ranking config selects at least 3 claims for a context packet (minimumSelectedClaims = 3).", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/config.ts", "symbol": "rankingConfig" }] + }, + { + "id": "claim.default_embedding_model", + "kind": "fact", + "text": "Graph context defaults to the local all-mpnet-base-v2 embedding model with 768 dimensions.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/config.ts", "symbol": "graphContextConfig" }] + }, + { + "id": "claim.max_code_anchors", + "kind": "fact", + "text": "Proposal validation allows a claim at most 3 code anchors (maxCodeAnchorsPerClaim = 3).", + "truth": "code_verified", "intent": "intended", "about": ["component.proposal_validation"], + "code_anchors": [{ "file": "libs/knowledge-graph/validate-proposal.ts", "symbol": "maxCodeAnchorsPerClaim" }] + }, + { + "id": "claim.bm25_normalization", + "kind": "fact", + "text": "scoreBm25 min-max normalizes results by dividing every document score by the top score, so the highest-ranked document scores 1.0.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "scoreBm25" }] + }, + { + "id": "claim.tokenize_stemming", + "kind": "fact", + "text": "Tokenization adds stemming variants for the suffixes -ing, -ed, -es, -s, and -ation.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "tokenVariants" }] + }, + { + "id": "claim.selection_rule", + "kind": "fact", + "text": "selectRankedDocuments keeps a document if it is within the minimumSelected count OR its score is at least the selection threshold.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "selectRankedDocuments" }] + }, + { + "id": "claim.install_platforms", + "kind": "fact", + "text": "Greplica supports 8 install platforms: codex, claude, copilot, cursor, opencode, openhands, factory-droid, and antigravity.", + "truth": "code_verified", "intent": "intended", "about": ["component.install"], + "code_anchors": [{ "file": "libs/install/paths.ts", "symbol": "installPlatforms" }] + }, + { + "id": "claim.round_score", + "kind": "fact", + "text": "roundScore rounds a ranking score to 3 decimal places.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "roundScore" }] + }, + { + "id": "claim.session_update_thresholds", + "kind": "fact", + "text": "Background memory updates trigger after 7 Stop hooks or after 40 minutes of uncovered activity, with a 5-minute grace window.", + "truth": "code_verified", "intent": "intended", "about": ["component.session_hooks"], + "code_anchors": [{ "file": "libs/hooks/session-state.ts", "symbol": "defaultTimeThresholdMinutes" }] + }, + { + "id": "claim.cfamily_fallback", + "kind": "fact", + "text": "When a file has no tree-sitter grammar, fallbackSymbolForFile locates a symbol by a C-family text search over the source.", + "truth": "code_verified", "intent": "intended", "about": ["component.code_anchors"], + "code_anchors": [{ "file": "libs/knowledge-graph/code-anchors/resolver.ts", "symbol": "fallbackSymbolForFile" }] + }, + { + "id": "claim.index_scores_helper", + "kind": "fact", + "text": "indexScores builds a lookup Map from a document id to its score entry.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/rank.ts", "symbol": "indexScores" }] + }, + { + "id": "claim.split_camel_case", + "kind": "fact", + "text": "splitCamelCase splits an identifier on camelCase, PascalCase, and letter/digit boundaries.", + "truth": "code_verified", "intent": "intended", "about": ["component.retrieval"], + "code_anchors": [{ "file": "libs/knowledge-graph/graph-context/bm25.ts", "symbol": "splitCamelCase" }] + }, + { + "id": "claim.default_database_path", + "kind": "fact", + "text": "The SQLite graph database lives at graph.db inside the Greplica home directory.", + "truth": "code_verified", "intent": "intended", "about": ["component.storage"], + "code_anchors": [{ "file": "libs/storage/sqlite/db.ts", "symbol": "defaultDatabasePath" }] + }, + { + "id": "claim.bundled_skills", + "kind": "fact", + "text": "Greplica bundles 3 skills: greplica-bootstrap, greplica-update-working-memory, and greplica-fast-session-bootstrap.", + "truth": "code_verified", "intent": "intended", "about": ["component.install"], + "code_anchors": [{ "file": "libs/install/paths.ts", "symbol": "skillNames" }] + } + ], + "sources": [], + "edges": [] + } +} diff --git a/evals/cases/anchor-drift-quality/rubric.json b/evals/cases/anchor-drift-quality/rubric.json new file mode 100644 index 0000000..5e3f52f --- /dev/null +++ b/evals/cases/anchor-drift-quality/rubric.json @@ -0,0 +1,52 @@ +{ + "case_id": "anchor-drift-quality", + "base_commit": "ec947c7e4728c2e6ef84868cc9428d473e22326e", + "score": { + "pass_threshold": 65, + "dimension_weights": { + "detection": 0.25, + "correctness": 0.35, + "completeness": 0.15, + "preservation": 0.15, + "anchor_accuracy": 0.1, + "restraint": 1.0 + }, + "difficulty_weights": { "easy": 1, "medium": 2, "hard": 3, "trap": 1.5 } + }, + "patch_summary": "One refactor-style patch to graph-context/config.ts, bm25.ts, rank.ts, validate-proposal.ts, install/paths.ts, and hooks/session-state.ts. Value changes: minimumSelectedClaims 3->5; embedding all-mpnet-base-v2/768 -> all-MiniLM-L6-v2/384; maxCodeAnchorsPerClaim 3->4; defaultTimeThresholdMinutes 40->30. Behavior: scoreBm25 no longer divides by the top score (returns raw bm25 scores); selectRankedDocuments now keeps a document only when it is within minimumSelected AND its score meets the threshold. Rename: tokenVariants -> stemVariants. Removal: indexScores deleted and inlined into rankContextDocuments. installPlatforms gains 'windsurf' (now 9 platforms).", + "cases": [ + { "id": "claim.minimum_selected_claims", "category": "value", "difficulty": "easy", "action": "supersede", + "expected": "Replacement states the ranking config selects at least 5 claims (minimumSelectedClaims = 5).", + "dimensions": ["detection", "correctness"] }, + { "id": "claim.default_embedding_model", "category": "value", "difficulty": "easy", "action": "supersede", + "expected": "Replacement states the default local embedding is all-MiniLM-L6-v2 with 384 dimensions (both the model name and the dimension).", + "dimensions": ["detection", "correctness", "completeness"] }, + { "id": "claim.max_code_anchors", "category": "value", "difficulty": "easy", "action": "supersede", + "expected": "Replacement states a claim may have at most 4 code anchors (maxCodeAnchorsPerClaim = 4).", + "dimensions": ["detection", "correctness"] }, + { "id": "claim.bm25_normalization", "category": "behavior", "difficulty": "medium", "action": "supersede", + "expected": "Replacement reflects that scoreBm25 no longer min-max normalizes; it returns the raw BM25 scores and the top document is no longer forced to 1.0.", + "dimensions": ["detection", "correctness"] }, + { "id": "claim.tokenize_stemming", "category": "rename", "difficulty": "hard", "action": "supersede", + "expected": "The stemming-suffix fact stays accurate (behavior is unchanged) and the claim is re-anchored to the renamed symbol stemVariants. The value here is fixing the broken anchor, not rewriting the text.", + "expected_anchor_symbol": "stemVariants", + "dimensions": ["detection", "anchor_accuracy", "preservation"] }, + { "id": "claim.selection_rule", "category": "behavior", "difficulty": "medium", "action": "supersede", + "expected": "Replacement reflects that selectRankedDocuments now keeps a document only when it is within the minimumSelected count AND its score meets the selection threshold (both conditions), not either one.", + "dimensions": ["detection", "correctness"] }, + { "id": "claim.install_platforms", "category": "enum", "difficulty": "medium", "action": "supersede", + "expected": "Replacement lists 9 install platforms, adding windsurf to the previous 8.", + "dimensions": ["detection", "correctness", "completeness"] }, + { "id": "claim.session_update_thresholds", "category": "half_wrong", "difficulty": "hard", "action": "supersede", + "expected": "Replacement corrects only the time threshold to 30 minutes while PRESERVING the 7 Stop hooks and the 5-minute grace window. Dropping or changing the still-correct 7 and 5 is a preservation failure.", + "dimensions": ["detection", "correctness", "completeness", "preservation"] }, + { "id": "claim.index_scores_helper", "category": "removal", "difficulty": "hard", "action": "retire", + "expected": "indexScores was removed (its logic was inlined into rankContextDocuments). The claim should be retired/superseded and the agent must NOT keep asserting indexScores still exists or fabricate a replacement helper.", + "dimensions": ["detection", "correctness"] }, + { "id": "claim.round_score", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] }, + { "id": "claim.cfamily_fallback", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] }, + { "id": "claim.split_camel_case", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] }, + { "id": "claim.default_database_path", "category": "fresh", "difficulty": "easy", "action": "leave", "dimensions": ["restraint"] }, + { "id": "claim.bundled_skills", "category": "fresh", "difficulty": "trap", "action": "leave", "dimensions": ["restraint"] } + ] +} diff --git a/evals/cases/anchor-drift-quality/run.ts b/evals/cases/anchor-drift-quality/run.ts new file mode 100644 index 0000000..08e4e5e --- /dev/null +++ b/evals/cases/anchor-drift-quality/run.ts @@ -0,0 +1,381 @@ +import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { + type CommandResult, + findRepoRoot, + readJson, + run, + runOrThrow, + timestamp, + valueAfter, + writeJson, +} from "../../lib/common.js"; +import { + buildJudgeCases, + judgeOutputSchema, + scoreQuality, + type QualityJudgeOutput, + type QualityRubric, + type QualityScore, +} from "../../lib/drift-quality-scoring.js"; +import { runCodexAgent } from "../../../libs/agent-runner/codex.js"; +import type { AgentRunResult } from "../../../libs/agent-runner/types.js"; +import { loadRepoEnv } from "../../../libs/env/load-local-env.js"; + +// Quality marker for anchor-drift memory repair. Seeds claims across a difficulty +// ladder, applies one realistic patch that drifts about half of them in varied +// ways, runs the update-working-memory flow, and grades the agent's repair with +// partial credit on several analytic dimensions so the score spreads. + +const caseId = "anchor-drift-quality"; +const baseCommit = "ec947c7e4728c2e6ef84868cc9428d473e22326e"; + +interface Args { + agentModel?: string; + judge?: "openai"; + judgeModel?: string; +} + +interface RunContext { + repoRoot: string; + fixtureDir: string; + runDir: string; + targetRepoDir: string; + targetRepoUrl: string; + greplicaHomeDir: string; + codexHomeDir: string; + seedProposalPath: string; + sessionPatchPath: string; + updateProposalPath: string; + graphReadPath: string; + rubricPath: string; + greplicaCommand: string[]; +} + +interface EvalResult { + case_id: string; + target_repo_url: string; + base_commit: string; + run_dir: string; + update_proposal_path: string; + success: boolean; + setup_commands: CommandResult[]; + patch_command?: CommandResult; + generation?: AgentRunResult; + update_commands: CommandResult[]; + graph_read_command?: CommandResult; + judge?: { + model: string; + judge_input_path: string; + judge_output_path: string; + score: QualityScore; + }; +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const context = prepareRun(); + copyFixtures(context); + prepareTargetRepo(context); + prepareGreplicaHome(context); + + const setupCommands = seedBootstrapMemory(context); + const setupSucceeded = setupCommands.every((command) => command.exit_code === 0); + const patchCommand = setupSucceeded ? applySessionPatch(context) : undefined; + const patchSucceeded = patchCommand?.exit_code === 0; + const generation = patchSucceeded ? await runUpdateAgent(context, args) : undefined; + const proposalCreated = existsSync(context.updateProposalPath); + const updateCommands = generation?.exit_code === 0 && proposalCreated ? applyUpdateProposal(context) : []; + const graphReadCommand = updateCommands.every((command) => command.exit_code === 0) ? readFinalGraph(context) : undefined; + + const rubric = readJson(context.rubricPath); + const proposal = proposalCreated ? readJson(context.updateProposalPath) : undefined; + const judge = graphReadCommand?.exit_code === 0 && proposal !== undefined && args.judge === "openai" + ? await runOpenAiJudge(context, rubric, proposal, args) + : undefined; + + const success = + setupSucceeded && + patchSucceeded && + generation?.exit_code === 0 && + proposalCreated && + updateCommands.every((command) => command.exit_code === 0) && + graphReadCommand?.exit_code === 0 && + (judge === undefined || judge.score.passed); + + writeResult(context, setupCommands, patchCommand, generation, updateCommands, graphReadCommand, judge, success); + printSummary(context, judge, success); + process.exitCode = success ? 0 : 1; +} + +function prepareRun(): RunContext { + const repoRoot = findRepoRoot(import.meta.url); + loadRepoEnv(repoRoot); + const fixtureDir = resolve(repoRoot, "evals/cases/anchor-drift-quality"); + const runDir = resolve(repoRoot, "eval-runs", timestamp(), caseId); + mkdirSync(runDir, { recursive: true }); + + return { + repoRoot, + fixtureDir, + runDir, + targetRepoDir: resolve(runDir, "target-repo"), + targetRepoUrl: process.env.GREPLICA_EVAL_TARGET_REPO_URL ?? repoRoot, + greplicaHomeDir: resolve(runDir, "greplica-home"), + codexHomeDir: resolve(runDir, "codex-home"), + seedProposalPath: resolve(runDir, "bootstrap-seed.proposal.json"), + sessionPatchPath: resolve(runDir, "session.patch"), + updateProposalPath: resolve(runDir, "update-proposal.json"), + graphReadPath: resolve(runDir, "final-graph.txt"), + rubricPath: resolve(fixtureDir, "rubric.json"), + greplicaCommand: ["node", resolve(repoRoot, "dist/apps/cli/main.js")], + }; +} + +function copyFixtures(context: RunContext): void { + copyFileSync(resolve(context.fixtureDir, "bootstrap-seed.proposal.json"), context.seedProposalPath); + copyFileSync(resolve(context.fixtureDir, "session.patch"), context.sessionPatchPath); +} + +function prepareTargetRepo(context: RunContext): void { + runOrThrow(["git", "clone", context.targetRepoUrl, context.targetRepoDir], context.repoRoot); + runOrThrow(["git", "checkout", baseCommit], context.targetRepoDir); +} + +function prepareGreplicaHome(context: RunContext): void { + mkdirSync(context.greplicaHomeDir, { recursive: true }); + mkdirSync(context.codexHomeDir, { recursive: true }); + seedCodexRuntimeHome(context.codexHomeDir); +} + +function seedCodexRuntimeHome(codexHomeDir: string): void { + const sourceHome = resolve(homedir(), ".codex"); + for (const file of ["auth.json", "config.toml", "models_cache.json", ".codex-global-state.json", "installation_id"]) { + const source = resolve(sourceHome, file); + if (existsSync(source)) copyFileSync(source, resolve(codexHomeDir, file)); + } +} + +function seedBootstrapMemory(context: RunContext): CommandResult[] { + return [ + runProductCommand(context, "install", "--platform", "codex", "--embedding", "local"), + runProductCommand(context, "proposal", "validate", context.seedProposalPath), + runProductCommand(context, "proposal", "apply", context.seedProposalPath), + ]; +} + +function applySessionPatch(context: RunContext): CommandResult { + return run(["git", "apply", context.sessionPatchPath], context.targetRepoDir, process.env); +} + +async function runUpdateAgent(context: RunContext, args: Args): Promise { + const result = await runCodexAgent({ + cwd: context.targetRepoDir, + env: { ...process.env, CODEX_HOME: context.codexHomeDir, GREPLICA_HOME: context.greplicaHomeDir }, + model: args.agentModel ?? "gpt-5.4-mini", + prompt: codexUpdatePrompt(context), + transcriptPath: resolve(context.runDir, "agent-events.jsonl"), + finalMessagePath: resolve(context.runDir, "agent-final-message.txt"), + proposalPath: context.updateProposalPath, + }); + + if (result.exit_code !== 0) throw new Error(`Codex agent failed with exit code ${String(result.exit_code)}.`); + if (!existsSync(context.updateProposalPath)) throw new Error(`Codex agent did not create proposal at ${context.updateProposalPath}.`); + return result; +} + +function applyUpdateProposal(context: RunContext): CommandResult[] { + return [ + runProductCommand(context, "proposal", "validate", context.updateProposalPath), + runProductCommand(context, "proposal", "apply", context.updateProposalPath), + ]; +} + +function readFinalGraph(context: RunContext): CommandResult { + const command = runProductCommand(context, "graph", "read"); + writeFileSync(context.graphReadPath, command.stdout ?? ""); + return command; +} + +function runProductCommand(context: RunContext, ...args: string[]): CommandResult { + const env = { ...process.env, CODEX_HOME: context.codexHomeDir, GREPLICA_HOME: context.greplicaHomeDir }; + return run([...context.greplicaCommand, ...args], context.targetRepoDir, env); +} + +async function runOpenAiJudge( + context: RunContext, + rubric: QualityRubric, + proposal: unknown, + args: Args, +): Promise> { + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) throw new Error("OPENAI_API_KEY is required when using --judge openai."); + const model = args.judgeModel ?? process.env.OPENAI_MODEL; + if (!model) throw new Error("Set OPENAI_MODEL or pass --judge-model when using --judge openai."); + + const seedClaims = readSeedClaimText(context.seedProposalPath); + const judgeInput = { + task: "Grade how well the agent repaired each drifted claim. Return JSON only; score every dimension in [0, 1]. Do not compute an overall score.", + instructions: [ + "For each case, compare the agent_replacements to the expected_repair and the patch_summary.", + "Score correctness: is the replacement factually right about the patched code? A confident but wrong value scores 0.", + "Score completeness: did the replacement capture the whole change (e.g. both the model name and the dimension, or every added enum member)?", + "Score preservation: did the replacement keep the parts of the original claim that are still true, and avoid deleting or corrupting them? For a claim where only one part drifted, changing the still-true parts is a preservation failure.", + "For a rename case the text should stay the same; judge preservation on whether the fact was kept intact.", + "For a removal case, correctness means the replacement acknowledges the symbol is gone and does not fabricate that it still exists.", + "If agent_replacements is empty, all three scores are 0.", + "Reason briefly before the scores. Do not reward verbosity or confident tone.", + ], + patch_summary: rubric.patch_summary, + cases: buildJudgeCases(rubric, seedClaims, proposal), + }; + + const judgeInputPath = resolve(context.runDir, "judge-input.json"); + const judgeOutputPath = resolve(context.runDir, "judge-output.json"); + writeJson(judgeInputPath, judgeInput); + + const judgeOutput = await requestJudge(apiKey, model, judgeInput); + writeJson(judgeOutputPath, judgeOutput); + + return { + model, + judge_input_path: judgeInputPath, + judge_output_path: judgeOutputPath, + score: scoreQuality(rubric, proposal, judgeOutput), + }; +} + +function readSeedClaimText(seedProposalPath: string): Map { + const seed = readJson<{ creates?: { claims?: Array<{ id?: unknown; text?: unknown }> } }>(seedProposalPath); + const map = new Map(); + for (const claim of seed.creates?.claims ?? []) { + if (typeof claim.id === "string" && typeof claim.text === "string") map.set(claim.id, claim.text); + } + return map; +} + +function writeResult( + context: RunContext, + setupCommands: CommandResult[], + patchCommand: CommandResult | undefined, + generation: AgentRunResult | undefined, + updateCommands: CommandResult[], + graphReadCommand: CommandResult | undefined, + judge: EvalResult["judge"], + success: boolean, +): void { + const result: EvalResult = { + case_id: caseId, + target_repo_url: context.targetRepoUrl, + base_commit: baseCommit, + run_dir: context.runDir, + update_proposal_path: context.updateProposalPath, + success, + setup_commands: setupCommands, + patch_command: patchCommand, + generation, + update_commands: updateCommands, + graph_read_command: graphReadCommand, + judge, + }; + writeJson(resolve(context.runDir, "result.json"), result); +} + +function printSummary(context: RunContext, judge: EvalResult["judge"], success: boolean): void { + console.log(success ? "Anchor drift quality run passed." : "Anchor drift quality run failed."); + console.log(`Run directory: ${context.runDir}`); + if (judge) { + console.log(`Quality score: ${judge.score.final_score.toFixed(2)} / 100 (pass >= ${judge.score.pass_threshold})`); + console.log(`By category: ${JSON.stringify(judge.score.by_category)}`); + console.log(`By difficulty: ${JSON.stringify(judge.score.by_difficulty)}`); + } +} + +function parseArgs(args: string[]): Args { + const judge = valueAfter(args, "--judge"); + if (judge === undefined) { + throw new Error("Anchor drift quality scoring requires --judge openai."); + } + if (judge !== undefined && judge !== "openai") throw new Error("Only --judge openai is supported."); + return { agentModel: valueAfter(args, "--agent-model"), judge, judgeModel: valueAfter(args, "--judge-model") }; +} + +function codexUpdatePrompt(context: RunContext): string { + const skill = readFileSync(resolve(context.repoRoot, "skills/greplica-update-working-memory/SKILL.md"), "utf8"); + const greplica = context.greplicaCommand.join(" "); + + return `You are running a Greplica update-working-memory pass after a coding change. + +Use this exact user-facing skill as the workflow contract: + + +${skill} + + +Runtime facts for this eval: +- Current working directory is the target repository root. +- GREPLICA_HOME is already set to an isolated eval directory. +- Greplica memory has already been seeded with baseline claims about this repo. +- A code change has already been applied to this working tree as uncommitted changes. +- Use this greplica command exactly: ${greplica} +- Write the final update proposal JSON exactly here: ${context.updateProposalPath} + +Task: +1. Run the update-working-memory skill workflow for the uncommitted changes. +2. Inspect the change with git status, git diff, and file reads. +3. Query existing memory with greplica graph context and greplica graph audit anchors to find claims about the changed code. +4. For any existing claim whose code changed so that it is now inaccurate, create a superseding claim with corrected text and correct code anchors. If a claim's anchored symbol was renamed, re-anchor the superseding claim to the new symbol. If the anchored code was removed, retire the claim rather than inventing a replacement fact. +5. Leave claims whose code did not change unchanged; do not supersede still-accurate memory. +6. Create the update proposal JSON at ${context.updateProposalPath}. +7. Validate it with: ${greplica} proposal validate ${context.updateProposalPath} +8. Fix validation errors until valid. Do not apply the proposal; the eval runner applies it after you exit.`; +} + +async function requestJudge(apiKey: string, model: string, input: unknown): Promise { + const response = await fetch("https://api.openai.com/v1/responses", { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + model, + input: [ + { + role: "system", + content: + "You are an evaluator for Greplica anchor-drift memory repair. Return JSON only. Score each case's dimensions in [0, 1] by comparing the agent's replacement claims to the expected repair. Do not compute an overall score.", + }, + { role: "user", content: JSON.stringify(input) }, + ], + text: { format: { type: "json_schema", name: "anchor_drift_quality_judge", strict: true, schema: judgeOutputSchema() } }, + }), + }); + + const body = await response.json() as Record; + if (!response.ok) throw new Error(`OpenAI judge request failed: ${JSON.stringify(body)}`); + return JSON.parse(extractOutputText(body)) as QualityJudgeOutput; +} + +function extractOutputText(body: Record): string { + if (typeof body.output_text === "string") return body.output_text; + const output = body.output; + if (!Array.isArray(output)) throw new Error("OpenAI response did not include output text."); + const texts: string[] = []; + for (const item of output) { + if (!isRecord(item) || !Array.isArray(item.content)) continue; + for (const content of item.content) { + if (isRecord(content) && typeof content.text === "string") texts.push(content.text); + } + } + const text = texts.join(""); + if (text.length === 0) throw new Error("OpenAI response output text was empty."); + return text; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/evals/cases/anchor-drift-quality/session.patch b/evals/cases/anchor-drift-quality/session.patch new file mode 100644 index 0000000..220c81b --- /dev/null +++ b/evals/cases/anchor-drift-quality/session.patch @@ -0,0 +1,133 @@ +diff --git a/libs/hooks/session-state.ts b/libs/hooks/session-state.ts +index c64e0a8..b4d3f8a 100644 +--- a/libs/hooks/session-state.ts ++++ b/libs/hooks/session-state.ts +@@ -5,7 +5,7 @@ import type { InstallPlatform } from "../install/paths.js"; + import type { AgentSession, ClaimedMemoryUpdateAttempt, MarkMemoryCurrentInput, RecordHookInput, RecordHookResult } from "./types.js"; + + const defaultStopThreshold = 7; +-const defaultTimeThresholdMinutes = 40; ++const defaultTimeThresholdMinutes = 30; + const defaultCurrentGraceMinutes = 5; + + export class HookSessionStore { +diff --git a/libs/install/paths.ts b/libs/install/paths.ts +index 503708f..5d81f6c 100644 +--- a/libs/install/paths.ts ++++ b/libs/install/paths.ts +@@ -2,10 +2,10 @@ import { existsSync } from "node:fs"; + import { dirname, join } from "node:path"; + import { fileURLToPath } from "node:url"; + +-export type InstallPlatform = "codex" | "claude" | "opencode" | "openhands" | "factory-droid" | "copilot" | "antigravity" | "cursor"; ++export type InstallPlatform = "codex" | "claude" | "opencode" | "openhands" | "factory-droid" | "copilot" | "antigravity" | "cursor" | "windsurf"; + export type InstallEmbedding = "local" | "openai"; + +-export const installPlatforms = ["codex", "claude", "copilot", "cursor", "opencode", "openhands", "factory-droid", "antigravity"] as const satisfies readonly InstallPlatform[]; ++export const installPlatforms = ["codex", "claude", "copilot", "cursor", "opencode", "openhands", "factory-droid", "antigravity", "windsurf"] as const satisfies readonly InstallPlatform[]; + export const installPlatformUsage = installPlatforms.join("|"); + export const installCommandSuggestion = `greplica install --platform <${installPlatformUsage}> --embedding local`; + +diff --git a/libs/knowledge-graph/graph-context/bm25.ts b/libs/knowledge-graph/graph-context/bm25.ts +index 4c8d570..7b88008 100644 +--- a/libs/knowledge-graph/graph-context/bm25.ts ++++ b/libs/knowledge-graph/graph-context/bm25.ts +@@ -40,10 +40,9 @@ export function scoreBm25(query: string, documents: ContextDocument[], config: G + .filter((entry) => entry.score > 0) + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + +- const maxScore = scored[0]?.score ?? 1; + return scored.map((entry, index) => ({ + id: entry.id, +- score: entry.score / maxScore, ++ score: entry.score, + raw_score: entry.score, + rank: index + 1, + })); +@@ -60,7 +59,7 @@ export function tokenize(text: string): string[] { + .map((token) => token.toLowerCase()) + .filter((token) => token.length > 1); + +- return tokens.flatMap((token) => [token, ...tokenVariants(token)]); ++ return tokens.flatMap((token) => [token, ...stemVariants(token)]); + } + + function expandIdentifierTokens(token: string): string[] { +@@ -86,7 +85,7 @@ function splitCamelCase(identifier: string): string[] { + .filter((piece) => piece.length > 0); + } + +-function tokenVariants(token: string): string[] { ++function stemVariants(token: string): string[] { + if (!/^[a-z]+$/.test(token)) return []; + const variants: string[] = []; + if (token.endsWith("ing") && token.length > 5) variants.push(token.slice(0, -3)); +diff --git a/libs/knowledge-graph/graph-context/config.ts b/libs/knowledge-graph/graph-context/config.ts +index 6903dbe..55b4936 100644 +--- a/libs/knowledge-graph/graph-context/config.ts ++++ b/libs/knowledge-graph/graph-context/config.ts +@@ -66,7 +66,7 @@ const rankingConfig: RankingConfig = { + selectionThreshold: 0.8, + packetMinimumScore: 0.15, + packetAdditionalDirectScoreFloor: 0.15, +- minimumSelectedClaims: 3, ++ minimumSelectedClaims: 5, + weights: { + semantic: 0.73, + bm25: 0.384, +@@ -127,8 +127,8 @@ export const graphContextConfig: GraphContextConfig = { + version: "graph-context-v33-precision-anchors", + embedding: { + provider: "local", +- model: "all-mpnet-base-v2", +- dimensions: 768, ++ model: "all-MiniLM-L6-v2", ++ dimensions: 384, + batchSize: 16, + }, + ranking: rankingConfig, +diff --git a/libs/knowledge-graph/graph-context/rank.ts b/libs/knowledge-graph/graph-context/rank.ts +index 5f2f6e9..e4d2720 100644 +--- a/libs/knowledge-graph/graph-context/rank.ts ++++ b/libs/knowledge-graph/graph-context/rank.ts +@@ -17,8 +17,8 @@ export function rankContextDocuments( + bm25: ScoreEntry[], + config: GraphContextConfig, + ): RankedContextDocument[] { +- const semanticById = indexScores(semantic); +- const bm25ById = indexScores(bm25); ++ const semanticById = new Map(semantic.map((score) => [score.id, score])); ++ const bm25ById = new Map(bm25.map((score) => [score.id, score])); + + const ranked = documents + .map((document) => { +@@ -78,7 +78,7 @@ export function selectRankedDocuments( + ): RankedContextDocument[] { + return ranked.filter( + (document, index) => +- index < (options.minimumSelected ?? 0) || ++ index < (options.minimumSelected ?? 0) && + document.score >= config.ranking.selectionThreshold, + ); + } +@@ -86,7 +86,3 @@ export function selectRankedDocuments( + export function roundScore(value: number): number { + return Math.round(value * 1000) / 1000; + } +- +-function indexScores(scores: ScoreEntry[]): Map { +- return new Map(scores.map((score) => [score.id, score])); +-} +diff --git a/libs/knowledge-graph/validate-proposal.ts b/libs/knowledge-graph/validate-proposal.ts +index 7cdad63..50608de 100644 +--- a/libs/knowledge-graph/validate-proposal.ts ++++ b/libs/knowledge-graph/validate-proposal.ts +@@ -9,7 +9,7 @@ const claimIntents = new Set(["intended", "accidental", "unknown"]); + const sourceKinds = new Set(["session"]); + const edgeKinds = new Set(["about", "contains", "touches", "supersedes", "evidenced_by"]); + const graphObjectTypes = new Set(["component", "flow", "claim", "edge", "source"]); +-const maxCodeAnchorsPerClaim = 3; ++const maxCodeAnchorsPerClaim = 4; + + export interface ExistingSubjectLookup { + subjectExists(type: GraphObjectType, id: string): boolean; diff --git a/evals/lib/drift-quality-scoring.ts b/evals/lib/drift-quality-scoring.ts new file mode 100644 index 0000000..3a688db --- /dev/null +++ b/evals/lib/drift-quality-scoring.ts @@ -0,0 +1,327 @@ +import { round } from "./common.js"; + +// Analytic scoring for the anchor-drift-quality eval. Each seeded claim is a +// "case" graded on several 0-1 dimensions with partial credit, then combined +// with a difficulty weight so the final score spreads instead of being pass/fail. +// Structural dimensions (detection, restraint, anchor_accuracy) are computed +// deterministically from the agent's proposal; the semantic dimensions +// (correctness, completeness, preservation) come from the LLM judge. + +export type DriftDimension = + | "detection" + | "correctness" + | "completeness" + | "preservation" + | "anchor_accuracy" + | "restraint"; + +export type CaseAction = "supersede" | "retire" | "leave"; +export type CaseDifficulty = "easy" | "medium" | "hard" | "trap"; + +export interface QualityCase { + id: string; + category: string; + difficulty: CaseDifficulty; + action: CaseAction; + expected?: string; + expected_anchor_symbol?: string; + dimensions: DriftDimension[]; +} + +export interface QualityRubric { + case_id: string; + base_commit: string; + score: { + pass_threshold: number; + dimension_weights: Record; + difficulty_weights: Record; + }; + patch_summary: string; + cases: QualityCase[]; +} + +// Per drift case the judge scores the three semantic dimensions in [0, 1]. +export interface JudgeCaseScore { + case_id: string; + correctness: number; + completeness: number; + preservation: number; + reason: string; +} + +export interface QualityJudgeOutput { + cases: JudgeCaseScore[]; +} + +export interface CaseScore { + id: string; + category: string; + difficulty: CaseDifficulty; + weight: number; + superseded: boolean; + dimensions: Partial>; + score: number; +} + +export interface QualityScore { + final_score: number; + pass_threshold: number; + passed: boolean; + cases: CaseScore[]; + by_category: Record; + by_difficulty: Record; +} + +export interface ReplacementClaim { + id: string; + text: string; + code_anchors: Array<{ file: string; symbol?: string }>; +} + +/** Claims in the proposal that supersede `oldClaimId`, via compact field or edge. */ +export function replacementClaimsFor(proposal: unknown, oldClaimId: string): ReplacementClaim[] { + const creates = proposalCreates(proposal); + if (creates === undefined) return []; + + const supersededByEdge = new Set( + proposalEdges(creates) + .filter((edge) => edge.kind === "supersedes" && edgeTo(edge) === oldClaimId) + .map((edge) => edgeFrom(edge)), + ); + + return proposalClaimObjects(creates).filter( + (claim) => stringArray(claim.supersedes).includes(oldClaimId) || supersededByEdge.has(claim.id), + ); +} + +export function isSuperseded(proposal: unknown, oldClaimId: string): boolean { + return replacementClaimsFor(proposal, oldClaimId).length > 0; +} + +function anchorAccuracy(replacements: ReplacementClaim[], expectedSymbol: string): number { + const matched = replacements.some((claim) => claim.code_anchors.some((anchor) => anchor.symbol === expectedSymbol)); + return matched ? 1 : 0; +} + +export function scoreQuality(rubric: QualityRubric, proposal: unknown, judge: QualityJudgeOutput): QualityScore { + validateJudgeOutput(rubric, judge); + const judgeById = new Map(judge.cases.map((entry) => [entry.case_id, entry])); + const dimensionWeights = rubric.score.dimension_weights; + + const cases: CaseScore[] = rubric.cases.map((rubricCase) => { + const superseded = isSuperseded(proposal, rubricCase.id); + const replacements = replacementClaimsFor(proposal, rubricCase.id); + const judged = judgeById.get(rubricCase.id); + const dimensions: Partial> = {}; + + for (const dimension of rubricCase.dimensions) { + dimensions[dimension] = scoreDimension(dimension, rubricCase, superseded, replacements, judged); + } + + return { + id: rubricCase.id, + category: rubricCase.category, + difficulty: rubricCase.difficulty, + weight: rubric.score.difficulty_weights[rubricCase.difficulty], + superseded, + dimensions, + score: weightedAverage(dimensions, dimensionWeights), + }; + }); + + const totalWeight = cases.reduce((sum, entry) => sum + entry.weight, 0); + const finalScore = totalWeight === 0 ? 0 : (cases.reduce((sum, entry) => sum + entry.weight * entry.score, 0) / totalWeight) * 100; + + return { + final_score: round(finalScore, 2), + pass_threshold: rubric.score.pass_threshold, + passed: finalScore >= rubric.score.pass_threshold, + cases, + by_category: groupedAverage(cases, (entry) => entry.category), + by_difficulty: groupedAverage(cases, (entry) => entry.difficulty), + }; +} + +function validateJudgeOutput(rubric: QualityRubric, judge: QualityJudgeOutput): void { + const expectedIds = rubric.cases + .filter((rubricCase) => rubricCase.action !== "leave") + .map((rubricCase) => rubricCase.id) + .sort(); + const actualIds = judge.cases.map((entry) => entry.case_id).sort(); + const uniqueIds = new Set(actualIds); + if ( + uniqueIds.size !== actualIds.length || + actualIds.length !== expectedIds.length || + actualIds.some((id, index) => id !== expectedIds[index]) + ) { + throw new Error( + `Judge output case IDs did not match the rubric. Expected [${expectedIds.join(", ")}], got [${actualIds.join(", ")}].`, + ); + } +} + +// A semantic dimension only earns credit when the claim was actually superseded: +// a correct replacement cannot exist for a claim the agent never touched. +function scoreDimension( + dimension: DriftDimension, + rubricCase: QualityCase, + superseded: boolean, + replacements: ReplacementClaim[], + judged: JudgeCaseScore | undefined, +): number { + switch (dimension) { + case "detection": + return superseded ? 1 : 0; + case "restraint": + return superseded ? 0 : 1; + case "anchor_accuracy": + return superseded && rubricCase.expected_anchor_symbol !== undefined + ? anchorAccuracy(replacements, rubricCase.expected_anchor_symbol) + : 0; + case "correctness": + return superseded ? clamp01(judged?.correctness) : 0; + case "completeness": + return superseded ? clamp01(judged?.completeness) : 0; + case "preservation": + return superseded ? clamp01(judged?.preservation) : 0; + } +} + +function weightedAverage( + dimensions: Partial>, + weights: Record, +): number { + let weightedSum = 0; + let weightTotal = 0; + for (const [dimension, value] of Object.entries(dimensions) as Array<[DriftDimension, number]>) { + const weight = weights[dimension]; + weightedSum += weight * value; + weightTotal += weight; + } + return weightTotal === 0 ? 0 : round(weightedSum / weightTotal, 4); +} + +function groupedAverage(cases: CaseScore[], key: (entry: CaseScore) => string): Record { + const buckets = new Map(); + for (const entry of cases) { + const bucket = buckets.get(key(entry)) ?? []; + bucket.push(entry.score); + buckets.set(key(entry), bucket); + } + const result: Record = {}; + for (const [name, scores] of buckets) { + result[name] = round(scores.reduce((sum, value) => sum + value, 0) / scores.length, 3); + } + return result; +} + +// Judge input: only the drift cases (supersede/retire) need semantic grading; +// each carries the original claim, the expected repair, and the agent's actual +// replacement claims so the judge can compare against ground truth. +export interface JudgeCaseInput { + case_id: string; + category: string; + original_claim: string; + expected_repair: string; + agent_replacements: ReplacementClaim[]; +} + +export function buildJudgeCases(rubric: QualityRubric, seedClaims: Map, proposal: unknown): JudgeCaseInput[] { + return rubric.cases + .filter((rubricCase) => rubricCase.action !== "leave") + .map((rubricCase) => ({ + case_id: rubricCase.id, + category: rubricCase.category, + original_claim: seedClaims.get(rubricCase.id) ?? "", + expected_repair: rubricCase.expected ?? "", + agent_replacements: replacementClaimsFor(proposal, rubricCase.id), + })); +} + +export function judgeOutputSchema(): Record { + const caseItem = { + type: "object", + additionalProperties: false, + properties: { + case_id: { type: "string" }, + correctness: { type: "number" }, + completeness: { type: "number" }, + preservation: { type: "number" }, + reason: { type: "string" }, + }, + required: ["case_id", "correctness", "completeness", "preservation", "reason"], + }; + return { + type: "object", + additionalProperties: false, + properties: { cases: { type: "array", items: caseItem } }, + required: ["cases"], + }; +} + +interface ProposalClaimObject { + id: string; + text: string; + supersedes?: unknown; + code_anchors: Array<{ file: string; symbol?: string }>; +} + +interface ProposalEdge { + kind?: unknown; + from?: unknown; + from_id?: unknown; + to?: unknown; + to_id?: unknown; +} + +function proposalCreates(proposal: unknown): Record | undefined { + if (!isRecord(proposal) || !isRecord(proposal.creates)) return undefined; + return proposal.creates; +} + +function proposalClaimObjects(creates: Record): ProposalClaimObject[] { + if (!Array.isArray(creates.claims)) return []; + return creates.claims.flatMap((claim) => { + if (!isRecord(claim) || typeof claim.id !== "string") return []; + return [{ + id: claim.id, + text: typeof claim.text === "string" ? claim.text : "", + supersedes: claim.supersedes, + code_anchors: parseAnchors(claim.code_anchors), + }]; + }); +} + +function parseAnchors(value: unknown): Array<{ file: string; symbol?: string }> { + if (!Array.isArray(value)) return []; + return value.flatMap((anchor) => { + if (!isRecord(anchor) || typeof anchor.file !== "string") return []; + return [{ file: anchor.file, symbol: typeof anchor.symbol === "string" ? anchor.symbol : undefined }]; + }); +} + +function proposalEdges(creates: Record): ProposalEdge[] { + if (!Array.isArray(creates.edges)) return []; + return creates.edges.flatMap((edge) => (isRecord(edge) ? [{ kind: edge.kind, from: edge.from, from_id: edge.from_id, to: edge.to, to_id: edge.to_id }] : [])); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; +} + +function edgeFrom(edge: ProposalEdge): string { + return typeof edge.from === "string" ? edge.from : typeof edge.from_id === "string" ? edge.from_id : ""; +} + +function edgeTo(edge: ProposalEdge): string { + return typeof edge.to === "string" ? edge.to : typeof edge.to_id === "string" ? edge.to_id : ""; +} + +function clamp01(value: number | undefined): number { + if (typeof value !== "number" || Number.isNaN(value)) return 0; + return Math.max(0, Math.min(1, value)); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/package.json b/package.json index 171aaab..b7163d6 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "smoke:copilot": "npm run build && node scripts/smoke-copilot-install.mjs", "smoke:opencode": "npm run build && node scripts/smoke-opencode-install.mjs", "smoke:cursor": "npm run build && node scripts/smoke-cursor-install.mjs", - "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-repo-installations.js && node scripts/check-managed-cli.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-source-memberships.js && node scripts/check-proposal-validate.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js && node scripts/check-find-similar-claims.js && node scripts/check-apply-proposal-dedupe.js && node scripts/check-opencode-sqlite-transcript.js", + "test": "npm run build && node scripts/check-transcript-bundle.js && node scripts/check-repo-context.js && node scripts/check-install-options.js && node scripts/check-repo-installations.js && node scripts/check-managed-cli.js && node scripts/check-graph-view.js && node scripts/check-graph-view-offline-browser.js && node scripts/check-source-memberships.js && node scripts/check-proposal-validate.js && node scripts/check-bm25-tokenizer.js && node scripts/check-anchor-drift.js && node scripts/check-find-similar-claims.js && node scripts/check-apply-proposal-dedupe.js && node scripts/check-opencode-sqlite-transcript.js && node scripts/check-drift-quality-scoring.js", "test:repo-installations": "npm run build && node scripts/check-repo-installations.js", "test:transcript-bundle": "npm run build && node scripts/check-transcript-bundle.js", "test:repo-context": "npm run build && node scripts/check-repo-context.js", @@ -40,6 +40,7 @@ "eval:claim-dedupe-threshold-calibration": "npm run build && node dist/evals/cases/claim-dedupe-threshold-calibration/run.js", "eval:update-working-memory-source-evidence": "npm run build && node dist/evals/cases/update-working-memory-source-evidence-at-0438915/run.js", "eval:anchor-drift-supersede": "npm run build && node dist/evals/cases/anchor-drift-supersede/run.js", + "eval:anchor-drift-quality": "npm run build && node dist/evals/cases/anchor-drift-quality/run.js", "smoke:factory-droid": "npm run build && node scripts/smoke-factory-droid.mjs", "bench:contextbench": "npm run build && node dist/evals/cases/contextbench-retrieval-from-sqlite/run.js", "optimize:search-ranking": "npm run build && node dist/evals/ranking-optimizer/train.js", diff --git a/scripts/check-drift-quality-scoring.js b/scripts/check-drift-quality-scoring.js new file mode 100644 index 0000000..3c367f1 --- /dev/null +++ b/scripts/check-drift-quality-scoring.js @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const root = new URL("..", import.meta.url); +const rubric = JSON.parse( + readFileSync(new URL("evals/cases/anchor-drift-quality/rubric.json", root), "utf8"), +); +const { + buildJudgeCases, + replacementClaimsFor, + scoreQuality, +} = await import(new URL("dist/evals/lib/drift-quality-scoring.js", root)); + +const driftCases = rubric.cases.filter(({ action }) => action !== "leave"); +const claims = []; +const edges = []; +for (const rubricCase of driftCases) { + const replacement = { + id: `replacement.${rubricCase.id}`, + kind: "fact", + text: rubricCase.expected, + truth: "code_verified", + intent: "intended", + code_anchors: [{ + file: "fixture.ts", + ...(rubricCase.expected_anchor_symbol === undefined + ? {} + : { symbol: rubricCase.expected_anchor_symbol }), + }], + }; + claims.push(replacement); + edges.push({ + id: `edge.${rubricCase.id}`, + from_id: replacement.id, + from_type: "claim", + to_id: rubricCase.id, + to_type: "claim", + kind: "supersedes", + }); +} +const perfectProposal = { creates: { claims, edges } }; +const perfectJudge = { + cases: driftCases.map(({ id }) => ({ + case_id: id, + correctness: 1, + completeness: 1, + preservation: 1, + reason: "Fixture-perfect replacement.", + })), +}; + +const perfect = scoreQuality(rubric, perfectProposal, perfectJudge); +assert.equal(perfect.final_score, 100); +assert.equal(perfect.passed, true); +assert.ok(Number.isFinite(perfect.final_score)); + +const noRepair = scoreQuality(rubric, { creates: {} }, perfectJudge); +assert.equal(noRepair.passed, false); +assert.ok(noRepair.final_score < rubric.score.pass_threshold); +assert.ok(noRepair.final_score >= 0); + +const trap = rubric.cases.find(({ action }) => action === "leave"); +assert.ok(trap); +const damagedProposal = { + creates: { + claims: [...claims, { + id: "replacement.trap", + text: "Needlessly replaced a fresh claim.", + code_anchors: [], + }], + edges: [...edges, { + kind: "supersedes", + from_id: "replacement.trap", + to_id: trap.id, + }], + }, +}; +const damaged = scoreQuality(rubric, damagedProposal, perfectJudge); +assert.ok(damaged.final_score < perfect.final_score); +assert.equal(damaged.cases.find(({ id }) => id === trap.id).dimensions.restraint, 0); + +const outOfRangeJudge = { + cases: perfectJudge.cases.map((entry, index) => ({ + ...entry, + correctness: index === 0 ? 2 : 1, + completeness: index === 1 ? -1 : 1, + preservation: index === 2 ? Number.NaN : 1, + })), +}; +const clamped = scoreQuality(rubric, perfectProposal, outOfRangeJudge); +assert.ok(Number.isFinite(clamped.final_score)); +assert.ok(clamped.final_score >= 0 && clamped.final_score <= 100); + +assert.throws( + () => scoreQuality(rubric, perfectProposal, { cases: perfectJudge.cases.slice(1) }), + /Judge output case IDs did not match the rubric/, +); +assert.throws( + () => scoreQuality(rubric, perfectProposal, { + cases: [...perfectJudge.cases, perfectJudge.cases[0]], + }), + /Judge output case IDs did not match the rubric/, +); + +const firstCase = driftCases[0]; +assert.equal(replacementClaimsFor(perfectProposal, firstCase.id).length, 1); +assert.deepEqual( + buildJudgeCases( + rubric, + new Map(rubric.cases.map(({ id }) => [id, `Original ${id}`])), + perfectProposal, + ).map(({ case_id }) => case_id), + driftCases.map(({ id }) => id), +); + +console.log("Drift quality scoring checks passed.");