diff --git a/docs/BUILD_LOG.md b/docs/BUILD_LOG.md index 5d13eff..662cc85 100644 --- a/docs/BUILD_LOG.md +++ b/docs/BUILD_LOG.md @@ -406,3 +406,13 @@ Evidence: `docs/v0.3/R8_COMBINED_PILOT_PROTOCOL.md` and updated experiment/execu An adversarial architecture review rejected a second mutable source and duplicate readiness state. The revised ADR makes “Dual Source” a falsifiable product metaphor over one event authority, separates stable node identity from immutable revision projections, keeps accountability failure off the production critical path, adds explicit R5.1/R5.2 runtime workstreams, and isolates live authority in R8 with matched prebuilt alternatives and separate speculative cost. Evidence: `docs/v0.3/ADR-010-DUAL-SOURCE-OPERATOR-SOURCE.md` and synchronized PRD, thesis, research, experiment, completion-audit, and handoff documents. This is a proposed projection and experiment, not implementation or skill-preservation evidence; full R7 remains the entry gate. + +## 2026-08-01 — Hash-verified R7 expert workspace + +- Added a local blind-rating CLI that verifies the frozen packet index and every packet hash before displaying a diff. +- Added resumable rater workspaces, exact-coverage export, strict categorical/confidence/reason validation, and refusal of unexpected material in the blind directory. +- Added an adjudication workspace that validates independent identities, carries both blinded source bundles, prefills only exact categorical consensus, and leaves disagreements for a distinct human panel decision. +- Added negative tests for tampered packets and outcome-like files plus coverage, resume, export, and adjudication tests. +- Verified the real frozen 18-packet index through CLI init/status/next, then passed `npm run check`, 114/114 runnable extension tests, production build, and VSIX packaging; two Docker-only tests remained skipped in the local Windows run. + +This removes clerical friction from the R7 human gate without replacing expert judgment or exposing automatic outcomes. Full R7 remains pending until two experienced TypeScript raters and the blinded adjudicator return real bundles. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 2ab8951..5d146b8 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -35,6 +35,7 @@ Branch `codex/shadow-cockpit-rnd` resets the product R&D thesis around **Dual-Co - `CONCEPT_LAB_CONTROLLABILITY.md` records a post-R7 category extension: compile an executable `observe → actuate → recover` human control surface, select takeover cut sets, and let a context-starved agent continue writing code from human-selected evidence and directives. Dissent cases and control dividends remain hypotheses with explicit falsifiers, not implemented features. - The preregistered R7 collector froze 30 eligible patches from six repositories after evaluating 457 bounded eligibility records. Manifest `a4ef6cbfa48c66cb9d384bcc2834ecbfae8ff08810abfd1863b395b8aa47d149` contains 12 development and 18 held-out patches; `docs/v0.3/results/R7_CORPUS_COLLECTION.md` reports repository and first-match exclusion counts. No compiler or human outcome influenced selection. - The frozen R7 automatic audit passed its preregistered automatic threshold: 17/18 held-out identities compiled and 16/18 were valid end-to-end. The frozen blind expert packet set and deterministic rating join exist, but two independent ratings and adjudication remain pending. Full R7 has not passed; R5/R6 stay gated. +- The R7 blind-rating path now includes a local hash-verified workspace CLI. It verifies the exact packet directory, supports resumable human drafts, refuses incomplete export, and prepares a blinded adjudication workspace by copying only exact categorical consensus. It automates no causal rating and exposes no compiler outcome; the two real expert bundles and panel decision are still pending. - ADR-007 proposes an Executable Operator Model and shadow-control protocol. ADR-008 adds a bidirectional Intent Ledger for artifact accountability. ADR-009 adds Decision Futures and a Takeover Envelope so an on-time pre-reveal human commitment can determine a live integrated branch while agents retain implementation. Together they cover artifact accountability, demonstrated control, and real decision authority; none is implementation evidence. - ADR-010 proposes Dual Source only as a falsifiable product metaphor: agents maintain the complete Software Source while a pure, sparse Operator Projection renders immutable human commitments and bounded control evidence. It owns no state or readiness authority and must beat the existing representation in a matched pilot before becoming core. - `docs/v0.3/R8_COMBINED_PILOT_PROTOCOL.md` now defines the draft four-condition human study needed to test the combined architecture against ordinary autonomous use and a fixed active-review comparator. It has no participants or measured outcomes and cannot be frozen until full R7 and the R5/R5.1/R5.2/R6 runtime exist. @@ -156,7 +157,7 @@ No external input blocks the repository-owned fixture R0–R4.5 mechanism in `do ## Next ordered actions -1. Obtain and adjudicate two independent blind expert ratings for packet index `54d78382b3ddbe15cba1f8153275e8149d32ddaa5192163f99ca5f43d903e8fe`. +1. Use `extension/scripts/r7-rater.mjs` to obtain and adjudicate two independent blind expert ratings for packet index `54d78382b3ddbe15cba1f8153275e8149d32ddaa5192163f99ca5f43d903e8fe`. 2. If full R7 passes, review ADR-007–010 and the R8 combined protocol together, then freeze the baseline ledger/model schemas and one offline Operator Source prototype. 3. Implement the local R5 evidence ledger and run Experiment 2.5 with identical evidence in both projections; make Operator Source the R6 default only if it improves causal localization per active minute without increasing false confidence. 4. Build the minimal cockpit with the surviving projection, then freeze and run the four-condition delayed-transfer pilot; run the longitudinal field pilot before making a sustained skill-retention claim. diff --git a/docs/v0.3/AGENT_EXECUTION.md b/docs/v0.3/AGENT_EXECUTION.md index d7584f8..80b1e03 100644 --- a/docs/v0.3/AGENT_EXECUTION.md +++ b/docs/v0.3/AGENT_EXECUTION.md @@ -402,6 +402,7 @@ If the fixture-only probe needs arbitrary participant code or model-generated te - Collect a development corpus and a separate held-out corpus of at least 30 eligible consented or open-source test-backed TypeScript patches in total; never redefine “supported” after seeing failures. - Freeze both the recovery compiler and semantic-probe compiler before running the held-out set and make no per-patch code changes. - Use two independent expert raters, blind to compiler outcome, for causal relevance and expected judge result; adjudicate disagreements and report inter-rater agreement. +- Use the hash-verified local rater workspace from `R7_EXPERT_RATING.md` so packet integrity, exact coverage, resume, bundle export, identity separation, and consensus-only adjudication prefill are machine-checked while causal decisions remain human. - Compile both a recovery episode and an Explain-to-Break probe for each eligible patch. Dynamic probes implement the Phase-B `SandboxControlProbe` contract: frozen sanitized snapshot/tree, immutable command-registry hash, approved command or controller-generated deterministic oracle, read-only mounts, and selected sandbox. They may never pass model output directly into code, tests, commands, arguments, paths, mounts, or environment. - Report valid-episode rate, valid-probe rate, capsule rejection/leakage results, false-pass rate, false-fail rate, and confidence intervals separately. - Run the held-out audit on Windows as well as Linux, including paths with spaces, concurrent twins, cancellation, locked files, and cleanup. diff --git a/docs/v0.3/R7_EXPERT_RATING.md b/docs/v0.3/R7_EXPERT_RATING.md index 048185c..b6e682e 100644 --- a/docs/v0.3/R7_EXPERT_RATING.md +++ b/docs/v0.3/R7_EXPERT_RATING.md @@ -25,7 +25,43 @@ Each rater returns one bundle containing exactly one record for every `packet-*. `confidence` is an integer from 1 to 5. A packet is expert-valid only when the adjudicated answer is `causalRelevance=yes`, `targetExpected=pass`, `rewindExpected=fail`, and `leakage=none`. Raters must not execute code or search commit history; this stage measures causal review of the disclosed participant surface, while automated sandbox receipts independently measure behavior. -After both files are returned, compare exact categorical agreement per field and adjudicate disagreements without exposing compiler outcomes. Save the full final 18-record decision as a third bundle with a distinct panel/adjudication ID. Then run: +## Hash-verified rater workspace + +Use the local workspace CLI so the engineer spends time on causal judgment rather than copying packet IDs or assembling JSON. The tool reads only the frozen packet directory. It verifies the index and every packet hash, refuses unexpected files or subdirectories, stores a resumable draft outside the packet directory, and will not export until all 18 packets have valid ratings. It has no command that accepts compiler results, audit summaries, commits, transcripts, or protected repairs. + +From `extension/`: + +```powershell +$packetDir = "..\docs\v0.3\results\held-out-rater-packets" +$ratingRoot = Join-Path $env:TEMP "pureflow-r7-expert-a" +New-Item -ItemType Directory -Path $ratingRoot +$workspace = Join-Path $ratingRoot "workspace.json" +$bundle = Join-Path $ratingRoot "expert-a.json" + +node scripts/r7-rater.mjs init $packetDir expert-a $workspace +node scripts/r7-rater.mjs next $packetDir $workspace +node scripts/r7-rater.mjs answer $packetDir $workspace yes pass fail none 4 "Reason grounded in the visible diff" +node scripts/r7-rater.mjs status $packetDir $workspace +node scripts/r7-rater.mjs export $packetDir $workspace $bundle +``` + +`next` prints the next unrated packet, visible diff, and frozen questions. `answer` may be repeated for a packet before export to correct a draft. Put neither the workspace nor the exported bundle inside `$packetDir`; the exact-content check deliberately rejects that. Give each rater a separate scratch directory and stable pseudonymous ID. + +After both files are returned, initialize a blind adjudication workspace. It validates both complete bundles, rejects reused identities, copies only exact four-field categorical consensus, and leaves every disagreement pending. `next` shows the frozen packet plus both blinded ratings and the fields that differ; it still has no access to compiler outcomes. + +```powershell +$panelRoot = Join-Path $env:TEMP "pureflow-r7-panel" +New-Item -ItemType Directory -Path $panelRoot +$panelWorkspace = Join-Path $panelRoot "workspace.json" +$adjudication = Join-Path $panelRoot "adjudication.json" + +node scripts/r7-rater.mjs init-adjudication $packetDir panel-chair $panelWorkspace +node scripts/r7-rater.mjs next $packetDir $panelWorkspace +node scripts/r7-rater.mjs answer $packetDir $panelWorkspace yes pass fail none 4 "Panel reason grounded in the visible diff" +node scripts/r7-rater.mjs export $packetDir $panelWorkspace $adjudication +``` + +Resolve all pending disagreements without exposing compiler outcomes. The exported adjudication is a full 18-record third bundle with a distinct panel ID. Then run: ```powershell cd extension diff --git a/extension/package.json b/extension/package.json index eb24e01..479e621 100644 --- a/extension/package.json +++ b/extension/package.json @@ -261,6 +261,7 @@ "fixture:runtime": "node scripts/provision-fixture-node.mjs", "r7:audit": "node scripts/r7-audit.mjs", "r7:corpus": "node scripts/r7-corpus.mjs", + "r7:rate": "node scripts/r7-rater.mjs", "test": "npm run fixture:runtime && vitest run", "package": "npm run build && vsce package --no-dependencies" }, diff --git a/extension/scripts/r7-rater.mjs b/extension/scripts/r7-rater.mjs new file mode 100644 index 0000000..f45a1c6 --- /dev/null +++ b/extension/scripts/r7-rater.mjs @@ -0,0 +1,22 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { build } from "esbuild"; + +const root = await mkdtemp(join(tmpdir(), "pureflow-r7-rater-cli-")); +const outfile = join(root, "cli.cjs"); + +try { + await build({ + entryPoints: ["src/rating/workspace-cli.ts"], + bundle: true, + platform: "node", + format: "cjs", + target: "node22", + outfile, + }); + await import(pathToFileURL(outfile).href); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/extension/src/rating/r7.ts b/extension/src/rating/r7.ts index 3a58695..f6ad515 100644 --- a/extension/src/rating/r7.ts +++ b/extension/src/rating/r7.ts @@ -59,10 +59,10 @@ export function summarizeR7Ratings( second: RatingBundle, adjudication: RatingBundle, ): R7RatingSummary { - const ids = validateIndex(index); - validateBundle(first, ids); - validateBundle(second, ids); - validateBundle(adjudication, ids); + const ids = validateRatingIndex(index); + validateRatingBundle(first, ids); + validateRatingBundle(second, ids); + validateRatingBundle(adjudication, ids); if (first.raterId === second.raterId) throw new Error("R7 requires two independent rater IDs"); if (adjudication.raterId === first.raterId || adjudication.raterId === second.raterId) { throw new Error("R7 adjudicator ID must be distinct from both raters"); @@ -101,7 +101,7 @@ export function summarizeR7Ratings( return { ...core, summarySha256: canonicalHash("r7-rating-summary", core) }; } -function validateIndex(index: RatingIndex): string[] { +export function validateRatingIndex(index: RatingIndex): string[] { if ( index.schemaVersion !== 1 || index.protocol !== "r7-blind-expert-v1" @@ -129,22 +129,34 @@ function validateIndex(index: RatingIndex): string[] { return ids; } -function validateBundle(bundle: RatingBundle, expectedIds: string[]): void { - if (bundle.schemaVersion !== 1 || bundle.protocol !== "r7-blind-expert-v1" || !/^[A-Za-z0-9][A-Za-z0-9._-]{2,63}$/.test(bundle.raterId)) { +export function validateRaterId(raterId: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{2,63}$/.test(raterId)) { + throw new Error("Invalid R7 rater ID"); + } +} + +export function validateExpertRating(rating: ExpertRating): void { + if (!/^[0-9a-f]{24}$/.test(rating.packetId)) throw new Error(`Invalid rating packet ID: ${rating.packetId}`); + if (!relevance.includes(rating.causalRelevance) || !expectation.includes(rating.targetExpected) || !expectation.includes(rating.rewindExpected)) { + throw new Error(`Invalid categorical rating: ${rating.packetId}`); + } + if (!leakage.includes(rating.leakage) || !Number.isInteger(rating.confidence) || rating.confidence < 1 || rating.confidence > 5) { + throw new Error(`Invalid leakage or confidence rating: ${rating.packetId}`); + } + if (rating.reason.trim().length < 10 || rating.reason.length > 1_000) throw new Error(`Invalid rating reason: ${rating.packetId}`); +} + +export function validateRatingBundle(bundle: RatingBundle, expectedIds: string[]): void { + if (bundle.schemaVersion !== 1 || bundle.protocol !== "r7-blind-expert-v1") { throw new Error("Invalid R7 rating bundle identity"); } + validateRaterId(bundle.raterId); const ids = bundle.ratings.map(({ packetId }) => packetId).sort(compareUtf8); if (ids.length !== expectedIds.length || ids.some((id, index) => id !== expectedIds[index])) { throw new Error(`Rating bundle ${bundle.raterId} does not cover the exact packet index`); } for (const rating of bundle.ratings) { - if (!relevance.includes(rating.causalRelevance) || !expectation.includes(rating.targetExpected) || !expectation.includes(rating.rewindExpected)) { - throw new Error(`Invalid categorical rating: ${rating.packetId}`); - } - if (!leakage.includes(rating.leakage) || !Number.isInteger(rating.confidence) || rating.confidence < 1 || rating.confidence > 5) { - throw new Error(`Invalid leakage or confidence rating: ${rating.packetId}`); - } - if (rating.reason.trim().length < 10 || rating.reason.length > 1_000) throw new Error(`Invalid rating reason: ${rating.packetId}`); + validateExpertRating(rating); } } diff --git a/extension/src/rating/workspace-cli.ts b/extension/src/rating/workspace-cli.ts new file mode 100644 index 0000000..83985c8 --- /dev/null +++ b/extension/src/rating/workspace-cli.ts @@ -0,0 +1,107 @@ +import type { ExpertRating } from "./r7"; +import { adjudicationDisagreement, createAdjudicationWorkspace, createRaterWorkspace, exportRatingBundle, ratingProgress, saveRating, type RaterPacket } from "./workspace"; +import { loadPacketSet, readRatingBundle, readWorkspace, replaceJson, writeNewJson } from "./workspace-files"; + +void main(); + +async function main(): Promise { + try { + const [command, packetDir, ...args] = process.argv.slice(2); + if (!command || !packetDir) return usage(); + const set = await loadPacketSet(packetDir); + if (command === "init") { + if (args.length !== 2) return usage(); + await writeNewJson(args[1]!, createRaterWorkspace(set.index, args[0]!)); + return printProgress(set.index.packetCount, 0, set.index.packets[0]?.packetId ?? null); + } + if (command === "init-adjudication") { + if (args.length !== 4) return usage(); + const [first, second] = await Promise.all([readRatingBundle(args[0]!), readRatingBundle(args[1]!)]); + const workspace = createAdjudicationWorkspace(set.index, first, second, args[2]!); + await writeNewJson(args[3]!, workspace); + const progress = ratingProgress(set.index, workspace); + return printProgress(progress.total, progress.complete, progress.nextPacketId); + } + if (command === "status" || command === "next") { + if (args.length !== 1) return usage(); + const workspace = await readWorkspace(args[0]!, set.index); + const progress = ratingProgress(set.index, workspace); + if (command === "status") return printProgress(progress.total, progress.complete, progress.nextPacketId); + if (!progress.nextPacketId) { + process.stdout.write("All packets are rated; export the bundle.\n"); + return; + } + const entry = set.packets.get(progress.nextPacketId)!; + process.stdout.write(renderPacket(entry.filename, entry.packet, adjudicationDisagreement(workspace, progress.nextPacketId))); + return; + } + if (command === "answer") { + if (args.length < 8) return usage(); + const [workspacePath, packetId, causalRelevance, targetExpected, rewindExpected, leakage, confidence, ...reason] = args; + const workspace = await readWorkspace(workspacePath!, set.index); + const rating = { + packetId: packetId!, + causalRelevance, + targetExpected, + rewindExpected, + leakage, + confidence: Number(confidence), + reason: reason.join(" ").trim(), + } as ExpertRating; + const updated = saveRating(set.index, workspace, rating); + await replaceJson(workspacePath!, updated); + const progress = ratingProgress(set.index, updated); + return printProgress(progress.total, progress.complete, progress.nextPacketId); + } + if (command === "export") { + if (args.length !== 2) return usage(); + const workspace = await readWorkspace(args[0]!, set.index); + await writeNewJson(args[1]!, exportRatingBundle(set.index, workspace)); + process.stdout.write(`Exported complete blind rating bundle for ${workspace.raterId}.\n`); + return; + } + usage(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} + +function renderPacket(filename: string, packet: RaterPacket, disagreement?: { first: ExpertRating; second: ExpertRating; fields: string[] }): string { + const comparison = disagreement + ? `\nADJUDICATION — disagreed fields: ${disagreement.fields.join(", ")}\nRater 1: ${JSON.stringify(disagreement.first)}\nRater 2: ${JSON.stringify(disagreement.second)}\n` + : ""; + return [ + `${filename} — ${packet.packetId}`, + `${packet.projectAlias} / ${packet.seam.path} / ${packet.seam.symbol} (${packet.seam.kind})`, + `Writable: ${packet.writablePaths.join(", ")}`, + `Tests: ${packet.attributedTestPaths.join(", ")}`, + "", + packet.mutationRule, + packet.observation, + comparison, + "SOURCE AND TEST DIFF", + packet.sourceAndTestDiff, + "QUESTIONS", + ...packet.questions.map((question, index) => `${index + 1}. ${question}`), + "", + ].join("\n"); +} + +function printProgress(total: number, complete: number, nextPacketId: string | null): void { + process.stdout.write(`Rated ${complete}/${total}.${nextPacketId ? ` Next packet: ${nextPacketId}.` : " Complete."}\n`); +} + +function usage(): void { + process.stderr.write([ + "Usage:", + " node scripts/r7-rater.mjs init ", + " node scripts/r7-rater.mjs next ", + " node scripts/r7-rater.mjs answer <1-5> ", + " node scripts/r7-rater.mjs status ", + " node scripts/r7-rater.mjs export ", + " node scripts/r7-rater.mjs init-adjudication ", + "", + ].join("\n")); + process.exitCode = 2; +} diff --git a/extension/src/rating/workspace-files.ts b/extension/src/rating/workspace-files.ts new file mode 100644 index 0000000..d01dbb6 --- /dev/null +++ b/extension/src/rating/workspace-files.ts @@ -0,0 +1,46 @@ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { validateRatingIndex, type RatingBundle, type RatingIndex } from "./r7"; +import { validatePacket, validateWorkspace, type RaterPacket, type RatingWorkspace } from "./workspace"; + +export interface PacketSet { + dir: string; + index: RatingIndex; + packets: Map; +} + +export async function loadPacketSet(packetDir: string): Promise { + const dir = resolve(packetDir); + const names = await readdir(dir, { withFileTypes: true }); + const index = JSON.parse(await readFile(join(dir, "index.json"), "utf8")) as RatingIndex; + validateRatingIndex(index); + const allowed = new Set(["index.json", ...index.packets.map(({ filename }) => filename)]); + const unexpected = names.filter((entry) => !entry.isFile() || !allowed.has(entry.name)); + if (unexpected.length) throw new Error(`Blind packet directory contains unexpected content: ${unexpected.map(({ name }) => name).join(", ")}`); + + const packets = new Map(); + for (const entry of index.packets) { + const packet = JSON.parse(await readFile(join(dir, entry.filename), "utf8")) as RaterPacket; + validatePacket(index, packet, entry.filename); + packets.set(packet.packetId, { filename: entry.filename, packet }); + } + return { dir, index, packets }; +} + +export async function readWorkspace(path: string, index: RatingIndex): Promise { + const workspace = JSON.parse(await readFile(resolve(path), "utf8")) as RatingWorkspace; + validateWorkspace(index, workspace); + return workspace; +} + +export async function readRatingBundle(path: string): Promise { + return JSON.parse(await readFile(resolve(path), "utf8")) as RatingBundle; +} + +export async function writeNewJson(path: string, value: unknown): Promise { + await writeFile(resolve(path), `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" }); +} + +export async function replaceJson(path: string, value: unknown): Promise { + await writeFile(resolve(path), `${JSON.stringify(value, null, 2)}\n`, { flag: "w" }); +} diff --git a/extension/src/rating/workspace.ts b/extension/src/rating/workspace.ts new file mode 100644 index 0000000..8388b4d --- /dev/null +++ b/extension/src/rating/workspace.ts @@ -0,0 +1,174 @@ +import { canonicalHash, compareUtf8 } from "../rnd/canonical"; +import { + validateExpertRating, + validateRaterId, + validateRatingBundle, + validateRatingIndex, + type ExpertRating, + type RatingBundle, + type RatingIndex, +} from "./r7"; + +const ratingFields = ["causalRelevance", "targetExpected", "rewindExpected", "leakage"] as const; + +export interface RaterPacket { + schemaVersion: 1; + protocol: "r7-blind-expert-v1"; + packetId: string; + projectAlias: string; + seam: { path: string; symbol: string; kind: string }; + writablePaths: string[]; + attributedTestPaths: string[]; + mutationRule: string; + observation: string; + sourceAndTestDiff: string; + questions: string[]; + packetSha256: string; +} + +export interface RatingWorkspace { + schemaVersion: 1; + protocol: "r7-blind-expert-v1"; + mode: "rater" | "adjudicator"; + packetIndexSha256: string; + raterId: string; + sourceBundles?: [RatingBundle, RatingBundle]; + ratings: ExpertRating[]; +} + +export function createRaterWorkspace(index: RatingIndex, raterId: string): RatingWorkspace { + validateRatingIndex(index); + validateRaterId(raterId); + return { + schemaVersion: 1, + protocol: "r7-blind-expert-v1", + mode: "rater", + packetIndexSha256: index.indexSha256, + raterId, + ratings: [], + }; +} + +export function createAdjudicationWorkspace( + index: RatingIndex, + first: RatingBundle, + second: RatingBundle, + adjudicatorId: string, +): RatingWorkspace { + const ids = validateRatingIndex(index); + validateRatingBundle(first, ids); + validateRatingBundle(second, ids); + validateRaterId(adjudicatorId); + if (first.raterId === second.raterId) throw new Error("R7 requires two independent rater IDs"); + if (adjudicatorId === first.raterId || adjudicatorId === second.raterId) { + throw new Error("R7 adjudicator ID must be distinct from both raters"); + } + + const left = new Map(first.ratings.map((rating) => [rating.packetId, rating])); + const right = new Map(second.ratings.map((rating) => [rating.packetId, rating])); + const ratings: ExpertRating[] = []; + for (const packetId of ids) { + const firstRating = left.get(packetId)!; + const secondRating = right.get(packetId)!; + const changed = ratingFields.filter((field) => firstRating[field] !== secondRating[field]); + if (changed.length) continue; + const lowerConfidence = firstRating.confidence <= secondRating.confidence ? firstRating : secondRating; + ratings.push({ + ...lowerConfidence, + reason: `Independent raters agreed. ${lowerConfidence.reason}`.slice(0, 1_000), + }); + } + + return { + schemaVersion: 1, + protocol: "r7-blind-expert-v1", + mode: "adjudicator", + packetIndexSha256: index.indexSha256, + raterId: adjudicatorId, + sourceBundles: [first, second].sort((a, b) => compareUtf8(a.raterId, b.raterId)) as [RatingBundle, RatingBundle], + ratings: ratings.sort((a, b) => compareUtf8(a.packetId, b.packetId)), + }; +} + +export function validatePacket(index: RatingIndex, packet: RaterPacket, filename: string): void { + validateRatingIndex(index); + const entry = index.packets.find((candidate) => candidate.filename === filename); + if (!entry || entry.packetId !== packet.packetId || entry.packetSha256 !== packet.packetSha256) { + throw new Error(`Packet is not bound to the frozen index: ${filename}`); + } + if (packet.schemaVersion !== 1 || packet.protocol !== "r7-blind-expert-v1") { + throw new Error(`Invalid R7 packet envelope: ${filename}`); + } + const { packetSha256, ...core } = packet; + if (canonicalHash("r7-rater-packet", core) !== packetSha256) { + throw new Error(`R7 packet hash mismatch: ${filename}`); + } +} + +export function validateWorkspace(index: RatingIndex, workspace: RatingWorkspace): void { + const ids = new Set(validateRatingIndex(index)); + validateRaterId(workspace.raterId); + if ( + workspace.schemaVersion !== 1 + || workspace.protocol !== "r7-blind-expert-v1" + || !["rater", "adjudicator"].includes(workspace.mode) + || workspace.packetIndexSha256 !== index.indexSha256 + ) { + throw new Error("Rating workspace does not match the frozen packet index"); + } + const seen = new Set(); + for (const rating of workspace.ratings) { + validateExpertRating(rating); + if (!ids.has(rating.packetId) || seen.has(rating.packetId)) throw new Error(`Invalid workspace coverage: ${rating.packetId}`); + seen.add(rating.packetId); + } + if (workspace.mode === "adjudicator") { + if (!workspace.sourceBundles || new Set([...workspace.sourceBundles.map(({ raterId }) => raterId), workspace.raterId]).size !== 3) { + throw new Error("Adjudication workspace requires three distinct identities"); + } + for (const bundle of workspace.sourceBundles) validateRatingBundle(bundle, [...ids].sort(compareUtf8)); + } +} + +export function adjudicationDisagreement(workspace: RatingWorkspace, packetId: string): { first: ExpertRating; second: ExpertRating; fields: string[] } | undefined { + if (workspace.mode !== "adjudicator" || !workspace.sourceBundles) return undefined; + const first = workspace.sourceBundles[0].ratings.find((rating) => rating.packetId === packetId); + const second = workspace.sourceBundles[1].ratings.find((rating) => rating.packetId === packetId); + if (!first || !second) throw new Error(`Missing adjudication source rating: ${packetId}`); + const fields = ratingFields.filter((field) => first[field] !== second[field]); + return fields.length ? { first, second, fields } : undefined; +} + +export function saveRating(index: RatingIndex, workspace: RatingWorkspace, rating: ExpertRating): RatingWorkspace { + validateWorkspace(index, workspace); + validateExpertRating(rating); + if (!index.packets.some(({ packetId }) => packetId === rating.packetId)) throw new Error(`Unknown R7 packet: ${rating.packetId}`); + const ratings = workspace.ratings.filter(({ packetId }) => packetId !== rating.packetId); + ratings.push(rating); + return { ...workspace, ratings: ratings.sort((a, b) => compareUtf8(a.packetId, b.packetId)) }; +} + +export function ratingProgress(index: RatingIndex, workspace: RatingWorkspace): { + complete: number; + total: number; + nextPacketId: string | null; + pendingPacketIds: string[]; +} { + validateWorkspace(index, workspace); + const complete = new Set(workspace.ratings.map(({ packetId }) => packetId)); + const pendingPacketIds = index.packets.map(({ packetId }) => packetId).filter((packetId) => !complete.has(packetId)); + return { complete: complete.size, total: index.packetCount, nextPacketId: pendingPacketIds[0] ?? null, pendingPacketIds }; +} + +export function exportRatingBundle(index: RatingIndex, workspace: RatingWorkspace): RatingBundle { + const ids = validateRatingIndex(index); + validateWorkspace(index, workspace); + const bundle: RatingBundle = { + schemaVersion: 1, + protocol: "r7-blind-expert-v1", + raterId: workspace.raterId, + ratings: [...workspace.ratings].sort((a, b) => compareUtf8(a.packetId, b.packetId)), + }; + validateRatingBundle(bundle, ids); + return bundle; +} diff --git a/extension/test/r7-rating-workspace.test.ts b/extension/test/r7-rating-workspace.test.ts new file mode 100644 index 0000000..ebe0890 --- /dev/null +++ b/extension/test/r7-rating-workspace.test.ts @@ -0,0 +1,128 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { canonicalHash } from "../src/rnd/canonical"; +import type { ExpertRating, RatingBundle, RatingIndex } from "../src/rating/r7"; +import { + adjudicationDisagreement, + createAdjudicationWorkspace, + createRaterWorkspace, + exportRatingBundle, + ratingProgress, + saveRating, + validatePacket, + type RaterPacket, +} from "../src/rating/workspace"; +import { loadPacketSet } from "../src/rating/workspace-files"; + +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +describe("R7 blind rating workspace", () => { + it("resumes partial work and exports only exact complete coverage", () => { + const { index } = packetSet(); + let workspace = createRaterWorkspace(index, "expert-one"); + + workspace = saveRating(index, workspace, rating(index.packets[0]!.packetId)); + expect(ratingProgress(index, workspace)).toMatchObject({ complete: 1, total: 2, nextPacketId: index.packets[1]!.packetId }); + expect(() => exportRatingBundle(index, workspace)).toThrow("exact packet index"); + + workspace = saveRating(index, workspace, rating(index.packets[1]!.packetId)); + const bundle = exportRatingBundle(index, workspace); + expect(bundle.raterId).toBe("expert-one"); + expect(bundle.ratings.map(({ packetId }) => packetId)).toEqual(index.packets.map(({ packetId }) => packetId).sort()); + }); + + it("rejects a packet changed after the frozen index was built", () => { + const { index, packets } = packetSet(); + const packet = structuredClone(packets[0]!); + packet.sourceAndTestDiff = "tampered"; + expect(() => validatePacket(index, packet, index.packets[0]!.filename)).toThrow("packet hash mismatch"); + }); + + it("prefills categorical consensus and leaves disagreements for blinded adjudication", () => { + const { index } = packetSet(); + const first = bundle("expert-one", index.packets.map(({ packetId }) => rating(packetId))); + const secondRatings = index.packets.map(({ packetId }) => rating(packetId)); + secondRatings[1] = { ...secondRatings[1]!, causalRelevance: "uncertain", reason: "The visible seam may not control the asserted behavior." }; + const second = bundle("expert-two", secondRatings); + + const workspace = createAdjudicationWorkspace(index, first, second, "panel-chair"); + + expect(workspace.ratings).toHaveLength(1); + expect(ratingProgress(index, workspace).nextPacketId).toBe(index.packets[1]!.packetId); + expect(adjudicationDisagreement(workspace, index.packets[1]!.packetId)?.fields).toEqual(["causalRelevance"]); + expect(workspace.sourceBundles?.map(({ raterId }) => raterId)).toEqual(["expert-one", "expert-two"]); + }); + + it("verifies every packet and rejects an outcome-like JSON file in the blind directory", async () => { + const { index, packets } = packetSet(); + const root = await mkdtemp(join(tmpdir(), "pureflow-rater-workspace-")); + roots.push(root); + await writeFile(join(root, "index.json"), JSON.stringify(index)); + for (let i = 0; i < packets.length; i += 1) { + await writeFile(join(root, index.packets[i]!.filename), JSON.stringify(packets[i])); + } + + const loaded = await loadPacketSet(root); + expect(loaded.packets.size).toBe(2); + + await writeFile(join(root, "held-out-summary.json"), "{}\n"); + await expect(loadPacketSet(root)).rejects.toThrow("unexpected content"); + }); +}); + +function packetSet(): { index: RatingIndex; packets: RaterPacket[] } { + const packets = [packet("a".repeat(24), "src/a.ts"), packet("b".repeat(24), "src/b.ts")]; + const core = { + schemaVersion: 1 as const, + protocol: "r7-blind-expert-v1" as const, + cohort: "held-out" as const, + corpusManifestSha256: "d".repeat(64), + sourcePlanFileSha256: "e".repeat(64), + packetCount: packets.length, + packets: packets.map((value, index) => ({ + filename: `packet-${String(index + 1).padStart(3, "0")}.json`, + packetId: value.packetId, + packetSha256: value.packetSha256, + })), + }; + return { index: { ...core, indexSha256: canonicalHash("r7-rater-index", core) }, packets }; +} + +function packet(packetId: string, path: string): RaterPacket { + const core = { + schemaVersion: 1 as const, + protocol: "r7-blind-expert-v1" as const, + packetId, + projectAlias: "project-12345678", + seam: { path, symbol: "run", kind: "function" }, + writablePaths: [path], + attributedTestPaths: [path.replace("src", "test")], + mutationRule: "Replace the changed source with its adjacent base version.", + observation: "Run the preregistered check and classify its process outcome.", + sourceAndTestDiff: `diff --git a/${path} b/${path}`, + questions: ["Is the proposed rewind causally relevant?"], + }; + return { ...core, packetSha256: canonicalHash("r7-rater-packet", core) }; +} + +function rating(packetId: string): ExpertRating { + return { + packetId, + causalRelevance: "yes", + targetExpected: "pass", + rewindExpected: "fail", + leakage: "none", + confidence: 4, + reason: "The visible test directly covers the changed behavior.", + }; +} + +function bundle(raterId: string, ratings: ExpertRating[]): RatingBundle { + return { schemaVersion: 1, protocol: "r7-blind-expert-v1", raterId, ratings }; +}