From 272e762eacd094e7e3a8c954aa503bde05818ab2 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Wed, 21 Oct 2026 10:01:22 +0200 Subject: [PATCH] chore(agents): review replay harness, and the 2026-10-21 process review Pins a PR's review snapshot and its ledger on origin (refs/replay), replays the review or one lens on it with stage=replay, and scores the findings against the ledger. Replayed #752 round 1 and #775 rounds 1 and 2; results in docs/internal/PROCESS-REVIEW-2026-10-21.md, with the findings over the deliver and spec runs after #739 and the 22 changes they call for. --- .agents/replay/752.json | 22 ++ .agents/replay/775.json | 39 ++ .agents/replay/results.jsonl | 12 + .agents/scripts/replay-freeze.mjs | 134 +++++++ .agents/scripts/review-inputs.sh | 57 ++- .agents/scripts/review-metrics.mjs | 8 +- .agents/scripts/review-replay.sh | 82 ++++ .agents/scripts/review-score.mjs | 221 +++++++++++ .claude/agents/reviewer.md | 10 +- docs/internal/PROCESS-REVIEW-2026-10-21.md | 434 +++++++++++++++++++++ docs/internal/REVIEW.md | 8 +- docs/internal/agent-rules/toolchain.md | 3 +- 12 files changed, 1013 insertions(+), 17 deletions(-) create mode 100644 .agents/replay/752.json create mode 100644 .agents/replay/775.json create mode 100644 .agents/replay/results.jsonl create mode 100755 .agents/scripts/replay-freeze.mjs create mode 100755 .agents/scripts/review-replay.sh create mode 100644 .agents/scripts/review-score.mjs create mode 100644 docs/internal/PROCESS-REVIEW-2026-10-21.md diff --git a/.agents/replay/752.json b/.agents/replay/752.json new file mode 100644 index 000000000..463826975 --- /dev/null +++ b/.agents/replay/752.json @@ -0,0 +1,22 @@ +{ + "pr": 752, + "issue": 699, + "rounds": { + "1": { + "head": "a1c5a6f6ab9c62a088681cc47152c3bec0b9ea10", + "base": "85d3311cbebd44e1695e53604ac3909676b1ff4e", + "frozenAt": "2026-10-21T07:20:09.447Z", + "baseline": { + "runs": 4, + "seconds": 637, + "costUsd": 1.3761999999999999, + "lenses": [ + "reviewer/both:claude-sonnet-5-5", + "code/both:claude-opus-5-5", + "spec/-:claude-sonnet-5-5", + "claims/-:claude-sonnet-5-5" + ] + } + } + } +} diff --git a/.agents/replay/775.json b/.agents/replay/775.json new file mode 100644 index 000000000..7fdc4039e --- /dev/null +++ b/.agents/replay/775.json @@ -0,0 +1,39 @@ +{ + "pr": 775, + "issue": 761, + "rounds": { + "1": { + "head": "551e6ec6f369a99d58c11d67fb0bf820621ad735", + "base": "f0eec44111297d0b7451181058b33384f8fa5fcf", + "frozenAt": "2026-10-21T07:37:20.878Z", + "baseline": { + "runs": 5, + "seconds": 1907, + "costUsd": 6.6301000000000005, + "lenses": [ + "reviewer/-:claude-sonnet-5-5", + "spec/-:claude-sonnet-5-5", + "code/behaviour:claude-opus-5-5", + "code/tests:claude-opus-5-5", + "claims/-:claude-sonnet-5-5" + ] + } + }, + "2": { + "head": "6ae6e9932e97cdcc89bfdd296b149e49df8d1c06", + "base": "f0eec44111297d0b7451181058b33384f8fa5fcf", + "previous": "551e6ec6f369a99d58c11d67fb0bf820621ad735", + "frozenAt": "2026-10-21T07:37:41.502Z", + "baseline": { + "runs": 3, + "seconds": 803, + "costUsd": 1.7179, + "lenses": [ + "reviewer/-:claude-sonnet-5-5", + "code/both:claude-opus-5-5", + "claims/-:claude-sonnet-5-5" + ] + } + } + } +} diff --git a/.agents/replay/results.jsonl b/.agents/replay/results.jsonl new file mode 100644 index 000000000..ed11a0640 --- /dev/null +++ b/.agents/replay/results.jsonl @@ -0,0 +1,12 @@ +{"at":"2026-10-21T07:19:20.477Z","pr":752,"round":"1","variant":"current-contaminated","agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":727,"costUsd":1.4061000000000001},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:20:17.416Z","pr":752,"round":"1","variant":"current-contaminated-rescored","agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":727,"costUsd":1.4061000000000001},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:21:48.692Z","pr":752,"round":"1","variant":"current-run2","valid":true,"agentsCommit":"baddf403","recall":"1/6","found":["R2"],"missed":["R1","R3","R4","R5","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec decision not implemented: pid-less staging logged]"],"nonDefectHits":0,"replay":{"runs":5,"seconds":764,"costUsd":1.5218},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:24:57.861Z","pr":752,"round":"1","variant":"current-run3","valid":false,"agentsCommit":"baddf403","recall":"0/6","found":[],"missed":["R1","R2","R3","R4","R5","R6"],"unmatched":["R1 src/core/write-leftovers.ts [issue decision not implemented]","R2 src/core/write-leftovers.ts [testing 3]","R3 src/core/write-leftovers.ts [legacy staging name parsed as pid]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":95,"costUsd":0.2261},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:32:48.776Z","pr":752,"round":"1","variant":"current-run4","valid":false,"agentsCommit":"baddf403","recall":"4/6","found":["R2","R3","R4","R5"],"missed":["R1","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec line undelivered]","R2 src/daemon/main.test.ts [testing: Tests line missing]","R5 src/core/write-leftovers.ts [architecture 10]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":432,"costUsd":0.2902},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:34:16.082Z","pr":752,"round":"1","variant":"current-run4","valid":true,"agentsCommit":"baddf403","recall":"4/6","found":["R2","R3","R4","R5"],"missed":["R1","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec line undelivered]","R2 src/daemon/main.test.ts [testing: Tests line missing]","R5 src/core/write-leftovers.ts [architecture 10]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":829,"costUsd":1.6874},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:39:13.181Z","pr":752,"round":"1","variant":"current-contaminated-rescored2","valid":true,"agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":7,"seconds":1291,"costUsd":2.0381},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:40:37.400Z","pr":752,"round":"1","variant":"debug","valid":true,"agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":7,"seconds":1291,"costUsd":2.0381},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:41:43.556Z","pr":752,"round":"1","variant":"lens-spec-check","valid":false,"agentsCommit":"baddf403","recall":"1/6","found":["R5"],"missed":["R1","R2","R3","R4","R6"],"unmatched":["F2 src/core/write-leftovers.ts:161 [triage Risk breached]","F3 src/core/write-leftovers.ts:139 [assumption decides user-visible behaviour; contradicts issue example]","F4 src/core/write-leftovers.ts:130 [`STAGING_DIRECTORY` needs a digits segment in the name. A pi]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":26,"costUsd":0.1228},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:42:08.995Z","pr":752,"round":"1","variant":"lens-spec-check","valid":true,"agentsCommit":"baddf403","recall":"1/1","found":["R5"],"missed":[],"unmatched":["F2 src/core/write-leftovers.ts:161 [triage Risk breached]","F3 src/core/write-leftovers.ts:139 [assumption decides user-visible behaviour; contradicts issue example]","F4 src/core/write-leftovers.ts:130 [`STAGING_DIRECTORY` needs a digits segment in the name. A pi]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":26,"costUsd":0.1228},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:50:03.995Z","pr":775,"round":"1","variant":"current-run1","valid":true,"agentsCommit":"baddf403","recall":"3/10","found":["R1","R2","R7"],"missed":["R3","R4","R5","R6","R10","R11","R12"],"unmatched":["R1 src/daemon/main.ts [spec line undelivered: daemon socket probe]","R4 src/daemon/main.test.ts [testing 1]","R5 src/core/registry.test.ts [testing 2]","R6 src/core/registry.ts [comment names nonexistent table or column]","R7 src/core/AGENTS.md [manual claims drivers own tables]"],"nonDefectHits":0,"replay":{"runs":5,"seconds":2058,"costUsd":6.4711},"baseline":{"runs":5,"seconds":1907,"costUsd":6.6301000000000005,"lenses":["reviewer/-:claude-sonnet-5-5","spec/-:claude-sonnet-5-5","code/behaviour:claude-opus-5-5","code/tests:claude-opus-5-5","claims/-:claude-sonnet-5-5"]}} +{"at":"2026-10-21T07:59:13.178Z","pr":775,"round":"2","variant":"current-run1","valid":true,"agentsCommit":"baddf403","recall":"6/10","found":["R1","R3","R4","R5","R10","R11"],"missed":["R2","R6","R7","R12"],"unmatched":["R2 unknown [claim false (round 1 text not carried)]","R6 unknown [claim false (round 1 text not carried)]","R7 unknown [claim false (round 1 text not carried)]","R8 unknown [note]","R9 unknown [note]","R11 src/core/registry.test.ts [testing 3, testing 9]"],"nonDefectHits":0,"replay":{"runs":3,"seconds":833,"costUsd":1.5988999999999998},"baseline":{"runs":3,"seconds":803,"costUsd":1.7179,"lenses":["reviewer/-:claude-sonnet-5-5","code/both:claude-opus-5-5","claims/-:claude-sonnet-5-5"]}} diff --git a/.agents/scripts/replay-freeze.mjs b/.agents/scripts/replay-freeze.mjs new file mode 100755 index 000000000..7afa93331 --- /dev/null +++ b/.agents/scripts/replay-freeze.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node +// `node .agents/scripts/replay-freeze.mjs --pr --round <1|2|fix-check> --head --base +// [--previous ] [--issue ]`: freezes one review snapshot of a PR so review-replay.sh can +// review it again later, with today's or tomorrow's agents, and review-score.mjs can score the +// result against what the real review found. +// +// Writes .agents/replay/.json (one file per PR, one entry per round): the SHAs and the real +// review's measured runs for the round as the baseline. Each SHA must be reachable from +// refs/pull//head or from the PR's base history; the script then pins them on origin as +// refs/replay///{head,base,previous}, a ref namespace no branch or tag shows, so the +// snapshot survives branch deletion and GC on every machine. +// +// The ground truth, the PR's `## Review ledger` findings (every outcome kept: `fixed` and `open` +// are the defects, the others known non-defects), goes to origin too, as the blob +// refs/replay//truth, and never into the checkout: the first replay's coordinator opened the +// snapshot file and read the answers before its lenses ran. review-score.mjs fetches the blob. +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const args = process.argv.slice(2); +const option = (name) => { + const at = args.indexOf(name); + return at === -1 ? null : args[at + 1]; +}; +const pr = option("--pr"); +const round = option("--round"); +const head = option("--head"); +const base = option("--base"); +const previous = option("--previous"); +const issue = option("--issue"); +if (!/^\d+$/.test(pr ?? "") || !round || !head || !base) { + console.error( + "usage: replay-freeze.mjs --pr --round <1|2|fix-check> --head --base [--previous ] [--issue ]", + ); + process.exit(64); +} + +const root = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const git = (...a) => execFileSync("git", a, { cwd: root, encoding: "utf8" }).trim(); +const gh = (...a) => execFileSync("gh", a, { cwd: root, encoding: "utf8" }); + +git("fetch", "-q", "origin", `+refs/pull/${pr}/head:refs/replay-freeze/${pr}`); +const full = {}; +for (const [name, sha] of Object.entries({ head, base, previous })) { + if (!sha) continue; + try { + full[name] = git("rev-parse", "--verify", `${sha}^{commit}`); + } catch { + console.error(`${name} ${sha} is not a local commit; fetch the PR below in the stack first`); + process.exit(65); + } +} +try { + git("merge-base", "--is-ancestor", full.head, `refs/replay-freeze/${pr}`); +} catch { + console.error(`head ${head} is not an ancestor of refs/pull/${pr}/head`); + process.exit(65); +} +git("update-ref", "-d", `refs/replay-freeze/${pr}`); + +const nwo = + process.env.GH_REPO ?? + git("remote", "get-url", "origin") + .replace(/\.git$/, "") + .replace(/.*[/:]([^/:]+\/[^/]+)$/, "$1"); +const comments = JSON.parse( + gh("api", "-X", "GET", `repos/${nwo}/issues/${pr}/comments`, "-f", "per_page=100"), +); +let findings = null; +for (const c of comments) { + if (!/^## Review ledger/m.test(c.body ?? "")) continue; + const m = c.body.match(/```json\s*(\{[\s\S]*?\})\s*```/); + if (m) findings = JSON.parse(m[1]).findings; +} +if (findings === null) + console.error( + `PR #${pr} has no Review ledger comment with a findings block; ground truth is empty`, + ); + +let baseline = null; +try { + const metrics = JSON.parse( + execFileSync( + "node", + [ + join(root, ".agents/scripts/review-metrics.mjs"), + "--pr", + pr, + ...(issue ? ["--issue", issue] : []), + "--json", + ], + { encoding: "utf8" }, + ), + ); + const runs = metrics.runs.filter((r) => r.stage === "review" && r.round === round); + const sum = (k) => + runs.some((r) => r[k] === null) ? null : runs.reduce((a, r) => a + (r[k] ?? 0), 0); + baseline = { + runs: runs.length, + seconds: sum("seconds"), + costUsd: sum("costUsd"), + lenses: runs.map((r) => `${r.role}/${r.lens}:${r.models.join("+")}`), + }; +} catch {} + +const dir = join(root, ".agents/replay"); +mkdirSync(dir, { recursive: true }); +const file = join(dir, `${pr}.json`); +const snapshot = existsSync(file) + ? JSON.parse(readFileSync(file, "utf8")) + : { pr: Number(pr), issue: issue ? Number(issue) : null, rounds: {} }; +snapshot.rounds[round] = { ...full, frozenAt: new Date().toISOString(), baseline }; +delete snapshot.findings; +writeFileSync(file, JSON.stringify(snapshot, null, 2) + "\n"); + +const refs = Object.entries(full).map(([name, sha]) => `${sha}:refs/replay/${pr}/${round}/${name}`); +if (findings !== null) { + const blob = execFileSync("git", ["hash-object", "-w", "--stdin"], { + cwd: root, + input: JSON.stringify({ pr: Number(pr), findings }, null, 2) + "\n", + encoding: "utf8", + }).trim(); + refs.push(`${blob}:refs/replay/${pr}/truth`); +} +git("push", "-q", "origin", ...refs); +console.log( + `frozen PR #${pr} round ${round}: ${Object.entries(full) + .map(([k, v]) => `${k} ${v.slice(0, 8)}`) + .join( + ", ", + )}; ${findings?.length ?? 0} ground-truth findings; pinned ${refs.length} refs on origin; ${file}`, +); diff --git a/.agents/scripts/review-inputs.sh b/.agents/scripts/review-inputs.sh index 91b87334d..d3a168eb7 100755 --- a/.agents/scripts/review-inputs.sh +++ b/.agents/scripts/review-inputs.sh @@ -35,7 +35,8 @@ # assumptions.md the PR body's `Assumption:` lines (coding rule 2) # # Nothing else: never the rest of the PR body, commit messages, or any other comment. Rules, ADRs -# and briefs come from the base branch, so a PR cannot rewrite what it is judged by. Whatever it looks +# and briefs come from the base branch, so a PR cannot rewrite what it is judged by. REVIEW_HEAD, +# REVIEW_BASE and REVIEW_RULES (below) replay a frozen snapshot. Whatever it looks # for and cannot find, it says on stderr, with one summary line at the end. The repo is GH_REPO, # else the origin remote's owner/name. set -eu @@ -64,11 +65,41 @@ pr_body=$(api "pulls/$pr" -q '.body // ""' | tr -d '\r') # The PR head through GitHub's pull ref, which outlives a deleted branch. The temporary ref is # per PR, so two reviews running at once do not overwrite each other's. -git fetch -q origin "+refs/heads/$base_ref:refs/remotes/origin/$base_ref" -git fetch -q origin "+refs/pull/$pr/head:refs/review-inputs/$pr" -head=$(git rev-parse "refs/review-inputs/$pr") -git update-ref -d "refs/review-inputs/$pr" -base=$(git rev-parse "origin/$base_ref") + +# why: a replay (REVIEW_HEAD and REVIEW_BASE, full SHAs, both or neither) reviews a frozen +# snapshot of a PR that may be merged and whose branches may be gone, so the SHAs must already be +# local (review-replay.sh fetches refs/replay//...). REVIEW_RULES names the commit rules, +# briefs and the protocol come from instead of the base, so an old snapshot can be reviewed with +# new instructions; the ADRs and the spec still come from the base, as the PR was judged then. +if [ -n "${REVIEW_HEAD:-}${REVIEW_BASE:-}" ]; then + if [ -z "${REVIEW_HEAD:-}" ] || [ -z "${REVIEW_BASE:-}" ]; then + say "REVIEW_HEAD and REVIEW_BASE go together" + exit 64 + fi + head=$(git rev-parse --verify "$REVIEW_HEAD^{commit}" 2>/dev/null) || { + say "REVIEW_HEAD $REVIEW_HEAD is not a local commit" + exit 65 + } + base=$(git rev-parse --verify "$REVIEW_BASE^{commit}" 2>/dev/null) || { + say "REVIEW_BASE $REVIEW_BASE is not a local commit" + exit 65 + } + say "replay: head $head, base $base" +else + git fetch -q origin "+refs/heads/$base_ref:refs/remotes/origin/$base_ref" + git fetch -q origin "+refs/pull/$pr/head:refs/review-inputs/$pr" + head=$(git rev-parse "refs/review-inputs/$pr") + git update-ref -d "refs/review-inputs/$pr" + base=$(git rev-parse "origin/$base_ref") +fi +rules_ref=${REVIEW_RULES:-$base} +if [ -n "${REVIEW_RULES:-}" ]; then + rules_ref=$(git rev-parse --verify "$REVIEW_RULES^{commit}" 2>/dev/null) || { + say "REVIEW_RULES $REVIEW_RULES is not a local commit" + exit 65 + } + say "rules, briefs and the protocol from $rules_ref" +fi fork=$(git merge-base "$base" "$head") if [ -n "$previous" ]; then previous=$(git rev-parse --verify "$previous^{commit}" 2>/dev/null) || { @@ -90,8 +121,8 @@ echo "$head" >"$out/commit" echo "$base" >"$out/base" echo "$fork" >"$out/merge-base" [ -z "$previous" ] || echo "$previous" >"$out/previous" -if git cat-file -e "$base:docs/internal/REVIEW.md" 2>/dev/null; then - git show "$base:docs/internal/REVIEW.md" >"$out/review-protocol.md" +if git cat-file -e "$rules_ref:docs/internal/REVIEW.md" 2>/dev/null; then + git show "$rules_ref:docs/internal/REVIEW.md" >"$out/review-protocol.md" fi node "$(dirname "$0")/diff-kind.mjs" "$fork" "$head" >"$out/kind" if [ -n "$previous" ]; then @@ -101,14 +132,14 @@ fi # Always written, so a reviewer can tell "the sweep found nothing" from "the sweep did not run". [ -s "$out/stale-refs.txt" ] || echo "(the sweep found no line naming anything the diff removed)" >"$out/stale-refs.txt" mkdir "$out/briefs" -for f in $(git ls-tree --name-only "$base" .agents/briefs/); do - git show "$base:$f" >"$out/briefs/$(basename "$f")" +for f in $(git ls-tree --name-only "$rules_ref" .agents/briefs/); do + git show "$rules_ref:$f" >"$out/briefs/$(basename "$f")" done -for f in $(git ls-tree --name-only "$base" docs/internal/agent-rules/); do - git show "$base:$f" >"$out/rules/$(basename "$f")" +for f in $(git ls-tree --name-only "$rules_ref" docs/internal/agent-rules/); do + git show "$rules_ref:$f" >"$out/rules/$(basename "$f")" done -git show "$base:docs/internal/adr/README.md" >"$out/rules/adr-index.md" +git show "$rules_ref:docs/internal/adr/README.md" >"$out/rules/adr-index.md" issue= case $head_ref in diff --git a/.agents/scripts/review-metrics.mjs b/.agents/scripts/review-metrics.mjs index 9850ceba5..9cb340345 100644 --- a/.agents/scripts/review-metrics.mjs +++ b/.agents/scripts/review-metrics.mjs @@ -11,6 +11,9 @@ // with the same id). Cost comes from the price table below; a model it does not list makes the // cost unknown, never zero. // +// --replay reports the runs tagged stage=replay instead (review-replay.sh); without it they are +// left out, so a replay never counts as the PR's delivery work. +// // --findings validates the coordinator's findings file (`{"findings": [...]}`, each with id, // role, class, outcome, path) and adds the outcome and class counts. --json prints the ledger // (runs, totals, findings) instead of the summary lines. @@ -37,7 +40,7 @@ const issue = option("--issue"); const findingsFile = option("--findings"); if (pr === null || !/^\d+$/.test(pr) || (issue !== null && !/^\d+$/.test(issue))) { console.error( - "usage: review-metrics.mjs --pr [--issue ] [--findings ] [--json]", + "usage: review-metrics.mjs --pr [--issue ] [--findings ] [--json] [--replay]", ); process.exit(64); } @@ -46,6 +49,9 @@ const root = join(process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"), " const runs = transcripts(root) .map((file) => ({ file, tag: tagOf(file) })) .filter(({ tag }) => tag !== null && (tag.pr === pr || (issue !== null && tag.issue === issue))) + // why: a replay (stage=replay, review-replay.sh) re-reviews a frozen snapshot of a merged PR; + // it must never count as that PR's delivery work. --replay reports the replays alone. + .filter(({ tag }) => (tag.stage === "replay") === args.includes("--replay")) .map(({ file, tag }) => run(file, tag)) .sort((a, b) => Date.parse(a.startedAt ?? "") - Date.parse(b.startedAt ?? "")); const findings = diff --git a/.agents/scripts/review-replay.sh b/.agents/scripts/review-replay.sh new file mode 100755 index 000000000..8a87ec00b --- /dev/null +++ b/.agents/scripts/review-replay.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env sh +# Re-reviews a frozen snapshot of a PR (replay-freeze.mjs) with the agents of this checkout, or +# with the rules, briefs and protocol of another commit: +# +# .agents/scripts/review-replay.sh [] +# .agents/scripts/review-replay.sh --lens [] +# REPLAY_LABEL=