Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .agents/replay/752.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"pr": 752,
"issue": 699,
"rounds": {
"1": {
"head": "a1c5a6f6ab9c62a088681cc47152c3bec0b9ea10",
"base": "85d3311cbebd44e1695e53604ac3909676b1ff4e",
"frozenAt": "2026-10-21T07:20:09.447Z",
"baseline": {
"runs": 4,
"seconds": 637,
"costUsd": 1.3761999999999999,
"lenses": [
"reviewer/both:claude-sonnet-5-5",
"code/both:claude-opus-5-5",
"spec/-:claude-sonnet-5-5",
"claims/-:claude-sonnet-5-5"
]
}
}
}
}
39 changes: 39 additions & 0 deletions .agents/replay/775.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"pr": 775,
"issue": 761,
"rounds": {
"1": {
"head": "551e6ec6f369a99d58c11d67fb0bf820621ad735",
"base": "f0eec44111297d0b7451181058b33384f8fa5fcf",
"frozenAt": "2026-10-21T07:37:20.878Z",
"baseline": {
"runs": 5,
"seconds": 1907,
"costUsd": 6.6301000000000005,
"lenses": [
"reviewer/-:claude-sonnet-5-5",
"spec/-:claude-sonnet-5-5",
"code/behaviour:claude-opus-5-5",
"code/tests:claude-opus-5-5",
"claims/-:claude-sonnet-5-5"
]
}
},
"2": {
"head": "6ae6e9932e97cdcc89bfdd296b149e49df8d1c06",
"base": "f0eec44111297d0b7451181058b33384f8fa5fcf",
"previous": "551e6ec6f369a99d58c11d67fb0bf820621ad735",
"frozenAt": "2026-10-21T07:37:41.502Z",
"baseline": {
"runs": 3,
"seconds": 803,
"costUsd": 1.7179,
"lenses": [
"reviewer/-:claude-sonnet-5-5",
"code/both:claude-opus-5-5",
"claims/-:claude-sonnet-5-5"
]
}
}
}
}
12 changes: 12 additions & 0 deletions .agents/replay/results.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{"at":"2026-10-21T07:19:20.477Z","pr":752,"round":"1","variant":"current-contaminated","agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":727,"costUsd":1.4061000000000001},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:20:17.416Z","pr":752,"round":"1","variant":"current-contaminated-rescored","agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":727,"costUsd":1.4061000000000001},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:21:48.692Z","pr":752,"round":"1","variant":"current-run2","valid":true,"agentsCommit":"baddf403","recall":"1/6","found":["R2"],"missed":["R1","R3","R4","R5","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec decision not implemented: pid-less staging logged]"],"nonDefectHits":0,"replay":{"runs":5,"seconds":764,"costUsd":1.5218},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:24:57.861Z","pr":752,"round":"1","variant":"current-run3","valid":false,"agentsCommit":"baddf403","recall":"0/6","found":[],"missed":["R1","R2","R3","R4","R5","R6"],"unmatched":["R1 src/core/write-leftovers.ts [issue decision not implemented]","R2 src/core/write-leftovers.ts [testing 3]","R3 src/core/write-leftovers.ts [legacy staging name parsed as pid]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":95,"costUsd":0.2261},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:32:48.776Z","pr":752,"round":"1","variant":"current-run4","valid":false,"agentsCommit":"baddf403","recall":"4/6","found":["R2","R3","R4","R5"],"missed":["R1","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec line undelivered]","R2 src/daemon/main.test.ts [testing: Tests line missing]","R5 src/core/write-leftovers.ts [architecture 10]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":432,"costUsd":0.2902},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:34:16.082Z","pr":752,"round":"1","variant":"current-run4","valid":true,"agentsCommit":"baddf403","recall":"4/6","found":["R2","R3","R4","R5"],"missed":["R1","R6"],"unmatched":["R1 src/core/write-leftovers.ts [spec line undelivered]","R2 src/daemon/main.test.ts [testing: Tests line missing]","R5 src/core/write-leftovers.ts [architecture 10]"],"nonDefectHits":0,"replay":{"runs":4,"seconds":829,"costUsd":1.6874},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:39:13.181Z","pr":752,"round":"1","variant":"current-contaminated-rescored2","valid":true,"agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":7,"seconds":1291,"costUsd":2.0381},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:40:37.400Z","pr":752,"round":"1","variant":"debug","valid":true,"agentsCommit":"baddf403","recall":"3/6","found":["R2","R4","R5"],"missed":["R1","R3","R6"],"unmatched":["R2 src/daemon/main.test.ts [Tests line unproven]","R4 src/drivers/ios/index.ts [testing 3]","R5 src/core/write-leftovers.ts [architecture 10]","R7 src/core/write-leftovers.ts [testing 3]","R8 src/core/device-root.test.ts [testing 7]"],"nonDefectHits":0,"replay":{"runs":7,"seconds":1291,"costUsd":2.0381},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:41:43.556Z","pr":752,"round":"1","variant":"lens-spec-check","valid":false,"agentsCommit":"baddf403","recall":"1/6","found":["R5"],"missed":["R1","R2","R3","R4","R6"],"unmatched":["F2 src/core/write-leftovers.ts:161 [triage Risk breached]","F3 src/core/write-leftovers.ts:139 [assumption decides user-visible behaviour; contradicts issue example]","F4 src/core/write-leftovers.ts:130 [`STAGING_DIRECTORY` needs a digits segment in the name. A pi]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":26,"costUsd":0.1228},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:42:08.995Z","pr":752,"round":"1","variant":"lens-spec-check","valid":true,"agentsCommit":"baddf403","recall":"1/1","found":["R5"],"missed":[],"unmatched":["F2 src/core/write-leftovers.ts:161 [triage Risk breached]","F3 src/core/write-leftovers.ts:139 [assumption decides user-visible behaviour; contradicts issue example]","F4 src/core/write-leftovers.ts:130 [`STAGING_DIRECTORY` needs a digits segment in the name. A pi]"],"nonDefectHits":0,"replay":{"runs":1,"seconds":26,"costUsd":0.1228},"baseline":{"runs":4,"seconds":637,"costUsd":1.3761999999999999,"lenses":["reviewer/both:claude-sonnet-5-5","code/both:claude-opus-5-5","spec/-:claude-sonnet-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:50:03.995Z","pr":775,"round":"1","variant":"current-run1","valid":true,"agentsCommit":"baddf403","recall":"3/10","found":["R1","R2","R7"],"missed":["R3","R4","R5","R6","R10","R11","R12"],"unmatched":["R1 src/daemon/main.ts [spec line undelivered: daemon socket probe]","R4 src/daemon/main.test.ts [testing 1]","R5 src/core/registry.test.ts [testing 2]","R6 src/core/registry.ts [comment names nonexistent table or column]","R7 src/core/AGENTS.md [manual claims drivers own tables]"],"nonDefectHits":0,"replay":{"runs":5,"seconds":2058,"costUsd":6.4711},"baseline":{"runs":5,"seconds":1907,"costUsd":6.6301000000000005,"lenses":["reviewer/-:claude-sonnet-5-5","spec/-:claude-sonnet-5-5","code/behaviour:claude-opus-5-5","code/tests:claude-opus-5-5","claims/-:claude-sonnet-5-5"]}}
{"at":"2026-10-21T07:59:13.178Z","pr":775,"round":"2","variant":"current-run1","valid":true,"agentsCommit":"baddf403","recall":"6/10","found":["R1","R3","R4","R5","R10","R11"],"missed":["R2","R6","R7","R12"],"unmatched":["R2 unknown [claim false (round 1 text not carried)]","R6 unknown [claim false (round 1 text not carried)]","R7 unknown [claim false (round 1 text not carried)]","R8 unknown [note]","R9 unknown [note]","R11 src/core/registry.test.ts [testing 3, testing 9]"],"nonDefectHits":0,"replay":{"runs":3,"seconds":833,"costUsd":1.5988999999999998},"baseline":{"runs":3,"seconds":803,"costUsd":1.7179,"lenses":["reviewer/-:claude-sonnet-5-5","code/both:claude-opus-5-5","claims/-:claude-sonnet-5-5"]}}
134 changes: 134 additions & 0 deletions .agents/scripts/replay-freeze.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#!/usr/bin/env node
// `node .agents/scripts/replay-freeze.mjs --pr <M> --round <1|2|fix-check> --head <sha> --base <sha>
// [--previous <sha>] [--issue <N>]`: freezes one review snapshot of a PR so review-replay.sh can
// review it again later, with today's or tomorrow's agents, and review-score.mjs can score the
// result against what the real review found.
//
// Writes .agents/replay/<M>.json (one file per PR, one entry per round): the SHAs and the real
// review's measured runs for the round as the baseline. Each SHA must be reachable from
// refs/pull/<M>/head or from the PR's base history; the script then pins them on origin as
// refs/replay/<M>/<round>/{head,base,previous}, a ref namespace no branch or tag shows, so the
// snapshot survives branch deletion and GC on every machine.
//
// The ground truth, the PR's `## Review ledger` findings (every outcome kept: `fixed` and `open`
// are the defects, the others known non-defects), goes to origin too, as the blob
// refs/replay/<M>/truth, and never into the checkout: the first replay's coordinator opened the
// snapshot file and read the answers before its lenses ran. review-score.mjs fetches the blob.
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const args = process.argv.slice(2);
const option = (name) => {
const at = args.indexOf(name);
return at === -1 ? null : args[at + 1];
};
const pr = option("--pr");
const round = option("--round");
const head = option("--head");
const base = option("--base");
const previous = option("--previous");
const issue = option("--issue");
if (!/^\d+$/.test(pr ?? "") || !round || !head || !base) {
console.error(
"usage: replay-freeze.mjs --pr <M> --round <1|2|fix-check> --head <sha> --base <sha> [--previous <sha>] [--issue <N>]",
);
process.exit(64);
}

const root = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const git = (...a) => execFileSync("git", a, { cwd: root, encoding: "utf8" }).trim();
const gh = (...a) => execFileSync("gh", a, { cwd: root, encoding: "utf8" });

git("fetch", "-q", "origin", `+refs/pull/${pr}/head:refs/replay-freeze/${pr}`);
const full = {};
for (const [name, sha] of Object.entries({ head, base, previous })) {
if (!sha) continue;
try {
full[name] = git("rev-parse", "--verify", `${sha}^{commit}`);
} catch {
console.error(`${name} ${sha} is not a local commit; fetch the PR below in the stack first`);
process.exit(65);
}
}
try {
git("merge-base", "--is-ancestor", full.head, `refs/replay-freeze/${pr}`);
} catch {
console.error(`head ${head} is not an ancestor of refs/pull/${pr}/head`);
process.exit(65);
}
git("update-ref", "-d", `refs/replay-freeze/${pr}`);

const nwo =
process.env.GH_REPO ??
git("remote", "get-url", "origin")
.replace(/\.git$/, "")
.replace(/.*[/:]([^/:]+\/[^/]+)$/, "$1");
const comments = JSON.parse(
gh("api", "-X", "GET", `repos/${nwo}/issues/${pr}/comments`, "-f", "per_page=100"),
);
let findings = null;
for (const c of comments) {
if (!/^## Review ledger/m.test(c.body ?? "")) continue;
const m = c.body.match(/```json\s*(\{[\s\S]*?\})\s*```/);
if (m) findings = JSON.parse(m[1]).findings;
}
if (findings === null)
console.error(
`PR #${pr} has no Review ledger comment with a findings block; ground truth is empty`,
);

let baseline = null;
try {
const metrics = JSON.parse(
execFileSync(
"node",
[
join(root, ".agents/scripts/review-metrics.mjs"),
"--pr",
pr,
...(issue ? ["--issue", issue] : []),
"--json",
],
{ encoding: "utf8" },
),
);
const runs = metrics.runs.filter((r) => r.stage === "review" && r.round === round);
const sum = (k) =>
runs.some((r) => r[k] === null) ? null : runs.reduce((a, r) => a + (r[k] ?? 0), 0);
baseline = {
runs: runs.length,
seconds: sum("seconds"),
costUsd: sum("costUsd"),
lenses: runs.map((r) => `${r.role}/${r.lens}:${r.models.join("+")}`),
};
} catch {}

const dir = join(root, ".agents/replay");
mkdirSync(dir, { recursive: true });
const file = join(dir, `${pr}.json`);
const snapshot = existsSync(file)
? JSON.parse(readFileSync(file, "utf8"))
: { pr: Number(pr), issue: issue ? Number(issue) : null, rounds: {} };
snapshot.rounds[round] = { ...full, frozenAt: new Date().toISOString(), baseline };
delete snapshot.findings;
writeFileSync(file, JSON.stringify(snapshot, null, 2) + "\n");

const refs = Object.entries(full).map(([name, sha]) => `${sha}:refs/replay/${pr}/${round}/${name}`);
if (findings !== null) {
const blob = execFileSync("git", ["hash-object", "-w", "--stdin"], {
cwd: root,
input: JSON.stringify({ pr: Number(pr), findings }, null, 2) + "\n",
encoding: "utf8",
}).trim();
refs.push(`${blob}:refs/replay/${pr}/truth`);
}
git("push", "-q", "origin", ...refs);
console.log(
`frozen PR #${pr} round ${round}: ${Object.entries(full)
.map(([k, v]) => `${k} ${v.slice(0, 8)}`)
.join(
", ",
)}; ${findings?.length ?? 0} ground-truth findings; pinned ${refs.length} refs on origin; ${file}`,
);
57 changes: 44 additions & 13 deletions .agents/scripts/review-inputs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
# assumptions.md the PR body's `Assumption:` lines (coding rule 2)
#
# Nothing else: never the rest of the PR body, commit messages, or any other comment. Rules, ADRs
# and briefs come from the base branch, so a PR cannot rewrite what it is judged by. Whatever it looks
# and briefs come from the base branch, so a PR cannot rewrite what it is judged by. REVIEW_HEAD,
# REVIEW_BASE and REVIEW_RULES (below) replay a frozen snapshot. Whatever it looks
# for and cannot find, it says on stderr, with one summary line at the end. The repo is GH_REPO,
# else the origin remote's owner/name.
set -eu
Expand Down Expand Up @@ -64,11 +65,41 @@ pr_body=$(api "pulls/$pr" -q '.body // ""' | tr -d '\r')

# The PR head through GitHub's pull ref, which outlives a deleted branch. The temporary ref is
# per PR, so two reviews running at once do not overwrite each other's.
git fetch -q origin "+refs/heads/$base_ref:refs/remotes/origin/$base_ref"
git fetch -q origin "+refs/pull/$pr/head:refs/review-inputs/$pr"
head=$(git rev-parse "refs/review-inputs/$pr")
git update-ref -d "refs/review-inputs/$pr"
base=$(git rev-parse "origin/$base_ref")

# why: a replay (REVIEW_HEAD and REVIEW_BASE, full SHAs, both or neither) reviews a frozen
# snapshot of a PR that may be merged and whose branches may be gone, so the SHAs must already be
# local (review-replay.sh fetches refs/replay/<pr>/...). REVIEW_RULES names the commit rules,
# briefs and the protocol come from instead of the base, so an old snapshot can be reviewed with
# new instructions; the ADRs and the spec still come from the base, as the PR was judged then.
if [ -n "${REVIEW_HEAD:-}${REVIEW_BASE:-}" ]; then
if [ -z "${REVIEW_HEAD:-}" ] || [ -z "${REVIEW_BASE:-}" ]; then
say "REVIEW_HEAD and REVIEW_BASE go together"
exit 64
fi
head=$(git rev-parse --verify "$REVIEW_HEAD^{commit}" 2>/dev/null) || {
say "REVIEW_HEAD $REVIEW_HEAD is not a local commit"
exit 65
}
base=$(git rev-parse --verify "$REVIEW_BASE^{commit}" 2>/dev/null) || {
say "REVIEW_BASE $REVIEW_BASE is not a local commit"
exit 65
}
say "replay: head $head, base $base"
else
git fetch -q origin "+refs/heads/$base_ref:refs/remotes/origin/$base_ref"
git fetch -q origin "+refs/pull/$pr/head:refs/review-inputs/$pr"
head=$(git rev-parse "refs/review-inputs/$pr")
git update-ref -d "refs/review-inputs/$pr"
base=$(git rev-parse "origin/$base_ref")
fi
rules_ref=${REVIEW_RULES:-$base}
if [ -n "${REVIEW_RULES:-}" ]; then
rules_ref=$(git rev-parse --verify "$REVIEW_RULES^{commit}" 2>/dev/null) || {
say "REVIEW_RULES $REVIEW_RULES is not a local commit"
exit 65
}
say "rules, briefs and the protocol from $rules_ref"
fi
fork=$(git merge-base "$base" "$head")
if [ -n "$previous" ]; then
previous=$(git rev-parse --verify "$previous^{commit}" 2>/dev/null) || {
Expand All @@ -90,8 +121,8 @@ echo "$head" >"$out/commit"
echo "$base" >"$out/base"
echo "$fork" >"$out/merge-base"
[ -z "$previous" ] || echo "$previous" >"$out/previous"
if git cat-file -e "$base:docs/internal/REVIEW.md" 2>/dev/null; then
git show "$base:docs/internal/REVIEW.md" >"$out/review-protocol.md"
if git cat-file -e "$rules_ref:docs/internal/REVIEW.md" 2>/dev/null; then
git show "$rules_ref:docs/internal/REVIEW.md" >"$out/review-protocol.md"
fi
node "$(dirname "$0")/diff-kind.mjs" "$fork" "$head" >"$out/kind"
if [ -n "$previous" ]; then
Expand All @@ -101,14 +132,14 @@ fi
# Always written, so a reviewer can tell "the sweep found nothing" from "the sweep did not run".
[ -s "$out/stale-refs.txt" ] || echo "(the sweep found no line naming anything the diff removed)" >"$out/stale-refs.txt"
mkdir "$out/briefs"
for f in $(git ls-tree --name-only "$base" .agents/briefs/); do
git show "$base:$f" >"$out/briefs/$(basename "$f")"
for f in $(git ls-tree --name-only "$rules_ref" .agents/briefs/); do
git show "$rules_ref:$f" >"$out/briefs/$(basename "$f")"
done

for f in $(git ls-tree --name-only "$base" docs/internal/agent-rules/); do
git show "$base:$f" >"$out/rules/$(basename "$f")"
for f in $(git ls-tree --name-only "$rules_ref" docs/internal/agent-rules/); do
git show "$rules_ref:$f" >"$out/rules/$(basename "$f")"
done
git show "$base:docs/internal/adr/README.md" >"$out/rules/adr-index.md"
git show "$rules_ref:docs/internal/adr/README.md" >"$out/rules/adr-index.md"

issue=
case $head_ref in
Expand Down
Loading
Loading