diff --git a/.agents/scripts/awake.mjs b/.agents/scripts/awake.mjs new file mode 100644 index 000000000..7a05767e5 --- /dev/null +++ b/.agents/scripts/awake.mjs @@ -0,0 +1,16 @@ +// Milliseconds of awake time since boot, from a clock that stops while the computer sleeps +// (time.monotonic(): mach_absolute_time on macOS, CLOCK_MONOTONIC on Linux); null when python3 +// is missing. clock.mjs reports sleep with it, and budget-guard.mjs counts active time with it. +import { execFileSync } from "node:child_process"; + +export function awakeMs() { + try { + const out = execFileSync("python3", ["-I", "-c", "import time; print(time.monotonic())"], { + encoding: "utf8", + timeout: 2_000, + }); + return Number(out) * 1000; + } catch { + return null; + } +} diff --git a/.agents/scripts/budget-guard.mjs b/.agents/scripts/budget-guard.mjs index ab298e36d..2388495e1 100755 --- a/.agents/scripts/budget-guard.mjs +++ b/.agents/scripts/budget-guard.mjs @@ -16,7 +16,9 @@ // group is killed. // // Run time counts only active time: a gap of more than 30 minutes between two tool calls (a -// session waiting overnight for a person) is not counted. +// session waiting overnight for a person) is not counted, and neither is time the computer +// slept, measured on the same monotonic clock clock.mjs uses (a laptop with its lid closed +// wakes for seconds at a time, so wall-clock gaps stay short while hours pass). // // Limits come from the environment (set them in settings.local.json "env"): // CLAUDE_BUDGET_AGENT_TOKENS default 15000000 (0 disables the agent token limit) @@ -27,6 +29,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { execFileSync } from "node:child_process"; +import { awakeMs } from "./awake.mjs"; import { emptyState, scan, sessionFiles } from "./usage.mjs"; const LIMIT_AGENT_TOKENS = Number(process.env.CLAUDE_BUDGET_AGENT_TOKENS ?? 15_000_000); @@ -44,7 +47,7 @@ if (!input.transcript_path || !input.session_id) process.exit(0); const { main, subDir, files } = sessionFiles(input.transcript_path); const stateFile = path.join(os.tmpdir(), `claude-budget-${input.session_id}.json`); -let state = { activeMs: 0, lastSeen: null, files: {} }; +let state = { activeMs: 0, lastSeen: null, lastAwake: null, files: {} }; try { state = JSON.parse(fs.readFileSync(stateFile, "utf8")); } catch {} @@ -55,9 +58,16 @@ for (const file of files) { scan(file, s); } const now = Date.now(); -const gap = typeof state.lastSeen === "number" ? now - state.lastSeen : 0; +const awake = awakeMs(); +const wallGap = typeof state.lastSeen === "number" ? now - state.lastSeen : 0; +// why: the awake gap excludes sleep; the wall gap is the fallback when python3 is missing. +const gap = + awake !== null && typeof state.lastAwake === "number" + ? Math.max(0, awake - state.lastAwake) + : wallGap; if (gap > 0 && gap <= IDLE_GAP_MS) state.activeMs = (state.activeMs ?? 0) + gap; state.lastSeen = now; +state.lastAwake = awake; fs.writeFileSync(stateFile, JSON.stringify(state)); const fmt = (n) => `${(n / 1e6).toFixed(1)}M`; diff --git a/.agents/scripts/cli-args.mjs b/.agents/scripts/cli-args.mjs new file mode 100644 index 000000000..a3ef8b4ac --- /dev/null +++ b/.agents/scripts/cli-args.mjs @@ -0,0 +1,6 @@ +// The one-line option parser the measuring scripts share: `option("--pr")` is the word after +// `--pr` on the command line, or null when it is absent. +export function option(name, args = process.argv.slice(2)) { + const at = args.indexOf(name); + return at === -1 ? null : (args[at + 1] ?? null); +} diff --git a/.agents/scripts/clock.mjs b/.agents/scripts/clock.mjs index 267564fae..62146a929 100644 --- a/.agents/scripts/clock.mjs +++ b/.agents/scripts/clock.mjs @@ -9,10 +9,10 @@ // while the computer sleeps (mach_absolute_time on macOS, CLOCK_MONOTONIC on Linux), Date.now() // does not. A gap under a minute is noise and counts as none. State lives in // $TMPDIR/claude-clock-.json. Sub-agents get nothing: they print no status lines. -import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { awakeMs } from "./awake.mjs"; const NOISE_MS = 60_000; @@ -54,18 +54,6 @@ console.log( }), ); -function awakeMs() { - try { - const out = execFileSync("python3", ["-I", "-c", "import time; print(time.monotonic())"], { - encoding: "utf8", - timeout: 2_000, - }); - return Number(out) * 1000; - } catch { - return null; - } -} - function minutes(ms) { return Math.round(ms / 60_000); } diff --git a/.agents/scripts/delivery-counters.mjs b/.agents/scripts/delivery-counters.mjs new file mode 100644 index 000000000..7fdf32882 --- /dev/null +++ b/.agents/scripts/delivery-counters.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node +// `node .agents/scripts/delivery-counters.mjs --since YYYY-MM-DD`: the counters that judge the +// delivery changes in PROCESS-REVIEW-2026-10-21.md (items 4 to 12), one row per PR merged since +// the date, from the PR's tagged runs on this machine and its files on GitHub: +// +// ci lines `ci:` lines the orchestrator put into fix prompts (CI failures sent to a fix run) +// syncs fix runs asked to merge the base or sync the stack +// empty builds build runs that built no cycle, or ended with no report at all +// no-change fix runs whose report fixed nothing (`found 0 fixed 0` on every line) +// Tests lines lines under the issue's `### Tests` +// tests added `it(`/`test(` blocks the PR adds in *.test.ts and *.spec.ts +// test/src LOC lines added to test files over lines added to other .ts files +// +// Unknown values print as `-`. Replays (stage=replay) never count. The repo is GH_REPO, else +// the origin remote's owner/name. +import { execFileSync } from "node:child_process"; +import { runOf, taggedRuns } from "./transcripts.mjs"; + +const args = process.argv.slice(2); +const at = args.indexOf("--since"); +const since = at === -1 ? null : args[at + 1]; +if (since === null || Number.isNaN(Date.parse(since))) { + console.error("usage: delivery-counters.mjs --since YYYY-MM-DD"); + process.exit(64); +} + +const repo = + process.env.GH_REPO ?? + execFileSync("git", ["remote", "get-url", "origin"], { encoding: "utf8" }) + .trim() + .replace(/\.git$/, "") + .replace(/.*[/:]([^/:]+\/[^/]+)$/, "$1"); +const api = (path, ...extra) => + JSON.parse( + execFileSync("gh", ["api", "-X", "GET", `repos/${repo}/${path}`, ...extra], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }), + ); + +const merged = []; +for (let page = 1; page <= 5; page += 1) { + const pulls = api( + "pulls", + "-f", + "state=closed", + "-f", + "sort=updated", + "-f", + "direction=desc", + "-f", + "per_page=100", + "-f", + `page=${page}`, + ); + for (const p of pulls) if (p.merged_at && p.merged_at >= since) merged.push(p); + if (pulls.length < 100 || pulls.every((p) => (p.updated_at ?? "") < since)) break; +} +merged.sort((a, b) => a.number - b.number); + +const runs = taggedRuns().filter(({ tag }) => tag.stage !== "replay"); +const rows = []; +for (const p of merged) { + const issue = /^(?:bug|feature|task)\/(\d+)$/.exec(p.head.ref)?.[1] ?? null; + const mine = runs + .filter(({ tag }) => tag.pr === String(p.number) || (issue !== null && tag.issue === issue)) + .map(({ file, tag }) => ({ tag, ...runOf(file) })); + const fixes = mine.filter((r) => r.tag.stage === "fix"); + const ci = fixes.reduce((n, r) => n + (r.prompt.match(/^\s*-?\s*ci:/gm) ?? []).length, 0); + const syncs = fixes.filter((r) => + /gh stack sync|merge (?:the |its )?base|stack: /i.test(r.prompt), + ).length; + // why: a build that could not start (a base behind main, a missing brief) ends with no report + // at all, which is the other way a build run builds nothing. + const empty = mine.filter( + (r) => + r.tag.stage === "build" && (r.lastText.trim() === "" || /^cycles:\s*0\//m.test(r.lastText)), + ).length; + const noChange = fixes.filter((r) => { + const found = [...r.lastText.matchAll(/-> found (\d+) fixed (\d+)/g)]; + return found.length > 0 && found.every((m) => m[2] === "0"); + }).length; + + let testsLines = null; + if (issue !== null) { + const body = api(`issues/${issue}`).body ?? ""; + const section = /^### Tests\s*$([\s\S]*?)(?=^#|$(?![\s\S]))/m.exec(body)?.[1] ?? ""; + testsLines = (section.match(/^- /gm) ?? []).length; + } + let testsAdded = 0; + let testLoc = 0; + let srcLoc = 0; + for (let page = 1; page <= 3; page += 1) { + const files = api(`pulls/${p.number}/files`, "-f", "per_page=100", "-f", `page=${page}`); + for (const f of files) { + if (!f.filename.endsWith(".ts")) continue; + if (/\.(test|spec)\.ts$/.test(f.filename)) { + testLoc += f.additions; + testsAdded += (f.patch?.match(/^\+\s*(?:it|test)(?:\.each)?\(/gm) ?? []).length; + } else srcLoc += f.additions; + } + if (files.length < 100) break; + } + rows.push({ + pr: p.number, + issue, + runs: mine.length, + ci, + syncs, + empty, + noChange, + testsLines, + testsAdded, + testLoc, + srcLoc, + }); +} + +const show = (v) => (v === null ? "-" : String(v)); +const ratio = (a, b) => (b === 0 || b === null ? "-" : (a / b).toFixed(1)); +console.log(`# Delivery counters, PRs merged since ${since} (${rows.length})\n`); +console.log( + "| PR | Issue | Runs | ci lines | syncs | empty builds | no-change fixes | Tests lines | tests added | tests/line | test/src LOC |", +); +console.log("| --- | --- | --: | --: | --: | --: | --: | --: | --: | --: | --: |"); +for (const r of rows) { + console.log( + `| #${r.pr} | ${r.issue ? `#${r.issue}` : "-"} | ${r.runs} | ${r.ci} | ${r.syncs} | ${r.empty} | ${r.noChange} | ${show(r.testsLines)} | ${r.testsAdded} | ${ratio(r.testsAdded, r.testsLines)} | ${ratio(r.testLoc, r.srcLoc)} |`, + ); +} +const sum = (k) => rows.reduce((n, r) => n + (r[k] ?? 0), 0); +console.log( + `\nTotals: ci lines ${sum("ci")}, syncs ${sum("syncs")}, empty builds ${sum("empty")}, no-change fixes ${sum("noChange")}, tests added ${sum("testsAdded")} over ${sum("testsLines")} Tests lines, test/src LOC ${ratio(sum("testLoc"), sum("srcLoc"))}.`, +); diff --git a/.agents/scripts/instruction-reads.mjs b/.agents/scripts/instruction-reads.mjs new file mode 100644 index 000000000..1c720154e --- /dev/null +++ b/.agents/scripts/instruction-reads.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// `node .agents/scripts/instruction-reads.mjs [--since YYYY-MM-DD] [--role ]`: how often +// each role's runs open each instruction file, from Claude Code's transcripts on this machine. +// +// A run is a ledger-tagged transcript (`toolchain.md`, "Review measurement"); its role is +// `/`. A file counts as opened when any tool call of the run names it: a rule +// under docs/internal/agent-rules/, a brief under .agents/briefs/ (or the rules/ and briefs/ +// copies a review's inputs directory holds), REVIEW.md, EXPLORATION.md, a flow doc, or a +// module manual (AGENTS.md). The agent's own definition is not a file it opens: the harness +// loads it. Prints one Markdown table per role: runs, then each file with the runs that +// opened it, most opened first. This is the number that judges a rewrite of the rules, the +// briefs or the router (PROCESS-REVIEW-2026-10-21.md, items 16 to 18). +import { runOf, taggedRuns } from "./transcripts.mjs"; +import { option } from "./cli-args.mjs"; + +const since = option("--since"); +const only = option("--role"); +const sinceMs = since === null ? 0 : Date.parse(since); +if (since !== null && Number.isNaN(sinceMs)) { + console.error("usage: instruction-reads.mjs [--since YYYY-MM-DD] [--role ]"); + process.exit(64); +} + +const FILE = new RegExp( + String.raw`(?:docs/internal/agent-rules/[\w.-]+\.md|\.agents/briefs/[\w.-]+\.md|(?:^|["'\s/])(?:rules|briefs)/[\w.-]+\.md|review-protocol\.md|docs/internal/REVIEW\.md|docs/internal/EXPLORATION\.md|docs/internal/flows/[\w.-]+\.md|(?:src/[\w/.-]*)?AGENTS\.md)`, + "g", +); + +const byRole = new Map(); +for (const { file, tag } of taggedRuns()) { + if (tag.stage === "replay") continue; + const role = `${tag.stage}/${tag.role}`; + if (only !== null && role !== only) continue; + const run = runOf(file); + if (Date.parse(run.startedAt ?? "") < sinceMs) continue; + const opened = new Set(); + for (const input of run.toolInputs) { + for (const m of input.matchAll(FILE)) { + opened.add( + m[0] + .replace(/^["'\s/]/, "") + .replace(/^(?:rules|briefs)\//, (p) => + p === "rules/" ? "docs/internal/agent-rules/" : ".agents/briefs/", + ), + ); + } + } + const entry = byRole.get(role) ?? { runs: 0, files: new Map() }; + entry.runs += 1; + for (const f of opened) entry.files.set(f, (entry.files.get(f) ?? 0) + 1); + byRole.set(role, entry); +} + +if (byRole.size === 0) { + console.log(`No tagged runs${since ? ` since ${since}` : ""}.`); + process.exit(0); +} +console.log(`# Instruction files opened per role${since ? `, runs since ${since}` : ""}\n`); +for (const [role, { runs, files }] of [...byRole].sort()) { + console.log(`## ${role}: ${runs} runs\n`); + console.log("| File | Runs that opened it |\n| --- | --: |"); + for (const [f, n] of [...files].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))) { + console.log(`| ${f} | ${n}/${runs} |`); + } + console.log(""); +} diff --git a/.agents/scripts/pr-body-check.mjs b/.agents/scripts/pr-body-check.mjs index 8c8828ce7..ffc17cade 100755 --- a/.agents/scripts/pr-body-check.mjs +++ b/.agents/scripts/pr-body-check.mjs @@ -9,7 +9,9 @@ // (`skipped` in place of the counts for a review the diff's kind left out) (rule 14); // - a line under `Rejected:` is not tagged `spec:`, `code:` or `claims:` (rule 14); // - the body says "spec needs" anywhere outside a code block (rule 15); -// - on a / branch, `### Status` or `## Assumptions` is missing (coding rule 2). +// - on a / branch, `### Status` or `## Assumptions` is missing (coding rule 2); +// - on a / branch, the Review counts line names a review that ran and the PR has no +// comment headed `## Review ledger` (REVIEW.md, "Measurement"); checked only with a PR number. // // It warns (exit 0) when the Review section has no `Rounds: . Tokens: M.` line, or the // prose runs over 200 words: everything outside code blocks, checklist lines and the Status, @@ -28,6 +30,7 @@ const args = process.argv.slice(2); const draftOk = args.includes("--draft-ok"); let body; let branch = ""; +let prNumber = null; const fileAt = args.indexOf("--body-file"); if (fileAt !== -1) { body = readFileSync(args[fileAt + 1], "utf8"); @@ -49,6 +52,7 @@ if (fileAt !== -1) { ); body = pull.body ?? ""; branch = pull.head.ref; + prNumber = pr; } body = body.replace(/\r/g, ""); @@ -110,6 +114,41 @@ if (/^(bug|feature|task)\/\d+$/.test(branch)) { } } +// why: only a delivery PR (/) has a ledger; a routine's or a person's chore PR gets +// the reviewer's findings comment instead. +if ( + prNumber !== null && + /^(bug|feature|task)\/\d+$/.test(branch) && + lines.some( + (l) => + REVIEW_COUNTS.test(l) && + !/^(Spec|Code|Claims) review: skipped\. (Code|Claims) review: skipped\. Claims review: skipped\.$/.test( + l, + ), + ) +) { + const comments = JSON.parse( + execFileSync( + "gh", + [ + "api", + "-X", + "GET", + `repos/${repository()}/issues/${prNumber}/comments`, + "-f", + "per_page=100", + ], + { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }, + ), + ); + if (!comments.some((c) => /^## Review ledger\s*$/m.test(c.body ?? ""))) { + failures.push("no comment headed `## Review ledger` (REVIEW.md, Measurement)"); + } +} + const words = budgetedWords(lines); if (words > BUDGET) { warnings.push( diff --git a/.agents/scripts/replay-freeze.mjs b/.agents/scripts/replay-freeze.mjs index 7afa93331..b301811ba 100755 --- a/.agents/scripts/replay-freeze.mjs +++ b/.agents/scripts/replay-freeze.mjs @@ -18,12 +18,8 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { option } from "./cli-args.mjs"; -const args = process.argv.slice(2); -const option = (name) => { - const at = args.indexOf(name); - return at === -1 ? null : args[at + 1]; -}; const pr = option("--pr"); const round = option("--round"); const head = option("--head"); @@ -69,10 +65,23 @@ const comments = JSON.parse( gh("api", "-X", "GET", `repos/${nwo}/issues/${pr}/comments`, "-f", "per_page=100"), ); let findings = null; +const recordedRounds = {}; for (const c of comments) { if (!/^## Review ledger/m.test(c.body ?? "")) continue; const m = c.body.match(/```json\s*(\{[\s\S]*?\})\s*```/); if (m) findings = JSON.parse(m[1]).findings; + for (const [, r, sha] of c.body.matchAll(/^Round (\S+): ([0-9a-f]{7,40})/gm)) + recordedRounds[r] = sha; +} +if ( + recordedRounds[round] && + !recordedRounds[round].startsWith(head.slice(0, 7)) && + !head.startsWith(recordedRounds[round].slice(0, 7)) +) { + console.error( + `the ledger comment says round ${round} reviewed ${recordedRounds[round]}, not ${head}`, + ); + process.exit(65); } if (findings === null) console.error( diff --git a/.agents/scripts/review-inputs.sh b/.agents/scripts/review-inputs.sh index d3a168eb7..bebe26f48 100755 --- a/.agents/scripts/review-inputs.sh +++ b/.agents/scripts/review-inputs.sh @@ -2,7 +2,12 @@ # Builds the blind inputs for the pre-merge reviews of a PR (delivery rule 14) and prints the # directory's path as the last line on stdout. # -# .agents/scripts/review-inputs.sh [] +# .agents/scripts/review-inputs.sh [ []] +# +# A previous commit that is no ancestor of the head (the branch was rebased or the stack synced) +# is mapped to its rebased equivalent when its base is given: the same number of commits from +# the new base, each with the same patch-id as its old counterpart. Any difference (a conflict +# resolved, a commit dropped) stays incomplete, as a rebased history nobody can map is. # # The directory holds, when each exists: # @@ -10,6 +15,7 @@ # commit the PR head the directory was built from # base, merge-base immutable actual PR base tip and fork # previous validated ancestor for targeted checks +# previous-mapped-from the previous commit as given, when it was mapped across a rebase # review-protocol.md review policy from the pinned base, when present # kind what .agents/scripts/diff-kind.mjs calls the diff: adr-only, # docs-only, comments-only, tests-only, full or large; it sets which @@ -43,9 +49,10 @@ set -eu pr=${1:-} previous=${2:-} +previous_base=${3:-} case $pr in '' | *[!0-9]*) - echo "usage: review-inputs.sh []" >&2 + echo "usage: review-inputs.sh [ []]" >&2 exit 64 ;; esac @@ -106,10 +113,28 @@ if [ -n "$previous" ]; then say "previous commit is unavailable; assessment incomplete" exit 65 } - git merge-base --is-ancestor "$previous" "$head" || { - say "previous commit is not an ancestor of head; assessment incomplete" - exit 65 - } + mapped_from= + if ! git merge-base --is-ancestor "$previous" "$head"; then + if [ -z "$previous_base" ]; then + say "previous commit is not an ancestor of head (rebased?); pass its base to map it, else assessment incomplete" + exit 65 + fi + previous_base=$(git rev-parse --verify "$previous_base^{commit}" 2>/dev/null) || { + say "previous base is unavailable; assessment incomplete" + exit 65 + } + n=$(git rev-list --count --no-merges "$previous_base..$previous") + old_ids=$(git rev-list --reverse --no-merges "$previous_base..$previous" | while read -r c; do git show "$c" | git patch-id --stable | cut -d' ' -f1; done) + new_ids=$(git rev-list --reverse --no-merges "$base..$head" | head -n "$n" | while read -r c; do git show "$c" | git patch-id --stable | cut -d' ' -f1; done) + if [ "$n" -gt 0 ] && [ "$old_ids" = "$new_ids" ]; then + mapped_from=$previous + previous=$(git rev-list --reverse --no-merges "$base..$head" | sed -n "${n}p") + say "previous $mapped_from mapped across the rebase to $previous ($n commits, same patch-ids)" + else + say "previous commit is not an ancestor of head and its $n commits do not match the rebased history by patch-id; assessment incomplete" + exit 65 + fi + fi fi tmp=${TMPDIR:-/tmp} @@ -121,6 +146,7 @@ echo "$head" >"$out/commit" echo "$base" >"$out/base" echo "$fork" >"$out/merge-base" [ -z "$previous" ] || echo "$previous" >"$out/previous" +[ -z "${mapped_from:-}" ] || echo "$mapped_from" >"$out/previous-mapped-from" if git cat-file -e "$rules_ref:docs/internal/REVIEW.md" 2>/dev/null; then git show "$rules_ref:docs/internal/REVIEW.md" >"$out/review-protocol.md" fi diff --git a/.agents/scripts/review-metrics.mjs b/.agents/scripts/review-metrics.mjs index 9cb340345..20939624c 100644 --- a/.agents/scripts/review-metrics.mjs +++ b/.agents/scripts/review-metrics.mjs @@ -15,11 +15,11 @@ // left out, so a replay never counts as the PR's delivery work. // // --findings validates the coordinator's findings file (`{"findings": [...]}`, each with id, -// role, class, outcome, path) and adds the outcome and class counts. --json prints the ledger +// role, class, outcome, path and, when present, round) and adds the outcome and class counts. --json prints the ledger // (runs, totals, findings) instead of the summary lines. -import { existsSync, openSync, readFileSync, readSync, closeSync, readdirSync } from "node:fs"; -import { homedir } from "node:os"; -import { join } from "node:path"; +import { readFileSync } from "node:fs"; +import { parsed, projectsRoot, tagOf, transcripts } from "./transcripts.mjs"; +import { option } from "./cli-args.mjs"; // $ per million tokens: input, output, cache read, 5-minute cache write, 1-hour cache write. // https://platform.claude.com/docs/en/about-claude/pricing, read 2026-10-19. @@ -29,12 +29,9 @@ const PRICES = { "claude-haiku-5-5": [0.1, 0.5, 0.01, 0.125, 0.2], }; const OUTCOMES = ["open", "fixed", "duplicate", "incorrect", "intentional", "nonblocking"]; +const ROUNDS = ["1", "2", "fix-check"]; const args = process.argv.slice(2); -const option = (name) => { - const at = args.indexOf(name); - return at === -1 ? null : args[at + 1]; -}; const pr = option("--pr"); const issue = option("--issue"); const findingsFile = option("--findings"); @@ -45,7 +42,7 @@ if (pr === null || !/^\d+$/.test(pr) || (issue !== null && !/^\d+$/.test(issue)) process.exit(64); } -const root = join(process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"), "projects"); +const root = projectsRoot(); const runs = transcripts(root) .map((file) => ({ file, tag: tagOf(file) })) .filter(({ tag }) => tag !== null && (tag.pr === pr || (issue !== null && tag.issue === issue))) @@ -100,36 +97,6 @@ if (findings !== null) { console.log(`Finding outcomes: ${count("outcome")}. Classes: ${count("class")}.`); } -function transcripts(dir) { - if (!existsSync(dir)) return []; - const files = []; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const path = join(dir, entry.name); - if (entry.isDirectory()) files.push(...transcripts(path)); - else if (entry.name.endsWith(".jsonl")) files.push(path); - } - return files; -} - -// why: transcripts run to hundreds of megabytes; the tag is in the first user record, near the top. -function tagOf(file) { - const fd = openSync(file, "r"); - const head = Buffer.alloc(256 * 1024); - const read = readSync(fd, head, 0, head.length, 0); - closeSync(fd); - for (const line of head.subarray(0, read).toString("utf8").split("\n")) { - const rec = parsed(line); - if (rec?.type !== "user") continue; - const first = promptOf(rec).split("\n")[0].trim(); - if (!first.startsWith("ledger:")) return null; - const tag = { pr: "-", issue: "-", stage: "-", round: "-", role: "-", lens: "-" }; - for (const [, key, value] of first.matchAll(/(\w+)=(\S+)/g)) - if (key in tag) tag[key] = value.replace(/^#/, ""); - return tag; - } - return null; -} - function run(file, tag) { const { messages, startedAt, finishedAt } = read(file); const tokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }; @@ -202,23 +169,11 @@ function checkFinding(f, list) { const missing = ["role", "class", "path"].find((k) => typeof f[k] !== "string" || f[k] === ""); if (missing !== undefined) throw new Error(`finding ${f.id} has no ${missing}`); if (!OUTCOMES.includes(f.outcome)) throw new Error(`finding ${f.id} has an invalid outcome`); + if (f.round !== undefined && !ROUNDS.includes(String(f.round))) { + throw new Error(`finding ${f.id} has an invalid round (1, 2 or fix-check)`); + } const canonical = list.some((x) => x.id === f.canonicalId && x.outcome !== "duplicate"); if (f.outcome === "duplicate" && !canonical) { throw new Error(`duplicate ${f.id} must name a canonical finding`); } } - -function promptOf(rec) { - const content = rec.message?.content; - if (typeof content === "string") return content; - return Array.isArray(content) ? (content.find((b) => b.type === "text")?.text ?? "") : ""; -} - -function parsed(line) { - if (line === "") return null; - try { - return JSON.parse(line); - } catch { - return null; - } -} diff --git a/.agents/scripts/review-score.mjs b/.agents/scripts/review-score.mjs index 54849fba9..e10d8118c 100644 --- a/.agents/scripts/review-score.mjs +++ b/.agents/scripts/review-score.mjs @@ -16,12 +16,8 @@ import { execFileSync } from "node:child_process"; import { appendFileSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { option } from "./cli-args.mjs"; -const args = process.argv.slice(2); -const option = (name) => { - const at = args.indexOf(name); - return at === -1 ? null : args[at + 1]; -}; const pr = option("--pr"); const round = option("--round"); const file = option("--findings"); @@ -140,8 +136,15 @@ for (const r of replay) { matches.push({ replay: r.id, truth: hit.id, truthOutcome: hit.outcome }); } else unmatched.push(r); } -// why: a single lens is judged on the findings its role owned in the real review. -const defects = truth.filter((t) => t.defect && (role === null || t.role === role)); +// why: a single lens is judged on the findings its role owned in the real review, and a later +// round on the findings that round raised, when the ledger says which round that was. +const rounded = truth.some((t) => t.round !== undefined); +const defects = truth.filter( + (t) => + t.defect && + (role === null || t.role === role) && + (!rounded || String(t.round) === String(round)), +); const found = defects.filter((t) => taken.has(t.id)); const missed = defects.filter((t) => !taken.has(t.id)); const nonDefectHits = matches.filter((m) => !["fixed", "open"].includes(m.truthOutcome)); diff --git a/.agents/scripts/transcripts.mjs b/.agents/scripts/transcripts.mjs new file mode 100644 index 000000000..6ea0b676c --- /dev/null +++ b/.agents/scripts/transcripts.mjs @@ -0,0 +1,86 @@ +// Claude Code's transcripts, for the scripts that measure agent runs: review-metrics.mjs, +// instruction-reads.mjs and delivery-counters.mjs. A run is a transcript whose first user +// message starts with the ledger tag line (`toolchain.md`, "Review measurement"). +import { closeSync, existsSync, openSync, readFileSync, readSync, readdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +export const projectsRoot = () => + join(process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"), "projects"); + +export function transcripts(dir) { + if (!existsSync(dir)) return []; + const files = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) files.push(...transcripts(path)); + else if (entry.name.endsWith(".jsonl")) files.push(path); + } + return files; +} + +// why: transcripts run to hundreds of megabytes; the tag is in the first user record, near the top. +export function tagOf(file) { + const fd = openSync(file, "r"); + const head = Buffer.alloc(256 * 1024); + const read = readSync(fd, head, 0, head.length, 0); + closeSync(fd); + for (const line of head.subarray(0, read).toString("utf8").split("\n")) { + const rec = parsed(line); + if (rec?.type !== "user") continue; + const first = promptOf(rec).split("\n")[0].trim(); + if (!first.startsWith("ledger:")) return null; + const tag = { pr: "-", issue: "-", stage: "-", round: "-", role: "-", lens: "-" }; + for (const [, key, value] of first.matchAll(/(\w+)=(\S+)/g)) + if (key in tag) tag[key] = value.replace(/^#/, ""); + return tag; + } + return null; +} + +export function taggedRuns(root = projectsRoot()) { + return transcripts(root) + .map((file) => ({ file, tag: tagOf(file) })) + .filter(({ tag }) => tag !== null); +} + +export function runOf(file) { + const records = readFileSync(file, "utf8").split("\n").map(parsed).filter(Boolean); + const first = records.find((r) => r.type === "user"); + const blocks = assistantBlocks(records); + return { + prompt: first ? promptOf(first) : "", + toolInputs: blocks + .filter((b) => b.type === "tool_use") + .map((b) => JSON.stringify(b.input ?? {})), + lastText: blocks.filter((b) => b.type === "text" && b.text?.trim()).at(-1)?.text ?? "", + startedAt: records.map((r) => r.timestamp).find((t) => typeof t === "string") ?? null, + }; +} + +// why: one API message is written as several records with the same id; its blocks count once. +function assistantBlocks(records) { + const seen = new Set(); + const blocks = []; + for (const r of records) { + if (r.type !== "assistant" || !r.message?.id || seen.has(r.message.id)) continue; + seen.add(r.message.id); + blocks.push(...(r.message.content ?? [])); + } + return blocks; +} + +export function promptOf(rec) { + const content = rec.message?.content; + if (typeof content === "string") return content; + return Array.isArray(content) ? (content.find((b) => b.type === "text")?.text ?? "") : ""; +} + +export function parsed(line) { + if (line === "") return null; + try { + return JSON.parse(line); + } catch { + return null; + } +} diff --git a/.claude/agents/reviewer.md b/.claude/agents/reviewer.md index e6acf6ac9..cb5e7f3da 100644 --- a/.claude/agents/reviewer.md +++ b/.claude/agents/reviewer.md @@ -14,9 +14,11 @@ impacts, the round cap and metrics. This file adds only how you run it. Edit no code. Arguments: PR, round (`1`, `2`, `fix-check`), previous full SHA after round 1, prior resolution ledger, and optional `pilot: sonnet`. -1. **Inputs.** Run `.agents/scripts/review-inputs.sh []` - and record its base, merge-base and commit; a failed build is - incomplete. A `replay: true` argument comes with an inputs directory +1. **Inputs.** Run `.agents/scripts/review-inputs.sh [ +[]]` and record its base, merge-base and commit; a + failed build is incomplete. A `previous-base:` argument comes with a + rebased branch: pass it, and report the mapping the script prints; + never check a rebase by hand. A `replay: true` argument comes with an inputs directory already built by `.agents/scripts/review-replay.sh`: use it, do not run `review-inputs.sh`, ignore the PR's state on GitHub (it may be merged), post nothing anywhere, never open `.agents/replay/`, and write the diff --git a/.claude/agents/spec-checker.md b/.claude/agents/spec-checker.md index 21bc40b7b..284f648a9 100644 --- a/.claude/agents/spec-checker.md +++ b/.claude/agents/spec-checker.md @@ -91,8 +91,9 @@ _note_: - **Note**: anything else, above all a gap the implementer may close as an `Assumption:` line (`coding.md` rule 2): a wording, an order, a bound the spec implies. A test for a case the spec does not name, defensive - handling, a needless Depends on, a task slightly over size. At most five - notes, the ones most likely to matter; drop the rest. + handling, a needless Depends on, a task slightly over size. A note only + when the implementer would act on it; none is the usual count, and a + report never pads to a number. 1. **contradiction.** A line that breaks an agent rule or an accepted ADR, or two lines that cannot both hold. A Done when line the task's own diff --git a/.claude/skills/deliver/SKILL.md b/.claude/skills/deliver/SKILL.md index 9022644f0..2178410d1 100644 --- a/.claude/skills/deliver/SKILL.md +++ b/.claude/skills/deliver/SKILL.md @@ -159,7 +159,10 @@ they name; never build again or restart review at round 1. is neither a defect nor a pass. - Fix lines with `Rerun: round 2` or `fix-check`: one consolidated fix run with the immutable base/merge-base, `previous `, ledger - and findings, then `reviewer` with the round `Rerun` names. + and findings, then `reviewer` with the round `Rerun` names. After a + rebase or stack sync, the review also gets `previous-base: `, so the inputs script can map the + previous commit across the rebase. - `Rerun: none` with unresolved nonmechanical findings or unchecked fixes at the cap: park. Mechanical changes get a coordinator diff check. - Commits after reviews passed, including conflict resolutions and CI @@ -206,7 +209,9 @@ they name; never build again or restart review at round 1. the counts for a review the diff's kind left out (for an ADR-only diff: `Code review: skipped. Claims review: skipped.`). `` is the number of legacy full/targeted rounds, 1 or 2. Post the `## Review ledger` comment - as `docs/internal/REVIEW.md` "Measurement" says, fix-checks included. Add + as `docs/internal/REVIEW.md` "Measurement" says, fix-checks included: + one `Round : ` line per check first, and `round` on + every finding. Add `Assessment: complete` only when coverage is complete and no finding is open; otherwise `Assessment: incomplete` and park. `` is `tokens=` from `node .agents/scripts/usage.mjs`, this session's usage so diff --git a/docs/internal/DELIVERY.md b/docs/internal/DELIVERY.md index 6c45455d0..1598e63d7 100644 --- a/docs/internal/DELIVERY.md +++ b/docs/internal/DELIVERY.md @@ -131,8 +131,12 @@ Two transitions stay manual on purpose, one click each: `bug:new` → last five: blocking findings per review raised and fixed, notes, PRs needing a second round, median hours to merge, rounds and tokens, handoffs. `--since ` lists PRs; `--calibration` prints the recent -incorrect findings every reviewer reads. Change one thing in the pipeline -at a time and compare the rows before and after. +incorrect findings every reviewer reads. `delivery-counters.mjs --since +` prints the per-PR counters the process review judges delivery +changes by (CI lines sent to fix runs, stack syncs, empty builds, fix runs +that changed nothing, tests per Tests line), and `instruction-reads.mjs` +how often each role opens each rule, brief and manual. Change one thing in +the pipeline at a time and compare the rows before and after. Weekly routines keep the suite and the reviews in shape: `harden-tests`, `audit-tests`, `garden-comments` and `calibrate-reviews` diff --git a/docs/internal/REVIEW.md b/docs/internal/REVIEW.md index f85acfcec..a9cd9874d 100644 --- a/docs/internal/REVIEW.md +++ b/docs/internal/REVIEW.md @@ -4,9 +4,12 @@ Delivery rule 14 delegates its review procedure to this document. Review one immutable PR snapshot. Use `.agents/scripts/review-inputs.sh ` for round 1, and add the previous report's full commit SHA for later checks. The inputs record `base` (base tip), `merge-base`, `commit` and `previous`. -Never substitute `origin/main` for a stacked PR's base. Missing or -non-ancestor previous commits make assessment incomplete: reconcile the -history and rebuild inputs, do not silently perform a full review. +Never substitute `origin/main` for a stacked PR's base. A missing previous +commit makes assessment incomplete. After a rebase or stack sync the +previous commit is no ancestor of the head: give the script that round's +base as well, and it maps the commit to its rebased equivalent when every +commit's patch-id matches; otherwise assessment is incomplete, and nobody +silently performs a full review in its place. Rules, briefs and this protocol come from the pinned PR base, not a moving `main`. No reviewer sees author narration (PR body, commit messages) or another lens's findings before synthesis. @@ -107,9 +110,13 @@ transcripts. Unknown values print as unknown, never zero. The coordinator keeps one findings file per PR, `{"findings": [...]}`, one entry per canonical finding: `id`, `role`, `class` (the `[class]` the lens gave, a rule like `testing 2` or at most 8 words), `outcome`, `path`, -`fix` (the fix commit, once fixed) and `canonicalId` for a duplicate. Before the gate, post one PR comment headed -`## Review ledger`: the output of `review-metrics.mjs --pr --issue ---findings `, then the findings file in a `json` code block. The +`round` (the check that raised it: `1`, `2` or `fix-check`), `fix` (the +fix commit, once fixed) and `canonicalId` for a duplicate. Before the gate, +post one PR comment headed `## Review ledger`: one line `Round : +` per check in order, the output of `review-metrics.mjs --pr + --issue --findings `, then the findings file in a `json` +code block. The round lines are what lets a check be frozen and replayed +later (`replay-freeze.mjs`). The weekly `recurring-findings` routine reads these comments. Record escaped defects after merge. Do not call raised-minus-fixed a false-positive rate. diff --git a/src/pr-body-check.test.ts b/src/pr-body-check.test.ts index 995e39b65..9abd32734 100644 --- a/src/pr-body-check.test.ts +++ b/src/pr-body-check.test.ts @@ -33,14 +33,28 @@ function check(body: string, ...args: string[]) { } } -function fromGitHub(pull: object, options: { origin?: boolean } = {}) { +function fromGitHub( + pull: object, + options: { origin?: boolean; comments?: readonly string[] } = {}, +) { const dir = mkdtempSync(join(tmpdir(), "pr-body-gh-")); writeFileSync(join(dir, "pull.json"), JSON.stringify(pull)); + // why: the comments endpoint answers only when the test says so, so a case with no + // `comments` proves the script never asked for them (a chore branch has no ledger). + if (options.comments !== undefined) { + writeFileSync( + join(dir, "comments.json"), + JSON.stringify(options.comments.map((body) => ({ body }))), + ); + } writeFileSync( join(dir, "gh"), `#!/bin/sh -[ "$*" = "api -X GET repos/o/r/pulls/7" ] || { echo "fake gh: unexpected: $*" >&2; exit 9; } -cat ${JSON.stringify(join(dir, "pull.json"))} +case "$*" in + "api -X GET repos/o/r/pulls/7") cat ${JSON.stringify(join(dir, "pull.json"))} ;; + "api -X GET repos/o/r/issues/7/comments -f per_page=100") cat ${JSON.stringify(join(dir, "comments.json"))} 2>/dev/null || { echo "fake gh: comments not expected" >&2; exit 9; } ;; + *) echo "fake gh: unexpected: $*" >&2; exit 9 ;; +esac `, ); chmodSync(join(dir, "gh"), 0o755); @@ -158,9 +172,10 @@ describe("pr-body-check.mjs", () => { const bare = `## Review\n\n${COUNTS}\nRounds: 1. Tokens: 1.0M.\n\n${FOOTER}\n`; expect(fromGitHub({ body: bare, head: { ref: "chore/x" } })).toEqual({ status: 0, lines: [] }); - expect(fromGitHub({ body: bare, head: { ref: "bug/7" } }).lines).toEqual([ + expect(fromGitHub({ body: bare, head: { ref: "bug/7" } }, { comments: [] }).lines).toEqual([ "pr-body: no `### Status` section", expect.stringContaining("no `## Assumptions` section"), + expect.stringContaining("no comment headed `## Review ledger`"), ]); expect(fromGitHub({ body: null, head: { ref: "chore/x" } }).lines).toEqual([ expect.stringContaining('the last line is not "*Written by an agent.*"'), @@ -168,6 +183,24 @@ describe("pr-body-check.mjs", () => { ]); }); + it("requires the Review ledger comment on a delivery branch whose reviews ran, and only there", () => { + const body = valid; + const ledger = '## Review ledger\n\nRound 1: abc\n```json\n{"findings": []}\n```'; + expect(fromGitHub({ body, head: { ref: "task/7" } }, { comments: ["hi", ledger] })).toEqual({ + status: 0, + lines: [], + }); + expect(fromGitHub({ body, head: { ref: "task/7" } }, { comments: ["hi"] }).lines).toEqual([ + "pr-body: no comment headed `## Review ledger` (REVIEW.md, Measurement)", + ]); + const skipped = body.replace( + COUNTS, + "Spec review: skipped. Code review: skipped. Claims review: skipped.", + ); + expect(fromGitHub({ body: skipped, head: { ref: "task/7" } }).status).toBe(0); + expect(fromGitHub({ body, head: { ref: "chore/x" } }).status).toBe(0); + }); + it("takes the repo from the origin remote when GH_REPO is unset", () => { const bare = `## Review\n\n${COUNTS}\nRounds: 1. Tokens: 1.0M.\n\n${FOOTER}\n`; const result = fromGitHub({ body: bare, head: { ref: "chore/x" } }, { origin: true });