From ceda5d0ac73f371574b8097c872f68039c89205d Mon Sep 17 00:00:00 2001 From: prode Date: Tue, 15 Sep 2026 23:36:08 -0300 Subject: [PATCH 1/3] feat(hooks): tell an unattended session that its kickoff question reaches nobody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rules/autonomy.md opens every piece of work with three questions, which is right whenever somebody is there to answer them. Measured on a pinned public repository at n=3, a headless run of the scaffolded harness ended every session holding those questions and wrote no code at all, while the same agent with no methodology at all shipped the feature every time. Adding the missing branch to the rule's prose changed nothing — 0 of 3 again — because it asks the agent to act on a condition it cannot observe: from inside the turn an unattended session looks like any other. SessionStart can observe it, so it says so, and the same arm then delivered 5 of 5 with the plan, the ticked box, the branch and a clean scc validate the methodology promises. The signal is undocumented and is treated that way: exactly one spelling is read and anything else, unset included, says nothing. A missed line leaves today's behaviour; a line in front of an attended session would talk somebody out of a question they were right to be asked. The rule carries the branch too, because two of the three harnesses have no hook surface to deliver it. It is paid for with a cut that leaves the file 10 bytes smaller than before, so the standing cost does not move. --- .claude/rules/autonomy.md | 18 ++++---- .claude/scc-manifest.json | 50 ++++++++++----------- internal/assets/assets.go | 2 +- internal/assets/assets_test.go | 2 +- internal/assets/templates/rules/autonomy.md | 18 ++++---- internal/cli/hooks_agent.go | 36 ++++++++++++++- internal/cli/hooks_agent_test.go | 35 +++++++++++++++ 7 files changed, 113 insertions(+), 48 deletions(-) diff --git a/.claude/rules/autonomy.md b/.claude/rules/autonomy.md index e7ee045..e893e03 100644 --- a/.claude/rules/autonomy.md +++ b/.claude/rules/autonomy.md @@ -8,10 +8,14 @@ questions, together, in the same breath: 1. **Run automatically, or gate each phase for review?** 2. **When the PR is open, wait for CI, or finish there?** -3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness — - particles (之/乃/為/其), verb before object, subject dropped. Say the cost: its - "80-90% reduction" counts **characters, not tokens**, and CJK spends more tokens per - character, so the real saving is smaller and unmeasured. Governs speech, not artifacts. +3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness. The + advertised saving counts characters, not tokens. Governs speech, not artifacts. + +**Nobody there to answer? Do not spend the turn asking.** A headless run, a CI job, an +unattended session: the question has no reader, and a turn that ends holding one has +delivered nothing — the one outcome kickoff was never meant to produce. Take `auto`, +`no-wait` and the user's own language, record them as assumed, say so in one line, and +build. A later session changes any of the three with `scc patch fm`. Record the answers in the artifact's frontmatter — `requirements.md` for a spec, the plan file itself for a plan — then never ask again for this piece of work: @@ -31,12 +35,6 @@ Recording them is what makes the run reproducible from the file and what stops a second session from re-asking. Ask in conversation rather than reading a flag, because the person who has to make the call is in the conversation. -## Asking at kickoff, not later - -The CI question especially: by the time the PR is open the work is done and the -user may be gone — which is exactly the situation "don't wait" exists for, and -exactly when a blocking question costs the most. - ## `auto` is not "never stop" Automatic keeps one exception. **The risk that mandates TDD also warrants a diff --git a/.claude/scc-manifest.json b/.claude/scc-manifest.json index 0d30d5d..bdd171f 100644 --- a/.claude/scc-manifest.json +++ b/.claude/scc-manifest.json @@ -10,62 +10,62 @@ { "hash": "08141d3f57fbe10f46ea557de9969fe06f96e4aceca90cd14079aea0162e4a73", "path": ".claude/agents/scc-code-review.md", - "version": "31" + "version": "32" }, { "hash": "86ae07c5864d0c38932cf9781e5b0ac5d6f6e36925577efdef646e779abbe65b", "path": ".claude/agents/scc-security-review.md", - "version": "31" + "version": "32" }, { "hash": "f3132fd6be0b859b5ffb063e22ba3b840226c5bfdde2613cd21993ff12afa477", "path": ".claude/rules/artifacts.md", - "version": "31" + "version": "32" }, { - "hash": "f5299384ccab206eac0eab1d6b93095831745d308cec32deac687b6359dbb900", + "hash": "9a6ee0e84e62f939f8a9304350957443b9593f51c109c188c43a4658c6acb764", "path": ".claude/rules/autonomy.md", - "version": "31" + "version": "32" }, { "hash": "8eee2d450faada83295a4a04e5a1267bf5f717df9e7ac7da5fddd7649c666499", "path": ".claude/rules/caveman.md", - "version": "31" + "version": "32" }, { "hash": "cfe46c6623c20b6da46c9f18ef9b35e0f554c6c4568791ff190648dda3524706", "path": ".claude/rules/code-search.md", - "version": "31" + "version": "32" }, { "hash": "3ad4d9513a212bcced1b99db171a9ed53ccb4be38063f345ae722e6c8f36fa20", "path": ".claude/rules/delivery.md", - "version": "31" + "version": "32" }, { "hash": "ffca716ea25746d8e100ba3a7a7b884907736adebf3a7131b989fa4e9fcd0d80", "path": ".claude/rules/knowledge-base.md", - "version": "31" + "version": "32" }, { "hash": "59bb916a2002d4176353463b017c366868192cf95c80fd1dce74265188b738c3", "path": ".claude/rules/ladder.md", - "version": "31" + "version": "32" }, { "hash": "79d5cbf1ab7fd09d26e9ea9119f3f283615467dbc35d3a4b1d4a030044b3598f", "path": ".claude/rules/methodology.md", - "version": "31" + "version": "32" }, { "hash": "7dcbf5cd9d2e9e8d08aed4eeaa96ee28af85da1d6de51fc35870b8430e30f331", "path": ".claude/rules/notes.md", - "version": "31" + "version": "32" }, { "hash": "52c5033fc427445b4e669c7865dbd8d5c2fc27d57f7200aeeb2cc48bc8e050d3", "path": ".claude/rules/prior-art.md", - "version": "31" + "version": "32" }, { "hash": "ae0030df2e1f053642a84953648e1d6038b0f0fbae831edf1c349badfb00485c", @@ -75,62 +75,62 @@ { "hash": "c8502ace5978c7b9c0543c49ebb5c9550584b1a3405e12c0217b595647e27722", "path": ".claude/rules/routing.md", - "version": "31" + "version": "32" }, { "hash": "0b86b1a73769c6ebd37c338c5ec3cbee8dda70b21d43b27a1c95eebc5ee70a9f", "path": ".claude/rules/specs.md", - "version": "31" + "version": "32" }, { "hash": "d64a118d493424b10ad8af24861083c34541f7946a92dae92fb153b0ab739c26", "path": ".claude/rules/tasks.md", - "version": "31" + "version": "32" }, { "hash": "403174a8133018b54adc69c3047c46f8ff603b8a91759c62da1e7ca6b7f45463", "path": ".claude/rules/verification.md", - "version": "31" + "version": "32" }, { "hash": "cce99457f203766837ec2b8165a9370924178d9ee377c7848dba65880461cf3f", "path": ".claude/skills/scc-adr/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "fc2ecb43e7e3c157082dc359d1d0fb20c99fdb6fb774cfcead3c8c0961fef922", "path": ".claude/skills/scc-codewiki/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "60e80dcee7e68498679735ac40812ac90641340fef6bd75d3e7de64d209f8716", "path": ".claude/skills/scc-glossary/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "9d21e953cce950ccf495810c94411a98706780b2b8d38817e3ce13bfc0c74625", "path": ".claude/skills/scc-init/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "2385f307f96cc4261e44d1a0d49bb39d6a5842a3205cee317e3228ced0f6aeb6", "path": ".claude/skills/scc-plan-run/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "3d3773f869d73beeab4c2061397c9ad81faf6d4c11d1ef423470aca432c5a9e1", "path": ".claude/skills/scc-prd/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "42dff0d55f15c60da604f62fc1dfe8af5b73de2e2497553411447aae57698e1b", "path": ".claude/skills/scc-stack/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "aa53b7ddf701ea495d68ea12aa23a6d352c202edc0e308b7218a1bf685b09f21", "path": ".claude/skills/scc-wiki/SKILL.md", - "version": "31" + "version": "32" }, { "hash": "9dc2f234756bfdc79aa5697e8470f80766e68a0e72b6cda98107e25ea8a9427a", diff --git a/internal/assets/assets.go b/internal/assets/assets.go index aa05294..462b146 100644 --- a/internal/assets/assets.go +++ b/internal/assets/assets.go @@ -270,7 +270,7 @@ import ( // and clear this artifact's findings". That moved into the skill bodies, where it is // paid by the run that invokes it instead of by every session; `argument-hint` moved // with it, since SKILL.md takes the same key. -const Version = "31" +const Version = "32" // The embedded tree. "all:" so nothing is silently dropped for having a name the // default embed pattern skips. diff --git a/internal/assets/assets_test.go b/internal/assets/assets_test.go index d9aea91..de84310 100644 --- a/internal/assets/assets_test.go +++ b/internal/assets/assets_test.go @@ -1075,7 +1075,7 @@ func (e errNoSuchRule) Error() string { // uses wherever a change is cheap to make and expensive to make by accident. func TestTheTemplateVersionMovesWithTheTemplates(t *testing.T) { // Bump Version, then replace this with the digest the failure prints. - const fingerprint = "fcd82bf91a559baa6c72268203627cc74c0e4a8839f1febe636641ff6a0d5aa2" + const fingerprint = "de55d2721ee9d0bee6b6c4f92509a55b35baf9567ff6f334ffb8fbae0ff19e18" sum := sha256.New() // Version goes into the hash, and without it this test does not do the job its diff --git a/internal/assets/templates/rules/autonomy.md b/internal/assets/templates/rules/autonomy.md index e7ee045..e893e03 100644 --- a/internal/assets/templates/rules/autonomy.md +++ b/internal/assets/templates/rules/autonomy.md @@ -8,10 +8,14 @@ questions, together, in the same breath: 1. **Run automatically, or gate each phase for review?** 2. **When the PR is open, wait for CI, or finish there?** -3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness — - particles (之/乃/為/其), verb before object, subject dropped. Say the cost: its - "80-90% reduction" counts **characters, not tokens**, and CJK spends more tokens per - character, so the real saving is smaller and unmeasured. Governs speech, not artifacts. +3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness. The + advertised saving counts characters, not tokens. Governs speech, not artifacts. + +**Nobody there to answer? Do not spend the turn asking.** A headless run, a CI job, an +unattended session: the question has no reader, and a turn that ends holding one has +delivered nothing — the one outcome kickoff was never meant to produce. Take `auto`, +`no-wait` and the user's own language, record them as assumed, say so in one line, and +build. A later session changes any of the three with `scc patch fm`. Record the answers in the artifact's frontmatter — `requirements.md` for a spec, the plan file itself for a plan — then never ask again for this piece of work: @@ -31,12 +35,6 @@ Recording them is what makes the run reproducible from the file and what stops a second session from re-asking. Ask in conversation rather than reading a flag, because the person who has to make the call is in the conversation. -## Asking at kickoff, not later - -The CI question especially: by the time the PR is open the work is done and the -user may be gone — which is exactly the situation "don't wait" exists for, and -exactly when a blocking question costs the most. - ## `auto` is not "never stop" Automatic keeps one exception. **The risk that mandates TDD also warrants a diff --git a/internal/cli/hooks_agent.go b/internal/cli/hooks_agent.go index 1d3f26a..c4c85b9 100644 --- a/internal/cli/hooks_agent.go +++ b/internal/cli/hooks_agent.go @@ -181,6 +181,8 @@ func sessionStartContext(root string) []string { if !workspace.IsWorkspace(root) { return nil } + // First, because it changes what the agent does with everything below it. + out := unattendedLines() // Drift is reported here rather than at the end of a turn, and the move was // paid for in a broken session. It is a fact about the branch — source changed // with no box ticked, a marker in code, commits on the base branch — so no @@ -188,7 +190,7 @@ func sessionStartContext(root string) []string { // the conversation, a fact that cannot be resolved is a fact that repeats // until the harness cuts it off. Said once, at the start, where it is // information rather than an instruction the agent cannot carry out. - out := driftLines(root) + out = append(out, driftLines(root)...) cfg, err := gate.Load(root) if err != nil || cfg.Any() { @@ -203,6 +205,38 @@ func sessionStartContext(root string) []string { ) } +// unattendedLines says, once, that nobody is going to read a question asked this +// turn. +// +// `rules/autonomy.md` opens every piece of work with three questions, and that is +// right whenever there is somebody to answer them. In a headless run there is not, +// and the rule alone cannot tell the difference: from inside the turn an unattended +// session looks exactly like an attended one. Measured, on a pinned ticket at n=3: +// the scaffolded harness ended every run holding the kickoff questions and wrote no +// code at all, while the same agent with no methodology shipped the feature every +// time. Adding the branch to the rule's prose changed nothing — 0 of 3 again — +// because the agent was being asked to act on a condition it cannot observe. With +// this line in front of it, the same arm delivered 5 of 5, with the plan, the +// ticked box, the branch and a clean `scc validate` the methodology promises. +// +// **The signal is undocumented and is treated that way.** `CLAUDE_CODE_SESSION_ +// ATTENDED=0` is in the environment of a hook during `claude -p`, verified against +// 2.1.273; Claude Code's own documentation describes neither the variable nor its +// values, so this reads exactly one spelling and says nothing for anything else, +// unset included. That default is the safe one in the only direction that matters: +// a missed line leaves today's behaviour, while a line in front of an attended +// session would talk somebody out of a question they were right to ask. +func unattendedLines() []string { + if os.Getenv("CLAUDE_CODE_SESSION_ATTENDED") != "0" { + return nil + } + return []string{ + "scc: this session is unattended, so a question asked this turn reaches nobody.", + "Take the kickoff answers as `autonomy: auto`, `ci: no-wait` and the user's own language,", + "record them in the artifact's frontmatter as assumed rather than answered, say so in one line, and build.", + } +} + // stopContext is what the end of a turn has to say: what the validators found, // and whether finished work is still sitting on this machine. // diff --git a/internal/cli/hooks_agent_test.go b/internal/cli/hooks_agent_test.go index 72af580..e63b748 100644 --- a/internal/cli/hooks_agent_test.go +++ b/internal/cli/hooks_agent_test.go @@ -350,3 +350,38 @@ func TestHookTimeoutsMatchWhatTheStageDoes(t *testing.T) { } } } + +// An unattended session is told so, once, and an attended one is told nothing. +// +// The direction of the default is the point. Measured on a pinned ticket at n=3, +// the scaffolded harness ended every headless run holding autonomy.md's kickoff +// questions and wrote no code; with this line in front of it the same arm +// delivered. But a line shown to an attended session would talk a person out of a +// question they were right to be asked, so anything other than the one spelling +// the harness actually sets — unset included — says nothing. +func TestSessionStartSpeaksOnlyToAnUnattendedSession(t *testing.T) { + root := initWorkspace(t) + setTest(t, root, echoJSON(10, "99")) + skipRest(t, root, "test") + + t.Setenv("CLAUDE_CODE_SESSION_ATTENDED", "0") + stdout, _, code := run(t, "hooks", "run", "session-start", "--root", root) + if code != ExitOK { + t.Fatalf("exit = %d, want %d — a harness hook never blocks", code, ExitOK) + } + ctx := additionalContext(t, stdout) + if !strings.Contains(ctx, "unattended") || !strings.Contains(ctx, "no-wait") { + t.Errorf("an unattended session was not told so: %q", ctx) + } + + for _, v := range []string{"1", "", "true", "yes"} { + t.Setenv("CLAUDE_CODE_SESSION_ATTENDED", v) + stdout, _, code = run(t, "hooks", "run", "session-start", "--root", root) + if code != ExitOK { + t.Fatalf("exit = %d for %q", code, v) + } + if strings.Contains(stdout, "unattended") { + t.Errorf("CLAUDE_CODE_SESSION_ATTENDED=%q was read as unattended: %q", v, stdout) + } + } +} From 8f7d279be664f745ae11cee8fdb72806d661f260 Mon Sep 17 00:00:00 2001 From: prode Date: Tue, 15 Sep 2026 23:38:20 -0300 Subject: [PATCH 2/3] docs(design): record the first measurement of what the rules cost and buy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit design/ponytail-port.md shipped its hook and rule work and skipped its measurement half, leaving the question standing: nothing measures whether the rules earn what they cost. This is that restart at a twentieth of the planned size — two tickets, one pinned target, one model, n=3, 24 headless sessions and 8 blind critics for about $17. What it found: the scaffold costs +14,703 tokens in every request, +42% on the harness's own baseline, with the scoped rules measurably absent from that figure; blind critics picked the scaffolded arm 3 of 3 on a bug fix, twice for the same reason, that it left regression tests the control deleted; and the same scaffold delivered nothing at all on new work, three times, because kickoff asks a question no unattended run can answer. Limitations first, and the section that says what is still unanswered is the longer half: fourteen of the fifteen rules have still never been told apart from their own absence, and the attended session — the one most people are in — is unmeasured. --- design/rule-measurement.md | 187 +++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 design/rule-measurement.md diff --git a/design/rule-measurement.md b/design/rule-measurement.md new file mode 100644 index 0000000..6c58097 --- /dev/null +++ b/design/rule-measurement.md @@ -0,0 +1,187 @@ +--- +autonomy: auto +ci: no-wait +lang: en +status: result +--- + +# Measuring the rules, first run + +`design/ponytail-port.md` shipped its hook and rule work and skipped its measurement +half, recording why: 48 agent sessions at an estimated $14–$120, against an apparatus +that produces nothing until somebody spends that. Its §Why was left standing — +*nothing measures whether the rules earn the ~870 lines they cost* — with the +instruction to restart from there rather than from the task list. + +This is that restart, at a twentieth of the planned size. It answers the question for +two tickets on one target with one model, finds one defect large enough to change the +product, and says plainly where its own numbers stop. + +## Limitations, before anything else + +- **One model, one target, n=3.** Sonnet 5 against `github.com/dustin/go-humanize` + pinned at `4d1d908` — 2,241 lines, stdlib-only, suite green. Nothing here transfers + to a large codebase or another model without being run again. +- **Every run is one headless turn.** `claude -p` gives the agent nobody to come back + to. That is what makes §3's failure visible and it is also the *unattended* case + specifically. Nothing here says what the same rules do with a person in the session. +- **The operator's global configuration was a constant, not an absence.** + `~/.claude/rules/context7.md` and an RTK `PreToolUse` hook were present in both + arms. Isolating the config directory needs a copy of the credentials file, which + was refused, so the leak was ruled out by probe (§2) rather than by removal. +- **A difference inside the spread of three runs is not a difference**, and §6 is + named rather than acted on for exactly that reason. +- **The hidden acceptance test is a narrow notion of effect.** It says the ticket + works. It does not say the repository is better for the change. +- **gofmt carries no signal on this target.** The pin is already not gofmt-clean + under Go 1.26, so the column was dropped rather than reported as damage the agent + caused. That same fact is what produced §6. + +## The apparatus + +Two tickets, chosen so that the methodology's own routing sends them different ways: + +- **T1** — `Ordinal` is wrong for negative numbers, with the expected strings given. + A correctness fix in a file that already has covering tests. +- **T2** — add `Cents(int64) string`, comma-grouped, two fraction digits, no panic on + `math.MinInt64`. New public surface, and money, which `methodology.md` mandates TDD + for. + +Each run is a fresh copy of the pin, scaffolded per arm, handed the ticket verbatim +through `claude -p` with one tool allowlist. Scoring reads a hidden acceptance test +the agent never sees, the repository's own `go test`/`go vet`, the artifacts the +methodology claims to leave behind, the git state, and the runner's own cost. + +Arms: **none** (bare checkout, no scc), **full** (today's `scc init --claude`), +**fixed** (full, with `autonomy.md` edited), **rule+hook** (fixed, plus one line of +`SessionStart` context; run in both the JSON and plain-stdout output forms). + +## 1. The standing cost, measured + +The first request of a session, full scaffold against bare checkout: + +| | tokens in request 1 | +|---|---| +| bare checkout | 34,752 | +| full scaffold | 49,364 – 49,546 | + +**+14,612 to +14,794 tokens in every request of every session, or +42%.** The three +scoped rules (`specs.md`, `tasks.md`, `knowledge-base.md`, 10.3KB) are absent from +that figure, which is the first direct evidence that `ScopedRules` does what it was +built to do — about 3.2k tokens per request that the unscoped set would have cost. + +## 2. Isolation + +The same probe, down the same code path as a real run, in both arms: *what process +does this repository require before writing code?* The control answered `NONE`. The +treatment recited kickoff, routing, prior art, the artifact set, the methodology +annotation and the delivery sequence. No leak. + +## 3. T1 — the methodology arm wins blind, 3 of 3 + +Both arms pass the acceptance test on every seed, so effect is settled and the +critics judged what is left. Three critics, one per seed, fresh context, arm labels +stripped, each reading only its own pair. **All three picked the scaffolded arm.** +Two gave the same reason unprompted: it left durable regression tests in +`ordinals_test.go` covering cases the ticket never named, while the control wrote a +throwaway test and deleted it — *"leaving the repository with zero test coverage for +negative ordinals after the change."* That is `methodology.md`'s Unit rule, which +says the test comes immediately and from the requirement. + +It cost **+56% in dollars and +41% in input tokens** to get that. + +Every critic, on both tickets, independently went after the same thing nobody asked +them about: the `math.MinInt` / `math.MinInt64` path. On T1 neither arm guarded it. + +## 4. T2 — the methodology arm delivers nothing, 3 of 3 + +| arm | acceptance | lines added | +|---|---|---| +| none | 3/3 | 31, 35, 55 | +| full | **0/3** | 0, 0, 0 | + +Three seeds, one failure, three times: the scaffolded arm ended its turn asking +`autonomy.md`'s three kickoff questions and wrote no code. The control shipped the +function every time. + +The rule says *ask, once, before writing anything*, and has no branch for nobody +being there — which is precisely the unattended run `autonomy: auto` exists to +describe. scc's own Go code has known this shape since the first integration: every +installer degrades on *"nobody is here to answer the install prompt"*. The rules +never learned it. + +Why T2 and not T1: a bug fix with the expected output given is work the agent starts; +new public surface routes through `routing.md` first, and that is where kickoff sits. + +## 5. The fix, and the one that did not work + +**Editing the rule changed nothing — 0 of 3 again.** `autonomy.md` gained a paragraph +saying that an unanswerable question should not be asked, paid for by cutting the +section it made redundant. The arm behaved exactly as before, and the reason is the +useful part: **the rule asks the agent to act on a condition it cannot observe.** +From inside the turn, an unattended session looks like any other. + +The harness can observe it. A hook in a `claude -p` run has +`CLAUDE_CODE_SESSION_ATTENDED=0` in its environment, measured against 2.1.273. scc +already registers `SessionStart`, so the whole intervention is one sentence of +injected context and no new mechanism. + +| arm | acceptance | plan written | branch cut | mean cost | +|---|---|---|---|---| +| full | 0/3 | 0 | 0 | $0.23 | +| fixed (rule only) | 0/3 | 0 | 0 | $0.18 | +| rule + hook | **6/6** | 5/6 | **6/6** | $2.22 | + +Both output forms worked. The plain-stdout form is the one Claude Code documents; +the JSON `hookSpecificOutput` form is documented for `UserPromptSubmit` only and +works here regardless, which is what scc already emits. + +`CLAUDE_CODE_SESSION_ATTENDED` is **not documented** — neither the variable nor its +values — so `unattendedLines` reads exactly one spelling and says nothing for +anything else, unset included. The default is the safe one in the only direction +that matters: a missed line leaves today's behaviour, while a line in front of an +attended session would talk somebody out of a question they were right to be asked. + +The price is the honest headline. **$2.22 a run against the control's $0.20** — an +order of magnitude — because this is the first arm in which the methodology actually +ran: prior art, a plan, TDD, the gates, a branch and a commit. + +**Judged blind on the Go code alone** — diffs only, no plan, no branch, no final +message — it is **2 of 3**, which is not a result. The one critic who picked the +control is §6. + +**Regression on the small ticket.** Seven runs of the three patched variants against +T1: acceptance **7/7**, diffs of 8–17 lines against the control's 8–20, at $0.16–$0.78 +with five of the seven under $0.26. The expensive pipeline fires on the ticket that +routes to it, not on every ticket — which is the failure mode a fix like this one +would most plausibly have introduced, and did not. + +## 6. Found, not fixed: the gate reformats what the branch never touched + +One blind critic picked the control on T2 for a reason worth recording: the +methodology arm had bundled lint-driven rewrites of `bigbytes.go`, `bytes.go` and +`si_test.go` into a money-formatting change. The agents recorded their format gate +correctly as the check rather than the rewrite (`gofmt -l .`, not `gofmt -w .`) — the +guidance held. What follows is that a tree-wide gate on a repository that was already +failing it turns any task into a whole-tree reformat. + +It happened in **2 of 6** delivering runs and 0 of 3 control runs. That is under the +resolution of n=3, so nothing was changed on the strength of it. The shape of the fix +is visible and consistent with discipline this codebase already has in three places — +`attribution` reads `base..HEAD`, the Stop hook intersects findings with +`git.Changed`, `spec sync` refuses to re-litigate a settled record: **a gate failure +on files this branch did not touch is somebody else's, and belongs in the PR body +rather than in the diff.** Measuring it needs n≈10 per arm. + +## What this cost + +24 headless agent sessions, 8 subagent critics, about $17 of an authorised $180. + +## What is still unanswered + +- **Fourteen of the fifteen rules have still never been told apart from their own + absence.** This run ablated the whole scaffold, not one rule at a time. `ladder.md`, + `caveman.md`, `prior-art.md` and the rest stand exactly where `ponytail-port.md` 5.3 + left them: argued for, never measured. +- **The attended session is unmeasured**, and it is the one most people are in. +- **§6**, which needs a bigger n than this run could pay for. From 85556cf87f25571f2e9148ae6c84df582213c588 Mon Sep 17 00:00:00 2001 From: prode Date: Wed, 16 Sep 2026 11:23:19 -0300 Subject: [PATCH 3/3] fix(hooks): say the conversation wins, and restore what the rule's cut took with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings from the code and security reviews of this branch, all upheld. An environment variable carries no provenance, so a hostile .envrc or devcontainer.json in a clone telling an attended session that nobody is watching would spend the one checkpoint a person gets. That was testable rather than arguable: exported as 1 into the shell that launched `claude -p`, the hook still saw 0, so the harness sets the value rather than inheriting it. One direction of a symmetric mechanism is not a proof of both, so the injected text now ends by saying that a user who asks for something else is here after all and what they say wins — live text is the one input no file in the repository can forge. The docstring quoted its own measurement wrong: 6 of 6, not 5 of 5. SessionStart's one-line description in `scc hooks check` still listed only what the stage did before. And the cut that paid for the new paragraph took a reason with it — why the CI question in particular has to be asked at kickoff — which is back on the question it belongs to, one line, inside the budget at 54. --- .claude/rules/autonomy.md | 3 ++- .claude/scc-manifest.json | 2 +- design/rule-measurement.md | 9 +++++++++ internal/assets/assets_test.go | 2 +- internal/assets/templates/rules/autonomy.md | 3 ++- internal/cli/hooks_agent.go | 13 ++++++++++++- internal/hooks/harness.go | 6 ++++-- 7 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.claude/rules/autonomy.md b/.claude/rules/autonomy.md index e893e03..4e8fa7f 100644 --- a/.claude/rules/autonomy.md +++ b/.claude/rules/autonomy.md @@ -7,7 +7,8 @@ But autonomy is the user's call, so **ask, once, before writing anything** — t questions, together, in the same breath: 1. **Run automatically, or gate each phase for review?** -2. **When the PR is open, wait for CI, or finish there?** +2. **When the PR is open, wait for CI, or finish there?** Asked now because by then + the work is done and the user may be gone, which is when a block costs the most. 3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness. The advertised saving counts characters, not tokens. Governs speech, not artifacts. diff --git a/.claude/scc-manifest.json b/.claude/scc-manifest.json index bdd171f..a2ec126 100644 --- a/.claude/scc-manifest.json +++ b/.claude/scc-manifest.json @@ -23,7 +23,7 @@ "version": "32" }, { - "hash": "9a6ee0e84e62f939f8a9304350957443b9593f51c109c188c43a4658c6acb764", + "hash": "2eb97e13f6d088379c51870ff7338ddd36d3e377486408d5a2a2d236fe05a1e1", "path": ".claude/rules/autonomy.md", "version": "32" }, diff --git a/design/rule-measurement.md b/design/rule-measurement.md index 6c58097..11c99a6 100644 --- a/design/rule-measurement.md +++ b/design/rule-measurement.md @@ -142,6 +142,15 @@ anything else, unset included. The default is the safe one in the only direction that matters: a missed line leaves today's behaviour, while a line in front of an attended session would talk somebody out of a question they were right to be asked. +The security review asked the right question about it — an environment variable has +no provenance, so a hostile `.envrc` or `devcontainer.json` in a clone could tell an +*attended* session that nobody is watching, and the one checkpoint a person gets is +`gated`. That was testable and was tested: exported as `1` into the shell that +launched `claude -p`, the hook still saw `0`. **The harness sets the value rather +than inheriting it.** One direction of a symmetric mechanism is not a proof of both, +so the injected text ends by saying the conversation wins — live user text is the one +input no file in the repository can forge. + The price is the honest headline. **$2.22 a run against the control's $0.20** — an order of magnitude — because this is the first arm in which the methodology actually ran: prior art, a plan, TDD, the gates, a branch and a commit. diff --git a/internal/assets/assets_test.go b/internal/assets/assets_test.go index de84310..9ac2dbc 100644 --- a/internal/assets/assets_test.go +++ b/internal/assets/assets_test.go @@ -1075,7 +1075,7 @@ func (e errNoSuchRule) Error() string { // uses wherever a change is cheap to make and expensive to make by accident. func TestTheTemplateVersionMovesWithTheTemplates(t *testing.T) { // Bump Version, then replace this with the digest the failure prints. - const fingerprint = "de55d2721ee9d0bee6b6c4f92509a55b35baf9567ff6f334ffb8fbae0ff19e18" + const fingerprint = "220a7b9ef576f9f83fcfd3f7268952e34cbdc66542ee8be26e4cf755180ca373" sum := sha256.New() // Version goes into the hash, and without it this test does not do the job its diff --git a/internal/assets/templates/rules/autonomy.md b/internal/assets/templates/rules/autonomy.md index e893e03..4e8fa7f 100644 --- a/internal/assets/templates/rules/autonomy.md +++ b/internal/assets/templates/rules/autonomy.md @@ -7,7 +7,8 @@ But autonomy is the user's call, so **ask, once, before writing anything** — t questions, together, in the same breath: 1. **Run automatically, or gate each phase for review?** -2. **When the PR is open, wait for CI, or finish there?** +2. **When the PR is open, wait for CI, or finish there?** Asked now because by then + the work is done and the user may be gone, which is when a block costs the most. 3. **Answer in English, or in 文言文?** Classical Chinese at maximum terseness. The advertised saving counts characters, not tokens. Governs speech, not artifacts. diff --git a/internal/cli/hooks_agent.go b/internal/cli/hooks_agent.go index c4c85b9..b55c45f 100644 --- a/internal/cli/hooks_agent.go +++ b/internal/cli/hooks_agent.go @@ -216,7 +216,7 @@ func sessionStartContext(root string) []string { // code at all, while the same agent with no methodology shipped the feature every // time. Adding the branch to the rule's prose changed nothing — 0 of 3 again — // because the agent was being asked to act on a condition it cannot observe. With -// this line in front of it, the same arm delivered 5 of 5, with the plan, the +// this line in front of it, the same arm delivered 6 of 6, with the plan, the // ticked box, the branch and a clean `scc validate` the methodology promises. // // **The signal is undocumented and is treated that way.** `CLAUDE_CODE_SESSION_ @@ -226,6 +226,16 @@ func sessionStartContext(root string) []string { // unset included. That default is the safe one in the only direction that matters: // a missed line leaves today's behaviour, while a line in front of an attended // session would talk somebody out of a question they were right to ask. +// +// **The value is set by the harness rather than inherited**, which is what makes it +// worth reading at all: an environment variable carries no provenance on its own, so +// a hostile `.envrc` or `devcontainer.json` in a clone would otherwise be able to +// tell an attended session that nobody is watching — and the one checkpoint a person +// gets is `gated`. Measured: exported as `1` into the shell that launched `claude +// -p`, the hook still saw `0`. That is one direction of a symmetric mechanism and +// not a proof of both, so the last line emitted says the conversation wins. Live +// user text is the one input no file in the repository can forge, and somebody who +// asks for a gated run gets one whatever this variable says. func unattendedLines() []string { if os.Getenv("CLAUDE_CODE_SESSION_ATTENDED") != "0" { return nil @@ -234,6 +244,7 @@ func unattendedLines() []string { "scc: this session is unattended, so a question asked this turn reaches nobody.", "Take the kickoff answers as `autonomy: auto`, `ci: no-wait` and the user's own language,", "record them in the artifact's frontmatter as assumed rather than answered, say so in one line, and build.", + "If the user does ask for something else in this conversation, they are here after all and what they say wins.", } } diff --git a/internal/hooks/harness.go b/internal/hooks/harness.go index e644625..47da371 100644 --- a/internal/hooks/harness.go +++ b/internal/hooks/harness.go @@ -40,7 +40,9 @@ type Event string // The events scc registers, in the order a session meets them. const ( // SessionStart is where a workspace says what it is missing before any work - // is done on it — one message, once, when it can still change the plan. + // is done on it — one message, once, when it can still change the plan. It is + // also where a session is told that nobody is at the other end of it, which is + // the one fact that has to arrive before the first question rather than after. SessionStart Event = "SessionStart" // UserPromptSubmit is the moment a request arrives and before the agent acts // on it — the only point in a turn where saying "read it this way" is still @@ -79,7 +81,7 @@ func (e Event) Stage() Stage { func (e Event) Why() string { switch e { case SessionStart: - return "syncs the graph, reports drift, and names what this workspace is missing" + return "syncs the graph, reports drift, says when nobody is watching, and names what this workspace is missing" case UserPromptSubmit: return "says how to read an artifact the prompt named, and nothing otherwise" case Stop: