Measure LLM round 2: recall, repeatability, and .github visibility

README.md

@@ -292,6 +292,28 @@ against code down to the flag semantics. The parse gate also earned its keep
 live: it caught a truncated response (raising the token budget for a
 reasoning-model generation) and a deprecated request parameter, both loudly.
 
+Measured (round 2, 2026-08 — recall and repeatability, not just precision):
+against BLINDED claim inventories written before any run, on two real merged
+changes. Precision held: 115 of 115 proposals across six runs were accurate,
+zero hallucinated files or mechanisms. Recall — the number round 1 never
+measured — was 11/12 and 9/11: every core behavioral claim surfaced, and the
+misses concentrate where a reader would predict — a subtle negative (a code
+path deliberately NOT instrumented), a deliberate design omission (counts
+disclosed instead of path lists), a documentation-meta claim. Repeatability:
+across four runs on identical input, exact-text stability was ZERO — the
+model rewords every proposal every run, so the content-hashed ids never
+collide — while semantic stability was effectively complete (all four runs
+covered the same claim set; counts varied 18–20). Consequence, stated
+honestly: `[llm-proposed]` remainder rows are stable in MEANING but not in
+id or wording across runs; diffing two receipts' LLM rows textually will
+show churn that is not change. One targeted scope fix also shipped measured:
+`.github` is project-owned behavior (this repository's merge gate lives
+there), so its diff sections now reach the model — ordered AFTER every
+non-hidden section so they can never crowd shipped code out of the byte cap.
+Live A/B on a real workflow-touching change: pre-fix the workflow claim was
+structurally unreachable; post-fix the model minted it from the workflow
+section itself, exact to the step name and setting.
+
 ## Known limitations (found by dogfooding, stated honestly)
 
 correctful was run on itself and on a real 101-file production change on its

internal/llmextract/llmextract.go

@@ -113,19 +113,33 @@ func includedSections(patch string, files []string) (sections []string, read []s
 	for _, f := range files {
 		inChange[f] = true
 	}
-	total := 0
+	// Hidden directories hold installed tooling and its documentation, not
+	// the change's code — the same principle every mechanical harvester
+	// applies. Measured live: without the exclusion, a change's dot-dir
+	// methodology documents sorted first in the diff and consumed the
+	// ENTIRE byte cap — the model never saw a line of shipped code and
+	// re-extracted the docs' claims instead (extraction-over-prose).
+	//
+	// ONE dot-directory is project-owned behavior, not tooling: .github
+	// carries the repository's own CI contract (this project's merge gate
+	// lives there), and a change to it makes claims. Its sections are
+	// included by ORDERING, not by plain exemption: placed after every
+	// non-hidden section, .github can never crowd shipped code out of the
+	// byte cap — the exact failure the dot-dir rule was measured against.
+	var ordered, githubSecs []string
 	for _, sec := range splitSections(patch) {
-		if harvest.UnderDotDir(sectionFile(sec)) {
-			// The same principle every mechanical harvester applies: hidden
-			// directories hold installed tooling and its documentation, not
-			// the change's code. Measured live: without this, a change's
-			// dot-dir methodology documents sorted first in the diff and
-			// consumed the ENTIRE byte cap — the model never saw a line of
-			// shipped code and re-extracted the docs' claims instead
-			// (extraction-over-prose, the class the cap exists to feed code
-			// into, not documents).
-			continue
+		f := sectionFile(sec)
+		switch {
+		case !harvest.UnderDotDir(f):
+			ordered = append(ordered, sec)
+		case strings.HasPrefix(f, ".github/"):
+			githubSecs = append(githubSecs, sec)
 		}
+	}
+	ordered = append(ordered, githubSecs...)
+
+	total := 0
+	for _, sec := range ordered {
 		if len(sec) > maxPatchBytes {
 			continue // a section is sent whole or not at all — never truncated mid-hunk
 		}

internal/llmextract/llmextract_test.go

@@ -237,3 +237,55 @@ func TestDotDirSectionsNeverReachTheModel(t *testing.T) {
 		t.Error("prompt carries a dot-dir section")
 	}
 }
+
+// TestGithubSectionsIncludedAfterCode: .github is project-owned behavior (the
+// repository's own CI contract), not installed tooling — its sections reach
+// the model, but ONLY after every non-hidden section, so they can never
+// crowd shipped code out of the byte cap. Other dot-dirs stay excluded.
+func TestGithubSectionsIncludedAfterCode(t *testing.T) {
+	patch := "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n--- a/.github/workflows/ci.yml\n+++ b/.github/workflows/ci.yml\n@@ -0,0 +1 @@\n+      continue-on-error: true\n" +
+		"diff --git a/.tooling/ARCH.md b/.tooling/ARCH.md\n--- a/.tooling/ARCH.md\n+++ b/.tooling/ARCH.md\n@@ -0,0 +1 @@\n+### INV-001: docs\n" +
+		"diff --git a/pkg/gate.go b/pkg/gate.go\n--- a/pkg/gate.go\n+++ b/pkg/gate.go\n@@ -0,0 +1 @@\n+func Gate() {}\n"
+	client, reqBody, _ := fixtureServer(t, 200, apiFixture(`[]`))
+	res, err := Harvester{Patch: patch, Client: client}.Harvest("", []string{".github/workflows/ci.yml", ".tooling/ARCH.md", "pkg/gate.go"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	body := string(*reqBody)
+	codeAt := strings.Index(body, "pkg/gate.go")
+	ghAt := strings.Index(body, ".github/workflows/ci.yml")
+	if codeAt < 0 || ghAt < 0 {
+		t.Fatalf("prompt missing sections (code at %d, .github at %d):\n%.400s", codeAt, ghAt, body)
+	}
+	if ghAt < codeAt {
+		t.Error(".github section precedes shipped code — ordering must keep code first")
+	}
+	if strings.Contains(body, ".tooling/ARCH.md") {
+		t.Error("a non-.github dot-dir section reached the model")
+	}
+	if len(res.Read) != 2 {
+		t.Errorf("read = %v, want the code and .github sections", res.Read)
+	}
+}
+
+// TestGithubSectionCannotCrowdOutCode: when the cap is nearly spent by
+// shipped-code sections, the .github section is the one that gets dropped —
+// the ordering is load-bearing, not cosmetic.
+func TestGithubSectionCannotCrowdOutCode(t *testing.T) {
+	line := strings.Repeat("x", 1024)
+	var code strings.Builder
+	for i := 0; code.Len() < maxPatchBytes-2048; i++ {
+		code.WriteString("diff --git a/pkg/f" + string(rune('a'+i%26)) + ".go b/pkg/f" + string(rune('a'+i%26)) + ".go\n@@ -0,0 +1 @@\n+// " + line + "\n")
+	}
+	gh := "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n@@ -0,0 +1 @@\n+" + strings.Repeat("y", 4096) + "\n"
+	// .github FIRST in the diff — the ordering must still put it last and
+	// drop it at the cap.
+	sections, _ := includedSections(gh+code.String(), nil)
+	joined := strings.Join(sections, "")
+	if strings.Contains(joined, ".github/workflows/ci.yml") {
+		t.Error(".github section crowded shipped code at the cap boundary")
+	}
+	if !strings.Contains(joined, "pkg/f") {
+		t.Error("shipped-code sections missing")
+	}
+}