diff --git a/README.md b/README.md index df3dc39..fc23199 100644 --- a/README.md +++ b/README.md @@ -292,6 +292,28 @@ against code down to the flag semantics. The parse gate also earned its keep live: it caught a truncated response (raising the token budget for a reasoning-model generation) and a deprecated request parameter, both loudly. +Measured (round 2, 2026-08 — recall and repeatability, not just precision): +against BLINDED claim inventories written before any run, on two real merged +changes. Precision held: 115 of 115 proposals across six runs were accurate, +zero hallucinated files or mechanisms. Recall — the number round 1 never +measured — was 11/12 and 9/11: every core behavioral claim surfaced, and the +misses concentrate where a reader would predict — a subtle negative (a code +path deliberately NOT instrumented), a deliberate design omission (counts +disclosed instead of path lists), a documentation-meta claim. Repeatability: +across four runs on identical input, exact-text stability was ZERO — the +model rewords every proposal every run, so the content-hashed ids never +collide — while semantic stability was effectively complete (all four runs +covered the same claim set; counts varied 18–20). Consequence, stated +honestly: `[llm-proposed]` remainder rows are stable in MEANING but not in +id or wording across runs; diffing two receipts' LLM rows textually will +show churn that is not change. One targeted scope fix also shipped measured: +`.github` is project-owned behavior (this repository's merge gate lives +there), so its diff sections now reach the model — ordered AFTER every +non-hidden section so they can never crowd shipped code out of the byte cap. +Live A/B on a real workflow-touching change: pre-fix the workflow claim was +structurally unreachable; post-fix the model minted it from the workflow +section itself, exact to the step name and setting. + ## Known limitations (found by dogfooding, stated honestly) correctful was run on itself and on a real 101-file production change on its diff --git a/internal/llmextract/llmextract.go b/internal/llmextract/llmextract.go index d59950d..20dc3f6 100644 --- a/internal/llmextract/llmextract.go +++ b/internal/llmextract/llmextract.go @@ -113,19 +113,33 @@ func includedSections(patch string, files []string) (sections []string, read []s for _, f := range files { inChange[f] = true } - total := 0 + // Hidden directories hold installed tooling and its documentation, not + // the change's code — the same principle every mechanical harvester + // applies. Measured live: without the exclusion, a change's dot-dir + // methodology documents sorted first in the diff and consumed the + // ENTIRE byte cap — the model never saw a line of shipped code and + // re-extracted the docs' claims instead (extraction-over-prose). + // + // ONE dot-directory is project-owned behavior, not tooling: .github + // carries the repository's own CI contract (this project's merge gate + // lives there), and a change to it makes claims. Its sections are + // included by ORDERING, not by plain exemption: placed after every + // non-hidden section, .github can never crowd shipped code out of the + // byte cap — the exact failure the dot-dir rule was measured against. + var ordered, githubSecs []string for _, sec := range splitSections(patch) { - if harvest.UnderDotDir(sectionFile(sec)) { - // The same principle every mechanical harvester applies: hidden - // directories hold installed tooling and its documentation, not - // the change's code. Measured live: without this, a change's - // dot-dir methodology documents sorted first in the diff and - // consumed the ENTIRE byte cap — the model never saw a line of - // shipped code and re-extracted the docs' claims instead - // (extraction-over-prose, the class the cap exists to feed code - // into, not documents). - continue + f := sectionFile(sec) + switch { + case !harvest.UnderDotDir(f): + ordered = append(ordered, sec) + case strings.HasPrefix(f, ".github/"): + githubSecs = append(githubSecs, sec) } + } + ordered = append(ordered, githubSecs...) + + total := 0 + for _, sec := range ordered { if len(sec) > maxPatchBytes { continue // a section is sent whole or not at all — never truncated mid-hunk } diff --git a/internal/llmextract/llmextract_test.go b/internal/llmextract/llmextract_test.go index ffab0f3..d8bd287 100644 --- a/internal/llmextract/llmextract_test.go +++ b/internal/llmextract/llmextract_test.go @@ -237,3 +237,55 @@ func TestDotDirSectionsNeverReachTheModel(t *testing.T) { t.Error("prompt carries a dot-dir section") } } + +// TestGithubSectionsIncludedAfterCode: .github is project-owned behavior (the +// repository's own CI contract), not installed tooling — its sections reach +// the model, but ONLY after every non-hidden section, so they can never +// crowd shipped code out of the byte cap. Other dot-dirs stay excluded. +func TestGithubSectionsIncludedAfterCode(t *testing.T) { + patch := "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n--- a/.github/workflows/ci.yml\n+++ b/.github/workflows/ci.yml\n@@ -0,0 +1 @@\n+ continue-on-error: true\n" + + "diff --git a/.tooling/ARCH.md b/.tooling/ARCH.md\n--- a/.tooling/ARCH.md\n+++ b/.tooling/ARCH.md\n@@ -0,0 +1 @@\n+### INV-001: docs\n" + + "diff --git a/pkg/gate.go b/pkg/gate.go\n--- a/pkg/gate.go\n+++ b/pkg/gate.go\n@@ -0,0 +1 @@\n+func Gate() {}\n" + client, reqBody, _ := fixtureServer(t, 200, apiFixture(`[]`)) + res, err := Harvester{Patch: patch, Client: client}.Harvest("", []string{".github/workflows/ci.yml", ".tooling/ARCH.md", "pkg/gate.go"}) + if err != nil { + t.Fatal(err) + } + body := string(*reqBody) + codeAt := strings.Index(body, "pkg/gate.go") + ghAt := strings.Index(body, ".github/workflows/ci.yml") + if codeAt < 0 || ghAt < 0 { + t.Fatalf("prompt missing sections (code at %d, .github at %d):\n%.400s", codeAt, ghAt, body) + } + if ghAt < codeAt { + t.Error(".github section precedes shipped code — ordering must keep code first") + } + if strings.Contains(body, ".tooling/ARCH.md") { + t.Error("a non-.github dot-dir section reached the model") + } + if len(res.Read) != 2 { + t.Errorf("read = %v, want the code and .github sections", res.Read) + } +} + +// TestGithubSectionCannotCrowdOutCode: when the cap is nearly spent by +// shipped-code sections, the .github section is the one that gets dropped — +// the ordering is load-bearing, not cosmetic. +func TestGithubSectionCannotCrowdOutCode(t *testing.T) { + line := strings.Repeat("x", 1024) + var code strings.Builder + for i := 0; code.Len() < maxPatchBytes-2048; i++ { + code.WriteString("diff --git a/pkg/f" + string(rune('a'+i%26)) + ".go b/pkg/f" + string(rune('a'+i%26)) + ".go\n@@ -0,0 +1 @@\n+// " + line + "\n") + } + gh := "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n@@ -0,0 +1 @@\n+" + strings.Repeat("y", 4096) + "\n" + // .github FIRST in the diff — the ordering must still put it last and + // drop it at the cap. + sections, _ := includedSections(gh+code.String(), nil) + joined := strings.Join(sections, "") + if strings.Contains(joined, ".github/workflows/ci.yml") { + t.Error(".github section crowded shipped code at the cap boundary") + } + if !strings.Contains(joined, "pkg/f") { + t.Error("shipped-code sections missing") + } +}