diff --git a/README.md b/README.md index c48ab2d..df3dc39 100644 --- a/README.md +++ b/README.md @@ -228,8 +228,11 @@ claims that are *already written* into the change: one invariant becomes a probe of the same claim — all run, any can refute. When one of those tests is accept-polarity and another is reject-polarity (exact-match name vocabulary, same package), the two collapse into a compound - pair probe: one `go test` invocation, verified from verbose output that *both* - actually executed and passed, conferring T2. The polarity vocabulary is + pair probe: one `go test -json` invocation under the same event-stream + discipline as the single runner — the pair passes only on an explicit pass + event for *each* name (a skipped or renamed-away side exits 0 and must not + confer T2), a failing side refutes, and an infra failure is not-run, never a + refutation — conferring T2. The polarity vocabulary is deliberately conservative — a mis-paired probe would mint an unearned T2, and ambiguous names classify as neither polarity. diff --git a/internal/probe/gotest.go b/internal/probe/gotest.go index 2df58f2..5e98b06 100644 --- a/internal/probe/gotest.go +++ b/internal/probe/gotest.go @@ -168,43 +168,6 @@ func failDetail(out string) string { return "(no output)" } -// runGoTest invokes `go test` for a -run pattern in pkgDir under repoDir. -// -count=1 ensures a fresh execution rather than a cached verdict. Still used -// by the pair runner, whose verdict comes from parsing "--- PASS:" lines in -// the -v output (a skip prints "--- SKIP:" and so cannot false-pass there). -func runGoTest(ctx context.Context, repoDir, pkgDir, runPattern string, verbose bool) (string, error) { - args := []string{"test", "-run", runPattern, "-count=1"} - if verbose { - args = append(args, "-v") - } - args = append(args, pkgDir) - cmd := exec.CommandContext(ctx, "go", args...) - cmd.Dir = repoDir - out, err := cmd.CombinedOutput() - return string(out), err -} - -// couldNotRunMarkers are substrings that mean a plain-text probe run never -// validly executed (build error, missing target). Used by the pair runner; -// deliberately narrow: a marker that also appears in ordinary assertion -// output would let a real refutation be laundered into "did not run". -var couldNotRunMarkers = []string{ - "no tests to run", - "no test files", - "[build failed]", - "no required module provides", -} - -func couldNotRun(out string) bool { - l := strings.ToLower(out) - for _, m := range couldNotRunMarkers { - if strings.Contains(l, m) { - return true - } - } - return false -} - // firstMeaningfulLine returns a short summary line for the receipt: the first // FAIL/ok/error line, else the first non-empty line. func firstMeaningfulLine(out string) string { diff --git a/internal/probe/gotest_verdict_test.go b/internal/probe/gotest_verdict_test.go index 2a00a31..6f82f17 100644 --- a/internal/probe/gotest_verdict_test.go +++ b/internal/probe/gotest_verdict_test.go @@ -63,6 +63,88 @@ func TestGoTestVerdictsFromEventStream(t *testing.T) { } } +// writePairModule creates a real throwaway module with the test shapes the +// compound pair verdict must distinguish. +func writePairModule(t *testing.T) string { + t.Helper() + dir := t.TempDir() + files := map[string]string{ + "go.mod": "module example.com/pairs\n\ngo 1.22\n", + "p_test.go": `package pairs + +import "testing" + +func TestAcceptsGood(t *testing.T) {} +func TestRejectsBad(t *testing.T) {} +func TestRejectFails(t *testing.T) { t.Fatal("negative case was accepted: bad") } +func TestAcceptSkips(t *testing.T) { t.Skip("env missing") } +`, + } + for rel, content := range files { + if err := os.WriteFile(filepath.Join(dir, rel), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// TestGoTestPairVerdictsFromEventStream: the compound verdict comes from the +// -json event stream under the single runner's discipline. The pair passes +// only on an explicit pass event for EACH side; a failing side refutes with +// that side's own output; a skipped side, a missing side, and a build +// failure are all Ran=false — every one of them exits in a way exit-code +// trust would misread (skip and single-match exit 0; an infra failure exits +// non-zero, which the old text path recorded as a REFUTATION). +func TestGoTestPairVerdictsFromEventStream(t *testing.T) { + dir := writePairModule(t) + run := func(accept, reject string) schema.Evidence { + t.Helper() + pid := schema.GoTestPairProbeID(".", accept, reject) + return GoTestPairRunner{}.Run(context.Background(), dir, + schema.Claim{ID: "P"}, pid) + } + + if ev := run("TestAcceptsGood", "TestRejectsBad"); !ev.Ran || !ev.Passed || + ev.Tier != schema.T2Adversarial || !strings.Contains(ev.Detail, "pair ok") { + t.Errorf("pass/pass: %+v", ev) + } + if ev := run("TestAcceptsGood", "TestRejectFails"); !ev.Ran || ev.Passed || + !strings.Contains(ev.Detail, "reject side failed") || !strings.Contains(ev.Detail, "negative case was accepted") { + t.Errorf("fail side: %+v, want refutation carrying the failing side's own output", ev) + } + if ev := run("TestAcceptSkips", "TestRejectsBad"); ev.Ran || ev.Passed || + !strings.Contains(ev.Detail, "skip") { + t.Errorf("skip side: %+v — a skipped side exits 0 and must NOT verify a pair", ev) + } + if ev := run("TestAcceptsGood", "TestRenamedAway"); ev.Ran || ev.Passed || + !strings.Contains(ev.Detail, "TestRenamedAway did not run") { + t.Errorf("missing side: %+v — a single-match run exits 0 and must NOT verify a pair", ev) + } +} + +// TestGoTestPairBuildFailureIsNotARefutation: an infra failure (the package +// does not compile) exits non-zero; the old text-output path recorded that +// as Ran=true/Passed=false — a false refutation that would block a merge +// over a broken fixture. The event stream has no terminal test events, so +// the verdict is not-run. +func TestGoTestPairBuildFailureIsNotARefutation(t *testing.T) { + dir := t.TempDir() + files := map[string]string{ + "go.mod": "module example.com/broken\n\ngo 1.22\n", + "b_test.go": "package broken\n\nfunc TestA(t *testing.T) {} // missing testing import: does not compile\n", + } + for rel, content := range files { + if err := os.WriteFile(filepath.Join(dir, rel), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + pid := schema.GoTestPairProbeID(".", "TestA", "TestB") + ev := GoTestPairRunner{}.Run(context.Background(), dir, schema.Claim{ID: "P"}, pid) + if ev.Ran || ev.Passed { + t.Errorf("build failure: %+v — nothing executed, so nothing is refuted", ev) + } +} + // overTierRunner claims more tier than it declares — the misbehavior the // dispatcher must cap. type overTierRunner struct{} diff --git a/internal/probe/gotestpair.go b/internal/probe/gotestpair.go index a26986a..2a51f4a 100644 --- a/internal/probe/gotestpair.go +++ b/internal/probe/gotestpair.go @@ -13,12 +13,17 @@ import ( // A pass means the positive case held AND the negative case was rejected — the // adversarial shape — so a pass confers T2. // -// Both tests run in a single `go test -v` invocation, and the runner then -// verifies from the verbose output that EACH named test actually executed and -// passed. Exit status alone is not enough: `go test` exits 0 when a -run -// pattern matches only one of the two names (the other was renamed or -// deleted), and consuming that as a pair pass would confer T2 on evidence that -// was never produced — a gate running against unverified substrate. +// Both tests run in a single `go test -json` invocation, and the verdict +// comes from the EVENT STREAM under the same discipline as the single runner +// — never the exit status. The compound shape adds its own hazards on top of +// the single runner's measured ones: `go test` exits 0 when the -run pattern +// matches only one of the two names (the other renamed or deleted), and a +// skipped side also exits 0 — consuming either as a pair pass would confer +// T2 on evidence that was never produced. The pair passes only on an +// explicit "pass" event for EACH name; a "fail" event on either side refutes +// (refutation dominates the other side's state); and everything else — a +// skip, a missing side, a build failure, cancellation — is Ran=false: it +// verifies nothing and refutes nothing. type GoTestPairRunner struct{} func (GoTestPairRunner) CanRun(probeID string) bool { @@ -38,34 +43,58 @@ func (GoTestPairRunner) Run(ctx context.Context, repoDir string, claim schema.Cl pattern := "^(" + regexp.QuoteMeta(acceptName) + "|" + regexp.QuoteMeta(rejectName) + ")$" start := time.Now() - out, err := runGoTest(ctx, repoDir, "./"+pkgDir, pattern, true) + events := runGoTestJSON(ctx, repoDir, "./"+pkgDir, pattern, false, probeID) ev.Duration = time.Since(start).Round(time.Millisecond).String() - switch { - case couldNotRun(out): - ev.Ran = false - ev.Detail = firstMeaningfulLine(out) - case err != nil: - ev.Ran = true - ev.Passed = false - ev.Detail = firstMeaningfulLine(out) - case !testPassedIn(out, acceptName): - ev.Ran = false - ev.Detail = "pair incomplete: " + acceptName + " did not run" - case !testPassedIn(out, rejectName): - ev.Ran = false - ev.Detail = "pair incomplete: " + rejectName + " did not run" - default: - ev.Ran = true - ev.Passed = true - ev.Detail = "pair ok: accept=" + acceptName + " reject=" + rejectName - } + ev.Ran, ev.Passed, ev.Detail = pairVerdictFromEvents(events, acceptName, rejectName) return ev } -// testPassedIn reports whether verbose go test output records a top-level pass -// for exactly the named test. The trailing space before the duration excludes -// subtest lines ("--- PASS: TestX/case") and name-prefix collisions. -func testPassedIn(out, name string) bool { - return strings.Contains(out, "--- PASS: "+name+" ") +// pairVerdictFromEvents reduces one event stream to the compound verdict for +// the two EXACT test names (subtests carry "Parent/sub" names and never +// match). +func pairVerdictFromEvents(events []goTestEvent, acceptName, rejectName string) (ran, passed bool, detail string) { + terminal := map[string]string{} + testOut := map[string][]string{} + var otherOut []string + for _, e := range events { + switch { + case e.Test == acceptName || e.Test == rejectName: + switch e.Action { + case "pass", "fail", "skip": + terminal[e.Test] = e.Action + case "output": + testOut[e.Test] = append(testOut[e.Test], e.Output) + } + case e.Action == "output" || e.Action == "build-output": + otherOut = append(otherOut, e.Output) + } + } + + sides := []struct{ name, label string }{ + {acceptName, "accept"}, + {rejectName, "reject"}, + } + // Refutation dominates: a side that executed and failed refutes the + // claim regardless of what the other side did. + for _, s := range sides { + if terminal[s.name] == "fail" { + return true, false, s.label + " side failed: " + failDetail(strings.Join(testOut[s.name], "")) + } + } + for _, s := range sides { + switch terminal[s.name] { + case "pass": + case "skip": + return false, false, "pair incomplete: " + s.name + " skipped — a skip asserts nothing" + default: + if len(terminal) == 0 { + // Neither side produced a terminal event: build failure, no + // matching tests, or cancellation. Nothing executed. + return false, false, firstMeaningfulLine(strings.Join(otherOut, "")) + } + return false, false, "pair incomplete: " + s.name + " did not run" + } + } + return true, true, "pair ok: accept=" + acceptName + " reject=" + rejectName } diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index 6cb2caf..b3c33b0 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -241,27 +241,21 @@ func TestAlloyProbeIDRoundTrips(t *testing.T) { } } -// TestCouldNotRunDetectsBuildFailure: a build failure is "did not run", so it -// can never be recorded as a refutation. -func TestCouldNotRunDetectsBuildFailure(t *testing.T) { - if !couldNotRun("FAIL\t[build failed]") { - t.Error("build failure not detected as could-not-run") - } - if couldNotRun("--- FAIL: TestX (0.00s)") { - t.Error("a real test failure was misread as could-not-run") - } -} - -// TestPassedInExcludesSubtestsAndPrefixes: the pass check matches exactly the -// named top-level test — not its subtests, not a longer name sharing the -// prefix. A sloppy match here would let a pair pass on evidence from the wrong -// test. -func TestPassedInExcludesSubtestsAndPrefixes(t *testing.T) { - out := "=== RUN TestA\n--- PASS: TestA/sub (0.00s)\n--- PASS: TestABC (0.01s)\n" - if testPassedIn(out, "TestA") { - t.Error("subtest or prefix collision counted as a top-level pass") - } - if !testPassedIn("--- PASS: TestA (0.02s)\n", "TestA") { - t.Error("a genuine top-level pass was not recognized") +// TestPairVerdictMatchesExactNamesOnly: the compound verdict counts events +// for exactly the two named top-level tests — a subtest ("TestA/sub") or a +// name sharing the prefix ("TestABC") must never supply a side's pass. A +// sloppy match here would let a pair pass on evidence from the wrong test. +func TestPairVerdictMatchesExactNamesOnly(t *testing.T) { + events := []goTestEvent{ + {Action: "pass", Test: "TestA/sub"}, + {Action: "pass", Test: "TestABC"}, + {Action: "pass", Test: "TestB"}, + } + if ran, passed, _ := pairVerdictFromEvents(events, "TestA", "TestB"); ran || passed { + t.Error("subtest or prefix collision supplied a side's pass") + } + events = append(events, goTestEvent{Action: "pass", Test: "TestA"}) + if ran, passed, _ := pairVerdictFromEvents(events, "TestA", "TestB"); !ran || !passed { + t.Error("a genuine pair of top-level passes was not recognized") } }