Migrate the pair runner to go test -json event verdicts

README.md

@@ -228,8 +228,11 @@ claims that are *already written* into the change:
   one invariant becomes a probe of the same claim — all run, any can refute.
   When one of those tests is accept-polarity and another is reject-polarity
   (exact-match name vocabulary, same package), the two collapse into a compound
-  pair probe: one `go test` invocation, verified from verbose output that *both*
-  actually executed and passed, conferring T2. The polarity vocabulary is
+  pair probe: one `go test -json` invocation under the same event-stream
+  discipline as the single runner — the pair passes only on an explicit pass
+  event for *each* name (a skipped or renamed-away side exits 0 and must not
+  confer T2), a failing side refutes, and an infra failure is not-run, never a
+  refutation — conferring T2. The polarity vocabulary is
   deliberately conservative — a mis-paired probe would mint an unearned T2, and
   ambiguous names classify as neither polarity.
 

internal/probe/gotest.go

@@ -168,43 +168,6 @@ func failDetail(out string) string {
 	return "(no output)"
 }
 
-// runGoTest invokes `go test` for a -run pattern in pkgDir under repoDir.
-// -count=1 ensures a fresh execution rather than a cached verdict. Still used
-// by the pair runner, whose verdict comes from parsing "--- PASS:" lines in
-// the -v output (a skip prints "--- SKIP:" and so cannot false-pass there).
-func runGoTest(ctx context.Context, repoDir, pkgDir, runPattern string, verbose bool) (string, error) {
-	args := []string{"test", "-run", runPattern, "-count=1"}
-	if verbose {
-		args = append(args, "-v")
-	}
-	args = append(args, pkgDir)
-	cmd := exec.CommandContext(ctx, "go", args...)
-	cmd.Dir = repoDir
-	out, err := cmd.CombinedOutput()
-	return string(out), err
-}
-
-// couldNotRunMarkers are substrings that mean a plain-text probe run never
-// validly executed (build error, missing target). Used by the pair runner;
-// deliberately narrow: a marker that also appears in ordinary assertion
-// output would let a real refutation be laundered into "did not run".
-var couldNotRunMarkers = []string{
-	"no tests to run",
-	"no test files",
-	"[build failed]",
-	"no required module provides",
-}
-
-func couldNotRun(out string) bool {
-	l := strings.ToLower(out)
-	for _, m := range couldNotRunMarkers {
-		if strings.Contains(l, m) {
-			return true
-		}
-	}
-	return false
-}
-
 // firstMeaningfulLine returns a short summary line for the receipt: the first
 // FAIL/ok/error line, else the first non-empty line.
 func firstMeaningfulLine(out string) string {

internal/probe/gotest_verdict_test.go

@@ -63,6 +63,88 @@ func TestGoTestVerdictsFromEventStream(t *testing.T) {
 	}
 }
 
+// writePairModule creates a real throwaway module with the test shapes the
+// compound pair verdict must distinguish.
+func writePairModule(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	files := map[string]string{
+		"go.mod": "module example.com/pairs\n\ngo 1.22\n",
+		"p_test.go": `package pairs
+
+import "testing"
+
+func TestAcceptsGood(t *testing.T) {}
+func TestRejectsBad(t *testing.T)  {}
+func TestRejectFails(t *testing.T) { t.Fatal("negative case was accepted: bad") }
+func TestAcceptSkips(t *testing.T) { t.Skip("env missing") }
+`,
+	}
+	for rel, content := range files {
+		if err := os.WriteFile(filepath.Join(dir, rel), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+// TestGoTestPairVerdictsFromEventStream: the compound verdict comes from the
+// -json event stream under the single runner's discipline. The pair passes
+// only on an explicit pass event for EACH side; a failing side refutes with
+// that side's own output; a skipped side, a missing side, and a build
+// failure are all Ran=false — every one of them exits in a way exit-code
+// trust would misread (skip and single-match exit 0; an infra failure exits
+// non-zero, which the old text path recorded as a REFUTATION).
+func TestGoTestPairVerdictsFromEventStream(t *testing.T) {
+	dir := writePairModule(t)
+	run := func(accept, reject string) schema.Evidence {
+		t.Helper()
+		pid := schema.GoTestPairProbeID(".", accept, reject)
+		return GoTestPairRunner{}.Run(context.Background(), dir,
+			schema.Claim{ID: "P"}, pid)
+	}
+
+	if ev := run("TestAcceptsGood", "TestRejectsBad"); !ev.Ran || !ev.Passed ||
+		ev.Tier != schema.T2Adversarial || !strings.Contains(ev.Detail, "pair ok") {
+		t.Errorf("pass/pass: %+v", ev)
+	}
+	if ev := run("TestAcceptsGood", "TestRejectFails"); !ev.Ran || ev.Passed ||
+		!strings.Contains(ev.Detail, "reject side failed") || !strings.Contains(ev.Detail, "negative case was accepted") {
+		t.Errorf("fail side: %+v, want refutation carrying the failing side's own output", ev)
+	}
+	if ev := run("TestAcceptSkips", "TestRejectsBad"); ev.Ran || ev.Passed ||
+		!strings.Contains(ev.Detail, "skip") {
+		t.Errorf("skip side: %+v — a skipped side exits 0 and must NOT verify a pair", ev)
+	}
+	if ev := run("TestAcceptsGood", "TestRenamedAway"); ev.Ran || ev.Passed ||
+		!strings.Contains(ev.Detail, "TestRenamedAway did not run") {
+		t.Errorf("missing side: %+v — a single-match run exits 0 and must NOT verify a pair", ev)
+	}
+}
+
+// TestGoTestPairBuildFailureIsNotARefutation: an infra failure (the package
+// does not compile) exits non-zero; the old text-output path recorded that
+// as Ran=true/Passed=false — a false refutation that would block a merge
+// over a broken fixture. The event stream has no terminal test events, so
+// the verdict is not-run.
+func TestGoTestPairBuildFailureIsNotARefutation(t *testing.T) {
+	dir := t.TempDir()
+	files := map[string]string{
+		"go.mod":    "module example.com/broken\n\ngo 1.22\n",
+		"b_test.go": "package broken\n\nfunc TestA(t *testing.T) {} // missing testing import: does not compile\n",
+	}
+	for rel, content := range files {
+		if err := os.WriteFile(filepath.Join(dir, rel), []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	pid := schema.GoTestPairProbeID(".", "TestA", "TestB")
+	ev := GoTestPairRunner{}.Run(context.Background(), dir, schema.Claim{ID: "P"}, pid)
+	if ev.Ran || ev.Passed {
+		t.Errorf("build failure: %+v — nothing executed, so nothing is refuted", ev)
+	}
+}
+
 // overTierRunner claims more tier than it declares — the misbehavior the
 // dispatcher must cap.
 type overTierRunner struct{}

internal/probe/gotestpair.go

@@ -13,12 +13,17 @@ import (
 // A pass means the positive case held AND the negative case was rejected — the
 // adversarial shape — so a pass confers T2.
 //
-// Both tests run in a single `go test -v` invocation, and the runner then
-// verifies from the verbose output that EACH named test actually executed and
-// passed. Exit status alone is not enough: `go test` exits 0 when a -run
-// pattern matches only one of the two names (the other was renamed or
-// deleted), and consuming that as a pair pass would confer T2 on evidence that
-// was never produced — a gate running against unverified substrate.
+// Both tests run in a single `go test -json` invocation, and the verdict
+// comes from the EVENT STREAM under the same discipline as the single runner
+// — never the exit status. The compound shape adds its own hazards on top of
+// the single runner's measured ones: `go test` exits 0 when the -run pattern
+// matches only one of the two names (the other renamed or deleted), and a
+// skipped side also exits 0 — consuming either as a pair pass would confer
+// T2 on evidence that was never produced. The pair passes only on an
+// explicit "pass" event for EACH name; a "fail" event on either side refutes
+// (refutation dominates the other side's state); and everything else — a
+// skip, a missing side, a build failure, cancellation — is Ran=false: it
+// verifies nothing and refutes nothing.
 type GoTestPairRunner struct{}
 
 func (GoTestPairRunner) CanRun(probeID string) bool {
@@ -38,34 +43,58 @@ func (GoTestPairRunner) Run(ctx context.Context, repoDir string, claim schema.Cl
 
 	pattern := "^(" + regexp.QuoteMeta(acceptName) + "|" + regexp.QuoteMeta(rejectName) + ")$"
 	start := time.Now()
-	out, err := runGoTest(ctx, repoDir, "./"+pkgDir, pattern, true)
+	events := runGoTestJSON(ctx, repoDir, "./"+pkgDir, pattern, false, probeID)
 	ev.Duration = time.Since(start).Round(time.Millisecond).String()
 
-	switch {
-	case couldNotRun(out):
-		ev.Ran = false
-		ev.Detail = firstMeaningfulLine(out)
-	case err != nil:
-		ev.Ran = true
-		ev.Passed = false
-		ev.Detail = firstMeaningfulLine(out)
-	case !testPassedIn(out, acceptName):
-		ev.Ran = false
-		ev.Detail = "pair incomplete: " + acceptName + " did not run"
-	case !testPassedIn(out, rejectName):
-		ev.Ran = false
-		ev.Detail = "pair incomplete: " + rejectName + " did not run"
-	default:
-		ev.Ran = true
-		ev.Passed = true
-		ev.Detail = "pair ok: accept=" + acceptName + " reject=" + rejectName
-	}
+	ev.Ran, ev.Passed, ev.Detail = pairVerdictFromEvents(events, acceptName, rejectName)
 	return ev
 }
 
-// testPassedIn reports whether verbose go test output records a top-level pass
-// for exactly the named test. The trailing space before the duration excludes
-// subtest lines ("--- PASS: TestX/case") and name-prefix collisions.
-func testPassedIn(out, name string) bool {
-	return strings.Contains(out, "--- PASS: "+name+" ")
+// pairVerdictFromEvents reduces one event stream to the compound verdict for
+// the two EXACT test names (subtests carry "Parent/sub" names and never
+// match).
+func pairVerdictFromEvents(events []goTestEvent, acceptName, rejectName string) (ran, passed bool, detail string) {
+	terminal := map[string]string{}
+	testOut := map[string][]string{}
+	var otherOut []string
+	for _, e := range events {
+		switch {
+		case e.Test == acceptName || e.Test == rejectName:
+			switch e.Action {
+			case "pass", "fail", "skip":
+				terminal[e.Test] = e.Action
+			case "output":
+				testOut[e.Test] = append(testOut[e.Test], e.Output)
+			}
+		case e.Action == "output" || e.Action == "build-output":
+			otherOut = append(otherOut, e.Output)
+		}
+	}
+
+	sides := []struct{ name, label string }{
+		{acceptName, "accept"},
+		{rejectName, "reject"},
+	}
+	// Refutation dominates: a side that executed and failed refutes the
+	// claim regardless of what the other side did.
+	for _, s := range sides {
+		if terminal[s.name] == "fail" {
+			return true, false, s.label + " side failed: " + failDetail(strings.Join(testOut[s.name], ""))
+		}
+	}
+	for _, s := range sides {
+		switch terminal[s.name] {
+		case "pass":
+		case "skip":
+			return false, false, "pair incomplete: " + s.name + " skipped — a skip asserts nothing"
+		default:
+			if len(terminal) == 0 {
+				// Neither side produced a terminal event: build failure, no
+				// matching tests, or cancellation. Nothing executed.
+				return false, false, firstMeaningfulLine(strings.Join(otherOut, ""))
+			}
+			return false, false, "pair incomplete: " + s.name + " did not run"
+		}
+	}
+	return true, true, "pair ok: accept=" + acceptName + " reject=" + rejectName
 }

internal/probe/probe_test.go

@@ -241,27 +241,21 @@ func TestAlloyProbeIDRoundTrips(t *testing.T) {
 	}
 }
 
-// TestCouldNotRunDetectsBuildFailure: a build failure is "did not run", so it
-// can never be recorded as a refutation.
-func TestCouldNotRunDetectsBuildFailure(t *testing.T) {
-	if !couldNotRun("FAIL\t[build failed]") {
-		t.Error("build failure not detected as could-not-run")
-	}
-	if couldNotRun("--- FAIL: TestX (0.00s)") {
-		t.Error("a real test failure was misread as could-not-run")
-	}
-}
-
-// TestPassedInExcludesSubtestsAndPrefixes: the pass check matches exactly the
-// named top-level test — not its subtests, not a longer name sharing the
-// prefix. A sloppy match here would let a pair pass on evidence from the wrong
-// test.
-func TestPassedInExcludesSubtestsAndPrefixes(t *testing.T) {
-	out := "=== RUN   TestA\n--- PASS: TestA/sub (0.00s)\n--- PASS: TestABC (0.01s)\n"
-	if testPassedIn(out, "TestA") {
-		t.Error("subtest or prefix collision counted as a top-level pass")
-	}
-	if !testPassedIn("--- PASS: TestA (0.02s)\n", "TestA") {
-		t.Error("a genuine top-level pass was not recognized")
+// TestPairVerdictMatchesExactNamesOnly: the compound verdict counts events
+// for exactly the two named top-level tests — a subtest ("TestA/sub") or a
+// name sharing the prefix ("TestABC") must never supply a side's pass. A
+// sloppy match here would let a pair pass on evidence from the wrong test.
+func TestPairVerdictMatchesExactNamesOnly(t *testing.T) {
+	events := []goTestEvent{
+		{Action: "pass", Test: "TestA/sub"},
+		{Action: "pass", Test: "TestABC"},
+		{Action: "pass", Test: "TestB"},
+	}
+	if ran, passed, _ := pairVerdictFromEvents(events, "TestA", "TestB"); ran || passed {
+		t.Error("subtest or prefix collision supplied a side's pass")
+	}
+	events = append(events, goTestEvent{Action: "pass", Test: "TestA"})
+	if ran, passed, _ := pairVerdictFromEvents(events, "TestA", "TestB"); !ran || !passed {
+		t.Error("a genuine pair of top-level passes was not recognized")
 	}
 }