diff --git a/.gitignore b/.gitignore index 4ff0b2673..e341daad5 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/superpowers/ zero-termux-arm64 zero-linux-sandbox zero-seccomp + +# Generated benchmark reports are evidence for one configuration, never tree state +internal/perfbench/reports/*.json diff --git a/cmd/zero-perf-bench/main_test.go b/cmd/zero-perf-bench/main_test.go index c22d75347..2e2a6631b 100644 --- a/cmd/zero-perf-bench/main_test.go +++ b/cmd/zero-perf-bench/main_test.go @@ -4,6 +4,8 @@ import ( "bytes" "strings" "testing" + + "github.com/Gitlawb/zero/internal/perfbench" ) func TestParseArgsUsesCliAndEnvOverrides(t *testing.T) { @@ -92,3 +94,18 @@ func TestHelpText(t *testing.T) { func emptyEnv(string) string { return "" } + +func TestTurnExitCodeFailsWhenEveryTaskErrored(t *testing.T) { + var stderr bytes.Buffer + allErrored := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 3} + if code := turnExitCode(allErrored, &stderr); code != 1 { + t.Fatalf("exit=%d, want 1 when every task errored", code) + } + if !strings.Contains(stderr.String(), "no accepted benchmark sample") { + t.Fatalf("stderr missing explanation: %q", stderr.String()) + } + partial := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 2} + if code := turnExitCode(partial, &stderr); code != 0 { + t.Fatalf("exit=%d, want 0 for partial errors (summary surfaces them)", code) + } +} diff --git a/cmd/zero-perf-bench/turn.go b/cmd/zero-perf-bench/turn.go index 0286e8c0b..7009bebe0 100644 --- a/cmd/zero-perf-bench/turn.go +++ b/cmd/zero-perf-bench/turn.go @@ -86,9 +86,23 @@ func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error()) return 1 } - return 0 + return turnExitCode(result, stderr) } _, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result)) + return turnExitCode(result, stderr) +} + +// turnExitCode fails the command when every attempted task errored (no +// iteration produced an accepted benchmark sample): such a report measures +// nothing, and exiting 0 would let a broken configuration (missing binary, bad +// path, failing harness step) pass as a clean baseline. Partial errors keep +// exit 0 — the summary surfaces them loudly and the surviving samples are +// still valid measurements. +func turnExitCode(result perfbench.TurnBenchResult, stderr io.Writer) int { + if result.TasksAttempted > 0 && result.TasksErrored == result.TasksAttempted { + _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored with no accepted benchmark sample (see warnings); the report contains no valid measurements") + return 1 + } return 0 } diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 9c314b88b..2594d37b4 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -40,7 +40,19 @@ import ( // prevents a v2→v2 cross-version comparison from misreading the jump as a model // improvement. New counts: correctness 34 (edit 10 + fix 8 + nav 10 + refactor // 6), build 0, latency 14 (longproc 4 + longctx 4 + parallel 6). -const TurnSchemaVersion = 3 +// +// v4 adds tasksErrored: the number of tasks whose every iteration produced no +// accepted benchmark sample. That covers pre-run failures (missing binary, +// spawn error, process crash) and harness errors after a successful agent run +// (e.g. oracle stamping failed) alike — in every case the iteration yields no +// usable measurement. Errored tasks were previously visible only in warnings, +// so a run where nothing was actually measured still printed a clean-looking +// 0% pass summary and exited 0. tasksErrored makes that state first-class: +// the summary surfaces it, and the turn command exits nonzero when every +// attempted task errored. Accounting: an errored oracle task stays in its +// tier denominator as a failure; an errored latency-only task counts in +// latencyOnlyTasks and, as always, in no pass rate. +const TurnSchemaVersion = 4 // TurnRunner runs one benchmark task and reports its outcome plus the captured // per-turn trace. A non-nil Err means the run failed to execute (process crash); @@ -144,6 +156,7 @@ type TurnBenchResult struct { Commit string `json:"commit,omitempty"` Date string `json:"date"` TasksAttempted int `json:"tasksAttempted"` + TasksErrored int `json:"tasksErrored"` TasksVerified int `json:"tasksVerified"` TasksPassed int `json:"tasksPassed"` LatencyOnlyTasks int `json:"latencyOnlyTasks"` @@ -249,6 +262,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe // runner is cold-start, so a flaky pass on one iteration and a fail on // another is a real regression signal, not noise to average away. passedForTask := true + erroredIterations := 0 for iter := 0; iter < iterations; iter++ { outcome := cfg.Runner(ctx, task, rc) if outcome.TraceIssue != "" { @@ -266,6 +280,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err), }) passedForTask = false + erroredIterations++ continue } if !outcome.Passed { @@ -306,6 +321,14 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe // counters. A latency-only task (no verificationCommand) is never counted // in any pass rate even when the runner reports Passed — an exit-0 // read-only run proves the turn ran, not that the answer was right. + if erroredIterations == iterations { + // Every iteration produced no accepted sample (spawn failure or a + // harness error after the run). The task still counts in its tier + // below — an oracle task as a failure, a latency-only task in no + // pass rate as always — but the errored state is first-class so a + // broken run can never print a clean-looking summary. + result.TasksErrored++ + } hasOracle := len(task.VerificationCommand) > 0 switch { case !hasOracle: @@ -456,6 +479,20 @@ func FormatTurnBenchSummary(result TurnBenchResult) string { result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100, result.LatencyOnlyTasks, result.Iterations), } + if result.TasksErrored > 0 { + lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) produced no accepted benchmark sample (spawn/crash or harness error) — errored oracle tasks count as failures in their tier's pass rate; latency-only tasks stay out of pass rates as always", result.TasksErrored)) + shown := 0 + for _, warning := range result.Warnings { + if warning.Metric != "run" { + continue + } + lines = append(lines, " "+warning.Message) + shown++ + if shown == 3 { + break + } + } + } if result.Mode != "" { lines = append(lines, "mode: "+result.Mode) } @@ -747,7 +784,15 @@ func ResolveBinary(explicit string) (string, error) { if _, err := os.Stat(v); err != nil { return "", fmt.Errorf("trace binary not found: %w", err) } - return v, nil + // Pin the explicit path to an absolute one: the turn runner sets each + // child's cmd.Dir to a per-task fixture copy, so a relative path + // (./zero, the Makefile default) that stats fine here would fail to + // spawn from inside every fixture dir, silently erroring all tasks. + absolute, err := filepath.Abs(v) + if err != nil { + return "", fmt.Errorf("resolve trace binary path: %w", err) + } + return absolute, nil } if path, err := exec.LookPath("zero"); err == nil { return path, nil diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 28fc580bf..13be90a94 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -530,14 +531,15 @@ func TestRunTurnBenchBuildOnlyMechanism(t *testing.T) { } } -// TestTurnSchemaVersion3 pins the schema bump. v3 records the tier -// reclassification (refactor structural-positive and nav answer-oracles moved -// into correctnessPassRate), so a v2->v3 cross-version comparison cannot -// silently misread the jump as a model improvement — exactly the misread the -// tier system exists to prevent. -func TestTurnSchemaVersion3(t *testing.T) { - if TurnSchemaVersion != 3 { - t.Fatalf("TurnSchemaVersion = %d, want 3", TurnSchemaVersion) +// TestTurnSchemaVersion pins the schema bump so a shape change is a conscious +// decision. v3 recorded the tier reclassification (refactor structural-positive +// and nav answer-oracles moved into correctnessPassRate). v4 adds tasksErrored: +// tasks whose every iteration died before the agent produced a run are now +// first-class in the report instead of visible only in warnings, so a +// spawn-broken run cannot print a clean-looking summary or exit 0. +func TestTurnSchemaVersion(t *testing.T) { + if TurnSchemaVersion != 4 { + t.Fatalf("TurnSchemaVersion = %d, want 4", TurnSchemaVersion) } } @@ -1067,3 +1069,95 @@ printf '%s\n' '{"type":"run_end","exitCode":0}' `) assertVerifyFailed(t, "nav-08 fmt-only-no-testing", outcome) } + +func TestResolveBinaryAbsolutizesExplicitPath(t *testing.T) { + dir := t.TempDir() + name := "zero-probe.exe" + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(cwd) }) + + resolved, err := ResolveBinary("./" + name) + if err != nil { + t.Fatalf("ResolveBinary: %v", err) + } + if !filepath.IsAbs(resolved) { + t.Fatalf("ResolveBinary returned relative path %q; the turn runner sets cmd.Dir per task, so a relative binary fails to spawn from every fixture copy", resolved) + } +} + +func TestRunTurnBenchCountsErroredTasks(t *testing.T) { + set := TaskSet{ + ID: "errored-suite", + Tasks: []BenchTask{ + {ID: "t1", Class: "nav", Prompt: "p1", VerificationCommand: []string{"true"}}, + {ID: "t2", Class: "longproc", Prompt: "p2"}, + }, + } + cfg := TurnBenchConfig{ + Model: "fake-model", + Iterations: 1, + Runner: func(context.Context, BenchTask, RunContext) TurnTaskOutcome { + return TurnTaskOutcome{Err: errors.New("fork/exec ./zero: file does not exist")} + }, + Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + result, err := RunTurnBench(context.Background(), set, cfg) + if err != nil { + t.Fatalf("RunTurnBench: %v", err) + } + if result.TasksErrored != 2 { + t.Fatalf("TasksErrored=%d, want 2 (every iteration of both tasks errored)", result.TasksErrored) + } + if len(result.Warnings) != 2 { + t.Fatalf("warnings=%d, want 2 run warnings", len(result.Warnings)) + } + summary := FormatTurnBenchSummary(result) + if !strings.Contains(summary, "ERRORED: 2 task(s)") { + t.Fatalf("summary does not surface the errored tasks:\n%s", summary) + } + if !strings.Contains(summary, "fork/exec ./zero") { + t.Fatalf("summary does not echo the underlying run error:\n%s", summary) + } +} + +func TestRunTurnBenchPartialErrorStillCounts(t *testing.T) { + set := TaskSet{ + ID: "partial-suite", + Tasks: []BenchTask{ + {ID: "ok", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}}, + {ID: "dead", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}}, + }, + } + canned := map[string]*trace.TurnTrace{"ok": cannedTrace(100, 10, 1000)} + inner := fakeTurnRunner(canned) + cfg := TurnBenchConfig{ + Model: "fake-model", + Iterations: 1, + Runner: func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome { + if task.ID == "dead" { + return TurnTaskOutcome{Err: errors.New("spawn failed")} + } + return inner(ctx, task, rc) + }, + Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + result, err := RunTurnBench(context.Background(), set, cfg) + if err != nil { + t.Fatalf("RunTurnBench: %v", err) + } + if result.TasksErrored != 1 { + t.Fatalf("TasksErrored=%d, want 1", result.TasksErrored) + } + if result.TasksAttempted != 2 { + t.Fatalf("TasksAttempted=%d, want 2", result.TasksAttempted) + } +}