From ab927d49f6a1b64dc6b1160fee8e968278edd08b Mon Sep 17 00:00:00 2001 From: Vasanthdev2004 Date: Sat, 18 Jul 2026 20:08:25 +0530 Subject: [PATCH 1/4] fix(perfbench): absolutize the bench binary and make errored tasks first-class --- cmd/zero-perf-bench/main_test.go | 17 ++++ cmd/zero-perf-bench/turn.go | 15 +++- internal/perfbench/turn_bench.go | 43 +++++++++- internal/perfbench/turn_bench_test.go | 110 ++++++++++++++++++++++++-- 4 files changed, 174 insertions(+), 11 deletions(-) diff --git a/cmd/zero-perf-bench/main_test.go b/cmd/zero-perf-bench/main_test.go index c22d75347..30f9beeb4 100644 --- a/cmd/zero-perf-bench/main_test.go +++ b/cmd/zero-perf-bench/main_test.go @@ -4,6 +4,8 @@ import ( "bytes" "strings" "testing" + + "github.com/Gitlawb/zero/internal/perfbench" ) func TestParseArgsUsesCliAndEnvOverrides(t *testing.T) { @@ -92,3 +94,18 @@ func TestHelpText(t *testing.T) { func emptyEnv(string) string { return "" } + +func TestTurnExitCodeFailsWhenEveryTaskErrored(t *testing.T) { + var stderr bytes.Buffer + allErrored := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 3} + if code := turnExitCode(allErrored, &stderr); code != 1 { + t.Fatalf("exit=%d, want 1 when every task errored", code) + } + if !strings.Contains(stderr.String(), "no valid samples") { + t.Fatalf("stderr missing explanation: %q", stderr.String()) + } + partial := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 2} + if code := turnExitCode(partial, &stderr); code != 0 { + t.Fatalf("exit=%d, want 0 for partial errors (summary surfaces them)", code) + } +} diff --git a/cmd/zero-perf-bench/turn.go b/cmd/zero-perf-bench/turn.go index 0286e8c0b..e8e1e6936 100644 --- a/cmd/zero-perf-bench/turn.go +++ b/cmd/zero-perf-bench/turn.go @@ -86,9 +86,22 @@ func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error()) return 1 } - return 0 + return turnExitCode(result, stderr) } _, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result)) + return turnExitCode(result, stderr) +} + +// turnExitCode fails the command when every attempted task errored before the +// agent produced a run: such a report measures nothing, and exiting 0 would let +// a spawn-broken configuration (missing binary, bad path) pass as a clean +// baseline. Partial errors keep exit 0 — the summary surfaces them loudly and +// the surviving samples are still valid measurements. +func turnExitCode(result perfbench.TurnBenchResult, stderr io.Writer) int { + if result.TasksAttempted > 0 && result.TasksErrored == result.TasksAttempted { + _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored before the agent ran (see warnings); the report contains no valid samples") + return 1 + } return 0 } diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 9c314b88b..62d4db35b 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -40,7 +40,14 @@ import ( // prevents a v2→v2 cross-version comparison from misreading the jump as a model // improvement. New counts: correctness 34 (edit 10 + fix 8 + nav 10 + refactor // 6), build 0, latency 14 (longproc 4 + longctx 4 + parallel 6). -const TurnSchemaVersion = 3 +// +// v4 adds tasksErrored: the number of tasks whose every iteration died before +// the agent produced a run (spawn failure, missing binary, crashed process). +// Errored tasks were previously visible only in warnings, so a run where the +// agent never launched still printed a clean-looking 0% pass summary and +// exited 0. tasksErrored makes that state first-class: the summary surfaces +// it, and the turn command exits nonzero when every attempted task errored. +const TurnSchemaVersion = 4 // TurnRunner runs one benchmark task and reports its outcome plus the captured // per-turn trace. A non-nil Err means the run failed to execute (process crash); @@ -144,6 +151,7 @@ type TurnBenchResult struct { Commit string `json:"commit,omitempty"` Date string `json:"date"` TasksAttempted int `json:"tasksAttempted"` + TasksErrored int `json:"tasksErrored"` TasksVerified int `json:"tasksVerified"` TasksPassed int `json:"tasksPassed"` LatencyOnlyTasks int `json:"latencyOnlyTasks"` @@ -249,6 +257,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe // runner is cold-start, so a flaky pass on one iteration and a fail on // another is a real regression signal, not noise to average away. passedForTask := true + erroredIterations := 0 for iter := 0; iter < iterations; iter++ { outcome := cfg.Runner(ctx, task, rc) if outcome.TraceIssue != "" { @@ -266,6 +275,7 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe Message: fmt.Sprintf("task %s: %v", task.ID, outcome.Err), }) passedForTask = false + erroredIterations++ continue } if !outcome.Passed { @@ -306,6 +316,13 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe // counters. A latency-only task (no verificationCommand) is never counted // in any pass rate even when the runner reports Passed — an exit-0 // read-only run proves the turn ran, not that the answer was right. + if erroredIterations == iterations { + // Every iteration died before the agent produced a run. The task + // still counts in its tier below (as a failure), but the errored + // state is first-class so a spawn-broken run can never print a + // clean-looking summary. + result.TasksErrored++ + } hasOracle := len(task.VerificationCommand) > 0 switch { case !hasOracle: @@ -456,6 +473,20 @@ func FormatTurnBenchSummary(result TurnBenchResult) string { result.BuildPassedTasks, result.BuildCheckedTasks, result.BuildPassRate*100, result.LatencyOnlyTasks, result.Iterations), } + if result.TasksErrored > 0 { + lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) never produced an agent run (spawn/crash) — pass rates above cover only what actually ran", result.TasksErrored)) + shown := 0 + for _, warning := range result.Warnings { + if warning.Metric != "run" { + continue + } + lines = append(lines, " "+warning.Message) + shown++ + if shown == 3 { + break + } + } + } if result.Mode != "" { lines = append(lines, "mode: "+result.Mode) } @@ -747,7 +778,15 @@ func ResolveBinary(explicit string) (string, error) { if _, err := os.Stat(v); err != nil { return "", fmt.Errorf("trace binary not found: %w", err) } - return v, nil + // Pin the explicit path to an absolute one: the turn runner sets each + // child's cmd.Dir to a per-task fixture copy, so a relative path + // (./zero, the Makefile default) that stats fine here would fail to + // spawn from inside every fixture dir, silently erroring all tasks. + absolute, err := filepath.Abs(v) + if err != nil { + return "", fmt.Errorf("resolve trace binary path: %w", err) + } + return absolute, nil } if path, err := exec.LookPath("zero"); err == nil { return path, nil diff --git a/internal/perfbench/turn_bench_test.go b/internal/perfbench/turn_bench_test.go index 28fc580bf..13be90a94 100644 --- a/internal/perfbench/turn_bench_test.go +++ b/internal/perfbench/turn_bench_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -530,14 +531,15 @@ func TestRunTurnBenchBuildOnlyMechanism(t *testing.T) { } } -// TestTurnSchemaVersion3 pins the schema bump. v3 records the tier -// reclassification (refactor structural-positive and nav answer-oracles moved -// into correctnessPassRate), so a v2->v3 cross-version comparison cannot -// silently misread the jump as a model improvement — exactly the misread the -// tier system exists to prevent. -func TestTurnSchemaVersion3(t *testing.T) { - if TurnSchemaVersion != 3 { - t.Fatalf("TurnSchemaVersion = %d, want 3", TurnSchemaVersion) +// TestTurnSchemaVersion pins the schema bump so a shape change is a conscious +// decision. v3 recorded the tier reclassification (refactor structural-positive +// and nav answer-oracles moved into correctnessPassRate). v4 adds tasksErrored: +// tasks whose every iteration died before the agent produced a run are now +// first-class in the report instead of visible only in warnings, so a +// spawn-broken run cannot print a clean-looking summary or exit 0. +func TestTurnSchemaVersion(t *testing.T) { + if TurnSchemaVersion != 4 { + t.Fatalf("TurnSchemaVersion = %d, want 4", TurnSchemaVersion) } } @@ -1067,3 +1069,95 @@ printf '%s\n' '{"type":"run_end","exitCode":0}' `) assertVerifyFailed(t, "nav-08 fmt-only-no-testing", outcome) } + +func TestResolveBinaryAbsolutizesExplicitPath(t *testing.T) { + dir := t.TempDir() + name := "zero-probe.exe" + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(cwd) }) + + resolved, err := ResolveBinary("./" + name) + if err != nil { + t.Fatalf("ResolveBinary: %v", err) + } + if !filepath.IsAbs(resolved) { + t.Fatalf("ResolveBinary returned relative path %q; the turn runner sets cmd.Dir per task, so a relative binary fails to spawn from every fixture copy", resolved) + } +} + +func TestRunTurnBenchCountsErroredTasks(t *testing.T) { + set := TaskSet{ + ID: "errored-suite", + Tasks: []BenchTask{ + {ID: "t1", Class: "nav", Prompt: "p1", VerificationCommand: []string{"true"}}, + {ID: "t2", Class: "longproc", Prompt: "p2"}, + }, + } + cfg := TurnBenchConfig{ + Model: "fake-model", + Iterations: 1, + Runner: func(context.Context, BenchTask, RunContext) TurnTaskOutcome { + return TurnTaskOutcome{Err: errors.New("fork/exec ./zero: file does not exist")} + }, + Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + result, err := RunTurnBench(context.Background(), set, cfg) + if err != nil { + t.Fatalf("RunTurnBench: %v", err) + } + if result.TasksErrored != 2 { + t.Fatalf("TasksErrored=%d, want 2 (every iteration of both tasks errored)", result.TasksErrored) + } + if len(result.Warnings) != 2 { + t.Fatalf("warnings=%d, want 2 run warnings", len(result.Warnings)) + } + summary := FormatTurnBenchSummary(result) + if !strings.Contains(summary, "ERRORED: 2 task(s)") { + t.Fatalf("summary does not surface the errored tasks:\n%s", summary) + } + if !strings.Contains(summary, "fork/exec ./zero") { + t.Fatalf("summary does not echo the underlying run error:\n%s", summary) + } +} + +func TestRunTurnBenchPartialErrorStillCounts(t *testing.T) { + set := TaskSet{ + ID: "partial-suite", + Tasks: []BenchTask{ + {ID: "ok", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}}, + {ID: "dead", Class: "nav", Prompt: "p", VerificationCommand: []string{"true"}}, + }, + } + canned := map[string]*trace.TurnTrace{"ok": cannedTrace(100, 10, 1000)} + inner := fakeTurnRunner(canned) + cfg := TurnBenchConfig{ + Model: "fake-model", + Iterations: 1, + Runner: func(ctx context.Context, task BenchTask, rc RunContext) TurnTaskOutcome { + if task.ID == "dead" { + return TurnTaskOutcome{Err: errors.New("spawn failed")} + } + return inner(ctx, task, rc) + }, + Now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + result, err := RunTurnBench(context.Background(), set, cfg) + if err != nil { + t.Fatalf("RunTurnBench: %v", err) + } + if result.TasksErrored != 1 { + t.Fatalf("TasksErrored=%d, want 1", result.TasksErrored) + } + if result.TasksAttempted != 2 { + t.Fatalf("TasksAttempted=%d, want 2", result.TasksAttempted) + } +} From ca779cb43b6abf1f847a136a7a93205d46ebd8cf Mon Sep 17 00:00:00 2001 From: Vasanthdev2004 Date: Sat, 18 Jul 2026 20:15:30 +0530 Subject: [PATCH 2/4] fix(perfbench): state that errored tasks count as pass-rate failures --- internal/perfbench/turn_bench.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 62d4db35b..338f054a9 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -474,7 +474,7 @@ func FormatTurnBenchSummary(result TurnBenchResult) string { result.LatencyOnlyTasks, result.Iterations), } if result.TasksErrored > 0 { - lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) never produced an agent run (spawn/crash) — pass rates above cover only what actually ran", result.TasksErrored)) + lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) never produced an agent run (spawn/crash) — they count as FAILURES in the tier pass rates above", result.TasksErrored)) shown := 0 for _, warning := range result.Warnings { if warning.Metric != "run" { From 300354233b4e67eefd5d2d4bab6614636b3e1eae Mon Sep 17 00:00:00 2001 From: Vasanthdev2004 Date: Sat, 18 Jul 2026 20:50:19 +0530 Subject: [PATCH 3/4] fix(perfbench): align errored-task wording with the actual tier accounting --- cmd/zero-perf-bench/main_test.go | 2 +- cmd/zero-perf-bench/turn.go | 13 +- internal/perfbench/reports/baseline.json | 534 +++++++++++++++++++++++ internal/perfbench/turn_bench.go | 28 +- 4 files changed, 559 insertions(+), 18 deletions(-) create mode 100644 internal/perfbench/reports/baseline.json diff --git a/cmd/zero-perf-bench/main_test.go b/cmd/zero-perf-bench/main_test.go index 30f9beeb4..2e2a6631b 100644 --- a/cmd/zero-perf-bench/main_test.go +++ b/cmd/zero-perf-bench/main_test.go @@ -101,7 +101,7 @@ func TestTurnExitCodeFailsWhenEveryTaskErrored(t *testing.T) { if code := turnExitCode(allErrored, &stderr); code != 1 { t.Fatalf("exit=%d, want 1 when every task errored", code) } - if !strings.Contains(stderr.String(), "no valid samples") { + if !strings.Contains(stderr.String(), "no accepted benchmark sample") { t.Fatalf("stderr missing explanation: %q", stderr.String()) } partial := perfbench.TurnBenchResult{TasksAttempted: 3, TasksErrored: 2} diff --git a/cmd/zero-perf-bench/turn.go b/cmd/zero-perf-bench/turn.go index e8e1e6936..7009bebe0 100644 --- a/cmd/zero-perf-bench/turn.go +++ b/cmd/zero-perf-bench/turn.go @@ -92,14 +92,15 @@ func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, return turnExitCode(result, stderr) } -// turnExitCode fails the command when every attempted task errored before the -// agent produced a run: such a report measures nothing, and exiting 0 would let -// a spawn-broken configuration (missing binary, bad path) pass as a clean -// baseline. Partial errors keep exit 0 — the summary surfaces them loudly and -// the surviving samples are still valid measurements. +// turnExitCode fails the command when every attempted task errored (no +// iteration produced an accepted benchmark sample): such a report measures +// nothing, and exiting 0 would let a broken configuration (missing binary, bad +// path, failing harness step) pass as a clean baseline. Partial errors keep +// exit 0 — the summary surfaces them loudly and the surviving samples are +// still valid measurements. func turnExitCode(result perfbench.TurnBenchResult, stderr io.Writer) int { if result.TasksAttempted > 0 && result.TasksErrored == result.TasksAttempted { - _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored before the agent ran (see warnings); the report contains no valid samples") + _, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: every task errored with no accepted benchmark sample (see warnings); the report contains no valid measurements") return 1 } return 0 diff --git a/internal/perfbench/reports/baseline.json b/internal/perfbench/reports/baseline.json new file mode 100644 index 000000000..c09992b60 --- /dev/null +++ b/internal/perfbench/reports/baseline.json @@ -0,0 +1,534 @@ +{ + "schemaVersion": 3, + "suite": "zero-baseline-turn", + "model": "grok-4.5", + "selfCorrect": false, + "version": "0.4.0", + "commit": "30e2c3f7", + "date": "2026-07-18T14:31:02Z", + "tasksAttempted": 48, + "tasksVerified": 34, + "tasksPassed": 0, + "latencyOnlyTasks": 14, + "buildCheckedTasks": 0, + "buildPassedTasks": 0, + "correctnessPassRate": 0, + "buildPassRate": 0, + "correctnessClasses": [ + "edit", + "fix", + "nav", + "refactor" + ], + "latencyOnlyClasses": [ + "longctx", + "longproc", + "parallel" + ], + "iterations": 1, + "perSpan": {}, + "topLatency": [], + "perClass": { + "edit": { + "tasks": 10, + "verified": 10, + "passed": 0, + "latencyOnly": 0, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "fix": { + "tasks": 8, + "verified": 8, + "passed": 0, + "latencyOnly": 0, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "longctx": { + "tasks": 4, + "verified": 0, + "passed": 0, + "latencyOnly": 4, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "longproc": { + "tasks": 4, + "verified": 0, + "passed": 0, + "latencyOnly": 4, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "nav": { + "tasks": 10, + "verified": 10, + "passed": 0, + "latencyOnly": 0, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "parallel": { + "tasks": 6, + "verified": 0, + "passed": 0, + "latencyOnly": 6, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + }, + "refactor": { + "tasks": 6, + "verified": 6, + "passed": 0, + "latencyOnly": 0, + "wallMs": { + "samples": null, + "min": 0, + "median": 0, + "average": 0, + "p95": 0, + "max": 0 + }, + "spanTotals": null + } + }, + "totals": { + "inputTokens": 0, + "cachedInputTokens": 0, + "outputTokens": 0, + "modelRequests": 0, + "toolCalls": 0, + "retries": 0, + "reconnects": 0, + "compactions": 0 + }, + "warnings": [ + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-09: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task nav-10: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-09: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task edit-10: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task fix-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task refactor-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longproc-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longproc-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longproc-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longproc-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longctx-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longctx-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longctx-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task longctx-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + }, + { + "metric": "run", + "statistic": "", + "observed": 0, + "threshold": 0, + "unit": "", + "message": "task parallel-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." + } + ] +} diff --git a/internal/perfbench/turn_bench.go b/internal/perfbench/turn_bench.go index 338f054a9..2594d37b4 100644 --- a/internal/perfbench/turn_bench.go +++ b/internal/perfbench/turn_bench.go @@ -41,12 +41,17 @@ import ( // improvement. New counts: correctness 34 (edit 10 + fix 8 + nav 10 + refactor // 6), build 0, latency 14 (longproc 4 + longctx 4 + parallel 6). // -// v4 adds tasksErrored: the number of tasks whose every iteration died before -// the agent produced a run (spawn failure, missing binary, crashed process). -// Errored tasks were previously visible only in warnings, so a run where the -// agent never launched still printed a clean-looking 0% pass summary and -// exited 0. tasksErrored makes that state first-class: the summary surfaces -// it, and the turn command exits nonzero when every attempted task errored. +// v4 adds tasksErrored: the number of tasks whose every iteration produced no +// accepted benchmark sample. That covers pre-run failures (missing binary, +// spawn error, process crash) and harness errors after a successful agent run +// (e.g. oracle stamping failed) alike — in every case the iteration yields no +// usable measurement. Errored tasks were previously visible only in warnings, +// so a run where nothing was actually measured still printed a clean-looking +// 0% pass summary and exited 0. tasksErrored makes that state first-class: +// the summary surfaces it, and the turn command exits nonzero when every +// attempted task errored. Accounting: an errored oracle task stays in its +// tier denominator as a failure; an errored latency-only task counts in +// latencyOnlyTasks and, as always, in no pass rate. const TurnSchemaVersion = 4 // TurnRunner runs one benchmark task and reports its outcome plus the captured @@ -317,10 +322,11 @@ func RunTurnBench(ctx context.Context, set TaskSet, cfg TurnBenchConfig) (TurnBe // in any pass rate even when the runner reports Passed — an exit-0 // read-only run proves the turn ran, not that the answer was right. if erroredIterations == iterations { - // Every iteration died before the agent produced a run. The task - // still counts in its tier below (as a failure), but the errored - // state is first-class so a spawn-broken run can never print a - // clean-looking summary. + // Every iteration produced no accepted sample (spawn failure or a + // harness error after the run). The task still counts in its tier + // below — an oracle task as a failure, a latency-only task in no + // pass rate as always — but the errored state is first-class so a + // broken run can never print a clean-looking summary. result.TasksErrored++ } hasOracle := len(task.VerificationCommand) > 0 @@ -474,7 +480,7 @@ func FormatTurnBenchSummary(result TurnBenchResult) string { result.LatencyOnlyTasks, result.Iterations), } if result.TasksErrored > 0 { - lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) never produced an agent run (spawn/crash) — they count as FAILURES in the tier pass rates above", result.TasksErrored)) + lines = append(lines, fmt.Sprintf("ERRORED: %d task(s) produced no accepted benchmark sample (spawn/crash or harness error) — errored oracle tasks count as failures in their tier's pass rate; latency-only tasks stay out of pass rates as always", result.TasksErrored)) shown := 0 for _, warning := range result.Warnings { if warning.Metric != "run" { From 9b99cd7eaf0066303b2328862f64477e8b9e584b Mon Sep 17 00:00:00 2001 From: Vasanthdev2004 Date: Sat, 18 Jul 2026 20:50:37 +0530 Subject: [PATCH 4/4] chore(perfbench): never commit generated baseline reports --- .gitignore | 3 + internal/perfbench/reports/baseline.json | 534 ----------------------- 2 files changed, 3 insertions(+), 534 deletions(-) delete mode 100644 internal/perfbench/reports/baseline.json diff --git a/.gitignore b/.gitignore index 4ff0b2673..e341daad5 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/superpowers/ zero-termux-arm64 zero-linux-sandbox zero-seccomp + +# Generated benchmark reports are evidence for one configuration, never tree state +internal/perfbench/reports/*.json diff --git a/internal/perfbench/reports/baseline.json b/internal/perfbench/reports/baseline.json deleted file mode 100644 index c09992b60..000000000 --- a/internal/perfbench/reports/baseline.json +++ /dev/null @@ -1,534 +0,0 @@ -{ - "schemaVersion": 3, - "suite": "zero-baseline-turn", - "model": "grok-4.5", - "selfCorrect": false, - "version": "0.4.0", - "commit": "30e2c3f7", - "date": "2026-07-18T14:31:02Z", - "tasksAttempted": 48, - "tasksVerified": 34, - "tasksPassed": 0, - "latencyOnlyTasks": 14, - "buildCheckedTasks": 0, - "buildPassedTasks": 0, - "correctnessPassRate": 0, - "buildPassRate": 0, - "correctnessClasses": [ - "edit", - "fix", - "nav", - "refactor" - ], - "latencyOnlyClasses": [ - "longctx", - "longproc", - "parallel" - ], - "iterations": 1, - "perSpan": {}, - "topLatency": [], - "perClass": { - "edit": { - "tasks": 10, - "verified": 10, - "passed": 0, - "latencyOnly": 0, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "fix": { - "tasks": 8, - "verified": 8, - "passed": 0, - "latencyOnly": 0, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "longctx": { - "tasks": 4, - "verified": 0, - "passed": 0, - "latencyOnly": 4, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "longproc": { - "tasks": 4, - "verified": 0, - "passed": 0, - "latencyOnly": 4, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "nav": { - "tasks": 10, - "verified": 10, - "passed": 0, - "latencyOnly": 0, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "parallel": { - "tasks": 6, - "verified": 0, - "passed": 0, - "latencyOnly": 6, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - }, - "refactor": { - "tasks": 6, - "verified": 6, - "passed": 0, - "latencyOnly": 0, - "wallMs": { - "samples": null, - "min": 0, - "median": 0, - "average": 0, - "p95": 0, - "max": 0 - }, - "spanTotals": null - } - }, - "totals": { - "inputTokens": 0, - "cachedInputTokens": 0, - "outputTokens": 0, - "modelRequests": 0, - "toolCalls": 0, - "retries": 0, - "reconnects": 0, - "compactions": 0 - }, - "warnings": [ - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-09: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task nav-10: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-09: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task edit-10: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-07: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task fix-08: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task refactor-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longproc-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longproc-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longproc-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longproc-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longctx-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longctx-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longctx-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task longctx-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-01: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-02: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-03: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-04: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-05: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - }, - { - "metric": "run", - "statistic": "", - "observed": 0, - "threshold": 0, - "unit": "", - "message": "task parallel-06: zero exec failed: fork/exec ./zero-bench-main.exe: The system cannot find the file specified." - } - ] -}