Add per-turn tracing and a turn-benchmark harness (Phase 0 baseline)

Makefile

@@ -1,7 +1,7 @@
 # Zero build/test/lint targets. AGENTS.md says "Build with `make`" and "Run `make
 # lint` before opening a PR" — these targets back those instructions.
 .DEFAULT_GOAL := build
-.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean help
+.PHONY: build build-all test test-race vet fmt fmt-check lint tidy clean baseline help
 
 # Build the main CLI binary into ./zero.
 build:
@@ -40,5 +40,19 @@ clean:
 	rm -f zero
 	go clean ./...
 
+# Run the per-turn benchmark harness over the checked-in baseline manifest and
+# write the JSON result to internal/perfbench/reports/baseline.json. Requires a
+# built `zero` binary and a model; set ZERO_BENCH_MODEL (required) and
+# ZERO_BENCH_BINARY (defaults to ./zero) to configure the run. The report is
+# machine-specific and regenerated, not hand-edited.
+baseline: build
+	@if [ -z "$(ZERO_BENCH_MODEL)" ]; then echo "Set ZERO_BENCH_MODEL (and optionally ZERO_BENCH_BINARY) before running 'make baseline'"; exit 2; fi
+	@ZERO_BIN="$${ZERO_BENCH_BINARY:-./zero}"; \
+	go run ./cmd/zero-perf-bench turn \
+		--suite internal/perfbench/manifests/baseline.json \
+		--model $(ZERO_BENCH_MODEL) \
+		--binary "$$ZERO_BIN" \
+		--output internal/perfbench/reports/baseline.json
+
 help:
-	@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean"
+	@echo "Targets: build (default), build-all, test, test-quick, vet, fmt, fmt-check, lint, tidy, clean, baseline"

cmd/zero-perf-bench/main.go

@@ -39,6 +39,9 @@ func run(args []string, getenv func(string) string, stdout io.Writer, stderr io.
 	if len(args) > 0 && args[0] == "tasks" {
 		return runTasksCommand(args[1:], getenv, stdout, stderr)
 	}
+	if len(args) > 0 && args[0] == "turn" {
+		return runTurnCommand(args[1:], getenv, stdout, stderr)
+	}
 	options, err := parseArgs(args, getenv)
 	if err != nil {
 		_, _ = fmt.Fprintln(stderr, err.Error())
@@ -206,6 +209,7 @@ func helpText() string {
 	return strings.Join([]string{
 		"Usage: zero-perf-bench [options]",
 		"       zero-perf-bench tasks [options]   (Terminal-Bench-style task harness; see `tasks --help`)",
+		"       zero-perf-bench turn  [options]   (per-turn tracing benchmark; see `turn --help`)",
 		"",
 		"Options:",
 		"  --iterations <n>             Measured samples to collect (default: 5)",

cmd/zero-perf-bench/turn.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/Gitlawb/zero/internal/perfbench"
+)
+
+// turnOptions configures the `zero-perf-bench turn` subcommand: the per-turn
+// benchmark harness that runs ZERO headlessly with --trace, parses each turn's
+// NDJSON trace, and records per-span latency plus the top controllable latency
+// sources — the Phase 0 baseline's "do not proceed until" criterion.
+type turnOptions struct {
+	SuitePath   string
+	Model       string
+	Mode        string
+	SelfCorrect bool
+	Binary      string
+	Iterations  int
+	Version     string
+	Commit      string
+	Output      string
+	JSON        bool
+	DryRun      bool
+	Help        bool
+}
+
+func runTurnCommand(args []string, getenv func(string) string, stdout io.Writer, stderr io.Writer) int {
+	options, err := parseTurnArgs(args, getenv)
+	if err != nil {
+		_, _ = fmt.Fprintln(stderr, err.Error())
+		return 2
+	}
+	if options.Help {
+		_, _ = fmt.Fprint(stdout, turnHelpText())
+		return 0
+	}
+
+	set, err := perfbench.LoadTaskSet(options.SuitePath)
+	if err != nil {
+		_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
+		return 1
+	}
+
+	// The dry-run path records a zero-iteration run without a binary, so the
+	// manifest loads and the report path is exercised in CI without a model.
+	if options.DryRun {
+		_, _ = fmt.Fprintln(stdout, "[zero] turn benchmark: dry run (no agent invoked)")
+		return 0
+	}
+
+	binary, err := perfbench.ResolveBinary(options.Binary)
+	if err != nil {
+		_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
+		return 2
+	}
+
+	result, err := perfbench.RunTurnBench(context.Background(), set, perfbench.TurnBenchConfig{
+		Model:       options.Model,
+		Mode:        options.Mode,
+		SelfCorrect: options.SelfCorrect,
+		Version:     options.Version,
+		Commit:      options.Commit,
+		Iterations:  options.Iterations,
+		Runner:      perfbench.NewTurnExecRunner(binary),
+	})
+	if err != nil {
+		_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
+		return 1
+	}
+
+	if options.Output != "" {
+		if err := writeTurnReport(options.Output, result); err != nil {
+			_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
+			return 1
+		}
+	}
+	if options.JSON {
+		if err := perfbench.WriteTurnBenchJSON(stdout, result); err != nil {
+			_, _ = fmt.Fprintln(stderr, "[zero] Turn benchmark failed: "+err.Error())
+			return 1
+		}
+		return 0
+	}
+	_, _ = fmt.Fprintln(stdout, perfbench.FormatTurnBenchSummary(result))
+	return 0
+}
+
+func parseTurnArgs(args []string, getenv func(string) string) (turnOptions, error) {
+	options := turnOptions{
+		Iterations: 1,
+		Version:    strings.TrimSpace(getenv("ZERO_BENCH_VERSION")),
+		Commit:     strings.TrimSpace(getenv("ZERO_BENCH_COMMIT")),
+	}
+	for index := 0; index < len(args); index++ {
+		arg := args[index]
+		flag, inlineValue := splitFlagValue(arg)
+		switch flag {
+		case "--suite":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.SuitePath = value
+			index = next
+		case "--model":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Model = value
+			index = next
+		case "--mode":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Mode = value
+			index = next
+		case "--binary":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Binary = value
+			index = next
+		case "--iterations":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			parsed, err := parsePositiveInteger(flag, value)
+			if err != nil {
+				return options, err
+			}
+			options.Iterations = parsed
+			index = next
+		case "--version":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Version = value
+			index = next
+		case "--commit":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Commit = value
+			index = next
+		case "--output":
+			value, next, err := readOptionValue(args, inlineValue, index, flag)
+			if err != nil {
+				return options, err
+			}
+			options.Output = value
+			index = next
+		case "--self-correct":
+			if strings.Contains(arg, "=") {
+				return options, fmt.Errorf("%s does not accept a value", flag)
+			}
+			options.SelfCorrect = true
+		case "--json":
+			if strings.Contains(arg, "=") {
+				return options, fmt.Errorf("%s does not accept a value", flag)
+			}
+			options.JSON = true
+		case "--dry-run":
+			if strings.Contains(arg, "=") {
+				return options, fmt.Errorf("%s does not accept a value", flag)
+			}
+			options.DryRun = true
+		case "-h", "--help":
+			if strings.Contains(arg, "=") {
+				return options, fmt.Errorf("%s does not accept a value", flag)
+			}
+			options.Help = true
+		default:
+			return options, fmt.Errorf("unknown option: %s", arg)
+		}
+	}
+	if options.Help {
+		return options, nil
+	}
+	if strings.TrimSpace(options.SuitePath) == "" {
+		return options, fmt.Errorf("--suite is required")
+	}
+	if strings.TrimSpace(options.Model) == "" && !options.DryRun {
+		return options, fmt.Errorf("--model is required (or pass --dry-run)")
+	}
+	return options, nil
+}
+
+func writeTurnReport(path string, result perfbench.TurnBenchResult) error {
+	if dir := filepath.Dir(path); dir != "" {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return err
+		}
+	}
+	var buffer bytes.Buffer
+	if err := perfbench.WriteTurnBenchJSON(&buffer, result); err != nil {
+		return err
+	}
+	return os.WriteFile(path, buffer.Bytes(), 0o644)
+}
+
+func turnHelpText() string {
+	return strings.Join([]string{
+		"Usage: zero-perf-bench turn [options]",
+		"",
+		"Runs ZERO headlessly with --trace against a per-turn benchmark task set and",
+		"records per-span latency plus the top controllable latency sources (the",
+		"Phase 0 baseline's \"do not proceed until\" criterion). Each task is a fresh",
+		"`zero exec` process, so iterations are cold-start samples; a warm path needs",
+		"an in-process runner (future work).",
+		"",
+		"Options:",
+		"  --suite <path>      Task set JSON file (required)",
+		"  --model <model>     Model to run (required unless --dry-run)",
+		"  --mode <name>       Exec mode preset to apply",
+		"  --self-correct      Enable the post-edit verify-and-correct loop",
+		"  --binary <path>     Path to the `zero` binary (default: zero on PATH / repo root)",
+		"  --iterations <n>    Times to run each task (default: 1)",
+		"  --version <v>       Record the ZERO version (default: $ZERO_BENCH_VERSION)",
+		"  --commit <sha>      Record the ZERO commit (default: $ZERO_BENCH_COMMIT)",
+		"  --output <path>     Write the JSON result to path",
+		"  --json              Print only the JSON result",
+		"  --dry-run           Load the manifest and exit without invoking the agent",
+		"  -h, --help          Show this help",
+	}, "\n") + "\n"
+}

internal/agent/compaction.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"strings"
 
+	"github.com/Gitlawb/zero/internal/trace"
 	"github.com/Gitlawb/zero/internal/zeroruntime"
 )
 
@@ -430,6 +431,12 @@ func (state *compactionState) maybeCompact(
 		state.lowWaterMark = size
 		return messages
 	}
+	// Only count a compaction when it actually shrank the history, so the
+	// compaction counter reflects real context reductions rather than paid
+	// no-ops that left the token budget untouched.
+	if r := trace.FromContext(ctx); r != nil {
+		r.Counter(trace.CounterCompactionCount, 1)
+	}
 	state.lowWaterMark = newSize
 	return compacted
 }
@@ -479,7 +486,11 @@ func (state *compactionState) recover(
 	// one-shot budget now so a provider that keeps returning context-limit errors
 	// after a successful compaction can't loop forever. Store the low-water mark in
 	// the SAME combined (messages + tool-defs) domain maybeCompact uses, so the
-	// proactive shrink-guard compares like with like.
+	// proactive shrink-guard compares like with like. Count it now that the shrink
+	// is confirmed, so the counter mirrors maybeCompact's real-reduction policy.
+	if r := trace.FromContext(ctx); r != nil {
+		r.Counter(trace.CounterCompactionCount, 1)
+	}
 	state.reactiveAttempted = true
 	state.lowWaterMark = state.calibratedTokens(estimateTokens(result) + estimateToolDefTokens(tools))
 	return result, true, nil