From 569009d85162ce71b7adc915656368b79a591f9c Mon Sep 17 00:00:00 2001 From: Fadion Dashi Date: Fri, 28 Aug 2026 00:59:44 +0200 Subject: [PATCH] perf: benchmarks, and a number for the slot-scopes question The characterization suite is the arbiter of meaning and there was nothing playing the same role for cost. Aria was never trying to be fast, and docs/compatibility.md puts performance outside what a version promises, so these are not targets to hit. They are for noticing when a change costs something nobody meant to spend. Go's testing.B rather than a harness of our own: benchstat, -count and -benchmem for free, and it runs under `go test` like everything else. Three phases separately, because a regression should point at a stage rather than at "the number moved". The macro workloads are examples rather than fixtures written for the benchmark, so check-examples.sh already keeps them from rotting into something that measures nothing. The standard library had to be handled before any of this meant anything. Eval loads it on every call, at 1.3ms and 11,005 allocations against 308 for an empty program, so a workload measured through Eval is ninety-seven percent stdlib loading and three percent workload. It gets its own benchmark, being a real cost every `aria run` pays, and everything else builds it with the timer stopped. Checked that StopTimer excludes allocations and not merely time: a trivial program through the same path reports 0 allocs/op, so the numbers are the workload. Read allocs/op, not ns/op. Every collection operation returns a new value, so allocation is the design's dominant cost, and a deterministic program allocates the same number of times on a laptop and on a shared runner. Wall-clock does not, which is why CI runs these at -benchtime=1x to prove they still run and compares nothing. Gating on a number would need a stored baseline, which goes stale and cries wolf. The last Known Gap now has evidence. name-lookup-deep and name-lookup-shallow run the same loop and the same arithmetic, differing only in whether the names are three scopes up or beside the loop, so the gap between them is chain-walking alone. It costs about a fifth more time and ten more allocations out of eleven thousand: pointer chasing, not garbage. That is the ceiling on what slot indices could win, measured in a case built to make the chain as expensive as it gets. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 12 ++ CLAUDE.md | 14 ++ docs/architecture.md | 9 ++ internal/interp/bench_test.go | 272 ++++++++++++++++++++++++++++++++++ 4 files changed, 307 insertions(+) create mode 100644 internal/interp/bench_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3493c7..c3a6e95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,18 @@ jobs: exit 1 fi + # One iteration each, which measures nothing and is not meant to. It + # proves the benchmarks still compile and still run, so they cannot rot + # into something that fails the first time somebody reaches for them -- + # which is always while chasing a regression and never at a good moment. + # + # Nothing here compares numbers. Wall-clock on a shared runner is noise, + # and gating on it would need a stored baseline that goes stale. Comparing + # revisions is a local benchstat job; see internal/interp/bench_test.go. + - name: Benchmarks run + if: matrix.os == 'ubuntu-latest' + run: go test ./internal/interp/ -run=XXX -bench=. -benchtime=1x + # Static analysis beyond `go vet`. Ubuntu only: the linters are # platform-independent, and running them three times says the same thing # three times. diff --git a/CLAUDE.md b/CLAUDE.md index d29f0de..0560f5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,6 +46,20 @@ The characterization suite is the important one. `go test` covers the Go code; t covers *the language*, and it is the thing that catches a change you did not intend to make. +There are benchmarks too, in `internal/interp/bench_test.go`. CI runs them at +`-benchtime=1x`, which measures nothing and only proves they still work. To actually +compare two revisions: + +```bash +go test ./internal/interp/ -run=XXX -bench=. -benchmem -count=10 > old.txt +go test ./internal/interp/ -run=XXX -bench=. -benchmem -count=10 > new.txt +benchstat old.txt new.txt +``` + +Read `allocs/op`, not `ns/op`. Every collection operation returns a new value, so +allocation is the dominant cost and the count is deterministic; wall-clock on a shared +runner is not, which is why nothing gates on it. + After touching the scanner or parser, also run the fuzzers for 30 seconds each. They are cheap and they have both found real bugs: diff --git a/docs/architecture.md b/docs/architecture.md index fd15a40..f5a4c30 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -998,3 +998,12 @@ itself a compatibility break. - **Slot-based scopes.** The resolver computes slot indices and scope sizes that the evaluator ignores, walking a chain of name maps instead. Switching would make lookup an array index. + + There is now a number attached to this. `BenchmarkMicro/name-lookup-deep` and + `name-lookup-shallow` run the same loop and the same arithmetic, differing only in + whether the names come from three scopes up or from beside the loop, so the gap between + them is chain-walking and nothing else. On one machine the deep case costs around a fifth + more time and **ten more allocations out of eleven thousand** — the chain costs pointer + chasing, not garbage. That is the ceiling on what slots could win, in a case built to + make the chain as expensive as possible; a real program keeps its names closer. Measure + with benchstat before believing any of it, and before spending a rewrite on it. diff --git a/internal/interp/bench_test.go b/internal/interp/bench_test.go new file mode 100644 index 0000000..d837dae --- /dev/null +++ b/internal/interp/bench_test.go @@ -0,0 +1,272 @@ +package interp + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/fadion/aria/internal/ast" + "github.com/fadion/aria/internal/diag" + "github.com/fadion/aria/internal/parser" + "github.com/fadion/aria/internal/source" +) + +// Benchmarks are the arbiter of cost the way the characterization suite is the +// arbiter of meaning. Aria was never trying to be fast, and docs/compatibility.md +// puts performance outside what a version promises, so nothing here is a target +// to hit. What they are for is noticing when a change costs something nobody +// meant to spend. +// +// Read allocs/op, not ns/op. Every collection operation returns a new value, so +// allocation is the design's dominant cost, and the count is deterministic for a +// deterministic program: the same number on a laptop and on a shared CI runner. +// ns/op on a GitHub runner is noise wearing a number's clothes, which is why CI +// runs these with -benchtime=1x to prove they still work and compares nothing. +// +// To actually compare two revisions: +// +// go test ./internal/interp/ -run=XXX -bench=. -benchmem -count=10 > old.txt +// # ... make the change ... +// go test ./internal/interp/ -run=XXX -bench=. -benchmem -count=10 > new.txt +// benchstat old.txt new.txt +// +// benchstat is what says whether a difference is real. A single run of each +// says nothing at all. + +// The macro workloads are examples, not fixtures written for the benchmark. +// They are already deterministic, already realistic, and check-examples.sh +// already keeps them running, so they cannot quietly rot into something that +// measures nothing. +var workloads = []string{ + "mandelbrot", // float arithmetic in a tight nested loop + "brainfuck", // an interpreter loop: dictionary lookups and string building + "levenshtein", // nested folds over arrays + "quicksort", // recursion, and building arrays from other arrays + "word-frequency", // dictionary insertion and sorting +} + +// The micro workloads isolate what the examples exercise but do not separate. +const ( + // Name lookup through a chain of scopes, which is what the resolver's unused + // slot indices would replace with an array index. See the Known Gaps entry + // in docs/architecture.md: this benchmark is what turns that from an opinion + // into a number. + srcNameLookup = ` +let outer = 1 +let level1 = func () + let middle = 2 + let level2 = func () + let inner = 3 + let level3 = func () + var total = 0 + for i in 1..2000 + total += outer + middle + inner + end + total + end + level3() + end + level2() +end +level1() +` + + // The same loop and the same arithmetic with every name declared beside it, + // so the difference between this and the one above is the cost of walking + // the scope chain and nothing else. Without a shallow case to subtract, the + // deep one measures a loop and three additions as well. + srcNameLookupShallow = ` +let level3 = func () + let outer = 1 + let middle = 2 + let inner = 3 + var total = 0 + for i in 1..2000 + total += outer + middle + inner + end + total +end +level3() +` + + // Function call overhead on its own: a call per iteration and almost nothing + // else. + srcCalls = ` +let add = func (x, y) do x + y end +var total = 0 +for i in 1..5000 + total += add(i, 1) +end +total +` + + // Building a collection an element at a time, which is quadratic because + // every operation returns a new array. That is the immutability rule's bill, + // and the number worth watching if anything ever tries to optimise it. + srcCollections = ` +var xs = [] +for i in 1..2000 + xs = Enum.insert(xs, i) +end +Enum.size(xs) +` +) + +// prepared is a program with everything done to it except running: parsed, +// resolved, with the standard library loaded and imports evaluated. +type prepared struct { + interp *Interp + prog *ast.Program +} + +// prepare mirrors what Eval does up to the point of running, so a benchmark can +// build it with the timer stopped and measure only the part it cares about. +func prepare(tb testing.TB, name, src string) prepared { + tb.Helper() + + file := source.NewFile(name, []byte(src)) + bag := diag.New(file) + prog := parser.New(file, bag).Parse() + if bag.HasErrors() { + tb.Fatalf("%s: %s", name, bag.Render()) + } + + i := New(file, nil) + i.Out = io.Discard + i.Err = io.Discard + + var sink discard + if !i.loadStdlib(&sink) { + tb.Fatalf("%s: standard library failed to load: %s", name, sink.String()) + } + units, info, ok := i.compile(file, prog, bag, &sink) + if !ok { + tb.Fatalf("%s: %s", name, sink.String()) + } + i.info = info + if err := i.evalUnits(units); err != nil { + tb.Fatalf("%s: %v", name, err) + } + return prepared{interp: i, prog: prog} +} + +// readExample returns an example's source and the path it should be compiled +// under, so that a workload with an import resolves it the way `aria run` would. +func readExample(tb testing.TB, name string) (path, src string) { + tb.Helper() + path = filepath.Join("..", "..", "examples", name+".ari") + b, err := os.ReadFile(path) + if err != nil { + tb.Fatalf("reading %s: %v", path, err) + } + return path, string(b) +} + +// BenchmarkStdlibLoad is the fixed cost of starting up: parsing, resolving and +// evaluating the whole standard library before a line of anybody's program runs. +// It is separate because it dominates everything else -- around fifty times the +// allocations of an empty program -- so folding it into the workloads would mean +// measuring it five times over and the workloads not at all. +func BenchmarkStdlibLoad(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + i := New(source.NewFile("bench.ari", nil), nil) + i.Out = io.Discard + i.Err = io.Discard + var sink discard + if !i.loadStdlib(&sink) { + b.Fatalf("standard library failed to load: %s", sink.String()) + } + } +} + +// BenchmarkParse is the scanner and the parser, with nothing else attached. +func BenchmarkParse(b *testing.B) { + for _, name := range workloads { + path, src := readExample(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + file := source.NewFile(path, []byte(src)) + bag := diag.New(file) + prog := parser.New(file, bag).Parse() + if bag.HasErrors() { + b.Fatalf("%s", bag.Render()) + } + _ = prog + } + }) + } +} + +// BenchmarkResolve is name binding and the checks that go with it, over an +// already-parsed program with the standard library already in scope. +func BenchmarkResolve(b *testing.B) { + for _, name := range workloads { + path, src := readExample(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + b.StopTimer() + file := source.NewFile(path, []byte(src)) + bag := diag.New(file) + prog := parser.New(file, bag).Parse() + i := New(file, nil) + i.Out, i.Err = io.Discard, io.Discard + var sink discard + if !i.loadStdlib(&sink) { + b.Fatalf("standard library failed to load: %s", sink.String()) + } + b.StartTimer() + + if _, _, ok := i.compile(file, prog, bag, &sink); !ok { + b.Fatalf("%s", sink.String()) + } + } + }) + } +} + +// BenchmarkEval is the evaluator alone. Everything else is built with the timer +// stopped, which excludes it from the allocation counts as well as the time. +func BenchmarkEval(b *testing.B) { + for _, name := range workloads { + path, src := readExample(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + b.StopTimer() + p := prepare(b, path, src) + b.StartTimer() + + if _, err := p.interp.Run(p.prog); err != nil { + b.Fatalf("%s: %v", name, err) + } + } + }) + } +} + +// BenchmarkMicro isolates one cost each, where the workloads above mix them. +func BenchmarkMicro(b *testing.B) { + for _, tt := range []struct{ name, src string }{ + {"name-lookup-deep", srcNameLookup}, + {"name-lookup-shallow", srcNameLookupShallow}, + {"calls", srcCalls}, + {"collections", srcCollections}, + } { + b.Run(tt.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + b.StopTimer() + p := prepare(b, tt.name+".ari", tt.src) + b.StartTimer() + + if _, err := p.interp.Run(p.prog); err != nil { + b.Fatalf("%s: %v", tt.name, err) + } + } + }) + } +}