debug: allow supported targets to retain DWARF

.github/workflows/benchmark.yml

@@ -32,6 +32,15 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
+      - name: Check out pull request base benchmark source
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v7
+        with:
+          repository: ${{ github.event.pull_request.base.repo.full_name }}
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: .benchmark/source
+          persist-credentials: false
+
       - name: Install dependencies
         uses: ./.github/actions/setup-deps
         with:
@@ -40,63 +49,41 @@ jobs:
       - name: Set up Go
         uses: ./.github/actions/setup-go
 
-      - name: Build LLGo
-        run: |
-          set -euo pipefail
-          mkdir -p .benchmark
-          go build -p=1 -o .benchmark/llgo ./cmd/llgo
-
-      - name: Measure programs
+      - name: Measure pull request base
+        if: github.event_name == 'pull_request'
         run: |
-          set -euo pipefail
-          go run ./benchmark/baseline \
-            -llgo "$GITHUB_WORKSPACE/.benchmark/llgo" \
-            -out "$GITHUB_WORKSPACE/.benchmark/results"
+          benchmark/baseline/run.sh \
+            "$GITHUB_WORKSPACE/.benchmark/source" \
+            "$GITHUB_WORKSPACE/.benchmark/base-llgo" \
+            "$GITHUB_WORKSPACE/.benchmark/base-results"
 
-      - name: Run selected Go benchmarks
-        run: |
-          set -euo pipefail
-          results="$GITHUB_WORKSPACE/.benchmark/results/go.txt"
-          GOMAXPROCS=1 go test \
-            -run '^$' \
-            -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \
-            -benchtime=250ms \
-            -count=1 \
-            -cpu=1 \
-            ./internal/clang ./internal/build/funcinfo \
-            | tee "$results"
-
-          GOMAXPROCS=1 "$GITHUB_WORKSPACE/.benchmark/llgo" test \
-            -run '^$' \
-            -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \
-            -benchtime=250ms \
-            -count=1 \
-            ./test/llgoext \
-            | tee -a "$results"
-
-          # The current native backend creates one pthread per goroutine and
-          # intentionally has a bounded lifecycle stress limit. Keep creation
-          # monitoring deterministic instead of letting testing auto-calibrate
-          # to millions of host threads.
-          GOMAXPROCS=1 "$GITHUB_WORKSPACE/.benchmark/llgo" test \
-            -run '^$' \
-            -bench '^BenchmarkGoroutine$' \
-            -benchtime=100x \
-            -count=1 \
-            ./test/llgoext \
-            | tee -a "$results"
+      - name: Check out pull request head benchmark source
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v7
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: .benchmark/source
+          persist-credentials: false
 
-      - name: Validate and export benchmark artifact
+      - name: Measure current revision
         run: |
-          go run ./benchmark/baseline \
-            -mode export \
-            -out .benchmark/results \
-            -benchmark-output .benchmark/results/benchmark.txt
+          source_root="$GITHUB_WORKSPACE"
+          if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then
+            source_root="$GITHUB_WORKSPACE/.benchmark/source"
+          fi
+          benchmark/baseline/run.sh \
+            "$source_root" \
+            "$GITHUB_WORKSPACE/.benchmark/llgo" \
+            "$GITHUB_WORKSPACE/.benchmark/results"
 
       - name: Record benchmark result
-        uses: xgo-dev/setup-benchmark-go-action@v1
+        uses: xgo-dev/setup-benchmark-go-action@v1.0.3
         with:
           config: .github/llgo-benchmark.yml
           benchmark-file: .benchmark/results/benchmark.txt
+          baseline-benchmark-file: >-
+            ${{ github.event_name == 'pull_request' &&
+                '.benchmark/base-results/benchmark.txt' || '' }}
           platform-id: ${{ matrix.id }}
           platform-label: ${{ matrix.display }}

README.md

@@ -377,7 +377,7 @@ brew link --overwrite llvm@19 lld@19 libffi
 echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list
 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
 sudo apt-get update
-sudo apt-get install -y llvm-19-dev clang-19 libclang-19-dev lld-19 libunwind-19-dev libc++-19-dev pkg-config libgc-dev libssl-dev zlib1g-dev libcjson-dev libsqlite3-dev libuv1-dev
+sudo apt-get install -y llvm-19-dev clang-19 libclang-19-dev lld-19 libunwind-19-dev libc++-19-dev pkg-config libgc-dev libssl-dev zlib1g-dev libffi-dev libcjson-dev libsqlite3-dev libuv1-dev
 sudo apt-get install -y python3.12-dev # optional
 #curl https://raw.githubusercontent.com/xgo-dev/llgo/refs/heads/main/install.sh | bash
 ./install.sh

benchmark/baseline/README.md

@@ -14,18 +14,27 @@ The program workloads reuse:
 - `benchmark/binary_size/println`: only the built-in `println`;
 - `benchmark/binary_size/fmtprintf`: `fmt.Printf`.
 
-For each workload, the collector records median warm build time, median process
-time, file size, executable-code bytes, allocated non-executable data, and
-zero-filled data. On ELF, read-only constants are included in the data bucket;
-on Mach-O, `__TEXT` constants are included in the text bucket. The Go benchmark
-stream adds selected compiler helpers and LLGo-generated core-language
-operations: direct/interface calls, defer, goroutine creation, channels,
-`getg`, and global access.
+For each workload, the collector performs an unmeasured warm build, then records
+median build time, median process time, file size, executable-code bytes,
+allocated non-executable data, and zero-filled data. On ELF, read-only constants
+are included in the data bucket; on Mach-O, `__TEXT` constants are included in
+the text bucket. The Go benchmark stream records five samples of selected
+compiler helpers and LLGo-generated core-language operations: direct/interface
+calls, defer, goroutine creation, channels, `getg`, and global access.
 
-The trusted publisher compares each platform against the latest matching
-`main` data and commits the history and generated site to the `pages` branch of
-the configured data repository. Every LLGo repository defaults to
-`<owner>/llgo-benchmark-data`:
+For pull requests, each platform job checks out the recorded base and current
+commits into the same source path, then runs both suites sequentially on the same
+runner. The pull request comment compares that pair, avoiding differences from
+runner machines and embedded source paths. Dependency setup is shared, and Go's
+build cache can be reused by unchanged packages; main pushes still run the suite
+only once. Very small changes can still be scheduler, frequency, or thermal
+noise and should be confirmed by repeated workflow runs. If a workflow does not
+provide a paired result, the publisher falls back to the latest matching `main`
+data.
+
+The trusted publisher commits the current result history and generated site to
+the `pages` branch of the configured data repository. Every LLGo repository
+defaults to `<owner>/llgo-benchmark-data`:
 
 ```text
 llgo/baseline/series/main/main
@@ -55,15 +64,15 @@ results=.benchmark/results/go.txt
 GOMAXPROCS=1 go test \
   -run '^$' \
   -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \
-  -benchtime=250ms -count=1 -cpu=1 \
+  -benchtime=250ms -count=5 -cpu=1 \
   ./internal/clang ./internal/build/funcinfo | tee "$results"
 GOMAXPROCS=1 .benchmark/llgo test \
   -run '^$' \
   -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \
-  -benchtime=250ms -count=1 \
+  -benchtime=250ms -count=5 \
   ./test/llgoext | tee -a "$results"
 GOMAXPROCS=1 .benchmark/llgo test \
-  -run '^$' -bench '^BenchmarkGoroutine$' -benchtime=100x -count=1 \
+  -run '^$' -bench '^BenchmarkGoroutine$' -benchtime=100x -count=5 \
   ./test/llgoext | tee -a "$results"
 ```
 

benchmark/baseline/main.go

@@ -74,6 +74,8 @@ var expectedGoBenchmarks = []string{
 	"BenchmarkRuntimeGetG",
 }
 
+const goBenchmarkSamples = 5
+
 type footprint struct {
 	file uint64
 	text uint64
@@ -221,6 +223,11 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int
 	var sizes, timings []metric
 	for _, item := range workloads {
 		binary := filepath.Join(binDir, item.name)
+		// Keep first-use toolchain and filesystem caches out of the measured
+		// median so the first revision is not systematically disadvantaged.
+		if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(root, item.source)); err != nil {
+			return fmt.Errorf("warm build %s: %w", item.name, err)
+		}
 		buildDurations := make([]time.Duration, 0, buildRuns)
 		for range buildRuns {
 			start := time.Now()
@@ -433,9 +440,9 @@ func validateMetrics(path string, expected map[string]string) error {
 }
 
 func validateGoBenchmarks(r io.Reader) error {
-	expected := make(map[string]bool, len(expectedGoBenchmarks))
+	expected := make(map[string]int, len(expectedGoBenchmarks))
 	for _, name := range expectedGoBenchmarks {
-		expected[name] = false
+		expected[name] = 0
 	}
 	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
@@ -444,12 +451,12 @@ func validateGoBenchmarks(r io.Reader) error {
 			continue
 		}
 		name := cpuSuffix.ReplaceAllString(fields[0], "")
-		seen, ok := expected[name]
+		samples, ok := expected[name]
 		if !ok {
 			return fmt.Errorf("unexpected Go benchmark %q", name)
 		}
-		if seen {
-			return fmt.Errorf("duplicate Go benchmark %q", name)
+		if samples >= goBenchmarkSamples {
+			return fmt.Errorf("too many samples for Go benchmark %q", name)
 		}
 		if len(fields) < 4 || (len(fields)-2)%2 != 0 {
 			return fmt.Errorf("malformed Go benchmark line %q", scanner.Text())
@@ -474,20 +481,25 @@ func validateGoBenchmarks(r io.Reader) error {
 		if !hasTime {
 			return fmt.Errorf("benchmark %q has no ns/op result", name)
 		}
-		expected[name] = true
+		expected[name] = samples + 1
 	}
 	if err := scanner.Err(); err != nil {
 		return err
 	}
 	var missing []string
-	for name, seen := range expected {
-		if !seen {
+	for name, samples := range expected {
+		if samples == 0 {
 			missing = append(missing, name)
 		}
 	}
 	slices.Sort(missing)
 	if len(missing) != 0 {
 		return fmt.Errorf("missing Go benchmarks: %s", strings.Join(missing, ", "))
 	}
+	for name, samples := range expected {
+		if samples != goBenchmarkSamples {
+			return fmt.Errorf("Go benchmark %q has %d samples, want %d", name, samples, goBenchmarkSamples)
+		}
+	}
 	return nil
 }

benchmark/baseline/main_test.go

@@ -23,6 +23,7 @@ import (
 	"debug/macho"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"math"
 	"os"
@@ -157,8 +158,10 @@ func TestValidateMetricsRejectsInvalidData(t *testing.T) {
 func TestValidateGoBenchmarks(t *testing.T) {
 	var input strings.Builder
 	for _, name := range expectedGoBenchmarks {
-		input.WriteString(name)
-		input.WriteString("-1 100 12.5 ns/op\n")
+		for range goBenchmarkSamples {
+			input.WriteString(name)
+			input.WriteString("-1 100 12.5 ns/op\n")
+		}
 	}
 	if err := validateGoBenchmarks(strings.NewReader(input.String())); err != nil {
 		t.Fatal(err)
@@ -170,7 +173,9 @@ func TestValidateGoBenchmarksRejectsInvalidData(t *testing.T) {
 		var input strings.Builder
 		for _, name := range expectedGoBenchmarks {
 			if name != skip {
-				input.WriteString(name + "-1 100 12.5 ns/op\n")
+				for range goBenchmarkSamples {
+					input.WriteString(name + "-1 100 12.5 ns/op\n")
+				}
 			}
 		}
 		return input.String()
@@ -182,7 +187,8 @@ func TestValidateGoBenchmarksRejectsInvalidData(t *testing.T) {
 	}{
 		{"missing", valid(expectedGoBenchmarks[0]), "missing Go benchmarks"},
 		{"unknown", valid("") + "BenchmarkUnknown-1 1 1 ns/op\n", `unexpected Go benchmark "BenchmarkUnknown"`},
-		{"duplicate", valid("") + expectedGoBenchmarks[0] + "-1 1 1 ns/op\n", "duplicate Go benchmark"},
+		{"too many samples", valid("") + expectedGoBenchmarks[0] + "-1 1 1 ns/op\n", "too many samples"},
+		{"too few samples", strings.Replace(valid(""), expectedGoBenchmarks[0]+"-1 100 12.5 ns/op\n", "", 1), fmt.Sprintf("has %d samples, want %d", goBenchmarkSamples-1, goBenchmarkSamples)},
 		{"malformed", strings.Replace(valid(""), " 100 12.5 ns/op", " bad", 1), "malformed Go benchmark"},
 		{"iterations", strings.Replace(valid(""), " 100 ", " bad ", 1), "invalid iteration count"},
 		{"value", strings.Replace(valid(""), "12.5", "bad", 1), "invalid value"},
@@ -523,7 +529,9 @@ func writeValidArtifact(t *testing.T, dir string) {
 func makeGoBenchmarkText() string {
 	var input strings.Builder
 	for _, name := range expectedGoBenchmarks {
-		input.WriteString(name + "-1 100 12.5 ns/op\n")
+		for range goBenchmarkSamples {
+			input.WriteString(name + "-1 100 12.5 ns/op\n")
+		}
 	}
 	return input.String()
 }

benchmark/baseline/run.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ $# -ne 3 ]]; then
+  echo "usage: $0 <source-root> <llgo-output> <result-directory>" >&2
+  exit 2
+fi
+
+harness_root="$(cd "$(dirname "$0")/../.." && pwd)"
+
+source_root="$(cd "$1" && pwd)"
+mkdir -p "$(dirname "$2")" "$3"
+llgo_output="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"
+result_directory="$(cd "$3" && pwd)"
+
+(
+  cd "$source_root"
+  LLGO_ROOT="$source_root" go build -p=1 -o "$llgo_output" ./cmd/llgo
+)
+
+(
+  cd "$harness_root"
+  # Keep one current-checkout harness for both source revisions. Benchmark
+  # suite changes must therefore remain executable against the PR base.
+  LLGO_ROOT="$source_root" go run ./benchmark/baseline \
+    -root "$source_root" \
+    -llgo "$llgo_output" \
+    -out "$result_directory"
+)
+
+go_results="$result_directory/go.txt"
+: > "$go_results"
+(
+  cd "$source_root"
+  GOMAXPROCS=1 LLGO_ROOT="$source_root" go test \
+    -run '^$' \
+    -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \
+    -benchtime=250ms \
+    -count=5 \
+    -cpu=1 \
+    ./internal/clang ./internal/build/funcinfo
+) | tee -a "$go_results"
+
+(
+  cd "$source_root"
+  GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \
+    -run '^$' \
+    -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \
+    -benchtime=250ms \
+    -count=5 \
+    ./test/llgoext
+) | tee -a "$go_results"
+
+# The current native backend creates one pthread per goroutine and intentionally
+# has a bounded lifecycle stress limit. Keep creation monitoring deterministic
+# instead of letting testing auto-calibrate to millions of host threads.
+(
+  cd "$source_root"
+  GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \
+    -run '^$' \
+    -bench '^BenchmarkGoroutine$' \
+    -benchtime=100x \
+    -count=5 \
+    ./test/llgoext
+) | tee -a "$go_results"
+
+(
+  cd "$harness_root"
+  go run ./benchmark/baseline \
+    -mode export \
+    -out "$result_directory" \
+    -benchmark-output "$result_directory/benchmark.txt"
+)