diff --git a/.github/workflows/benchmark-publish.yml b/.github/workflows/benchmark-publish.yml index d78b6b0a11..d120948c4c 100644 --- a/.github/workflows/benchmark-publish.yml +++ b/.github/workflows/benchmark-publish.yml @@ -14,7 +14,7 @@ permissions: jobs: publish: if: github.event.workflow_run.conclusion == 'success' - uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1.0.4 + uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1.0.5 with: run_id: ${{ github.event.workflow_run.id }} config_path: .github/llgo-benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 68821b81fa..b323610e34 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,7 +25,7 @@ jobs: id: macos display: macOS runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 env: GOMAXPROCS: "2" LLGO_ROOT: ${{ github.workspace }} @@ -38,7 +38,16 @@ jobs: with: repository: ${{ github.event.pull_request.base.repo.full_name }} ref: ${{ github.event.pull_request.base.sha }} - path: .benchmark/source + path: .benchmark/base-source + persist-credentials: false + + - name: Check out pull request head benchmark source + if: github.event_name == 'pull_request' + uses: actions/checkout@v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: .benchmark/head-source persist-credentials: false - name: Install dependencies @@ -49,41 +58,34 @@ jobs: - name: Set up Go uses: ./.github/actions/setup-go - - name: Measure pull request base + - name: Measure paired pull request revisions if: github.event_name == 'pull_request' run: | benchmark/baseline/run.sh \ - "$GITHUB_WORKSPACE/.benchmark/source" \ + "$GITHUB_WORKSPACE/.benchmark/base-source" \ "$GITHUB_WORKSPACE/.benchmark/base-llgo" \ - "$GITHUB_WORKSPACE/.benchmark/base-results" - - - name: Check out pull request head benchmark source - if: github.event_name == 'pull_request' - uses: actions/checkout@v7 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - path: .benchmark/source - persist-credentials: false + "$GITHUB_WORKSPACE/.benchmark/base-results" \ + "$GITHUB_WORKSPACE/.benchmark/head-source" \ + "$GITHUB_WORKSPACE/.benchmark/llgo" \ + "$GITHUB_WORKSPACE/.benchmark/results" - name: Measure current revision + if: github.event_name != 'pull_request' run: | - source_root="$GITHUB_WORKSPACE" - if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then - source_root="$GITHUB_WORKSPACE/.benchmark/source" - fi benchmark/baseline/run.sh \ - "$source_root" \ + "$GITHUB_WORKSPACE" \ "$GITHUB_WORKSPACE/.benchmark/llgo" \ "$GITHUB_WORKSPACE/.benchmark/results" - name: Record benchmark result - uses: xgo-dev/setup-benchmark-go-action@v1.0.4 + uses: xgo-dev/setup-benchmark-go-action@v1.0.5 with: config: .github/llgo-benchmark.yml benchmark-file: .benchmark/results/benchmark.txt baseline-benchmark-file: >- ${{ github.event_name == 'pull_request' && '.benchmark/base-results/benchmark.txt' || '' }} + sample-pairing: >- + ${{ github.event_name == 'pull_request' && 'index' || '' }} platform-id: ${{ matrix.id }} platform-label: ${{ matrix.display }} diff --git a/benchmark/baseline/README.md b/benchmark/baseline/README.md index 32073268cf..0d70d3889e 100644 --- a/benchmark/baseline/README.md +++ b/benchmark/baseline/README.md @@ -1,7 +1,7 @@ # LLGo baseline benchmarks This suite is the lightweight performance gate for ordinary LLGo changes. It -uses fixed workloads and short calibrated benchmarks on Linux and macOS so it +uses fixed workloads and calibrated benchmarks on Linux and macOS so it can run on every `main` push and pull request. Branch-only series can be run explicitly with `workflow_dispatch`, avoiding duplicate push and pull-request jobs for the same commit. The two native jobs record normalized artifacts; a @@ -15,22 +15,33 @@ The program workloads reuse: - `benchmark/binary_size/fmtprintf`: `fmt.Printf`. For each workload, the collector performs an unmeasured warm build, then records -median build time, median process time, file size, executable-code bytes, -allocated non-executable data, and zero-filled data. On ELF, read-only constants -are included in the data bucket; on Mach-O, `__TEXT` constants are included in -the text bucket. The Go benchmark stream records five samples of selected -compiler helpers and LLGo-generated core-language operations: direct/interface -calls, defer, goroutine creation, channels, `getg`, and global access. +the median of six builds and eighteen process runs, file size, executable-code +bytes, allocated non-executable data, and zero-filled data. Workload order is +rotated between rounds to balance runner drift and cache position. On ELF, +read-only constants are included in the data bucket; on Mach-O, `__TEXT` +constants are included in the text bucket. The Go benchmark stream performs one +unrecorded warmup, then records seven one-second samples of compiler helpers and +LLGo-generated core-language operations: direct/interface calls, defer, +channels, `getg`, and global access. Goroutine creation keeps its bounded +100-iteration samples. -For pull requests, each platform job checks out the recorded base and current -commits into the same source path, then runs both suites sequentially on the same -runner. The pull request comment compares that pair, avoiding differences from -runner machines and embedded source paths. Dependency setup is shared, and Go's -build cache can be reused by unchanged packages; main pushes still run the suite -only once. Very small changes can still be scheduler, frequency, or thermal -noise and should be confirmed by repeated workflow runs. If a workflow does not -provide a paired result, the publisher falls back to the latest matching `main` -data. +The program table also includes standalone memory-profile workloads so the +whole-program no-consumer path is measurable independently of the retained +profile paths. Every process disables BDWGC, warms with two million allocations, +then internally times forty million escaping 16-byte allocations. The reported +duration is the median of eighteen processes. The workloads are +`memprofile-no-consumer`, `memprofile-rate0`, and `memprofile-default`. + +For pull requests, each platform job builds the recorded base and current +revisions on one runner, then alternates their measurements within every round. +Compiler and runtime benchmark binaries are built before sampling, so only their +execution is interleaved and each matching base/current sample remains adjacent. +This prevents a phase-wide frequency, thermal, or host load change from being +attributed entirely to one revision. Dependency setup is shared, and Go's build +cache can be reused by unchanged packages; main pushes still run the suite only +once. Very small changes can remain scheduler noise and should be confirmed by +repeated workflow runs. If a workflow does not provide a paired result, the +publisher falls back to the latest matching `main` data. The trusted publisher commits the current result history and generated site to the `pages` branch of the configured data repository. Every LLGo repository @@ -48,39 +59,13 @@ updated summary comment linking to their long-term trend page. If no matching `main` history exists yet, the pull-request report is still published and marks every metric as `new`. -Local collection: - -```sh -GOMAXPROCS=2 go build -o .benchmark/llgo ./cmd/llgo -go run ./benchmark/baseline \ - -llgo .benchmark/llgo \ - -out .benchmark/results -``` - -Write the selected Go benchmark output to `.benchmark/results/go.txt`: +Run the complete local collection with the same script as CI: ```sh -results=.benchmark/results/go.txt -GOMAXPROCS=1 go test \ - -run '^$' \ - -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \ - -benchtime=250ms -count=5 -cpu=1 \ - ./internal/clang ./internal/build/funcinfo | tee "$results" -GOMAXPROCS=1 .benchmark/llgo test \ - -run '^$' \ - -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \ - -benchtime=250ms -count=5 \ - ./test/llgoext | tee -a "$results" -GOMAXPROCS=1 .benchmark/llgo test \ - -run '^$' -bench '^BenchmarkGoroutine$' -benchtime=100x -count=5 \ - ./test/llgoext | tee -a "$results" +benchmark/baseline/run.sh \ + "$PWD" \ + "$PWD/.benchmark/llgo" \ + "$PWD/.benchmark/results" ``` -Then validate and export the complete artifact in standard Go benchmark format: - -```sh -go run ./benchmark/baseline \ - -mode export \ - -out .benchmark/results \ - -benchmark-output .benchmark/results/benchmark.txt -``` +The normalized artifact is `.benchmark/results/benchmark.txt`. diff --git a/benchmark/baseline/main.go b/benchmark/baseline/main.go index f9e4cbe19b..9c131de072 100644 --- a/benchmark/baseline/main.go +++ b/benchmark/baseline/main.go @@ -48,15 +48,37 @@ type metric struct { } type workload struct { - name string - source string - output string + name string + source string + output string + args []string + harnessSource bool + internalDuration bool } var workloads = []workload{ {name: "cprintf", source: "benchmark/binary_size/cprintf/main.go", output: "Hello, world\n"}, {name: "println", source: "benchmark/binary_size/println/main.go", output: "Hello, world\n"}, {name: "fmtprintf", source: "benchmark/binary_size/fmtprintf/main.go", output: "Hello, world\n"}, + { + name: "memprofile-no-consumer", + source: "benchmark/memprofile/noconsumer", + harnessSource: true, + internalDuration: true, + }, + { + name: "memprofile-rate0", + source: "benchmark/memprofile/enabled", + args: []string{"rate0"}, + harnessSource: true, + internalDuration: true, + }, + { + name: "memprofile-default", + source: "benchmark/memprofile/enabled", + harnessSource: true, + internalDuration: true, + }, } var expectedGoBenchmarks = []string{ @@ -74,7 +96,7 @@ var expectedGoBenchmarks = []string{ "BenchmarkRuntimeGetG", } -const goBenchmarkSamples = 5 +const goBenchmarkSamples = 7 type footprint struct { file uint64 @@ -95,12 +117,16 @@ func main() { func runCLI(ctx context.Context, args []string) error { flags := flag.NewFlagSet("llgo-baseline", flag.ContinueOnError) flags.SetOutput(io.Discard) - mode := flags.String("mode", "collect", "collect, validate, or export") + mode := flags.String("mode", "collect", "collect, collect-paired, validate, or export") root := flags.String("root", ".", "LLGo repository root") + harnessRoot := flags.String("harness-root", ".", "benchmark harness repository root") llgo := flags.String("llgo", "llgo", "LLGo command") out := flags.String("out", filepath.Join("benchmark", "baseline", "out"), "result directory") - buildRuns := flags.Int("build-runs", 3, "build repetitions per workload") - runRuns := flags.Int("run-runs", 7, "process repetitions per workload") + baseRoot := flags.String("base-root", "", "comparison LLGo repository root") + baseLLGo := flags.String("base-llgo", "", "comparison LLGo command") + baseOut := flags.String("base-out", "", "comparison result directory") + buildRuns := flags.Int("build-runs", 6, "build repetitions per workload") + runRuns := flags.Int("run-runs", 18, "process repetitions per workload") benchmarkOutput := flags.String( "benchmark-output", "", @@ -112,7 +138,18 @@ func runCLI(ctx context.Context, args []string) error { switch *mode { case "collect": - return collect(ctx, *root, *llgo, *out, *buildRuns, *runRuns) + return collectWithHarness(ctx, *root, *harnessRoot, *llgo, *out, *buildRuns, *runRuns) + case "collect-paired": + if *baseRoot == "" || *baseLLGo == "" || *baseOut == "" { + return errors.New("collect-paired mode requires base-root, base-llgo, and base-out") + } + return collectPaired( + ctx, + collectionSpec{name: "base", root: *baseRoot, harnessRoot: *harnessRoot, llgo: *baseLLGo, out: *baseOut}, + collectionSpec{name: "current", root: *root, harnessRoot: *harnessRoot, llgo: *llgo, out: *out}, + *buildRuns, + *runRuns, + ) case "validate": return validateArtifact(*out) case "export": @@ -196,51 +233,195 @@ func formatMetric(value float64) string { } func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int) error { + return collectWithHarness(ctx, root, root, llgo, out, buildRuns, runRuns) +} + +func collectWithHarness( + ctx context.Context, root, harnessRoot, llgo, out string, buildRuns, runRuns int, +) error { + return collectSpecs(ctx, []collectionSpec{{ + root: root, harnessRoot: harnessRoot, llgo: llgo, out: out, + }}, buildRuns, runRuns) +} + +type collectionSpec struct { + name string + root string + harnessRoot string + llgo string + out string +} + +type workloadMeasurement struct { + binary string + buildDurations []time.Duration + runDurations []time.Duration +} + +type collectionState struct { + collectionSpec + binDir string + env []string + measurements []workloadMeasurement +} + +func collectPaired( + ctx context.Context, base, current collectionSpec, buildRuns, runRuns int, +) error { + return collectSpecs(ctx, []collectionSpec{base, current}, buildRuns, runRuns) +} + +func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRuns int) error { if buildRuns <= 0 || runRuns <= 0 { return errors.New("build and run repetitions must be positive") } - root, err := filepath.Abs(root) - if err != nil { - return err + if len(specs) == 0 { + return errors.New("at least one collection is required") } - out, err = filepath.Abs(out) - if err != nil { - return err + states := make([]collectionState, len(specs)) + for index, spec := range specs { + root, err := filepath.Abs(spec.root) + if err != nil { + return err + } + harnessRoot, err := filepath.Abs(spec.harnessRoot) + if err != nil { + return err + } + out, err := filepath.Abs(spec.out) + if err != nil { + return err + } + binDir := filepath.Join(out, "bin") + if err := os.RemoveAll(out); err != nil { + return err + } + if err := os.MkdirAll(binDir, 0o755); err != nil { + return err + } + states[index] = collectionState{ + collectionSpec: collectionSpec{ + name: spec.name, root: root, harnessRoot: harnessRoot, llgo: spec.llgo, out: out, + }, + binDir: binDir, + env: append(os.Environ(), "GOMAXPROCS=2", "LLGO_ROOT="+root, "LLGO_FULL_RPATH=true"), + measurements: make([]workloadMeasurement, len(workloads)), + } } - binDir := filepath.Join(out, "bin") - if err := os.RemoveAll(out); err != nil { - return err + + // Warm every workload before measuring any of them so the first workload + // does not pay unique toolchain and filesystem cache costs. Paired + // collections alternate which revision runs first for adjacent workloads. + for workloadIndex, item := range workloads { + for stateOffset := range states { + state := &states[collectionStateIndex(0, workloadIndex, stateOffset, len(states))] + if err := buildWorkload(ctx, state, item); err != nil { + return collectionError(state, "warm build", item.name, err) + } + } } - if err := os.MkdirAll(binDir, 0o755); err != nil { - return err + + // Rotate both workload and revision order across rounds. In paired mode each + // base/current measurement is adjacent, so runner drift is not attributed to + // one complete revision phase. + for round := range buildRuns { + for offset := range workloads { + index := (round + offset) % len(workloads) + item := workloads[index] + for stateOffset := range states { + state := &states[collectionStateIndex(round, index, stateOffset, len(states))] + start := time.Now() + if err := buildWorkload(ctx, state, item); err != nil { + return collectionError(state, "build", item.name, err) + } + state.measurements[index].buildDurations = append( + state.measurements[index].buildDurations, time.Since(start), + ) + } + } } - env := append(os.Environ(), - "GOMAXPROCS=2", - "LLGO_ROOT="+root, - "LLGO_FULL_RPATH=true", - ) - var sizes, timings []metric - for _, item := range workloads { - binary := filepath.Join(binDir, item.name) - // Keep first-use toolchain and filesystem caches out of the measured - // median so the first revision is not systematically disadvantaged. - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(root, item.source)); err != nil { - return fmt.Errorf("warm build %s: %w", item.name, err) - } - buildDurations := make([]time.Duration, 0, buildRuns) - for range buildRuns { - start := time.Now() - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(root, item.source)); err != nil { - return fmt.Errorf("build %s: %w", item.name, err) + // Inspect final binaries and warm every execution path before timing. + for workloadIndex, item := range workloads { + for stateOffset := range states { + state := &states[collectionStateIndex(0, workloadIndex, stateOffset, len(states))] + binary := filepath.Join(state.binDir, item.name) + state.measurements[workloadIndex].binary = binary + if _, err := inspectExecutable(binary); err != nil { + return collectionError(state, "inspect", item.name, err) + } + if _, err := executeWorkload(ctx, state, item, binary); err != nil { + return collectionError(state, "execute", item.name, err) } - buildDurations = append(buildDurations, time.Since(start)) } - timings = append(timings, durationMetric("compile/"+item.name, buildDurations)) + } + + for round := range runRuns { + for offset := range workloads { + index := (round + offset) % len(workloads) + item := workloads[index] + for stateOffset := range states { + state := &states[collectionStateIndex(round, index, stateOffset, len(states))] + measurement := &state.measurements[index] + duration, err := executeWorkload(ctx, state, item, measurement.binary) + if err != nil { + return collectionError(state, "execute", item.name, err) + } + measurement.runDurations = append(measurement.runDurations, duration) + } + } + } + + for stateIndex := range states { + if err := writeCollection(&states[stateIndex]); err != nil { + return err + } + } + return nil +} + +func collectionStateIndex(round, workloadIndex, stateOffset, stateCount int) int { + return (round + workloadIndex + stateOffset) % stateCount +} + +func buildWorkload(ctx context.Context, state *collectionState, item workload) error { + binary := filepath.Join(state.binDir, item.name) + sourceRoot := state.root + if item.harnessSource { + sourceRoot = state.harnessRoot + } + return run(ctx, state.env, io.Discard, state.llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)) +} - size, err := inspectExecutable(binary) +func executeWorkload( + ctx context.Context, state *collectionState, item workload, binary string, +) (time.Duration, error) { + var output bytes.Buffer + start := time.Now() + if err := run(ctx, state.env, &output, binary, item.args...); err != nil { + return 0, err + } + duration := time.Since(start) + if item.internalDuration { + return parseInternalDuration(output.String()) + } + if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { + return 0, fmt.Errorf("output %q, want %q", got, item.output) + } + return duration, nil +} + +func writeCollection(state *collectionState) error { + var sizes, timings []metric + for index, item := range workloads { + measurement := &state.measurements[index] + timings = append(timings, + durationMetric("compile/"+item.name, measurement.buildDurations), + durationMetric("run/"+item.name, measurement.runDurations), + ) + size, err := inspectExecutable(measurement.binary) if err != nil { - return fmt.Errorf("inspect %s: %w", item.name, err) + return collectionError(state, "inspect", item.name, err) } sizes = append(sizes, byteMetric("binary/"+item.name+"/file", size.file), @@ -248,29 +429,28 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int byteMetric("binary/"+item.name+"/data", size.data), byteMetric("binary/"+item.name+"/bss", size.bss), ) + } + if err := writeMetrics(filepath.Join(state.out, "size.json"), sizes); err != nil { + return err + } + return writeMetrics(filepath.Join(state.out, "time.json"), timings) +} - var output bytes.Buffer - if err := run(ctx, env, &output, binary); err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) - } - if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { - return fmt.Errorf("execute %s: output %q, want %q", item.name, got, item.output) - } - runDurations := make([]time.Duration, 0, runRuns) - for range runRuns { - start := time.Now() - if err := run(ctx, env, io.Discard, binary); err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) - } - runDurations = append(runDurations, time.Since(start)) - } - timings = append(timings, durationMetric("run/"+item.name, runDurations)) +func collectionError(state *collectionState, operation, workload string, err error) error { + prefix := "" + if state.name != "" { + prefix = state.name + " " } + return fmt.Errorf("%s%s %s: %w", prefix, operation, workload, err) +} - if err := writeMetrics(filepath.Join(out, "size.json"), sizes); err != nil { - return err +func parseInternalDuration(output string) (time.Duration, error) { + value := strings.TrimSpace(output) + nanoseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || nanoseconds < 0 { + return 0, fmt.Errorf("invalid internal duration %q", value) } - return writeMetrics(filepath.Join(out, "time.json"), timings) + return time.Duration(nanoseconds), nil } func run(ctx context.Context, env []string, output io.Writer, name string, args ...string) error { @@ -298,7 +478,7 @@ func durationMetric(name string, values []time.Duration) metric { Value: median, Range: strconv.FormatInt(ordered[0].Nanoseconds(), 10) + ".." + strconv.FormatInt(ordered[len(ordered)-1].Nanoseconds(), 10), - Extra: fmt.Sprintf("median of %d consecutive runs", len(ordered)), + Extra: fmt.Sprintf("median of %d rotated runs", len(ordered)), } } diff --git a/benchmark/baseline/main_test.go b/benchmark/baseline/main_test.go index 2fe56a56e5..de6c5d834f 100644 --- a/benchmark/baseline/main_test.go +++ b/benchmark/baseline/main_test.go @@ -40,7 +40,7 @@ func TestDurationMetric(t *testing.T) { if got.Name != "compile/test" || got.Unit != "ns" || got.Value != 6 { t.Fatalf("durationMetric = %+v", got) } - if got.Range != "3..9" || got.Extra != "median of 3 consecutive runs" { + if got.Range != "3..9" || got.Extra != "median of 3 rotated runs" { t.Fatalf("duration metadata = %+v", got) } if !slices.Equal(values, []time.Duration{9, 3, 6}) { @@ -53,6 +53,18 @@ func TestDurationMetric(t *testing.T) { } } +func TestParseInternalDuration(t *testing.T) { + got, err := parseInternalDuration(" 123\n") + if err != nil || got != 123*time.Nanosecond { + t.Fatalf("parseInternalDuration = %v, %v", got, err) + } + for _, input := range []string{"", "not-a-duration", "-1"} { + if _, err := parseInternalDuration(input); err == nil { + t.Fatalf("parseInternalDuration(%q) unexpectedly succeeded", input) + } + } +} + func TestWriteAndValidateMetrics(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "metrics.json") @@ -97,6 +109,7 @@ func TestExportBenchmarks(t *testing.T) { "Unit file-bytes better=lower assume=exact", "Unit build-ns better=lower", "BenchmarkProgram/cprintf 1 1 file-bytes 1 text-bytes 1 data-bytes 1 bss-bytes 1 build-ns 1 run-ns", + "BenchmarkProgram/memprofile-no-consumer 1 1 file-bytes 1 text-bytes 1 data-bytes 1 bss-bytes 1 build-ns 1 run-ns", "BenchmarkRuntimeGetG-1 100 12.5 ns/op", } { if !strings.Contains(text, want) { @@ -293,7 +306,7 @@ func TestCollect(t *testing.T) { }) out := filepath.Join(root, "out") - if err := collect(context.Background(), root, fakeLLGo, out, 1, 1); err != nil { + if err := collect(context.Background(), root, fakeLLGo, out, 2, 2); err != nil { t.Fatal(err) } goText := makeGoBenchmarkText() @@ -305,6 +318,68 @@ func TestCollect(t *testing.T) { } } +func TestCollectPaired(t *testing.T) { + if os.PathSeparator != '/' { + t.Skip("fake compiler uses a POSIX shell") + } + root := t.TempDir() + baseRoot := filepath.Join(root, "base") + currentRoot := filepath.Join(root, "current") + if err := os.MkdirAll(baseRoot, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(currentRoot, 0o755); err != nil { + t.Fatal(err) + } + baseLLGo := writeFakeCompiler(t, baseRoot, "printf 'Hello, world\\n'") + currentLLGo := writeFakeCompiler(t, currentRoot, "printf 'Hello, world\\n'") + + oldInspect := inspectExecutable + inspectExecutable = func(path string) (footprint, error) { + info, err := os.Stat(path) + if err != nil { + return footprint{}, err + } + return footprint{file: uint64(info.Size()), text: 10, data: 2, bss: 1}, nil + } + t.Cleanup(func() { + inspectExecutable = oldInspect + }) + + baseOut := filepath.Join(root, "base-out") + currentOut := filepath.Join(root, "current-out") + err := collectPaired( + context.Background(), + collectionSpec{name: "base", root: baseRoot, harnessRoot: root, llgo: baseLLGo, out: baseOut}, + collectionSpec{name: "current", root: currentRoot, harnessRoot: root, llgo: currentLLGo, out: currentOut}, + 2, + 2, + ) + if err != nil { + t.Fatal(err) + } + for _, out := range []string{baseOut, currentOut} { + if err := os.WriteFile(filepath.Join(out, "go.txt"), []byte(makeGoBenchmarkText()), 0o644); err != nil { + t.Fatal(err) + } + if err := validateArtifact(out); err != nil { + t.Fatal(err) + } + } +} + +func TestCollectionStateIndexAlternatesEachWorkload(t *testing.T) { + for workloadIndex := range workloads { + first := collectionStateIndex(0, workloadIndex, 0, 2) + if second := collectionStateIndex(1, workloadIndex, 0, 2); second == first { + t.Fatalf("workload %d keeps state %d first across rounds", workloadIndex, first) + } + if peer := collectionStateIndex(0, workloadIndex, 1, 2); peer == first { + t.Fatalf("workload %d state order repeats %d within a round", workloadIndex, first) + } + } +} + func TestCollectRejectsInvalidRuns(t *testing.T) { err := collect(context.Background(), ".", "llgo", t.TempDir(), 0, 1) if err == nil || !strings.Contains(err.Error(), "must be positive") { @@ -456,6 +531,10 @@ func TestRunCLI(t *testing.T) { !strings.Contains(err.Error(), "unknown mode") { t.Fatalf("unknown CLI error = %v", err) } + if err := runCLI(context.Background(), []string{"-mode=collect-paired"}); err == nil || + !strings.Contains(err.Error(), "requires base-root") { + t.Fatalf("collect-paired CLI error = %v", err) + } if err := runCLI(context.Background(), []string{"-not-a-flag"}); err == nil { t.Fatal("runCLI unexpectedly accepted an unknown flag") } @@ -486,10 +565,20 @@ while [ "$#" -gt 0 ]; do *) shift ;; esac done -cat > "$out" <<'LLGO_BENCH_PROGRAM' +case "$out" in + *memprofile-*) + cat > "$out" <<'LLGO_BENCH_PROGRAM' +#!/bin/sh +printf '1\n' +LLGO_BENCH_PROGRAM + ;; + *) + cat > "$out" <<'LLGO_BENCH_PROGRAM' #!/bin/sh ` + program + ` LLGO_BENCH_PROGRAM + ;; +esac chmod +x "$out" ` return writeScript(t, filepath.Join(root, "llgo"), script) diff --git a/benchmark/baseline/run.sh b/benchmark/baseline/run.sh index c0ffdf02ec..0f7bf48fc9 100755 --- a/benchmark/baseline/run.sh +++ b/benchmark/baseline/run.sh @@ -2,73 +2,249 @@ set -euo pipefail -if [[ $# -ne 3 ]]; then - echo "usage: $0 " >&2 +if [[ $# -ne 3 && $# -ne 6 ]]; then + echo "usage: $0 [ ]" >&2 exit 2 fi harness_root="$(cd "$(dirname "$0")/../.." && pwd)" -source_root="$(cd "$1" && pwd)" -mkdir -p "$(dirname "$2")" "$3" -llgo_output="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" -result_directory="$(cd "$3" && pwd)" - -( - cd "$source_root" - LLGO_ROOT="$source_root" go build -p=1 -o "$llgo_output" ./cmd/llgo +benchmark_names=( + MergeCompilerFlags + MergeLinkerFlags + LookupPCRandom + RuntimeGetG + GlobalRead + GlobalWrite + DirectCall + InterfaceCall + Defer + ChannelBuffered + ChannelHandoff + Goroutine ) - -( - cd "$harness_root" - # Keep one current-checkout harness for both source revisions. Benchmark - # suite changes must therefore remain executable against the PR base. - LLGO_ROOT="$source_root" go run ./benchmark/baseline \ - -root "$source_root" \ - -llgo "$llgo_output" \ - -out "$result_directory" +benchmark_binaries=( + clang + clang + funcinfo + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext ) -go_results="$result_directory/go.txt" -: > "$go_results" -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" go test \ - -run '^$' \ - -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \ - -benchtime=250ms \ - -count=5 \ - -cpu=1 \ - ./internal/clang ./internal/build/funcinfo -) | tee -a "$go_results" - -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \ - -run '^$' \ - -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \ - -benchtime=250ms \ - -count=5 \ - ./test/llgoext -) | tee -a "$go_results" - -# The current native backend creates one pthread per goroutine and intentionally -# has a bounded lifecycle stress limit. Keep creation monitoring deterministic -# instead of letting testing auto-calibrate to millions of host threads. -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \ - -run '^$' \ - -bench '^BenchmarkGoroutine$' \ - -benchtime=100x \ - -count=5 \ - ./test/llgoext -) | tee -a "$go_results" - -( - cd "$harness_root" - go run ./benchmark/baseline \ - -mode export \ - -out "$result_directory" \ - -benchmark-output "$result_directory/benchmark.txt" -) +absolute_output() { + mkdir -p "$(dirname "$1")" + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +absolute_directory() { + mkdir -p "$1" + (cd "$1" && pwd) +} + +build_llgo() { + local source_root="$1" + local llgo_output="$2" + ( + cd "$source_root" + LLGO_ROOT="$source_root" go build -p=1 -o "$llgo_output" ./cmd/llgo + ) +} + +export_result() { + local result_directory="$1" + ( + cd "$harness_root" + go run ./benchmark/baseline \ + -mode export \ + -out "$result_directory" \ + -benchmark-output "$result_directory/benchmark.txt" + ) +} + +run_single() { + local source_root="$1" + local llgo_output="$2" + local result_directory="$3" + + build_llgo "$source_root" "$llgo_output" + ( + cd "$harness_root" + LLGO_ROOT="$source_root" go run ./benchmark/baseline \ + -root "$source_root" \ + -harness-root "$harness_root" \ + -llgo "$llgo_output" \ + -out "$result_directory" + ) + + local go_results="$result_directory/go.txt" + : > "$go_results" + + local binaries="$result_directory/tests" + build_benchmark_binaries "$source_root" "$llgo_output" "$binaries" + local index + for index in "${!benchmark_names[@]}"; do + local benchtime=1s + [[ "${benchmark_names[$index]}" != Goroutine ]] || benchtime=100x + run_benchmark_series \ + "$source_root" \ + "$binaries/${benchmark_binaries[$index]}.test" \ + "$go_results" \ + "^Benchmark${benchmark_names[$index]}$" \ + "$benchtime" + done + + export_result "$result_directory" +} + +build_benchmark_binaries() { + local source_root="$1" + local llgo_output="$2" + local binary_directory="$3" + mkdir -p "$binary_directory" + ( + cd "$source_root" + GOMAXPROCS=1 LLGO_ROOT="$source_root" go test -c \ + -o "$binary_directory/clang.test" ./internal/clang + GOMAXPROCS=1 LLGO_ROOT="$source_root" go test -c \ + -o "$binary_directory/funcinfo.test" ./internal/build/funcinfo + GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true \ + "$llgo_output" test -c -o "$binary_directory/llgoext.test" ./test/llgoext + ) +} + +run_benchmark_sample() { + local source_root="$1" + local binary="$2" + local pattern="$3" + local benchtime="$4" + local output="$5" + ( + cd "$source_root" + GOMAXPROCS=1 LLGO_ROOT="$source_root" "$binary" \ + -test.run '^$' \ + -test.bench "$pattern" \ + -test.benchtime "$benchtime" \ + -test.count=1 \ + -test.cpu=1 + ) | tee -a "$output" +} + +run_benchmark_series() { + local source_root="$1" + local binary="$2" + local output="$3" + local pattern="$4" + local benchtime="$5" + + run_benchmark_sample "$source_root" "$binary" "$pattern" "$benchtime" /dev/null + local round + for ((round = 0; round < 7; round++)); do + run_benchmark_sample "$source_root" "$binary" "$pattern" "$benchtime" "$output" + done +} + +run_benchmark_pair() { + local base_root="$1" + local base_binary="$2" + local base_output="$3" + local current_root="$4" + local current_binary="$5" + local current_output="$6" + local pattern="$7" + local benchtime="$8" + local pair_index="$9" + + # Warm both binaries without recording the result. Alternate the leading + # revision between groups, then alternate it again for every measured sample. + if (( pair_index % 2 == 0 )); then + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" /dev/null + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" /dev/null + else + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" /dev/null + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" /dev/null + fi + + local round + for ((round = 0; round < 7; round++)); do + if (( (round + pair_index) % 2 == 0 )); then + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" "$base_output" + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" "$current_output" + else + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" "$current_output" + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" "$base_output" + fi + done +} + +run_paired() { + local base_root="$1" + local base_llgo="$2" + local base_result="$3" + local current_root="$4" + local current_llgo="$5" + local current_result="$6" + + build_llgo "$base_root" "$base_llgo" + build_llgo "$current_root" "$current_llgo" + + # Keep one current-checkout harness for both revisions. Suite changes must + # therefore remain executable against the pull request base. + ( + cd "$harness_root" + go run ./benchmark/baseline \ + -mode collect-paired \ + -base-root "$base_root" \ + -base-llgo "$base_llgo" \ + -base-out "$base_result" \ + -root "$current_root" \ + -harness-root "$harness_root" \ + -llgo "$current_llgo" \ + -out "$current_result" + ) + + local base_binaries="$base_result/tests" + local current_binaries="$current_result/tests" + build_benchmark_binaries "$base_root" "$base_llgo" "$base_binaries" + build_benchmark_binaries "$current_root" "$current_llgo" "$current_binaries" + + local base_go="$base_result/go.txt" + local current_go="$current_result/go.txt" + : > "$base_go" + : > "$current_go" + + local index + for index in "${!benchmark_names[@]}"; do + local benchtime=1s + [[ "${benchmark_names[$index]}" != Goroutine ]] || benchtime=100x + local binary="${benchmark_binaries[$index]}.test" + run_benchmark_pair \ + "$base_root" "$base_binaries/$binary" "$base_go" \ + "$current_root" "$current_binaries/$binary" "$current_go" \ + "^Benchmark${benchmark_names[$index]}$" "$benchtime" "$index" + done + + export_result "$base_result" + export_result "$current_result" +} + +source_root="$(cd "$1" && pwd)" +llgo_output="$(absolute_output "$2")" +result_directory="$(absolute_directory "$3")" + +if [[ $# -eq 3 ]]; then + run_single "$source_root" "$llgo_output" "$result_directory" +else + current_source_root="$(cd "$4" && pwd)" + current_llgo_output="$(absolute_output "$5")" + current_result_directory="$(absolute_directory "$6")" + run_paired \ + "$source_root" "$llgo_output" "$result_directory" \ + "$current_source_root" "$current_llgo_output" "$current_result_directory" +fi diff --git a/benchmark/memprofile/enabled/main.go b/benchmark/memprofile/enabled/main.go new file mode 100644 index 0000000000..7276ca68b5 --- /dev/null +++ b/benchmark/memprofile/enabled/main.go @@ -0,0 +1,42 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + "runtime" + _ "unsafe" + + "github.com/xgo-dev/llgo/benchmark/memprofile/internal/runner" +) + +//go:linkname disableGC C.GC_disable +func disableGC() + +func main() { + if len(os.Args) == 2 && os.Args[1] == "rate0" { + runtime.MemProfileRate = 0 + } + // Keep the profiling consumer in this executable without including profile + // materialization in the timed allocation region. + runtime.MemProfile(nil, false) + disableGC() + elapsed := runner.Run() + println(elapsed.Nanoseconds()) +} diff --git a/benchmark/memprofile/internal/runner/runner.go b/benchmark/memprofile/internal/runner/runner.go new file mode 100644 index 0000000000..f9fe3d20aa --- /dev/null +++ b/benchmark/memprofile/internal/runner/runner.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runner + +import "time" + +const ( + iterations = 40_000_000 + warmup = 2_000_000 +) + +var sink [4096]*[16]byte + +//go:noinline +func allocate(n int) { + for i := 0; i < n; i++ { + sink[i&(len(sink)-1)] = new([16]byte) + } +} + +func Run() time.Duration { + allocate(warmup) + start := time.Now() + allocate(iterations) + return time.Since(start) +} diff --git a/benchmark/memprofile/noconsumer/main.go b/benchmark/memprofile/noconsumer/main.go new file mode 100644 index 0000000000..17cc3d9476 --- /dev/null +++ b/benchmark/memprofile/noconsumer/main.go @@ -0,0 +1,34 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + _ "unsafe" + + "github.com/xgo-dev/llgo/benchmark/memprofile/internal/runner" +) + +//go:linkname disableGC C.GC_disable +func disableGC() + +func main() { + disableGC() + elapsed := runner.Run() + println(elapsed.Nanoseconds()) +} diff --git a/cl/_testdata/method/in.go b/cl/_testdata/method/in.go index 70e2d04416..d9ae848268 100644 --- a/cl/_testdata/method/in.go +++ b/cl/_testdata/method/in.go @@ -25,7 +25,7 @@ func main() { // CHECK-LABEL: define i64 @"main.(*T).Add"(ptr %0, i64 %1){{.*}} { // CHECK: [[NIL:%[0-9]+]] = icmp eq ptr %0, null -// CHECK-NEXT: call void @"{{.*}}PanicWrapNilPointer"(i1 [[NIL]], %"{{.*}}String" { ptr @{{[0-9]+}}, i64 44 }, %"{{.*}}String" { ptr @{{[0-9]+}}, i64 3 }) +// CHECK-NEXT: call void @"{{.*}}PanicWrapNilPointer"(i1 [[NIL]], %"{{.*}}String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }, %"{{.*}}String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }) // CHECK-NEXT: [[RECEIVER:%[0-9]+]] = load i64, ptr %0 // CHECK-NEXT: [[WRAPPED_SUM:%[0-9]+]] = call i64 @main.T.Add(i64 [[RECEIVER]], i64 %1) // CHECK-NEXT: ret i64 [[WRAPPED_SUM]] diff --git a/cl/_testgo/localitycodegen/in.go b/cl/_testgo/localitycodegen/in.go index 100ef977ac..86eb136a9a 100644 --- a/cl/_testgo/localitycodegen/in.go +++ b/cl/_testgo/localitycodegen/in.go @@ -31,7 +31,8 @@ package main // CHECK-LABEL: define { i64, ptr, ptr } @main.values() // CHECK: call void @"main.__llgo_tls_init$ensure"() -// CHECK: load i64, ptr @main.scalar +// CHECK-NEXT: [[SCALAR_ADDR:%[0-9]+]] = call ptr @llvm.threadlocal.address{{.*}}(ptr @main.scalar) +// CHECK-NEXT: load i64, ptr [[SCALAR_ADDR]] // CHECK: call ptr @main.__llgo_local_block() // CHECK-LABEL: define ptr @"main._llgo_routine$1"(ptr diff --git a/cl/_testmeta/reflect_named/meta-expect.txt b/cl/_testmeta/reflect_named/meta-expect.txt index 31e4612586..60d47772f2 100644 --- a/cl/_testmeta/reflect_named/meta-expect.txt +++ b/cl/_testmeta/reflect_named/meta-expect.txt @@ -44,8 +44,8 @@ main.main: [UseIfaceMethod] main.main: - github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E - github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E + github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E + github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E [UseNamedMethod] main.main: @@ -61,7 +61,7 @@ _llgo_main.T: 1 main.m _llgo_func$2_iS07vIlF2_rZqWB5eU0IvP_9HviM4MYZNkXZDvbac main.(*T).m main.T.m [InterfaceInfo] -github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw: +github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY: Align _llgo_func$ETeB8WwW04JEq0ztcm-XPTJtuYvtpkjIsAc0-2NT9zA AssignableTo _llgo_func$Kxk9fspGkjXcoNWf2ucHG1vOQ5VHxVtYionfm-DnvWE Bits _llgo_func$ETeB8WwW04JEq0ztcm-XPTJtuYvtpkjIsAc0-2NT9zA @@ -101,6 +101,6 @@ github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFp PkgPath _llgo_func$zNDVRsWTIpUPKouNUS805RGX--IV9qVK8B31IZbg5to Size _llgo_func$1kITCsyu7hFLMxHLR7kDlvu4SOra_HtrtdFUQH9P13s String _llgo_func$zNDVRsWTIpUPKouNUS805RGX--IV9qVK8B31IZbg5to - reflect.common _llgo_func$w6XuV-1SmW103DbauPseXBpW50HpxXAEsUsGFibl0Uw - reflect.uncommon _llgo_func$iG49bujiXjI2lVflYdE0hPXlCAABL-XKRANSNJEKOio + reflect.common _llgo_func$_FtQJl7A5s1VI_ob-KR67FLMXbIG5TjcVaEFuqM5Lo4 + reflect.uncommon _llgo_func$tpkXXN4h9pcUPRsL9rq-8-w7qAx06dW7DgXtepA0U1E diff --git a/cl/_testrt/struct/in.go b/cl/_testrt/struct/in.go index 6fcd3cadd1..97fc4fd738 100644 --- a/cl/_testrt/struct/in.go +++ b/cl/_testrt/struct/in.go @@ -36,7 +36,7 @@ func main() { // CHECK-LABEL: define void @"main.(*Foo).Print"(ptr %0){{.*}} { // CHECK: [[WRAPPER_NIL:%[0-9]+]] = icmp eq ptr %0, null -// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.PanicWrapNilPointer"(i1 [[WRAPPER_NIL]], %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 44 }, %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 5 }) +// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.PanicWrapNilPointer"(i1 [[WRAPPER_NIL]], %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }, %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }) // CHECK-NEXT: [[WRAPPER_VALUE:%[0-9]+]] = load %main.Foo, ptr %0 // CHECK-NEXT: call void @main.Foo.Print(%main.Foo [[WRAPPER_VALUE]]) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index a7085718c9..4c4e2b98f1 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -4,6 +4,7 @@ package cl import ( + "fmt" "go/ast" "go/importer" "go/parser" @@ -14,6 +15,7 @@ import ( "github.com/goplus/gogen/packages" llssa "github.com/xgo-dev/llgo/ssa" + llabi "github.com/xgo-dev/llgo/ssa/abi" gossa "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" ) @@ -248,7 +250,7 @@ func plain() {} t.Fatal("Version should not be a runtime caller metadata function") } - rtpkg, _ := buildCallerFrameSSAPackage(t, "github.com/xgo-dev/llgo/runtime/internal/lib/runtime", `package runtime + rtpkg, _ := buildCallerFrameSSAPackage(t, llabi.PatchPathPrefix+"runtime", `package runtime func Caller(skip int) (uintptr, string, int, bool) { return 0, "", 0, false } func FuncForPC(pc uintptr) uintptr { return 0 } `) @@ -734,6 +736,35 @@ func leaf() {} } } +func TestCompileMemoryProfileAllocationPCLineMetadata(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/profileline", `package profileline + +var sink *int + +func init() { +//line profile_alloc.go:321 + sink = new(int) +} +`) + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprint(enabled), func(t *testing.T) { + prog := newLLSSAProgForTarget(t, &llssa.Target{GOOS: "linux", GOARCH: "amd64"}) + prog.EnableMemoryProfiling(enabled) + prog.EnableFuncInfoMetadata(true) + prog.EnableFuncInfoSites(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + got := strings.Contains(ir, "!llgo.pcline") && strings.Contains(ir, `!"profile_alloc.go"`) + if got != enabled { + t.Fatalf("allocation pcline present = %v, memory profiling = %v\n%s", got, enabled, ir) + } + }) + } +} + func TestCompileRuntimeCallerPCLineMetadata32Bit(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" @@ -1199,3 +1230,145 @@ func TestDirectiveFilename(t *testing.T) { t.Fatal("nil fset must pass through") } } + +// Packages that read the memory profile pin allocation paths, not unrelated +// helpers: per-site heap attribution needs only the observable stack. +func TestPackageReadsMemProfilePin(t *testing.T) { + ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/hp", `package main + +import "runtime" + +func allocLeaf() *[64]byte { return new([64]byte) } + +func allocWrapper() *[64]byte { return allocLeaf() } + +func plainHelper() int { return 1 } + +func main() { + runtime.MemProfileRate = 1 + _ = allocWrapper() + _ = plainHelper() + var r [4]runtime.MemProfileRecord + runtime.MemProfile(r[:], true) +} +`) + set := runtimeCallerFuncSet(NewCallerTracking(), ssapkg) + for _, name := range []string{"allocLeaf", "allocWrapper", "main"} { + if !set[ssapkg.Func(name)] { + t.Fatalf("%s must be pinned in a memprofile-reading package", name) + } + } + if set[ssapkg.Func("plainHelper")] { + t.Fatal("plainHelper must remain inlineable in a memprofile-reading package") + } + quiet, _ := buildCallerFrameSSAPackage(t, "example.com/quiet", `package q + +func Helper() int { return 2 } +`) + if set := runtimeCallerFuncSet(NewCallerTracking(), quiet); set[quiet.Func("Helper")] { + t.Fatal("quiet package must not be pinned") + } +} + +func TestMemoryProfileConvertMayAllocate(t *testing.T) { + stringType := types.Typ[types.String] + intType := types.Typ[types.Int] + tests := []struct { + name string + from, to types.Type + want bool + }{ + {"numeric", intType, types.Typ[types.Int64], false}, + {"string to bytes", stringType, types.NewSlice(types.Typ[types.Uint8]), true}, + {"runes to string", types.NewSlice(types.Typ[types.Int32]), stringType, true}, + {"rune to string", types.Typ[types.Int32], stringType, true}, + {"string to ints", stringType, types.NewSlice(intType), false}, + {"pointer", types.NewPointer(intType), types.Typ[types.UnsafePointer], false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := memoryProfileConvertMayAllocate(test.from, test.to); got != test.want { + t.Fatalf("memoryProfileConvertMayAllocate(%v, %v) = %v, want %v", test.from, test.to, got, test.want) + } + }) + } +} + +func TestPackageReadsMemProfileDetection(t *testing.T) { + tests := []struct { + name string + src string + want bool + }{ + { + name: "MemProfile call", + src: `package p +import "runtime" +func report() { + var records []runtime.MemProfileRecord + runtime.MemProfile(records, true) +}`, + want: true, + }, + { + name: "MemProfileRate write", + src: `package p +import "runtime" +func enable() { runtime.MemProfileRate = 1 }`, + want: true, + }, + { + name: "MemProfile function value", + src: `package p +import "runtime" +var report = runtime.MemProfile`, + want: true, + }, + { + name: "unrelated runtime use", + src: `package p +import "runtime" +func goos() string { return runtime.GOOS }`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkg, _ := buildCallerFrameSSAPackage(t, "example.com/"+tt.name, tt.src) + _, trackable := collectRuntimeCallerFunctions(pkg) + if got := packageReadsMemProfile(trackable); got != tt.want { + t.Fatalf("packageReadsMemProfile() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMemProfileConsumer(t *testing.T) { + quiet, _ := buildCallerFrameSSAPackage(t, "example.com/quiet", `package quiet +func Value() int { return 1 }`) + if got := MemProfileConsumer([]*gossa.Package{quiet}); got != "" { + t.Fatal("quiet program unexpectedly enabled memory profiling") + } + pprof, _ := buildCallerFrameSSAPackage(t, "runtime/pprof", `package pprof`) + if got := MemProfileConsumer([]*gossa.Package{quiet, pprof}); got != "runtime/pprof" { + t.Fatal("runtime/pprof did not enable memory profiling") + } +} + +func TestPackageReadsMemProfileSkipsFunctionWithoutPackage(t *testing.T) { + funcs := map[*gossa.Function]bool{new(gossa.Function): true} + if packageReadsMemProfile(funcs) { + t.Fatal("function without package metadata must not read the memory profile") + } +} + +func TestPublicRuntimePath(t *testing.T) { + for path, want := range map[string]bool{ + "runtime": true, + llabi.PatchPathPrefix + "runtime": true, + "runtime/debug": false, + } { + if got := isPublicRuntimePath(path); got != want { + t.Errorf("isPublicRuntimePath(%q) = %v, want %v", path, got, want) + } + } +} diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go index 4505862286..38ea8f3146 100644 --- a/cl/caller_tracking_precompute_test.go +++ b/cl/caller_tracking_precompute_test.go @@ -127,6 +127,35 @@ func Plain() { dep.Quiet() } } } +func TestCallerTrackingPrecomputePinsCrossPackageMemoryProfileAllocations(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +func Alloc() *[64]byte { return new([64]byte) } +func Plain() int { return 1 } +`, + "example.com/root", `package root +import ( + "example.com/dep" + "runtime" +) +func UseAlloc() *[64]byte { return dep.Alloc() } +func Plain() int { return dep.Plain() } +func Report(records []runtime.MemProfileRecord) { runtime.MemProfile(records, false) } +`) + tracking := NewCallerTracking() + tracking.SetMemoryProfileAttribution(true) + tracking.Precompute([]*gossa.Package{dep, root}) + if !runtimeCallerFuncSet(tracking, dep)[dep.Func("Alloc")] { + t.Fatal("cross-package allocation leaf was not pinned") + } + if !runtimeCallerFuncSet(tracking, root)[root.Func("UseAlloc")] { + t.Fatal("cross-package allocation wrapper was not pinned") + } + if runtimeCallerFuncSet(tracking, dep)[dep.Func("Plain")] || runtimeCallerFuncSet(tracking, root)[root.Func("Plain")] { + t.Fatal("unrelated cross-package path was pinned") + } +} + func TestCallerTrackingPrecomputeRejectsLatePackages(t *testing.T) { dep, root := buildCallerFrameSSAProgram(t, "example.com/dep", `package dep diff --git a/cl/compile.go b/cl/compile.go index 10be42368a..07776b402d 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -755,13 +755,13 @@ func needsRuntimeStackNoInline(pkg *types.Package, f *ssa.Function) bool { if pkg == nil || f == nil || f.Signature.Recv() != nil { return false } - switch pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + path := pkg.Path() + if isPublicRuntimePath(path) { switch f.Name() { case "Caller", "Callers", "callers": return true } - case "github.com/xgo-dev/llgo/runtime/internal/clite/debug": + } else if path == "github.com/xgo-dev/llgo/runtime/internal/clite/debug" { return f.Name() == "StackTrace" } return false @@ -1434,6 +1434,13 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue return } elem := p.type_(t.Elem(), llssa.InGo) + if v.Heap && p.prog.MemoryProfilingEnabled() { + // Heap allocations are memory-profile sample sites; give each + // one a statement anchor in tracked functions so sampled + // records attribute to the allocating line (heapsampling.go + // keys buckets by the leaf frame's exact line). + p.emitPCLineLabel(b, v.Pos()) + } ret = b.Alloc(elem, v.Heap) p.debugAlloc(b, v, ret) p.markRecoverSlot(v) @@ -1941,7 +1948,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr { } if p.options.DebugSymbols && p.localityAllowsGlobalDebug(v) { pos := p.fset.Position(v.Pos()) - b.DIGlobal(val, v.Name(), pos) + b.DIGlobal(p.localityGlobalDebugValue(v, val), v.Name(), pos) } return val case *ssa.Const: diff --git a/cl/instr.go b/cl/instr.go index 033afe39e7..0c1b313d55 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -31,6 +31,7 @@ import ( "golang.org/x/tools/go/ssa" llssa "github.com/xgo-dev/llgo/ssa" + llabi "github.com/xgo-dev/llgo/ssa/abi" ) var asmRegisterRegex = regexp.MustCompile(`\{[a-zA-Z]+\}`) @@ -907,7 +908,7 @@ func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { // runtimeCallerFuncSet is the per-package tracking set: functions that // must keep physical frames (noinline, no tail calls) and get statement -// anchors at their call sites. Five criteria feed it: +// anchors at their call sites. Six criteria feed it: // // 1. the function (transitively, within the package) reaches a // runtime.Caller/Callers call — it consumes caller pcs itself; @@ -923,6 +924,8 @@ func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { // 5. the function can run below a defer that consumes panic pcs — recover // exposes the panicked call chain after longjmp has removed those physical // frames, so the compiler must keep and annotate the possible callees. +// 6. the program reads the memory profile — functions on a path to an +// allocation keep physical frames for per-site attribution. // // Criterion 2 tests membership against the callee package's *base* set // (criterion 1 alone), so tracking extends exactly one call level past a @@ -952,7 +955,11 @@ func callerTrackingFuncSetsForPackage(c *CallerTracking, pkg *ssa.Package) calle } base := runtimeCallerBaseSet(c, pkg) funcs, trackable := collectRuntimeCallerFunctions(pkg) - sets := computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkg, funcs, base, trackable, func(dep *ssa.Package) map[*ssa.Function]bool { + var profileFrames map[*ssa.Function]bool + if packageReadsMemProfile(trackable) { + profileFrames = memoryProfileAllocationFrames(trackable) + } + sets := computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkg, funcs, base, trackable, profileFrames, func(dep *ssa.Package) map[*ssa.Function]bool { return runtimeCallerBaseSet(c, dep) }) c.extended[pkg] = sets @@ -964,15 +971,24 @@ type callerTrackingFuncSets struct { recoverPanicSites map[*ssa.Function]bool } -func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs, base, trackable map[*ssa.Function]bool, baseSet func(*ssa.Package) map[*ssa.Function]bool) callerTrackingFuncSets { +func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs, base, trackable, profileFrames map[*ssa.Function]bool, baseSet func(*ssa.Package) map[*ssa.Function]bool) callerTrackingFuncSets { frames := make(map[*ssa.Function]bool, len(base)) for fn := range base { frames[fn] = true } + // Criterion 6: retain only functions that can reach an allocation when the + // program reads the profile. This keeps allocation leaf and wrapper + // identities, including across packages, without disabling inlining for + // unrelated helpers. gc represents the same logical frames in its inline + // tree; LLGo currently keeps these selected physical frames. for fn := range trackable { if frames[fn] { continue } + if profileFrames[fn] { + frames[fn] = true + continue + } // Criterion 3: pin program-unique frames. main.main and package // init functions run once, so noinline is free — and they are the // bottom frames of almost every panic traceback, where an @@ -1009,6 +1025,95 @@ func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs return callerTrackingFuncSets{frames: frames, recoverPanicSites: recoverPanicSites} } +// memoryProfileAllocationFrames returns allocation-bearing functions and +// their static callers. Calls whose implementation is outside the analyzed +// function set are conservatively allocation-bearing: the external callee may +// use AllocZ/U, and the local wrapper is part of the exposed profile stack. +func memoryProfileAllocationFrames(funcs map[*ssa.Function]bool) map[*ssa.Function]bool { + frames := make(map[*ssa.Function]bool) + for fn := range funcs { + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if memoryProfileInstructionMayAllocate(instr, funcs) { + frames[fn] = true + break + } + } + if frames[fn] { + break + } + } + } + for changed := true; changed; { + changed = false + for fn := range funcs { + if frames[fn] { + continue + } + forEachCall(fn, func(call *ssa.CallCommon) { + if callee := call.StaticCallee(); callee != nil && frames[callee] { + frames[fn] = true + changed = true + } + }) + } + } + return frames +} + +func memoryProfileInstructionMayAllocate(instr ssa.Instruction, funcs map[*ssa.Function]bool) bool { + switch instr := instr.(type) { + case *ssa.Alloc: + return instr.Heap + case *ssa.MakeChan, *ssa.MakeClosure, *ssa.MakeInterface, *ssa.MakeMap, *ssa.MakeSlice, + *ssa.Defer, *ssa.Go, *ssa.MapUpdate, *ssa.Send: + return true + case *ssa.BinOp: + return instr.Op == token.ADD && types.Identical(instr.Type(), types.Typ[types.String]) + case *ssa.Convert: + return memoryProfileConvertMayAllocate(instr.X.Type(), instr.Type()) + case *ssa.MultiConvert: + // A type-parameter conversion can select an allocating string/slice + // conversion after instantiation. Keep it conservative until its type + // set is lowered to concrete alternatives here. + return true + case ssa.CallInstruction: + call := instr.Common() + if builtin, ok := call.Value.(*ssa.Builtin); ok { + return builtin.Name() == "append" + } + callee := call.StaticCallee() + return callee == nil || !funcs[callee] + } + return false +} + +func memoryProfileConvertMayAllocate(from, to types.Type) bool { + from = types.Unalias(from).Underlying() + to = types.Unalias(to).Underlying() + switch to := to.(type) { + case *types.Basic: + if to.Kind() != types.String { + return false + } + switch from := from.(type) { + case *types.Basic: + return from.Info()&types.IsInteger != 0 + case *types.Slice: + return isByteOrRune(from.Elem()) + } + case *types.Slice: + from, ok := from.(*types.Basic) + return ok && from.Kind() == types.String && isByteOrRune(to.Elem()) + } + return false +} + +func isByteOrRune(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && (basic.Kind() == types.Uint8 || basic.Kind() == types.Int32) +} + // addRecoverObservableCallees keeps the same-package synchronous call/defer // subtree below a defer that can inspect caller pcs. These frames are no // longer physically live when the deferred function runs after recover; @@ -1201,10 +1306,27 @@ func (a *runtimeCallerAnalysis) callTargets(fn *ssa.Function, call *ssa.CallComm // Precompute before workers start; recover facts also synchronize lazy queries // for nested and synthetic functions that are not package members. type CallerTracking struct { - base map[*ssa.Package]map[*ssa.Function]bool - extended map[*ssa.Package]callerTrackingFuncSets - recover *recoverFacts - precomputed bool + base map[*ssa.Package]map[*ssa.Function]bool + extended map[*ssa.Package]callerTrackingFuncSets + recover *recoverFacts + memoryProfileAttribution bool + memoryProfileConfigured bool + precomputed bool +} + +// SetMemoryProfileAttribution supplies the build-wide profiling decision. +// The build coordinator already computed it for allocator selection, so +// reusing it avoids a second program scan and covers externally callable +// library modes that have no visible profile consumer. +func (c *CallerTracking) SetMemoryProfileAttribution(enable bool) { + if c == nil { + return + } + if c.precomputed { + panic("memory-profile attribution configured after caller tracking") + } + c.memoryProfileAttribution = enable + c.memoryProfileConfigured = true } // Precompute resolves caller-tracking and recover data before package backends @@ -1255,8 +1377,22 @@ func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { analyses[i] = analyzeCallerTrackingPackage(pkgs[i], methods[pkgs[i]]) base[i] = analyses[i].base } + profileEnabled := c.memoryProfileAttribution + if !c.memoryProfileConfigured { + profileEnabled = MemProfileConsumer(pkgs) != "" + } + var profileFrames map[*ssa.Function]bool + if profileEnabled { + allTrackable := make(map[*ssa.Function]bool) + for i := range analyses { + for fn := range analyses[i].trackable { + allTrackable[fn] = true + } + } + profileFrames = memoryProfileAllocationFrames(allTrackable) + } for i := range pkgs { - extended[i] = computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkgs[i], analyses[i].funcs, base[i], analyses[i].trackable, func(dep *ssa.Package) map[*ssa.Function]bool { + extended[i] = computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkgs[i], analyses[i].funcs, base[i], analyses[i].trackable, profileFrames, func(dep *ssa.Package) map[*ssa.Function]bool { j, ok := index[dep] if !ok { panic("caller-tracking dependency was not precomputed") @@ -1357,6 +1493,108 @@ func NewCallerTracking() *CallerTracking { } } +// isPublicRuntimePath matches the public runtime package under both +// spellings: go/ssa unit builds see "runtime"; the real pipeline patches +// it to LLGo's implementation package. +func isPublicRuntimePath(path string) bool { + return path == "runtime" || + path == llabi.PatchPathPrefix+"runtime" +} + +func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { + // Cheap import pre-filter: scanning every instruction of every + // function costs real compile time across thousands of small + // packages (goroot shards); a package that never imports the public + // runtime cannot reference MemProfile. + imported := false + for fn := range funcs { + if fn.Pkg == nil || fn.Pkg.Pkg == nil { + continue + } + for _, imp := range fn.Pkg.Pkg.Imports() { + if isPublicRuntimePath(imp.Path()) { + imported = true + } + } + break + } + if !imported { + return false + } + for fn := range funcs { + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if call, ok := instr.(ssa.CallInstruction); ok { + if callee := call.Common().StaticCallee(); callee != nil && + callee.Pkg != nil && isPublicRuntimePath(callee.Pkg.Pkg.Path()) && + callee.Name() == "MemProfile" { + return true + } + } + rands := instr.Operands(nil) + for _, rand := range rands { + if rand == nil { + continue + } + switch value := (*rand).(type) { + case *ssa.Global: + if value.Pkg != nil && isPublicRuntimePath(value.Pkg.Pkg.Path()) && value.Name() == "MemProfileRate" { + return true + } + case *ssa.Function: + if value.Pkg != nil && isPublicRuntimePath(value.Pkg.Pkg.Path()) && value.Name() == "MemProfile" { + return true + } + } + } + } + } + } + return false +} + +// MemProfileConsumer returns the package that made whole-program allocation +// recording necessary, or an empty string when it is provably unused. +// It runs after Go SSA construction but before backend compilation, so the +// same decision applies to runtime and every dependency. Loading runtime/pprof +// itself is a consumer even though its runtime call uses go:linkname rather +// than a normal public-runtime reference. +func MemProfileConsumer(pkgs []*ssa.Package) string { + for _, pkg := range pkgs { + if pkg == nil || pkg.Pkg == nil { + continue + } + // The public runtime implementation wires MemProfileRate and the + // capture hook internally; those references exist in every program and + // are providers, not consumers. + if isPublicRuntimePath(pkg.Pkg.Path()) { + continue + } + if pkg.Pkg.Path() == "runtime/pprof" { + return pkg.Pkg.Path() + } + _, funcs := collectRuntimeCallerFunctions(pkg) + if packageReadsMemProfile(funcs) { + return pkg.Pkg.Path() + } + } + return "" +} + +func (p *context) omitMemProfileProviderCall(fn *ssa.Function) bool { + if p.prog.MemoryProfilingEnabled() || p.pkg == nil || fn == nil { + return false + } + path := p.pkg.Path() + switch { + case path == llssa.PkgRuntime: + return fn.Name() == "recordMemProfileAlloc" + case isPublicRuntimePath(path): + return fn.Name() == "installMemProfileHooks" + } + return false +} + func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { if fn == nil || fn.Parent() != nil { return false @@ -1823,10 +2061,10 @@ func isRuntimeCallerFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): return isRuntimeCallerName(fn.Name()) - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" default: return false @@ -1837,10 +2075,10 @@ func isRuntimeCallerFrameFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): return isRuntimeCallerFrameName(fn.Name()) - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" default: return false @@ -1851,13 +2089,13 @@ func isRuntimeCallerLookupFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): switch fn.Name() { case "Caller", "Callers", "Stack": return true } - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" } return false @@ -2446,6 +2684,12 @@ func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallComm args := p.compileValues(b, args, kind) ret = p.emitDo(b, act, ds, false, llssa.Builtin(fn), llssa.Builder.Call, args...) case *ssa.Function: + if p.omitMemProfileProviderCall(cv) { + // These provider calls have no result. Evaluate arguments for + // completeness, then omit the call itself even at O0. + p.compileValues(b, args, kind) + return + } aFn, pyFn, ftype := p.compileFunction(cv) // TODO(xsw): check ca != llssa.Call switch ftype { diff --git a/cl/locality_lower.go b/cl/locality_lower.go index 1825ec1d11..7e0a2e535d 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -63,6 +63,11 @@ type localEnsureCacheKey struct { kind locality.Kind } +type localTLSCacheKey struct { + block *ssa.BasicBlock + variable *localVariable +} + // localityLowering owns all compiler state for TLS/GLS lowering. Only this // value is embedded in the general compiler context. type localityLowering struct { @@ -75,6 +80,7 @@ type localityFunction struct { block *ssa.BasicBlock packageBases map[localBaseCacheKey]llssa.Expr packageEnsures map[localEnsureCacheKey]bool + tlsAddresses map[localTLSCacheKey]llssa.Expr entry *localEntryContext } @@ -180,6 +186,21 @@ func (p *context) localityAllowsGlobalDebug(global *ssa.Global) bool { return variable == nil || variable.planned.Storage == localitylayout.StorageNativeTLS } +// localityGlobalDebugValue keeps debug metadata on the native TLS global. +// Normal accesses use llvm.threadlocal.address instructions, which are values +// rather than GlobalObjects and therefore cannot carry DIGlobal metadata. +func (p *context) localityGlobalDebugValue(global *ssa.Global, value llssa.Expr) llssa.Expr { + variable := p.locality.variables[global] + if variable == nil || variable.planned.Storage != localitylayout.StorageNativeTLS { + return value + } + direct := variable.owner.direct[variable.planned.Name] + if direct == nil { + panic(fmt.Sprintf("missing native TLS storage for %s", variable.planned.Name)) + } + return direct.Expr +} + func (p *context) localTypesPackage(fullName string) *types.Package { matches := func(pkg *types.Package) bool { if pkg == nil { @@ -345,7 +366,26 @@ func (p *context) localVariableAddr(b llssa.Builder, v *ssa.Global, info llssa.V if direct == nil { panic(fmt.Sprintf("missing native TLS storage for %s", name)) } - return direct.Expr + // Metadata probes have no instruction builder. Real lowering + // materializes and dominance-caches the address so Darwin TLV and ELF + // TLS code generation resolve it only once along a control-flow path. + if b == nil { + return direct.Expr + } + state := &p.locality.function + for block := state.block; block != nil; block = block.Idom() { + if addr, ok := state.tlsAddresses[localTLSCacheKey{block: block, variable: variable}]; ok { + return addr + } + } + addr := b.ThreadLocalAddress(direct) + if state.block != nil { + if state.tlsAddresses == nil { + state.tlsAddresses = make(map[localTLSCacheKey]llssa.Expr) + } + state.tlsAddresses[localTLSCacheKey{block: state.block, variable: variable}] = addr + } + return addr } base := p.localPackageBase(b, variable.owner) return b.FieldAddr(base, variable.planned.Field) diff --git a/cl/locality_test.go b/cl/locality_test.go index f95f765c74..f953f74ed2 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -129,6 +129,24 @@ func value() *int { return pointer } } } +func TestNativeTLSAddressIsReusedWithinDominatingPath(t *testing.T) { + _, ir := compileLocalitySource(t, `package locality + +//llgo:tls +var counter int + +func add() int { + counter++ + counter++ + return counter +} +`) + add := llvmFunction(t, ir, "example.com/locality.add") + if got := strings.Count(add, "@llvm.threadlocal.address"); got != 1 { + t.Fatalf("native TLS address resolutions = %d, want one:\n%s", got, add) + } +} + func llvmFunction(t *testing.T, ir, name string) string { t.Helper() markerAt := strings.Index(ir, `@"`+name+`"(`) diff --git a/internal/build/build.go b/internal/build/build.go index e7ec8f3abf..01792b82c5 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -195,6 +195,10 @@ type Config struct { // for float-to-uint32 conversions. SaturatingFloatToUint32 bool + // memoryProfiling is derived from whole-program SSA before package cache + // lookup. Library build modes set it conservatively. + memoryProfiling bool + // PthreadStackSize sets a custom stack size, in bytes, for pthread-backed // goroutines. A zero value keeps the platform pthread default. PthreadStackSize int64 @@ -679,6 +683,20 @@ func Build(inv Invocation) ([]Package, error) { return nil, err } buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) + memProfileConsumer := cl.MemProfileConsumer(progSSA.AllPackages()) + conf.memoryProfiling = enableMemoryProfiling(conf.BuildMode, memProfileConsumer) + prog.EnableMemoryProfiling(conf.memoryProfiling) + if verbose { + reason := memProfileConsumer + if conf.BuildMode != BuildModeExe { + reason = string(conf.BuildMode) + } + if reason == "" { + reason = "not reachable" + } + fmt.Fprintf(os.Stderr, "memory profiling: %t (%s)\n", conf.memoryProfiling, reason) + } + ctx.callerTracking.SetMemoryProfileAttribution(conf.memoryProfiling) callerSpan := buildTrace.startCoordinator("precompute caller tracking", nil) ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) callerSpan.done() @@ -799,6 +817,10 @@ func Build(inv Invocation) ([]Package, error) { return allPkgs, nil } +func enableMemoryProfiling(mode BuildMode, consumer string) bool { + return mode != BuildModeExe || consumer != "" +} + // cHeaderPackages excludes the patched standard runtime implementation. Its // //export callbacks are linker implementation details and may use internal C // types that are deliberately not representable in a public generated header. diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 06e49164e4..f8c5a412d5 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -1340,12 +1340,13 @@ func TestCHeaderPackagesExcludesStandardRuntime(t *testing.T) { userLPkg := prog.NewPackage("example.com/p", "example.com/p") userLPkg.SetExport("example.com/p.Export", "Export") runtimeLPkg := prog.NewPackage("runtime", "runtime") - llgoRuntimeLPkg := prog.NewPackage("github.com/xgo-dev/llgo/runtime/internal/lib/runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime") + llgoRuntimePath := altPkgPathPrefix + "runtime" + llgoRuntimeLPkg := prog.NewPackage(llgoRuntimePath, llgoRuntimePath) dependencyLPkg := prog.NewPackage("example.com/dep", "example.com/dep") pkgs := []*aPackage{ {Package: &packages.Package{PkgPath: "example.com/p"}, LPkg: userLPkg}, {Package: &packages.Package{PkgPath: "runtime"}, LPkg: runtimeLPkg}, - {Package: &packages.Package{PkgPath: "github.com/xgo-dev/llgo/runtime/internal/lib/runtime"}, LPkg: llgoRuntimeLPkg}, + {Package: &packages.Package{PkgPath: llgoRuntimePath}, LPkg: llgoRuntimeLPkg}, {Package: &packages.Package{PkgPath: "example.com/dep"}, LPkg: dependencyLPkg}, nil, } diff --git a/internal/build/collect.go b/internal/build/collect.go index ea08d32fea..b212bf1afc 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -29,6 +29,7 @@ import ( "github.com/xgo-dev/llgo/internal/env" "github.com/xgo-dev/llgo/internal/meta" "github.com/xgo-dev/llgo/internal/packages" + llssa "github.com/xgo-dev/llgo/ssa" gopackages "golang.org/x/tools/go/packages" ) @@ -166,6 +167,9 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error m.pkg.PkgPath = p.PkgPath m.pkg.PkgID = p.ID + if p.PkgPath == llssa.PkgRuntime { + m.pkg.MemoryProfiling = c.buildConf.memoryProfiling + } // Go source files goFilesList, err := digestFilesWithOverlay(p.GoFiles, c.buildConf.Overlay) diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index f6c824ba28..312e39c59f 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -140,16 +140,17 @@ func (s *commonSection) empty() bool { } type packageSection struct { - PkgPath string `yaml:"pkg_path,omitempty"` - PkgID string `yaml:"pkg_id,omitempty"` - GoFiles []fileDigest `yaml:"go_files,omitempty"` - AltGoFiles []fileDigest `yaml:"alt_go_files,omitempty"` - OtherFiles []fileDigest `yaml:"other_files,omitempty"` - RewriteVars orderedStringMap `yaml:"rewrite_vars,omitempty"` + PkgPath string `yaml:"pkg_path,omitempty"` + PkgID string `yaml:"pkg_id,omitempty"` + MemoryProfiling bool `yaml:"memory_profiling,omitempty"` + GoFiles []fileDigest `yaml:"go_files,omitempty"` + AltGoFiles []fileDigest `yaml:"alt_go_files,omitempty"` + OtherFiles []fileDigest `yaml:"other_files,omitempty"` + RewriteVars orderedStringMap `yaml:"rewrite_vars,omitempty"` } func (s *packageSection) empty() bool { - return s.PkgPath == "" && s.PkgID == "" && len(s.GoFiles) == 0 && len(s.AltGoFiles) == 0 && len(s.OtherFiles) == 0 && len(s.RewriteVars) == 0 + return s.PkgPath == "" && s.PkgID == "" && !s.MemoryProfiling && len(s.GoFiles) == 0 && len(s.AltGoFiles) == 0 && len(s.OtherFiles) == 0 && len(s.RewriteVars) == 0 } // manifestBuilder builds manifest text with sorted sections. diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go index 3947925bed..50fbf0890f 100644 --- a/internal/build/fingerprint_test.go +++ b/internal/build/fingerprint_test.go @@ -174,6 +174,22 @@ func TestManifestBuilder_SaturatingFloatToUint32(t *testing.T) { } } +func TestManifestBuilder_MemoryProfiling(t *testing.T) { + plain := newManifestBuilder() + profiled := newManifestBuilder() + profiled.pkg.MemoryProfiling = true + if plain.Fingerprint() == profiled.Fingerprint() { + t.Fatal("memory profiling did not change the package fingerprint") + } + data, err := decodeManifest(profiled.Build()) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Package == nil || !data.Package.MemoryProfiling { + t.Fatalf("memory profiling missing from manifest: %+v", data.Package) + } +} + func TestManifestBuilder_EmptySections(t *testing.T) { m := newManifestBuilder() content := m.Build() diff --git a/internal/build/module_hook_test.go b/internal/build/module_hook_test.go index 99d2bde828..628416f4a7 100644 --- a/internal/build/module_hook_test.go +++ b/internal/build/module_hook_test.go @@ -4,7 +4,12 @@ package build import ( + "os" + "path/filepath" + "strings" "testing" + + llssa "github.com/xgo-dev/llgo/ssa" ) func TestModuleHookReceivesMainPackageModule(t *testing.T) { @@ -35,3 +40,103 @@ func TestModuleHookReceivesMainPackageModule(t *testing.T) { t.Fatalf("expected non-empty module snapshot for %s", mainPkg) } } + +func TestMemoryProfileConsumerSelectsAllocatorInstrumentation(t *testing.T) { + cacheDir := t.TempDir() + oldCacheRoot := cacheRootFunc + cacheRootFunc = func() string { return cacheDir } + defer func() { cacheRootFunc = oldCacheRoot }() + + plain := memoryProfileProviderIR(t, `package main +import "runtime" +func main() { println(runtime.GOOS) } +`) + if strings.Contains(plain.allocator, "recordMemProfileAlloc") { + t.Fatalf("plain executable allocator retained memory profiling:\n%s", plain.allocator) + } + if hasMemProfileHookInstall(plain.publicRuntime) { + t.Fatalf("plain executable runtime installed memory-profile hooks:\n%s", plain.publicRuntime) + } + + profiled := memoryProfileProviderIR(t, `package main +import "runtime" +func main() { runtime.MemProfile(nil, false) } +`) + if !strings.Contains(profiled.allocator, "recordMemProfileAlloc") { + t.Fatalf("memory-profile consumer allocator lost recording:\n%s", profiled.allocator) + } + if !hasMemProfileHookInstall(profiled.publicRuntime) { + t.Fatalf("memory-profile consumer runtime lost hook installation:\n%s", profiled.publicRuntime) + } +} + +func TestMemoryProfileLibraryModeSelection(t *testing.T) { + for _, mode := range []BuildMode{BuildModeCArchive, BuildModeCShared} { + if !enableMemoryProfiling(mode, "") { + t.Errorf("%s build disabled externally callable memory profiling", mode) + } + } + if enableMemoryProfiling(BuildModeExe, "") { + t.Error("plain executable enabled memory profiling") + } + if !enableMemoryProfiling(BuildModeExe, "runtime/pprof") { + t.Error("profile consumer did not enable executable memory profiling") + } +} + +type memoryProfileProviders struct { + allocator string + publicRuntime string +} + +func memoryProfileProviderIR(t *testing.T, source string) memoryProfileProviders { + t.Helper() + dir := t.TempDir() + mainFile := filepath.Join(dir, "main.go") + if err := os.WriteFile(mainFile, []byte(source), 0o644); err != nil { + t.Fatal(err) + } + conf := NewDefaultConf(ModeGen) + var providers memoryProfileProviders + conf.ModuleHook = func(pkg Package) { + if pkg.PkgPath == "runtime" || pkg.PkgPath == altPkgPathPrefix+"runtime" { + providers.publicRuntime = pkg.LPkg.String() + } + if pkg.PkgPath != llssa.PkgRuntime { + return + } + ir := pkg.LPkg.String() + marker := `define ptr @"` + llssa.PkgRuntime + `.AllocZ"(` + start := strings.Index(ir, marker) + if start < 0 { + return + } + end := strings.Index(ir[start:], "\n}") + if end >= 0 { + providers.allocator = ir[start : start+end+2] + } + } + pkgs, err := Do([]string{mainFile}, conf) + if err != nil { + t.Fatalf("generate memory-profile allocator IR: %v", err) + } + if len(pkgs) == 1 && pkgs[0].LPkg != nil { + defer pkgs[0].LPkg.Prog.Dispose() + } + if providers.allocator == "" { + t.Fatal("runtime AllocZ module was not observed") + } + if providers.publicRuntime == "" { + t.Fatal("public runtime module was not observed") + } + return providers +} + +func hasMemProfileHookInstall(ir string) bool { + for line := range strings.SplitSeq(ir, "\n") { + if strings.Contains(line, "call void") && strings.Contains(line, ".installMemProfileHooks(") { + return true + } + } + return false +} diff --git a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go index 172930c92b..8a84dfa437 100644 --- a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go +++ b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go @@ -2,7 +2,11 @@ package runtime -import _ "unsafe" +import ( + _ "unsafe" + + llrt "github.com/xgo-dev/llgo/runtime/internal/runtime" +) type pprofMemProfileRecord struct { AllocBytes, FreeBytes int64 @@ -12,6 +16,12 @@ type pprofMemProfileRecord struct { //go:linkname pprof_memProfileInternal runtime.pprof_memProfileInternal func pprof_memProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, ok bool) { + previous := llrt.MemProfilePause() + defer llrt.MemProfileResume(previous) + return pprofMemProfileInternal(p, inuseZero) +} + +func pprofMemProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, ok bool) { n, _ = MemProfile(nil, inuseZero) if len(p) < n { return n, false @@ -19,12 +29,17 @@ func pprof_memProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, if n == 0 { return 0, true } - var records [64]MemProfileRecord - if n > len(records) { - return n, false + // Size dynamically with slack and retry: a fixed cap makes pprof's + // retry-until-ok loop spin forever once the bucket set outgrows it. + records := make([]MemProfileRecord, n+n/4+16) + for attempt := 0; ; attempt++ { + n, ok = MemProfile(records, inuseZero) + if ok || attempt >= 3 { + break + } + records = make([]MemProfileRecord, n+n/4+16) } - n, ok = MemProfile(records[:n], inuseZero) - if !ok { + if !ok || len(p) < n { return n, false } for i := 0; i < n; i++ { diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 3a7f615a2b..83a0615840 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -29,6 +29,15 @@ type MemProfileRecord struct { Stack0 [32]uintptr } +// MemProfile reuses its caller's record storage for the internal snapshot. +// Keep that zero-copy conversion guarded if either definition changes. +var ( + _ [unsafe.Sizeof(MemProfileRecord{}) - unsafe.Sizeof(llrt.MemProfileRecord{})]byte + _ [unsafe.Sizeof(llrt.MemProfileRecord{}) - unsafe.Sizeof(MemProfileRecord{})]byte + _ [unsafe.Offsetof(MemProfileRecord{}.Stack0) - unsafe.Offsetof(llrt.MemProfileRecord{}.Stack0)]byte + _ [unsafe.Offsetof(llrt.MemProfileRecord{}.Stack0) - unsafe.Offsetof(MemProfileRecord{}.Stack0)]byte +) + func (r *MemProfileRecord) InUseBytes() int64 { return r.AllocBytes - r.FreeBytes } @@ -53,30 +62,53 @@ type BlockProfileRecord struct { Stack []uintptr } -func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { - n, _ = llrt.MemProfile(nil, inuseZero) - if len(p) < n { - return n, false +// trimMemProfileStack drops the allocator/runtime plumbing the physical +// capture recorded above the allocation site (AllocZ, the capture path +// itself) so record stacks start at user code like gc's. +func trimMemProfileStack(stk [32]uintptr) [32]uintptr { + i := 0 + for i < len(stk) && stk[i] != 0 { + if !isRuntimePlumbingFrame(stk[i]) { + break + } + i++ } - if n == 0 { - return 0, true + if i == 0 { + return stk } - var records [64]llrt.MemProfileRecord - if n > len(records) { - return n, false + var out [32]uintptr + copy(out[:], stk[i:]) + return out +} + +// isRuntimePlumbingFrame reports whether pc belongs to LLGo runtime +// plumbing (allocator and capture hooks). +func isRuntimePlumbingFrame(pc uintptr) bool { + name := frameSymbol(pc - 1).function + if name == "" { + return false } - n, ok = llrt.MemProfile(records[:n], inuseZero) + return hasPrefix(name, "github.com/xgo-dev/llgo/runtime/internal/") || + name == "runtime.captureMemProfileStack" +} + +func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + previous := llrt.MemProfilePause() + defer llrt.MemProfileResume(previous) + return memProfile(p, inuseZero) +} + +func memProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + // The public and core records deliberately have the same fixed layout. + // Reuse the caller buffer so reading a rate-1 profile does not recursively + // allocate progressively larger profile buffers and create more buckets. + records := unsafe.Slice((*llrt.MemProfileRecord)(unsafe.Pointer(unsafe.SliceData(p))), len(p)) + n, ok = llrt.MemProfile(records, inuseZero) if !ok { return n, false } for i := 0; i < n; i++ { - p[i] = MemProfileRecord{ - AllocBytes: records[i].AllocBytes, - FreeBytes: records[i].FreeBytes, - AllocObjects: records[i].AllocObjects, - FreeObjects: records[i].FreeObjects, - Stack0: records[i].Stack0, - } + p[i].Stack0 = trimMemProfileStack(p[i].Stack0) } return n, true } diff --git a/runtime/internal/lib/runtime/string_llgo.go b/runtime/internal/lib/runtime/string_llgo.go new file mode 100644 index 0000000000..034275bdb2 --- /dev/null +++ b/runtime/internal/lib/runtime/string_llgo.go @@ -0,0 +1,7 @@ +//go:build !baremetal + +package runtime + +func hasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} diff --git a/runtime/internal/lib/runtime/trace_stub_llgo.go b/runtime/internal/lib/runtime/trace_stub_llgo.go index 35f5512b3a..28aa8fed4d 100644 --- a/runtime/internal/lib/runtime/trace_stub_llgo.go +++ b/runtime/internal/lib/runtime/trace_stub_llgo.go @@ -11,7 +11,7 @@ func traceAdvance(stopTrace bool) {} func traceClockNow() uint64 { return 0 } //go:linkname runtime_readTrace runtime/trace.runtime_readTrace -func runtime_readTrace() []byte { return nil } +func runtime_readTrace() []byte { return ReadTrace() } //go:linkname trace_userTaskCreate runtime/trace.userTaskCreate func trace_userTaskCreate(id, parentID uint64, taskType string) {} diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index bdd3c15d73..0b24e040fa 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -3,6 +3,7 @@ package runtime import ( + latomic "sync/atomic" "unsafe" rtdebug "github.com/xgo-dev/llgo/runtime/internal/runtime" @@ -19,6 +20,18 @@ func init() { rtdebug.PanicRecovered = clearFaultTraceback rtdebug.PanicPCSnapshot = capturePanicPCs rtdebug.RecoverMark = recoverMark + installMemProfileHooks() +} + +// installMemProfileHooks is a separate call so whole-program executable +// builds that do not consume memory profiles can omit the complete setup and +// capture path without changing runtime APIs. +func installMemProfileHooks() { + // Table initialization may allocate. Complete it before installing the + // allocator hook so the first sample never initializes it from AllocZ/U. + initRuntimeFuncPCFrames() + rtdebug.MemProfileStackCapture = captureMemProfileStack + rtdebug.MemProfileRatePtr = &MemProfileRate } // recoverMark records the recovering deferred frame (and one above, for @@ -50,6 +63,22 @@ func capturePanicPCs() { rtdebug.StorePanicPCs(pcs[:n]) } +// captureMemProfileStack walks the physical stack at a sampled allocation. +// The leading allocator plumbing (AllocZ/AllocU, this capture path) is +// trimmed at read time by the MemProfile wrapper, where symbolization is +// safe and cached. +func captureMemProfileStack(pcs []uintptr) int { + if !fpUnwindAvailable() { + return 0 + } + // Defensively avoid waiting on the frame-table init latch if future init + // ordering changes. The table is normally ready before this hook is set. + if latomic.LoadUint32(&runtimeFuncPCInitState) == runtimeFuncInfoInitBusy { + return 0 + } + return fpCallers(0, pcs) +} + // panicSplicePCs returns the snapshot when it is observable: either the // panic is still in flight, or the deferred frame that recovered it is // still live on the physical chain (gc keeps panic frames on the stack @@ -222,10 +251,6 @@ func callersWithPanicSplice(skip int, pc []uintptr) int { return copy(pc, view[skip:]) } -func hasPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && s[:len(prefix)] == prefix -} - // panicTraceback prints a Go-style stack trace for an unrecovered panic: // one "function(...)" line plus an indented file:line per physical frame, // matching the shape of runtime.Stack and gc's panic output. Reports false diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go index c6221ca1dd..e393927d11 100644 --- a/runtime/internal/runtime/memprofile.go +++ b/runtime/internal/runtime/memprofile.go @@ -1,6 +1,10 @@ package runtime -// MemProfileRecord describes allocations aggregated by size class. +import ( + "unsafe" +) + +// MemProfileRecord describes allocations aggregated into one profile bucket. type MemProfileRecord struct { AllocBytes, FreeBytes int64 AllocObjects, FreeObjects int64 @@ -60,18 +64,115 @@ var memProfileBuckets = [...]memProfileBucket{ {size: 1073741824}, } -func recordMemProfileAlloc(size uintptr) { - if size == 0 { +// MemProfileStackCapture, set by the public runtime package, walks the +// physical stack at a sampled allocation. When present, MemProfile reports +// stack-keyed buckets (gc semantics: heapsampling.go attributes sampled +// bytes to call stacks); without it the legacy size-class counters remain +// (baremetal, wasm). +var MemProfileStackCapture func(pcs []uintptr) int + +// MemProfileRatePtr points at the public runtime.MemProfileRate variable +// (user-settable at any time, read per sampling decision). +var MemProfileRatePtr *int + +type memStackBucket struct { + next *memStackBucket + hash uintptr + size uintptr + allocBytes memProfileCounter + allocObjects memProfileCounter + nstk int32 + stk [32]uintptr +} + +type memProfileThreadState struct { + remaining uintptr + rand uint64 +} + +const memStackTabSize = 512 // power of two + +var ( + memStackTab [memStackTabSize]memProfileBucketHead + memStackTabLock memProfileLock +) + +func sampleMemProfileStack(size uintptr) { + // Tiny allocations occupy at least one 16-byte granule (bdwgc's + // minimum on 64-bit, matching gc's tiny size class): report the + // granule so bytes/objects ratios match what the allocator really + // spends — pprof consumers and the tiny-allocation tests key on it. + if size < 16 { + size = 16 + } + var pcs [32]uintptr + n := MemProfileStackCapture(pcs[:]) + if n <= 0 { + return + } + var h uintptr = 5381 + for i := 0; i < n; i++ { + h = h*33 + pcs[i] + } + // gc keys heap profile buckets by both stack and allocation size. Besides + // preserving that contract, keeping sizes separate lets the consumer's + // Poisson correction use the actual sample size instead of a mixed mean. + h = h*33 + size + slot := h & (memStackTabSize - 1) + if b := findMemStackBucket(memProfileLoadBucket(&memStackTab[slot]), h, size, pcs[:n]); b != nil { + memStackAdd(b, size) + return + } + + // Allocation re-enters recordMemProfileAlloc, where this physical + // thread's recursion guard suppresses the internal allocation. Keep the + // allocation outside the table lock so allocator or finalizer work cannot + // invert lock order with a concurrent MemProfile call. + b := (*memStackBucket)(AllocZ(unsafe.Sizeof(memStackBucket{}))) + b.hash = h + b.size = size + b.nstk = int32(n) + copy(b.stk[:], pcs[:n]) + + memStackTabLock.lock() + head := memProfileLoadBucket(&memStackTab[slot]) + if existing := findMemStackBucket(head, h, size, pcs[:n]); existing != nil { + memStackTabLock.unlock() + memStackAdd(existing, size) return } - size = memProfileSizeClass(size) - for i := range memProfileBuckets { - b := &memProfileBuckets[i] - if b.size == size { - memProfileAddObject(&b.objects) - return + memStackAdd(b, size) + b.next = head + memProfileStoreBucket(&memStackTab[slot], b) + memStackTabLock.unlock() +} + +// Buckets and their next links are immutable after the head is atomically +// published, so lookups need no lock. The insertion lock only resolves the +// rare race between two first samples for the same key. +func findMemStackBucket(head *memStackBucket, hash, size uintptr, pcs []uintptr) *memStackBucket { + for b := head; b != nil; b = b.next { + if b.hash == hash && b.size == size && int(b.nstk) == len(pcs) && memStackEqual(b, pcs) { + return b + } + } + return nil +} + +func memStackEqual(b *memStackBucket, pcs []uintptr) bool { + for i := range pcs { + if b.stk[i] != pcs[i] { + return false } } + return true +} + +// memStackAdd records one raw sampled allocation (gc semantics: no +// scaling here; readers un-bias with the sampling-rate correction). +func memStackAdd(b *memStackBucket, size uintptr) { + memProfileAddN(&b.allocObjects, 1) + memProfileAddN(&b.allocBytes, uint64(size)) } func memProfileSizeClass(size uintptr) uintptr { @@ -87,6 +188,9 @@ func memProfileSizeClass(size uintptr) uintptr { } func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + if MemProfileStackCapture != nil && MemProfileRatePtr != nil { + return memProfileStacks(p) + } for i := range memProfileBuckets { if memProfileLoadObjects(&memProfileBuckets[i].objects) != 0 { n++ @@ -110,3 +214,32 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { } return n, true } + +func memProfileStacks(p []MemProfileRecord) (n int, ok bool) { + // Save one atomic head per slot. Published chains are immutable, so this + // gives a stable snapshot without blocking allocation samples and without + // touching the caller thread's sampling state. + var heads [memStackTabSize]*memStackBucket + for i := range heads { + heads[i] = memProfileLoadBucket(&memStackTab[i]) + for b := heads[i]; b != nil; b = b.next { + n++ + } + } + if len(p) < n { + return n, false + } + j := 0 + for i := range heads { + for b := heads[i]; b != nil; b = b.next { + r := MemProfileRecord{ + AllocBytes: int64(memProfileLoadObjects(&b.allocBytes)), + AllocObjects: int64(memProfileLoadObjects(&b.allocObjects)), + } + copy(r.Stack0[:], b.stk[:b.nstk]) + p[j] = r + j++ + } + } + return n, true +} diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go index c0a1f1176d..9315d200e4 100644 --- a/runtime/internal/runtime/memprofile_atomic.go +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -2,14 +2,37 @@ package runtime -import "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" +import ( + "unsafe" + + "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" +) type memProfileCounter = uint64 +type memProfileBucketHead = unsafe.Pointer + +// Keep the hot-path fields in one TLS object so one address lookup serves the +// whole allocation decision. A sentinel countdown marks recursive sampling. +// +//llgo:tls +var memProfileState memProfileThreadState func memProfileAddObject(p *memProfileCounter) { atomic.Add(p, memProfileCounter(1)) } +func memProfileAddN(p *memProfileCounter, n uint64) { + atomic.Add(p, memProfileCounter(n)) +} + func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { return atomic.Load(p) } + +func memProfileLoadBucket(p *memProfileBucketHead) *memStackBucket { + return (*memStackBucket)(atomic.Load(p)) +} + +func memProfileStoreBucket(p *memProfileBucketHead, b *memStackBucket) { + atomic.Store(p, unsafe.Pointer(b)) +} diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go index 29ba9ab6d9..26565e81d9 100644 --- a/runtime/internal/runtime/memprofile_baremetal.go +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -2,7 +2,18 @@ package runtime +import "unsafe" + type memProfileCounter = uintptr +type memProfileBucketHead = unsafe.Pointer + +// Bare-metal runtimes have a single execution context and must not introduce +// native TLS relocations. +var memProfileState memProfileThreadState + +func memProfileAddN(p *memProfileCounter, n uint64) { + *p += memProfileCounter(n) +} func memProfileAddObject(p *memProfileCounter) { *p = *p + 1 @@ -11,3 +22,11 @@ func memProfileAddObject(p *memProfileCounter) { func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { return *p } + +func memProfileLoadBucket(p *memProfileBucketHead) *memStackBucket { + return (*memStackBucket)(*p) +} + +func memProfileStoreBucket(p *memProfileBucketHead, b *memStackBucket) { + *p = unsafe.Pointer(b) +} diff --git a/runtime/internal/runtime/memprofile_lock_native.go b/runtime/internal/runtime/memprofile_lock_native.go new file mode 100644 index 0000000000..969a832aa6 --- /dev/null +++ b/runtime/internal/runtime/memprofile_lock_native.go @@ -0,0 +1,59 @@ +//go:build llgo && !baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + psync "github.com/xgo-dev/llgo/runtime/internal/clite/pthread/sync" + "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" +) + +const ( + memProfileLockUninitialized uint32 = iota + memProfileLockInitializing + memProfileLockReady +) + +type memProfileLock struct { + state uint32 + mu psync.Mutex +} + +func (l *memProfileLock) lock() { + l.ensureInitialized() + l.mu.Lock() +} + +func (l *memProfileLock) unlock() { + l.mu.Unlock() +} + +func (l *memProfileLock) ensureInitialized() { + if atomic.Load(&l.state) == memProfileLockReady { + return + } + if _, won := atomic.CompareAndExchange(&l.state, memProfileLockUninitialized, memProfileLockInitializing); won { + if l.mu.Init(nil) != 0 { + panic("runtime: failed to initialize memory profile lock") + } + atomic.Store(&l.state, memProfileLockReady) + return + } + for atomic.Load(&l.state) != memProfileLockReady { + } +} diff --git a/runtime/internal/runtime/memprofile_lock_stub.go b/runtime/internal/runtime/memprofile_lock_stub.go new file mode 100644 index 0000000000..1d60cac74c --- /dev/null +++ b/runtime/internal/runtime/memprofile_lock_stub.go @@ -0,0 +1,25 @@ +//go:build !llgo || baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +// Host-tool tests and bare-metal targets are single-threaded at this layer. +type memProfileLock struct{} + +func (*memProfileLock) lock() {} +func (*memProfileLock) unlock() {} diff --git a/runtime/internal/runtime/memprofile_record_legacy.go b/runtime/internal/runtime/memprofile_record_legacy.go new file mode 100644 index 0000000000..b9ec0a4afe --- /dev/null +++ b/runtime/internal/runtime/memprofile_record_legacy.go @@ -0,0 +1,27 @@ +//go:build !llgo || baremetal || wasm + +package runtime + +func MemProfilePause() uintptr { + return 0 +} + +func MemProfileResume(uintptr) { +} + +// Legacy targets do not yet expose the physical native stack walk used by the +// sampled profiler. Keep their existing size-class accounting separate so the +// native allocator hot path has no mode dispatch. +func recordMemProfileAlloc(size uintptr) { + if size == 0 { + return + } + sizeClass := memProfileSizeClass(size) + for i := range memProfileBuckets { + b := &memProfileBuckets[i] + if b.size == sizeClass { + memProfileAddObject(&b.objects) + return + } + } +} diff --git a/runtime/internal/runtime/memprofile_record_native.go b/runtime/internal/runtime/memprofile_record_native.go new file mode 100644 index 0000000000..e5334724ac --- /dev/null +++ b/runtime/internal/runtime/memprofile_record_native.go @@ -0,0 +1,118 @@ +//go:build llgo && !baremetal && !wasm + +package runtime + +import ( + "unsafe" + + "github.com/xgo-dev/llgo/runtime/internal/clite/bitcast" +) + +const memProfileSampling = ^uintptr(0) + +// MemProfilePause suppresses allocations made while materializing a profile. +// Without it, rate-1 profiling would observe its own symbolization and buffer +// work and recursively grow the bucket table. The returned value supports +// nested readers and is restored by MemProfileResume. +func MemProfilePause() uintptr { + state := &memProfileState + previous := state.remaining + state.remaining = memProfileSampling + return previous +} + +func MemProfileResume(previous uintptr) { + memProfileState.remaining = previous +} + +// recordMemProfileAlloc is deliberately small enough to inline into AllocZ/U. +// The usual enabled path does one rate load, one TLS address calculation, and +// one countdown update. Random threshold generation and stack capture stay on +// the sampled slow path. +func recordMemProfileAlloc(size uintptr) { + if size == 0 || MemProfileRatePtr == nil { + return + } + rate := *MemProfileRatePtr + if rate <= 0 { + return + } + state := &memProfileState + remaining := state.remaining + if remaining == memProfileSampling { + return + } + if rate != 1 && size < remaining { + state.remaining = remaining - size + return + } + recordMemProfileAllocSlow(state, size, rate) +} + +//go:noinline +func recordMemProfileAllocSlow(state *memProfileThreadState, size uintptr, rate int) { + if MemProfileStackCapture == nil { + return + } + next := state.remaining + if rate != 1 { + // A zero countdown initializes the stream. A non-zero countdown reached + // here because this allocation crossed the next sampling point. + if next == 0 { + next = memProfileNextThreshold(state, rate) + if size < next { + state.remaining = next - size + return + } + } + next = memProfileNextThreshold(state, rate) + } + state.remaining = memProfileSampling + sampleMemProfileStack(size) + state.remaining = next +} + +func memProfileNextThreshold(state *memProfileThreadState, rate int) uintptr { + // Exponentially distributed with mean rate, like gc's fastexprand: + // the memoryless property is required — with any bounded-support + // distribution a near-periodic allocation pattern phase-locks the + // sampling points onto the large sites and skews per-site estimates. + x := state.rand + if x == 0 { + x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(state))) + if x == 0 { + x = 0x9e3779b97f4a7c15 + } + } + x ^= x >> 12 + x ^= x << 25 + x ^= x >> 27 + state.rand = x + r := (x * 0x2545f4914f6cdd1d) >> 11 // 53 random bits + u := float64(r) / (1 << 53) + if u < 1e-12 { + u = 1e-12 + } + t := -lnApprox(u) * float64(rate) + if t < 1 { + t = 1 + } + if max := float64(rate) * 64; t > max { + t = max + } + return uintptr(t) +} + +// lnApprox computes ln(u) for u in (0,1] via exponent split and an +// atanh series on the mantissa. Its error is far below sampling noise. +func lnApprox(u float64) float64 { + const ln2 = 0.6931471805599453 + bits := uint64(bitcast.FromFloat64(u)) + e := int((bits>>52)&0x7ff) - 1023 + mbits := (bits &^ (uint64(0x7ff) << 52)) | (uint64(1023) << 52) + m := bitcast.ToFloat64(int64(mbits)) // in [1, 2) + z := (m - 1) / (m + 1) + z2 := z * z + lnm := 2 * z * (1 + z2/3 + z2*z2/5 + z2*z2*z2/7) + return float64(e)*ln2 + lnm +} diff --git a/ssa/backend_program_test.go b/ssa/backend_program_test.go index 06df50fdc3..dea6fe7216 100644 --- a/ssa/backend_program_test.go +++ b/ssa/backend_program_test.go @@ -28,6 +28,7 @@ func TestNewBackendProgramSharesPreparedGoState(t *testing.T) { coordinator.DisableBoundsChecks(true) coordinator.EnableGoGlobalDCE(true) coordinator.EnableDeadcodeDrop(true) + coordinator.EnableMemoryProfiling(true) coordinator.SetPthreadStackSize(4096) coordinator.EnableLTOPluginMarkers(true) coordinator.EnableFuncInfoMetadata(true) @@ -77,7 +78,7 @@ func TestNewBackendProgramSharesPreparedGoState(t *testing.T) { if backend.python() != nil { t.Fatal("backend Program changed the prepared optional Python package") } - if !backend.disableBoundsChecks || !backend.enableGoGlobalDCE || !backend.enableDeadcodeDrop || + if !backend.disableBoundsChecks || !backend.enableGoGlobalDCE || !backend.enableDeadcodeDrop || !backend.MemoryProfilingEnabled() || backend.pthreadStackSize != 4096 || !backend.enableLTOPluginMarker || !backend.enableFuncInfoMetadata || !backend.enableFuncInfoSites || backend.debugInfoOptimized { t.Fatal("backend Program did not preserve coordinator configuration") diff --git a/ssa/memory.go b/ssa/memory.go index 3241d6cd34..ea90562a47 100644 --- a/ssa/memory.go +++ b/ssa/memory.go @@ -24,6 +24,8 @@ import ( "github.com/xgo-dev/llvm" ) +var threadLocalAddressIntrinsic = llvm.LookupIntrinsicID("llvm.threadlocal.address") + // ----------------------------------------------------------------------------- func (b Builder) aggregateAllocU(t Type, flds ...llvm.Value) llvm.Value { @@ -167,6 +169,19 @@ func (b Builder) AllocaT(t Type) (ret Expr) { return } +// ThreadLocalAddress materializes the address of a native TLS global. Keeping +// this as an SSA value lets a caller reuse one target TLS resolver result for +// all dominated accesses instead of resolving the global at every use. +func (b Builder) ThreadLocalAddress(v Global) Expr { + if threadLocalAddressIntrinsic == 0 { + panic("ssa: llvm.threadlocal.address is unavailable") + } + return Expr{ + b.impl.CreateIntrinsic(v.impl.Type(), threadLocalAddressIntrinsic, []llvm.Value{v.impl}, ""), + v.Type, + } +} + /* TODO(xsw): // AllocaU allocates uninitialized space for n*sizeof(elem) bytes. func (b Builder) AllocaU(elem Type, n ...int64) (ret Expr) { diff --git a/ssa/package.go b/ssa/package.go index dda1ec7940..1aff74c816 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -235,6 +235,7 @@ type aProgram struct { enableGoGlobalDCE bool enableDeadcodeDrop bool disableBoundsChecks bool + memoryProfiling bool pthreadStackSize uint64 enableLTOPluginMarker bool @@ -348,6 +349,7 @@ func (p Program) NewBackendProgram() Program { backend.enableGoGlobalDCE = p.enableGoGlobalDCE backend.enableDeadcodeDrop = p.enableDeadcodeDrop backend.disableBoundsChecks = p.disableBoundsChecks + backend.memoryProfiling = p.memoryProfiling backend.pthreadStackSize = p.pthreadStackSize backend.enableLTOPluginMarker = p.enableLTOPluginMarker backend.enableFuncInfoMetadata = p.enableFuncInfoMetadata @@ -391,6 +393,17 @@ func (p Program) EnableGoGlobalDCE(enable bool) { p.enableGoGlobalDCE = enable } +// EnableMemoryProfiling selects whether allocator recording calls are emitted +// for this whole-program build. The build coordinator sets it before package +// backends start. +func (p Program) EnableMemoryProfiling(enable bool) { + p.memoryProfiling = enable +} + +func (p Program) MemoryProfilingEnabled() bool { + return p.memoryProfiling +} + func (p Program) EnableDeadcodeDrop(enable bool) { p.enableDeadcodeDrop = enable } diff --git a/test/go/caller_acceptance_test.go b/test/go/caller_acceptance_test.go index 3e5112d716..c263e9a2a5 100644 --- a/test/go/caller_acceptance_test.go +++ b/test/go/caller_acceptance_test.go @@ -698,3 +698,80 @@ func runLLGoInModule(t *testing.T, dir string, args ...string) (string, error) { out, err := cmd.CombinedOutput() return string(out), err } + +// Scenario: runtime.MemProfile attributes sampled allocations to exact +// call stacks and statement lines (gc semantics: raw sampled counts, +// physical frames). rate=1 samples everything, so counts are exact. +func TestCallerAcceptanceMemProfileAttribution(t *testing.T) { + dir := t.TempDir() + const src = `package main + +import ( + "os" + "runtime" + "strconv" +) + +var s1, s2 *[2048]byte + +func allocOne() { s1 = new([2048]byte) } // MEMA_MARK + +func allocTwo() { s2 = new([2048]byte) } // MEMB_MARK + +func main() { + runtime.MemProfileRate = 1 + for i := 0; i < 50; i++ { + allocOne() + allocTwo() + } + var recs [128]runtime.MemProfileRecord + n, ok := runtime.MemProfile(recs[:], true) + if !ok { + panic("MemProfile failed: n=" + strconv.Itoa(n)) + } + var oneObjs, twoObjs int64 + for _, r := range recs[:n] { + frames := runtime.CallersFrames(r.Stack()) + first := true + for { + f, more := frames.Next() + if first { + first = false + switch { + case f.Function == "main.allocOne" && f.Line == MEMA_LINE: + oneObjs += r.AllocObjects + case f.Function == "main.allocTwo" && f.Line == MEMB_LINE: + twoObjs += r.AllocObjects + } + } + if !more { + break + } + } + } + if oneObjs != 50 || twoObjs != 50 { + panic("bad attribution: one=" + strconv.Itoa(int(oneObjs)) + " two=" + strconv.Itoa(int(twoObjs))) + } + os.Stdout.WriteString("MEMPROF_OK\n") +} +` + source := src + for _, name := range []string{"MEMA", "MEMB"} { + line := markerLine(source, name+"_MARK") + if line == 0 { + t.Fatalf("marker %s_MARK not found", name) + } + source = strings.ReplaceAll(source, name+"_LINE", strconv.Itoa(line)) + } + writeCallerAcceptanceModule(t, dir, map[string]string{ + "main.go": source, + "go.mod": "module memprof\n\ngo 1.21\n", + }) + out, err := runLLGoInModule(t, dir, "run", ".") + if err != nil { + t.Fatalf("memprofile probe failed: %v\n%s", err, out) + } + if !strings.Contains(out, "MEMPROF_OK") { + t.Fatalf("memprofile probe missing marker:\n%s", out) + } +} diff --git a/test/go/memprofile/concurrent_llgo_test.go b/test/go/memprofile/concurrent_llgo_test.go new file mode 100644 index 0000000000..f392f0ecb4 --- /dev/null +++ b/test/go/memprofile/concurrent_llgo_test.go @@ -0,0 +1,85 @@ +//go:build llgo + +package memprofile + +import ( + "runtime" + "strings" + "sync" + "testing" +) + +var concurrentSink [][]*int + +func TestConcurrentSamplingAndSnapshots(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + const ( + workers = 16 + allocations = 256 + ) + concurrentSink = make([][]*int, workers) + for i := range concurrentSink { + concurrentSink[i] = make([]*int, allocations) + } + before := memProfileObjectsForFunction(t, "concurrentProfileAlloc") + + start := make(chan struct{}) + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + <-start + for i := 0; i < 256; i++ { + runtime.MemProfile(nil, false) + } + }() + + var wg sync.WaitGroup + wg.Add(workers) + for i := range concurrentSink { + go func(dst []*int) { + defer wg.Done() + <-start + concurrentProfileAlloc(dst) + }(concurrentSink[i]) + } + close(start) + wg.Wait() + <-readerDone + + after := memProfileObjectsForFunction(t, "concurrentProfileAlloc") + if got, want := after-before, int64(workers*allocations); got != want { + t.Fatalf("concurrent sampled allocations = %d, want %d", got, want) + } +} + +func concurrentProfileAlloc(dst []*int) { + for i := range dst { + p := new(int) + *p = i + dst[i] = p + } +} + +func memProfileObjectsForFunction(t *testing.T, suffix string) int64 { + t.Helper() + var total int64 + for _, record := range readMemProfile(t) { + frames := runtime.CallersFrames(record.Stack()) + for { + frame, more := frames.Next() + if strings.HasSuffix(frame.Function, "."+suffix) { + total += record.AllocObjects + break + } + if !more { + break + } + } + } + return total +} diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go index 2e4a9d4644..938419bcb7 100644 --- a/test/go/memprofile/memprofile_test.go +++ b/test/go/memprofile/memprofile_test.go @@ -3,12 +3,41 @@ package memprofile import ( "bytes" "fmt" + "reflect" "runtime" "runtime/pprof" + "strings" "testing" ) var tinySink []*int32 +var mixedSizeSink [][]byte +var memProfileWarmup []byte + +type profiledClosure struct { + fn func(int) int +} + +func makeProfiledClosure(base int) profiledClosure { + return profiledClosure{fn: func(v int) int { return base + v }} +} + +func TestSamplingPreservesReflectCallFrames(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + makeFn := reflect.ValueOf(makeProfiledClosure) + for i := 0; i < 64; i++ { + out := makeFn.Call([]reflect.Value{reflect.ValueOf(i)}) + closure := out[0].Interface().(profiledClosure) + if got, want := closure.fn(2), i+2; got != want { + t.Fatalf("sampled reflect call returned %d, want %d", got, want) + } + } +} func TestRuntimeMemProfileReportsTinyAllocations(t *testing.T) { oldRate := runtime.MemProfileRate @@ -45,6 +74,59 @@ func TestRuntimeMemProfileReportsTinyAllocations(t *testing.T) { t.Fatalf("MemProfile did not report tiny allocations totaling at least %d bytes: %#v", wantBytes, records) } +func TestRuntimeMemProfileSeparatesSizesAtOneStack(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + // gc caches the active rate in each mcache. Allocate enough to force the + // current cache to observe the change before checking the interesting + // allocation stacks (the Go runtime's own profiler tests do the same). + for range 1024 { + memProfileWarmup = make([]byte, 1024) + } + + mixedSizeSink = make([][]byte, 128) + mixedSizeProfileAlloc(mixedSizeSink) + // The gc runtime publishes heap profile samples up to two GC cycles + // after allocation. Materialize both sizes before reading them. + runtime.GC() + runtime.GC() + sizes := make(map[int64]bool) + for _, record := range readMemProfile(t) { + if record.AllocObjects == 0 { + continue + } + frames := runtime.CallersFrames(record.Stack()) + for { + frame, more := frames.Next() + if strings.HasSuffix(frame.Function, ".mixedSizeProfileAlloc") { + sizes[record.AllocBytes/record.AllocObjects] = true + break + } + if !more { + break + } + } + } + if !sizes[64] || !sizes[256] { + t.Fatalf("same-stack memory profile sizes = %v, want 64 and 256", sizes) + } +} + +//go:noinline +func mixedSizeProfileAlloc(dst [][]byte) { + for i := range dst { + size := 64 + if i&1 != 0 { + size = 256 + } + dst[i] = make([]byte, size) + } +} + func TestRuntimePprofHeapProfileReportsTinyAllocations(t *testing.T) { oldRate := runtime.MemProfileRate runtime.MemProfileRate = 1 diff --git a/test/goroot/xfail.yaml b/test/goroot/xfail.yaml index ce01c845ef..32b3e97811 100644 --- a/test/goroot/xfail.yaml +++ b/test/goroot/xfail.yaml @@ -1263,19 +1263,3 @@ xfails: directive: errorcheck case: bombad.go reason: llgo parser recovery emits additional byte-order-mark diagnostics - - - platform: darwin/arm64 - directive: run - case: heapsampling.go - reason: latest main goroot run failure on darwin/arm64 - - version: go1.24 - platform: linux/amd64 - directive: run - case: heapsampling.go - reason: go1.24 goroot run failure on linux/amd64 - - version: go1.26 - platform: linux/amd64 - directive: run - case: heapsampling.go - reason: go1.26 goroot run failure on linux/amd64 -