From 2ee07443f3daabd64b37c016e390bc873de1056c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sat, 4 Jul 2026 15:23:25 +0800 Subject: [PATCH 01/26] runtime,cl: stack-keyed sampled memory profiling with gc semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the size-class counters with gc-shaped heap profiling: sampled allocations are attributed to physical call stacks at exact statement lines, and records hold RAW sampled counts — consumers (pprof, goroot heapsampling.go) apply the Poisson correction themselves, exactly as with gc. - Sampling mirrors gc's mcache.nextSample: bytes count down to an exponentially distributed threshold (mean MemProfileRate), sample once on crossing, redraw. The memoryless distribution is load-bearing: with any bounded-support threshold a near-periodic allocation pattern phase-locks the sample points onto the large sites (observed 1.6x per-site skew on heapsampling's interleaved sizes). ln() is a small local approximation — the runtime core cannot import math. - Stacks come from the FP walk at sample time (fpCallers via a hook the public runtime registers), bucketed by stack hash; allocator plumbing (including __llgo_stub. wrapper frames of the hook) is trimmed at read time. A reentrancy flag spans the whole decision path: threshold drawing and bucket allocation themselves allocate, and a recursive sample overflows the stack. - Heap allocations get statement anchors in tracked functions, and a package that reads the memory profile (runtime.MemProfile / MemProfileRate under either the "runtime" or the patched lib-runtime spelling) pins all its trackable functions: per-site attribution loses sites to inlining otherwise. Profiling packages are rare and accuracy beats inlining there; gc gets both via its inline tree (P4). goroot heapsampling.go passes on darwin/arm64 and linux/arm64 (the latter was a pre-existing platform gap). Depends on --icf=none from the line-directive PR: heapsampling's three identical wrapper functions must keep distinct pcs. Co-Authored-By: Claude Fable 5 --- cl/compile.go | 7 + cl/instr.go | 66 ++++++ .../runtime/pprof_memprofile_go123_llgo.go | 15 +- .../lib/runtime/pprof_runtime_stub_llgo.go | 59 ++++- runtime/internal/lib/runtime/unwind_llgo.go | 13 ++ runtime/internal/runtime/memprofile.go | 213 +++++++++++++++++- runtime/internal/runtime/memprofile_atomic.go | 4 + .../internal/runtime/memprofile_baremetal.go | 4 + 8 files changed, 363 insertions(+), 18 deletions(-) diff --git a/cl/compile.go b/cl/compile.go index 10be42368a..f3c3dad644 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -1434,6 +1434,13 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue return } elem := p.type_(t.Elem(), llssa.InGo) + if v.Heap { + // Heap allocations are memory-profile sample sites; give each + // one a statement anchor in tracked functions so sampled + // records attribute to the allocating line (heapsampling.go + // keys buckets by the leaf frame's exact line). + p.emitPCLineLabel(b, v.Pos()) + } ret = b.Alloc(elem, v.Heap) p.debugAlloc(b, v, ret) p.markRecoverSlot(v) diff --git a/cl/instr.go b/cl/instr.go index 033afe39e7..bafb7ecbcc 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -969,10 +969,22 @@ func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs for fn := range base { frames[fn] = true } + // Criterion 5: a package that reads the memory profile gets every + // trackable function pinned. Heap records attribute sampled + // allocations to physical frames at exact statement lines; inlining + // any function in such a package would merge its allocation sites + // into the caller and lose per-site attribution (goroot + // heapsampling.go). Profiling packages are rare and accuracy beats + // inlining there — gc keeps per-site attribution via its inline tree. + pinAll := packageReadsMemProfile(trackable) for fn := range trackable { if frames[fn] { continue } + if pinAll { + out[fn] = true + continue + } // Criterion 3: pin program-unique frames. main.main and package // init functions run once, so noinline is free — and they are the // bottom frames of almost every panic traceback, where an @@ -1357,6 +1369,60 @@ func NewCallerTracking() *CallerTracking { } } +// isPublicRuntimePath matches the public runtime package under both +// spellings: go/ssa unit builds see "runtime"; the real pipeline patches +// it to LLGo's implementation package. +func isPublicRuntimePath(path string) bool { + return path == "runtime" || + path == "github.com/goplus/llgo/runtime/internal/lib/runtime" +} + +func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { + // Cheap import pre-filter: scanning every instruction of every + // function costs real compile time across thousands of small + // packages (goroot shards); a package that never imports the public + // runtime cannot reference MemProfile. + imported := false + for fn := range funcs { + if fn.Pkg == nil || fn.Pkg.Pkg == nil { + continue + } + for _, imp := range fn.Pkg.Pkg.Imports() { + if isPublicRuntimePath(imp.Path()) { + imported = true + } + } + break + } + if !imported { + return false + } + for fn := range funcs { + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if call, ok := instr.(ssa.CallInstruction); ok { + if callee := call.Common().StaticCallee(); callee != nil && + callee.Pkg != nil && isPublicRuntimePath(callee.Pkg.Pkg.Path()) && + callee.Name() == "MemProfile" { + return true + } + } + rands := instr.Operands(nil) + for _, rand := range rands { + if rand == nil { + continue + } + if g, ok := (*rand).(*ssa.Global); ok && g.Pkg != nil && + isPublicRuntimePath(g.Pkg.Pkg.Path()) && g.Name() == "MemProfileRate" { + return true + } + } + } + } + } + return false +} + func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { if fn == nil || fn.Parent() != nil { return false diff --git a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go index 172930c92b..0cc01acadc 100644 --- a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go +++ b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go @@ -19,12 +19,17 @@ func pprof_memProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, if n == 0 { return 0, true } - var records [64]MemProfileRecord - if n > len(records) { - return n, false + // Size dynamically with slack and retry: a fixed cap makes pprof's + // retry-until-ok loop spin forever once the bucket set outgrows it. + records := make([]MemProfileRecord, n+n/4+16) + for attempt := 0; ; attempt++ { + n, ok = MemProfile(records, inuseZero) + if ok || attempt >= 3 { + break + } + records = make([]MemProfileRecord, n+n/4+16) } - n, ok = MemProfile(records[:n], inuseZero) - if !ok { + if !ok || len(p) < n { return n, false } for i := 0; i < n; i++ { diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 3a7f615a2b..75937270fe 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -53,29 +53,66 @@ type BlockProfileRecord struct { Stack []uintptr } +// trimMemProfileStack drops the allocator/runtime plumbing the physical +// capture recorded above the allocation site (AllocZ, the capture path +// itself) so record stacks start at user code like gc's. +func trimMemProfileStack(stk [32]uintptr) [32]uintptr { + i := 0 + for i < len(stk) && stk[i] != 0 { + if !isRuntimePlumbingFrame(stk[i]) { + break + } + i++ + } + if i == 0 { + return stk + } + var out [32]uintptr + copy(out[:], stk[i:]) + return out +} + +// isRuntimePlumbingFrame reports whether pc belongs to LLGo runtime +// plumbing (allocator, capture hooks — including their __llgo_stub. +// wrappers, which is how a hook held in a function variable is entered). +func isRuntimePlumbingFrame(pc uintptr) bool { + name := frameSymbol(pc - 1).function + if name == "" { + return false + } + const stub = "__llgo_stub." + if hasPrefix(name, stub) { + name = name[len(stub):] + } + return hasPrefix(name, "github.com/goplus/llgo/runtime/internal/") || + name == "runtime.captureMemProfileStack" +} + func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { - n, _ = llrt.MemProfile(nil, inuseZero) - if len(p) < n { + // Size dynamically with slack and retry: sampling between a sizing + // call and its fill call can grow the bucket set, and a fixed cap + // would make callers that retry-until-ok (pprof) loop forever. + records := make([]llrt.MemProfileRecord, 64) + for attempt := 0; ; attempt++ { + n, ok = llrt.MemProfile(records, inuseZero) + if ok || attempt >= 3 { + break + } + records = make([]llrt.MemProfileRecord, n+n/4+16) + } + if !ok || len(p) < n { return n, false } if n == 0 { return 0, true } - var records [64]llrt.MemProfileRecord - if n > len(records) { - return n, false - } - n, ok = llrt.MemProfile(records[:n], inuseZero) - if !ok { - return n, false - } for i := 0; i < n; i++ { p[i] = MemProfileRecord{ AllocBytes: records[i].AllocBytes, FreeBytes: records[i].FreeBytes, AllocObjects: records[i].AllocObjects, FreeObjects: records[i].FreeObjects, - Stack0: records[i].Stack0, + Stack0: trimMemProfileStack(records[i].Stack0), } } return n, true diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index bdd3c15d73..6f025cf224 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -19,6 +19,8 @@ func init() { rtdebug.PanicRecovered = clearFaultTraceback rtdebug.PanicPCSnapshot = capturePanicPCs rtdebug.RecoverMark = recoverMark + rtdebug.MemProfileStackCapture = captureMemProfileStack + rtdebug.MemProfileRatePtr = &MemProfileRate } // recoverMark records the recovering deferred frame (and one above, for @@ -50,6 +52,17 @@ func capturePanicPCs() { rtdebug.StorePanicPCs(pcs[:n]) } +// captureMemProfileStack walks the physical stack at a sampled allocation. +// The leading allocator plumbing (AllocZ/AllocU, this capture path) is +// trimmed at read time by the MemProfile wrapper, where symbolization is +// safe and cached. +func captureMemProfileStack(pcs []uintptr) int { + if !fpUnwindAvailable() { + return 0 + } + return fpCallers(0, pcs) +} + // panicSplicePCs returns the snapshot when it is observable: either the // panic is still in flight, or the deferred frame that recovered it is // still live on the physical chain (gc keeps panic frames on the stack diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go index c6221ca1dd..dad06088aa 100644 --- a/runtime/internal/runtime/memprofile.go +++ b/runtime/internal/runtime/memprofile.go @@ -1,5 +1,7 @@ package runtime +import "unsafe" + // MemProfileRecord describes allocations aggregated by size class. type MemProfileRecord struct { AllocBytes, FreeBytes int64 @@ -60,20 +62,186 @@ var memProfileBuckets = [...]memProfileBucket{ {size: 1073741824}, } +// MemProfileStackCapture, set by the public runtime package, walks the +// physical stack at a sampled allocation. When present, MemProfile reports +// stack-keyed buckets (gc semantics: heapsampling.go attributes sampled +// bytes to call stacks); without it the legacy size-class counters remain +// (baremetal, wasm). +var MemProfileStackCapture func(pcs []uintptr) int + +// MemProfileRatePtr points at the public runtime.MemProfileRate variable +// (user-settable at any time, read per sampling decision). +var MemProfileRatePtr *int + +type memStackBucket struct { + next *memStackBucket + hash uintptr + allocBytes memProfileCounter + allocObjects memProfileCounter + nstk int32 + stk [32]uintptr +} + +const memStackTabSize = 512 // power of two + +var memStackTab [memStackTabSize]*memStackBucket + +// memProfileRemaining counts down allocated bytes to the next sample. +// Thresholds are drawn uniformly from [1, 2*rate) — mean rate — because a +// deterministic stride starves small allocation sites when they interleave +// with larger ones (every crossing lands on the big sites; gc randomizes +// for the same reason). Benign races: sampling is statistical. +var ( + memProfileRemaining uintptr + memProfileRandState uint64 = 0x9e3779b97f4a7c15 +) + +func memProfileNextThreshold(rate int) uintptr { + // Exponentially distributed with mean rate, like gc's fastexprand: + // the memoryless property is required — with any bounded-support + // distribution a near-periodic allocation pattern phase-locks the + // sampling points onto the large sites and skews per-site estimates + // (observed 1.6x on goroot heapsampling's interleaved sizes). + x := memProfileRandState + x ^= x >> 12 + x ^= x << 25 + x ^= x >> 27 + memProfileRandState = x + r := (x * 0x2545f4914f6cdd1d) >> 11 // 53 random bits + u := float64(r) / (1 << 53) + if u < 1e-12 { + u = 1e-12 + } + t := -lnApprox(u) * float64(rate) + if t < 1 { + t = 1 + } + if max := float64(rate) * 64; t > max { + t = max + } + return uintptr(t) +} + +// lnApprox computes ln(u) for u in (0,1] via exponent split and an +// atanh series on the mantissa — a few 1e-6s of relative error, far +// below sampling noise. +func lnApprox(u float64) float64 { + const ln2 = 0.6931471805599453 + bits := *(*uint64)(unsafe.Pointer(&u)) + e := int((bits>>52)&0x7ff) - 1023 + mbits := (bits &^ (uint64(0x7ff) << 52)) | (uint64(1023) << 52) + m := *(*float64)(unsafe.Pointer(&mbits)) // in [1, 2) + z := (m - 1) / (m + 1) + z2 := z * z + lnm := 2 * z * (1 + z2/3 + z2*z2/5 + z2*z2*z2/7) + return float64(e)*ln2 + lnm +} + +// memProfileInSample breaks the recursion: allocating a bucket node (and +// anything the capture path allocates) re-enters recordMemProfileAlloc. +// Benign-racy flag — a concurrent thread skipping one sample is fine. +var memProfileInSample bool + func recordMemProfileAlloc(size uintptr) { if size == 0 { return } - size = memProfileSizeClass(size) + if MemProfileStackCapture != nil && MemProfileRatePtr != nil { + // The guard covers the whole decision path: threshold drawing and + // stack capture may themselves allocate (escaping locals, the + // bucket node), and a recursive sample would overflow the stack. + if memProfileInSample { + return + } + memProfileInSample = true + rate := *MemProfileRatePtr + if rate <= 0 { + memProfileInSample = false + return + } + if rate == 1 { + sampleMemProfileStack(size) + memProfileInSample = false + return + } + // Mirror gc's mcache.nextSample: subtract, sample once on + // crossing, redraw. Records hold RAW sampled counts — consumers + // (pprof, goroot heapsampling.go) apply the Poisson correction + // (scaleHeapSample) themselves, exactly like with gc. + if memProfileRemaining == 0 { + memProfileRemaining = memProfileNextThreshold(rate) + } + if size < memProfileRemaining { + memProfileRemaining -= size + memProfileInSample = false + return + } + memProfileRemaining = memProfileNextThreshold(rate) + sampleMemProfileStack(size) + memProfileInSample = false + return + } + sizeClass := memProfileSizeClass(size) for i := range memProfileBuckets { b := &memProfileBuckets[i] - if b.size == size { + if b.size == sizeClass { memProfileAddObject(&b.objects) return } } } +func sampleMemProfileStack(size uintptr) { + // Tiny allocations occupy at least one 16-byte granule (bdwgc's + // minimum on 64-bit, matching gc's tiny size class): report the + // granule so bytes/objects ratios match what the allocator really + // spends — pprof consumers and the tiny-allocation tests key on it. + if size < 16 { + size = 16 + } + var pcs [32]uintptr + n := MemProfileStackCapture(pcs[:]) + if n <= 0 { + return + } + var h uintptr = 5381 + for i := 0; i < n; i++ { + h = h*33 + pcs[i] + } + slot := h & (memStackTabSize - 1) + for b := memStackTab[slot]; b != nil; b = b.next { + if b.hash == h && int(b.nstk) == n && memStackEqual(b, pcs[:n]) { + memStackAdd(b, size) + return + } + } + b := (*memStackBucket)(AllocZ(unsafe.Sizeof(memStackBucket{}))) + b.hash = h + b.nstk = int32(n) + copy(b.stk[:], pcs[:n]) + memStackAdd(b, size) + // Benign-racy publish: a lost insert loses one sample, never corrupts + // (nodes are immutable once linked and the list is prepend-only). + b.next = memStackTab[slot] + memStackTab[slot] = b +} + +func memStackEqual(b *memStackBucket, pcs []uintptr) bool { + for i := range pcs { + if b.stk[i] != pcs[i] { + return false + } + } + return true +} + +// memStackAdd records one raw sampled allocation (gc semantics: no +// scaling here; readers un-bias with the sampling-rate correction). +func memStackAdd(b *memStackBucket, size uintptr) { + memProfileAddN(&b.allocObjects, 1) + memProfileAddN(&b.allocBytes, uint64(size)) +} + func memProfileSizeClass(size uintptr) uintptr { if size <= 16 { return 16 @@ -87,6 +255,9 @@ func memProfileSizeClass(size uintptr) uintptr { } func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + if MemProfileStackCapture != nil && MemProfileRatePtr != nil { + return memProfileStacks(p) + } for i := range memProfileBuckets { if memProfileLoadObjects(&memProfileBuckets[i].objects) != 0 { n++ @@ -110,3 +281,41 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { } return n, true } + +func memProfileStacks(p []MemProfileRecord) (n int, ok bool) { + // Freeze sampling while enumerating so the count cannot grow between + // a caller's sizing call and its fill call. Explicit reset instead of + // defer: the wasm backend crashes in instruction selection on the + // deferred closure here. + memProfileInSample = true + n, ok = memProfileStacksLocked(p) + memProfileInSample = false + return n, ok +} + +func memProfileStacksLocked(p []MemProfileRecord) (n int, ok bool) { + for i := range memStackTab { + for b := memStackTab[i]; b != nil; b = b.next { + n++ + } + } + if len(p) < n { + return n, false + } + j := 0 + for i := range memStackTab { + for b := memStackTab[i]; b != nil; b = b.next { + if j >= len(p) { + break + } + r := MemProfileRecord{ + AllocBytes: int64(memProfileLoadObjects(&b.allocBytes)), + AllocObjects: int64(memProfileLoadObjects(&b.allocObjects)), + } + copy(r.Stack0[:], b.stk[:b.nstk]) + p[j] = r + j++ + } + } + return n, true +} diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go index c0a1f1176d..300305c541 100644 --- a/runtime/internal/runtime/memprofile_atomic.go +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -10,6 +10,10 @@ func memProfileAddObject(p *memProfileCounter) { atomic.Add(p, memProfileCounter(1)) } +func memProfileAddN(p *memProfileCounter, n uint64) { + atomic.Add(p, memProfileCounter(n)) +} + func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { return atomic.Load(p) } diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go index 29ba9ab6d9..967288c312 100644 --- a/runtime/internal/runtime/memprofile_baremetal.go +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -4,6 +4,10 @@ package runtime type memProfileCounter = uintptr +func memProfileAddN(p *memProfileCounter, n uint64) { + *p += memProfileCounter(n) +} + func memProfileAddObject(p *memProfileCounter) { *p = *p + 1 } From d9c3ae0506ac803797121b9198567d8905d422cc Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sat, 4 Jul 2026 15:23:26 +0800 Subject: [PATCH 02/26] test: memprofile attribution regressions; retire heapsampling xfails An acceptance regression asserts exact per-line attribution at rate=1 (raw counts are exact there), and a cl unit test covers the memprofile-package pinning criterion under both runtime spellings. Co-Authored-By: Claude Fable 5 --- cl/caller_frame_test.go | 34 ++++++++++++++ test/go/caller_acceptance_test.go | 77 +++++++++++++++++++++++++++++++ test/goroot/xfail.yaml | 16 ------- 3 files changed, 111 insertions(+), 16 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index a7085718c9..058aefbead 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -1199,3 +1199,37 @@ func TestDirectiveFilename(t *testing.T) { t.Fatal("nil fset must pass through") } } + +// Packages that read the memory profile pin every trackable function: +// per-site heap attribution needs frames and allocation-site anchors. +func TestPackageReadsMemProfilePin(t *testing.T) { + ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/hp", `package main + +import "runtime" + +func allocLeaf() *[64]byte { return new([64]byte) } + +func plainHelper() int { return 1 } + +func main() { + runtime.MemProfileRate = 1 + _ = allocLeaf() + _ = plainHelper() + var r [4]runtime.MemProfileRecord + runtime.MemProfile(r[:], true) +} +`) + set := runtimeCallerFuncSet(NewCallerTracking(), ssapkg) + for _, name := range []string{"allocLeaf", "plainHelper", "main"} { + if !set[ssapkg.Func(name)] { + t.Fatalf("%s must be pinned in a memprofile-reading package", name) + } + } + quiet, _ := buildCallerFrameSSAPackage(t, "example.com/quiet", `package q + +func Helper() int { return 2 } +`) + if set := runtimeCallerFuncSet(NewCallerTracking(), quiet); set[quiet.Func("Helper")] { + t.Fatal("quiet package must not be pinned") + } +} diff --git a/test/go/caller_acceptance_test.go b/test/go/caller_acceptance_test.go index 3e5112d716..c263e9a2a5 100644 --- a/test/go/caller_acceptance_test.go +++ b/test/go/caller_acceptance_test.go @@ -698,3 +698,80 @@ func runLLGoInModule(t *testing.T, dir string, args ...string) (string, error) { out, err := cmd.CombinedOutput() return string(out), err } + +// Scenario: runtime.MemProfile attributes sampled allocations to exact +// call stacks and statement lines (gc semantics: raw sampled counts, +// physical frames). rate=1 samples everything, so counts are exact. +func TestCallerAcceptanceMemProfileAttribution(t *testing.T) { + dir := t.TempDir() + const src = `package main + +import ( + "os" + "runtime" + "strconv" +) + +var s1, s2 *[2048]byte + +func allocOne() { s1 = new([2048]byte) } // MEMA_MARK + +func allocTwo() { s2 = new([2048]byte) } // MEMB_MARK + +func main() { + runtime.MemProfileRate = 1 + for i := 0; i < 50; i++ { + allocOne() + allocTwo() + } + var recs [128]runtime.MemProfileRecord + n, ok := runtime.MemProfile(recs[:], true) + if !ok { + panic("MemProfile failed: n=" + strconv.Itoa(n)) + } + var oneObjs, twoObjs int64 + for _, r := range recs[:n] { + frames := runtime.CallersFrames(r.Stack()) + first := true + for { + f, more := frames.Next() + if first { + first = false + switch { + case f.Function == "main.allocOne" && f.Line == MEMA_LINE: + oneObjs += r.AllocObjects + case f.Function == "main.allocTwo" && f.Line == MEMB_LINE: + twoObjs += r.AllocObjects + } + } + if !more { + break + } + } + } + if oneObjs != 50 || twoObjs != 50 { + panic("bad attribution: one=" + strconv.Itoa(int(oneObjs)) + " two=" + strconv.Itoa(int(twoObjs))) + } + os.Stdout.WriteString("MEMPROF_OK\n") +} +` + source := src + for _, name := range []string{"MEMA", "MEMB"} { + line := markerLine(source, name+"_MARK") + if line == 0 { + t.Fatalf("marker %s_MARK not found", name) + } + source = strings.ReplaceAll(source, name+"_LINE", strconv.Itoa(line)) + } + writeCallerAcceptanceModule(t, dir, map[string]string{ + "main.go": source, + "go.mod": "module memprof\n\ngo 1.21\n", + }) + out, err := runLLGoInModule(t, dir, "run", ".") + if err != nil { + t.Fatalf("memprofile probe failed: %v\n%s", err, out) + } + if !strings.Contains(out, "MEMPROF_OK") { + t.Fatalf("memprofile probe missing marker:\n%s", out) + } +} diff --git a/test/goroot/xfail.yaml b/test/goroot/xfail.yaml index ce01c845ef..32b3e97811 100644 --- a/test/goroot/xfail.yaml +++ b/test/goroot/xfail.yaml @@ -1263,19 +1263,3 @@ xfails: directive: errorcheck case: bombad.go reason: llgo parser recovery emits additional byte-order-mark diagnostics - - - platform: darwin/arm64 - directive: run - case: heapsampling.go - reason: latest main goroot run failure on darwin/arm64 - - version: go1.24 - platform: linux/amd64 - directive: run - case: heapsampling.go - reason: go1.24 goroot run failure on linux/amd64 - - version: go1.26 - platform: linux/amd64 - directive: run - case: heapsampling.go - reason: go1.26 goroot run failure on linux/amd64 - From f8c9798292dd36bdddc6a53f030b46763a36c0ef Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sun, 5 Jul 2026 09:14:08 +0800 Subject: [PATCH 03/26] runtime: never wait on the frame-table init latch from the memprofile capture path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frame-table init allocates; when one of those allocations crossed the sampling threshold, captureMemProfileStack -> fpCallers re-entered initRuntimeFuncPCFramesSlow on the thread that already held the Busy latch and usleep-spun forever. First testing.callerName call of a test binary triggers the init, so whole test binaries hung at startup — which packages hit it depends on the deterministic threshold sequence meeting the binary's pre-init allocation volume: net/rpc and net/rpc/jsonrpc under go1.24 stdlib, net/http/expvar/cookiejar under go1.26 (CI shard timeouts on ubuntu, both attempts). Entering an Uninit latch from the capture path is safe (the whole sample runs under memProfileInSample, so init's own allocations cannot re-sample); only Busy must not be waited on. Drop that one sample. --- runtime/internal/lib/runtime/unwind_llgo.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index 6f025cf224..544673eec5 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -3,6 +3,7 @@ package runtime import ( + latomic "sync/atomic" "unsafe" rtdebug "github.com/xgo-dev/llgo/runtime/internal/runtime" @@ -60,6 +61,16 @@ func captureMemProfileStack(pcs []uintptr) int { if !fpUnwindAvailable() { return 0 } + // The frame-table init itself allocates; when such an allocation is + // sampled, walking from here would wait on the init latch this very + // thread holds (fpCallers -> init busy -> usleep forever — observed + // hanging whole net/rpc and net/http test binaries at the first + // testing.callerName). Drop the sample instead of waiting. Uninit is + // fine to enter: memProfileInSample is set for the whole sample, so + // the init's own allocations cannot re-sample. + if latomic.LoadUint32(&runtimeFuncPCInitState) == runtimeFuncInfoInitBusy { + return 0 + } return fpCallers(0, pcs) } From 685ca1443c7e1528ec69c20d3a3e11cdfab04685 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 20 Jul 2026 20:19:26 +0800 Subject: [PATCH 04/26] test: cover memprofile package detection paths --- cl/caller_frame_test.go | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 058aefbead..1e1939ef6d 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -1233,3 +1233,56 @@ func Helper() int { return 2 } t.Fatal("quiet package must not be pinned") } } + +func TestPackageReadsMemProfileDetection(t *testing.T) { + tests := []struct { + name string + src string + want bool + }{ + { + name: "MemProfile call", + src: `package p +import "runtime" +func report() { + var records []runtime.MemProfileRecord + runtime.MemProfile(records, true) +}`, + want: true, + }, + { + name: "MemProfileRate write", + src: `package p +import "runtime" +func enable() { runtime.MemProfileRate = 1 }`, + want: true, + }, + { + name: "unrelated runtime use", + src: `package p +import "runtime" +func goos() string { return runtime.GOOS }`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkg, _ := buildCallerFrameSSAPackage(t, "example.com/"+tt.name, tt.src) + _, trackable := collectRuntimeCallerFunctions(pkg) + if got := packageReadsMemProfile(trackable); got != tt.want { + t.Fatalf("packageReadsMemProfile() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPublicRuntimePath(t *testing.T) { + for path, want := range map[string]bool{ + "runtime": true, + "github.com/goplus/llgo/runtime/internal/lib/runtime": true, + "runtime/debug": false, + } { + if got := isPublicRuntimePath(path); got != want { + t.Errorf("isPublicRuntimePath(%q) = %v, want %v", path, got, want) + } + } +} From b83f8e39c3c48fa25ecf8b9b3bb854fea7bf496c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 20 Jul 2026 21:14:31 +0800 Subject: [PATCH 05/26] runtime: initialize frame metadata before memory sampling --- runtime/internal/lib/runtime/unwind_llgo.go | 12 ++++------ test/go/memprofile/memprofile_test.go | 26 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index 544673eec5..5890e6f938 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -20,6 +20,9 @@ func init() { rtdebug.PanicRecovered = clearFaultTraceback rtdebug.PanicPCSnapshot = capturePanicPCs rtdebug.RecoverMark = recoverMark + // Table initialization may allocate. Complete it before installing the + // allocator hook so the first sample never initializes it from AllocZ/U. + initRuntimeFuncPCFrames() rtdebug.MemProfileStackCapture = captureMemProfileStack rtdebug.MemProfileRatePtr = &MemProfileRate } @@ -61,13 +64,8 @@ func captureMemProfileStack(pcs []uintptr) int { if !fpUnwindAvailable() { return 0 } - // The frame-table init itself allocates; when such an allocation is - // sampled, walking from here would wait on the init latch this very - // thread holds (fpCallers -> init busy -> usleep forever — observed - // hanging whole net/rpc and net/http test binaries at the first - // testing.callerName). Drop the sample instead of waiting. Uninit is - // fine to enter: memProfileInSample is set for the whole sample, so - // the init's own allocations cannot re-sample. + // Defensively avoid waiting on the frame-table init latch if future init + // ordering changes. The table is normally ready before this hook is set. if latomic.LoadUint32(&runtimeFuncPCInitState) == runtimeFuncInfoInitBusy { return 0 } diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go index 2e4a9d4644..37e499a4e2 100644 --- a/test/go/memprofile/memprofile_test.go +++ b/test/go/memprofile/memprofile_test.go @@ -3,6 +3,7 @@ package memprofile import ( "bytes" "fmt" + "reflect" "runtime" "runtime/pprof" "testing" @@ -10,6 +11,31 @@ import ( var tinySink []*int32 +type profiledClosure struct { + fn func(int) int +} + +func makeProfiledClosure(base int) profiledClosure { + return profiledClosure{fn: func(v int) int { return base + v }} +} + +func TestSamplingPreservesReflectCallFrames(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + makeFn := reflect.ValueOf(makeProfiledClosure) + for i := 0; i < 64; i++ { + out := makeFn.Call([]reflect.Value{reflect.ValueOf(i)}) + closure := out[0].Interface().(profiledClosure) + if got, want := closure.fn(2), i+2; got != want { + t.Fatalf("sampled reflect call returned %d, want %d", got, want) + } + } +} + func TestRuntimeMemProfileReportsTinyAllocations(t *testing.T) { oldRate := runtime.MemProfileRate runtime.MemProfileRate = 1 From b2406f0a2532611cc2193fcc94b9d1b52db69606 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 20 Jul 2026 21:45:55 +0800 Subject: [PATCH 06/26] test: cover memprofile detection without package metadata --- cl/caller_frame_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 1e1939ef6d..492909e6b0 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -1275,6 +1275,13 @@ func goos() string { return runtime.GOOS }`, } } +func TestPackageReadsMemProfileSkipsFunctionWithoutPackage(t *testing.T) { + funcs := map[*gossa.Function]bool{new(gossa.Function): true} + if packageReadsMemProfile(funcs) { + t.Fatal("function without package metadata must not read the memory profile") + } +} + func TestPublicRuntimePath(t *testing.T) { for path, want := range map[string]bool{ "runtime": true, From 3fb591c35d3733d1873b674823f33abff71ae609 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 29 Jul 2026 23:25:14 +0800 Subject: [PATCH 07/26] runtime: share prefix helper with wasm --- runtime/internal/lib/runtime/string_llgo.go | 7 +++++++ runtime/internal/lib/runtime/unwind_llgo.go | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 runtime/internal/lib/runtime/string_llgo.go diff --git a/runtime/internal/lib/runtime/string_llgo.go b/runtime/internal/lib/runtime/string_llgo.go new file mode 100644 index 0000000000..034275bdb2 --- /dev/null +++ b/runtime/internal/lib/runtime/string_llgo.go @@ -0,0 +1,7 @@ +//go:build !baremetal + +package runtime + +func hasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index 5890e6f938..99ffd8c86c 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -244,10 +244,6 @@ func callersWithPanicSplice(skip int, pc []uintptr) int { return copy(pc, view[skip:]) } -func hasPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && s[:len(prefix)] == prefix -} - // panicTraceback prints a Go-style stack trace for an unrecovered panic: // one "function(...)" line plus an indented file:line per physical frame, // matching the shape of runtime.Stack and gc's panic output. Reports false From c75decdb4f654aacfac1c57eaf4fef1d1dd3a343 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 10 Aug 2026 08:51:31 +0800 Subject: [PATCH 08/26] runtime: make memory profile sampling concurrency-safe --- runtime/internal/runtime/memprofile.go | 79 +++++++++++------ runtime/internal/runtime/memprofile_atomic.go | 11 +++ .../internal/runtime/memprofile_baremetal.go | 8 ++ .../runtime/memprofile_lock_native.go | 59 +++++++++++++ .../internal/runtime/memprofile_lock_stub.go | 25 ++++++ test/go/memprofile/concurrent_llgo_test.go | 85 +++++++++++++++++++ 6 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 runtime/internal/runtime/memprofile_lock_native.go create mode 100644 runtime/internal/runtime/memprofile_lock_stub.go create mode 100644 test/go/memprofile/concurrent_llgo_test.go diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go index dad06088aa..3e3299c3ad 100644 --- a/runtime/internal/runtime/memprofile.go +++ b/runtime/internal/runtime/memprofile.go @@ -84,18 +84,16 @@ type memStackBucket struct { const memStackTabSize = 512 // power of two -var memStackTab [memStackTabSize]*memStackBucket - -// memProfileRemaining counts down allocated bytes to the next sample. -// Thresholds are drawn uniformly from [1, 2*rate) — mean rate — because a -// deterministic stride starves small allocation sites when they interleave -// with larger ones (every crossing lands on the big sites; gc randomizes -// for the same reason). Benign races: sampling is statistical. var ( - memProfileRemaining uintptr - memProfileRandState uint64 = 0x9e3779b97f4a7c15 + memStackTab [memStackTabSize]*memStackBucket + memStackTabLock memProfileLock ) +// memProfileRemaining counts down allocated bytes to the next sample. +// It and the random state are physical-thread local, matching gc's per-M +// sampler: independent allocation streams must not race or phase-lock each +// other. See memprofile_atomic.go for the native TLS declarations. + func memProfileNextThreshold(rate int) uintptr { // Exponentially distributed with mean rate, like gc's fastexprand: // the memoryless property is required — with any bounded-support @@ -103,6 +101,14 @@ func memProfileNextThreshold(rate int) uintptr { // sampling points onto the large sites and skews per-site estimates // (observed 1.6x on goroot heapsampling's interleaved sizes). x := memProfileRandState + if x == 0 { + // Mix the physical stack address into the first state so independently + // initialized threads do not replay the same threshold stream. + x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(&rate))) + if x == 0 { + x = 0x9e3779b97f4a7c15 + } + } x ^= x >> 12 x ^= x << 25 x ^= x >> 27 @@ -137,11 +143,6 @@ func lnApprox(u float64) float64 { return float64(e)*ln2 + lnm } -// memProfileInSample breaks the recursion: allocating a bucket node (and -// anything the capture path allocates) re-enters recordMemProfileAlloc. -// Benign-racy flag — a concurrent thread skipping one sample is fine. -var memProfileInSample bool - func recordMemProfileAlloc(size uintptr) { if size == 0 { return @@ -209,21 +210,43 @@ func sampleMemProfileStack(size uintptr) { h = h*33 + pcs[i] } slot := h & (memStackTabSize - 1) - for b := memStackTab[slot]; b != nil; b = b.next { - if b.hash == h && int(b.nstk) == n && memStackEqual(b, pcs[:n]) { - memStackAdd(b, size) - return - } + memStackTabLock.lock() + if b := findMemStackBucket(slot, h, pcs[:n]); b != nil { + memStackAdd(b, size) + memStackTabLock.unlock() + return } + memStackTabLock.unlock() + + // Allocation re-enters recordMemProfileAlloc, where this physical + // thread's recursion guard suppresses the internal allocation. Keep the + // allocation outside the table lock so allocator or finalizer work cannot + // invert lock order with a concurrent MemProfile call. b := (*memStackBucket)(AllocZ(unsafe.Sizeof(memStackBucket{}))) b.hash = h b.nstk = int32(n) copy(b.stk[:], pcs[:n]) + + memStackTabLock.lock() + if existing := findMemStackBucket(slot, h, pcs[:n]); existing != nil { + memStackAdd(existing, size) + memStackTabLock.unlock() + return + } memStackAdd(b, size) - // Benign-racy publish: a lost insert loses one sample, never corrupts - // (nodes are immutable once linked and the list is prepend-only). b.next = memStackTab[slot] memStackTab[slot] = b + memStackTabLock.unlock() +} + +// findMemStackBucket is called with memStackTabLock held. +func findMemStackBucket(slot, hash uintptr, pcs []uintptr) *memStackBucket { + for b := memStackTab[slot]; b != nil; b = b.next { + if b.hash == hash && int(b.nstk) == len(pcs) && memStackEqual(b, pcs) { + return b + } + } + return nil } func memStackEqual(b *memStackBucket, pcs []uintptr) bool { @@ -283,13 +306,17 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { } func memProfileStacks(p []MemProfileRecord) (n int, ok bool) { - // Freeze sampling while enumerating so the count cannot grow between - // a caller's sizing call and its fill call. Explicit reset instead of - // defer: the wasm backend crashes in instruction selection on the - // deferred closure here. + // Protect both list publication and enumeration. The public wrappers + // tolerate a bucket appearing between their sizing and fill calls, but + // each individual snapshot must see a valid immutable list. Snapshot + // construction may itself allocate, so suppress recursive sampling only + // on this physical thread while the table lock is held. + wasInSample := memProfileInSample memProfileInSample = true + memStackTabLock.lock() n, ok = memProfileStacksLocked(p) - memProfileInSample = false + memStackTabLock.unlock() + memProfileInSample = wasInSample return n, ok } diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go index 300305c541..9e76ffa578 100644 --- a/runtime/internal/runtime/memprofile_atomic.go +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -6,6 +6,17 @@ import "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" type memProfileCounter = uint64 +// Native memory-profile sampling state is per physical thread, matching gc's +// per-M sampling state and keeping recursive allocator entry local to the +// thread that is currently capturing a stack. +// +//llgo:tls +var ( + memProfileRemaining uintptr + memProfileRandState uint64 + memProfileInSample bool +) + func memProfileAddObject(p *memProfileCounter) { atomic.Add(p, memProfileCounter(1)) } diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go index 967288c312..31704ea177 100644 --- a/runtime/internal/runtime/memprofile_baremetal.go +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -4,6 +4,14 @@ package runtime type memProfileCounter = uintptr +// Bare-metal runtimes have a single execution context and must not introduce +// native TLS relocations. +var ( + memProfileRemaining uintptr + memProfileRandState uint64 + memProfileInSample bool +) + func memProfileAddN(p *memProfileCounter, n uint64) { *p += memProfileCounter(n) } diff --git a/runtime/internal/runtime/memprofile_lock_native.go b/runtime/internal/runtime/memprofile_lock_native.go new file mode 100644 index 0000000000..96196fac83 --- /dev/null +++ b/runtime/internal/runtime/memprofile_lock_native.go @@ -0,0 +1,59 @@ +//go:build llgo && !baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + psync "github.com/goplus/llgo/runtime/internal/clite/pthread/sync" + "github.com/goplus/llgo/runtime/internal/clite/sync/atomic" +) + +const ( + memProfileLockUninitialized uint32 = iota + memProfileLockInitializing + memProfileLockReady +) + +type memProfileLock struct { + state uint32 + mu psync.Mutex +} + +func (l *memProfileLock) lock() { + l.ensureInitialized() + l.mu.Lock() +} + +func (l *memProfileLock) unlock() { + l.mu.Unlock() +} + +func (l *memProfileLock) ensureInitialized() { + if atomic.Load(&l.state) == memProfileLockReady { + return + } + if _, won := atomic.CompareAndExchange(&l.state, memProfileLockUninitialized, memProfileLockInitializing); won { + if l.mu.Init(nil) != 0 { + panic("runtime: failed to initialize memory profile lock") + } + atomic.Store(&l.state, memProfileLockReady) + return + } + for atomic.Load(&l.state) != memProfileLockReady { + } +} diff --git a/runtime/internal/runtime/memprofile_lock_stub.go b/runtime/internal/runtime/memprofile_lock_stub.go new file mode 100644 index 0000000000..1d60cac74c --- /dev/null +++ b/runtime/internal/runtime/memprofile_lock_stub.go @@ -0,0 +1,25 @@ +//go:build !llgo || baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +// Host-tool tests and bare-metal targets are single-threaded at this layer. +type memProfileLock struct{} + +func (*memProfileLock) lock() {} +func (*memProfileLock) unlock() {} diff --git a/test/go/memprofile/concurrent_llgo_test.go b/test/go/memprofile/concurrent_llgo_test.go new file mode 100644 index 0000000000..f392f0ecb4 --- /dev/null +++ b/test/go/memprofile/concurrent_llgo_test.go @@ -0,0 +1,85 @@ +//go:build llgo + +package memprofile + +import ( + "runtime" + "strings" + "sync" + "testing" +) + +var concurrentSink [][]*int + +func TestConcurrentSamplingAndSnapshots(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + const ( + workers = 16 + allocations = 256 + ) + concurrentSink = make([][]*int, workers) + for i := range concurrentSink { + concurrentSink[i] = make([]*int, allocations) + } + before := memProfileObjectsForFunction(t, "concurrentProfileAlloc") + + start := make(chan struct{}) + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + <-start + for i := 0; i < 256; i++ { + runtime.MemProfile(nil, false) + } + }() + + var wg sync.WaitGroup + wg.Add(workers) + for i := range concurrentSink { + go func(dst []*int) { + defer wg.Done() + <-start + concurrentProfileAlloc(dst) + }(concurrentSink[i]) + } + close(start) + wg.Wait() + <-readerDone + + after := memProfileObjectsForFunction(t, "concurrentProfileAlloc") + if got, want := after-before, int64(workers*allocations); got != want { + t.Fatalf("concurrent sampled allocations = %d, want %d", got, want) + } +} + +func concurrentProfileAlloc(dst []*int) { + for i := range dst { + p := new(int) + *p = i + dst[i] = p + } +} + +func memProfileObjectsForFunction(t *testing.T, suffix string) int64 { + t.Helper() + var total int64 + for _, record := range readMemProfile(t) { + frames := runtime.CallersFrames(record.Stack()) + for { + frame, more := frames.Next() + if strings.HasSuffix(frame.Function, "."+suffix) { + total += record.AllocObjects + break + } + if !more { + break + } + } + } + return total +} From e544e08ab7284313ad6605238c89a969a0ca8b2f Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 17 Aug 2026 22:14:15 +0800 Subject: [PATCH 09/26] runtime,cl: adapt memory profiling to current main --- cl/caller_frame_test.go | 2 +- cl/instr.go | 10 ++++++---- .../internal/lib/runtime/pprof_runtime_stub_llgo.go | 9 ++------- runtime/internal/runtime/memprofile_lock_native.go | 4 ++-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 492909e6b0..c0748c2b39 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -1285,7 +1285,7 @@ func TestPackageReadsMemProfileSkipsFunctionWithoutPackage(t *testing.T) { func TestPublicRuntimePath(t *testing.T) { for path, want := range map[string]bool{ "runtime": true, - "github.com/goplus/llgo/runtime/internal/lib/runtime": true, + "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": true, "runtime/debug": false, } { if got := isPublicRuntimePath(path); got != want { diff --git a/cl/instr.go b/cl/instr.go index bafb7ecbcc..7006f1b571 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -907,7 +907,7 @@ func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { // runtimeCallerFuncSet is the per-package tracking set: functions that // must keep physical frames (noinline, no tail calls) and get statement -// anchors at their call sites. Five criteria feed it: +// anchors at their call sites. Six criteria feed it: // // 1. the function (transitively, within the package) reaches a // runtime.Caller/Callers call — it consumes caller pcs itself; @@ -923,6 +923,8 @@ func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { // 5. the function can run below a defer that consumes panic pcs — recover // exposes the panicked call chain after longjmp has removed those physical // frames, so the compiler must keep and annotate the possible callees. +// 6. the package reads the memory profile — exact per-site allocation +// attribution requires physical frames for all trackable functions. // // Criterion 2 tests membership against the callee package's *base* set // (criterion 1 alone), so tracking extends exactly one call level past a @@ -969,7 +971,7 @@ func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs for fn := range base { frames[fn] = true } - // Criterion 5: a package that reads the memory profile gets every + // Criterion 6: a package that reads the memory profile gets every // trackable function pinned. Heap records attribute sampled // allocations to physical frames at exact statement lines; inlining // any function in such a package would merge its allocation sites @@ -982,7 +984,7 @@ func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs continue } if pinAll { - out[fn] = true + frames[fn] = true continue } // Criterion 3: pin program-unique frames. main.main and package @@ -1374,7 +1376,7 @@ func NewCallerTracking() *CallerTracking { // it to LLGo's implementation package. func isPublicRuntimePath(path string) bool { return path == "runtime" || - path == "github.com/goplus/llgo/runtime/internal/lib/runtime" + path == "github.com/xgo-dev/llgo/runtime/internal/lib/runtime" } func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 75937270fe..799824db05 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -73,18 +73,13 @@ func trimMemProfileStack(stk [32]uintptr) [32]uintptr { } // isRuntimePlumbingFrame reports whether pc belongs to LLGo runtime -// plumbing (allocator, capture hooks — including their __llgo_stub. -// wrappers, which is how a hook held in a function variable is entered). +// plumbing (allocator and capture hooks). func isRuntimePlumbingFrame(pc uintptr) bool { name := frameSymbol(pc - 1).function if name == "" { return false } - const stub = "__llgo_stub." - if hasPrefix(name, stub) { - name = name[len(stub):] - } - return hasPrefix(name, "github.com/goplus/llgo/runtime/internal/") || + return hasPrefix(name, "github.com/xgo-dev/llgo/runtime/internal/") || name == "runtime.captureMemProfileStack" } diff --git a/runtime/internal/runtime/memprofile_lock_native.go b/runtime/internal/runtime/memprofile_lock_native.go index 96196fac83..969a832aa6 100644 --- a/runtime/internal/runtime/memprofile_lock_native.go +++ b/runtime/internal/runtime/memprofile_lock_native.go @@ -19,8 +19,8 @@ package runtime import ( - psync "github.com/goplus/llgo/runtime/internal/clite/pthread/sync" - "github.com/goplus/llgo/runtime/internal/clite/sync/atomic" + psync "github.com/xgo-dev/llgo/runtime/internal/clite/pthread/sync" + "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" ) const ( From 0dbef809a9957947c5c3fa5b71259706b62023c0 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 17 Aug 2026 22:14:20 +0800 Subject: [PATCH 10/26] runtime: reduce memory profiler TLS hot-path overhead --- runtime/internal/runtime/memprofile.go | 74 +++++++++++-------- runtime/internal/runtime/memprofile_atomic.go | 12 +-- .../internal/runtime/memprofile_baremetal.go | 6 +- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go index 3e3299c3ad..9b04e52e55 100644 --- a/runtime/internal/runtime/memprofile.go +++ b/runtime/internal/runtime/memprofile.go @@ -1,6 +1,10 @@ package runtime -import "unsafe" +import ( + "unsafe" + + "github.com/xgo-dev/llgo/runtime/internal/clite/bitcast" +) // MemProfileRecord describes allocations aggregated by size class. type MemProfileRecord struct { @@ -82,6 +86,12 @@ type memStackBucket struct { stk [32]uintptr } +type memProfileThreadState struct { + remaining uintptr + rand uint64 + inSample bool +} + const memStackTabSize = 512 // power of two var ( @@ -89,22 +99,22 @@ var ( memStackTabLock memProfileLock ) -// memProfileRemaining counts down allocated bytes to the next sample. -// It and the random state are physical-thread local, matching gc's per-M -// sampler: independent allocation streams must not race or phase-lock each -// other. See memprofile_atomic.go for the native TLS declarations. +// The sampling state is physical-thread local, LLGo's analogue of gc's per-P +// countdown. Independent allocation streams must not race or phase-lock each +// other. See memprofile_atomic.go for the native TLS declaration. -func memProfileNextThreshold(rate int) uintptr { +func memProfileNextThreshold(state *memProfileThreadState, rate int) uintptr { // Exponentially distributed with mean rate, like gc's fastexprand: // the memoryless property is required — with any bounded-support // distribution a near-periodic allocation pattern phase-locks the // sampling points onto the large sites and skews per-site estimates // (observed 1.6x on goroot heapsampling's interleaved sizes). - x := memProfileRandState + x := state.rand if x == 0 { - // Mix the physical stack address into the first state so independently - // initialized threads do not replay the same threshold stream. - x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(&rate))) + // Mix the TLS state address into the first state so independently + // initialized threads do not replay the same threshold stream. Using + // the existing state pointer also avoids making a local escape here. + x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(state))) if x == 0 { x = 0x9e3779b97f4a7c15 } @@ -112,7 +122,7 @@ func memProfileNextThreshold(rate int) uintptr { x ^= x >> 12 x ^= x << 25 x ^= x >> 27 - memProfileRandState = x + state.rand = x r := (x * 0x2545f4914f6cdd1d) >> 11 // 53 random bits u := float64(r) / (1 << 53) if u < 1e-12 { @@ -133,10 +143,10 @@ func memProfileNextThreshold(rate int) uintptr { // below sampling noise. func lnApprox(u float64) float64 { const ln2 = 0.6931471805599453 - bits := *(*uint64)(unsafe.Pointer(&u)) + bits := uint64(bitcast.FromFloat64(u)) e := int((bits>>52)&0x7ff) - 1023 mbits := (bits &^ (uint64(0x7ff) << 52)) | (uint64(1023) << 52) - m := *(*float64)(unsafe.Pointer(&mbits)) // in [1, 2) + m := bitcast.ToFloat64(int64(mbits)) // in [1, 2) z := (m - 1) / (m + 1) z2 := z * z lnm := 2 * z * (1 + z2/3 + z2*z2/5 + z2*z2*z2/7) @@ -148,38 +158,37 @@ func recordMemProfileAlloc(size uintptr) { return } if MemProfileStackCapture != nil && MemProfileRatePtr != nil { - // The guard covers the whole decision path: threshold drawing and - // stack capture may themselves allocate (escaping locals, the - // bucket node), and a recursive sample would overflow the stack. - if memProfileInSample { - return - } - memProfileInSample = true + // Check the public rate before touching TLS so disabled profiling has + // no per-thread lookup cost. rate := *MemProfileRatePtr if rate <= 0 { - memProfileInSample = false + return + } + state := &memProfileState + if state.inSample { return } if rate == 1 { + state.inSample = true sampleMemProfileStack(size) - memProfileInSample = false + state.inSample = false return } // Mirror gc's mcache.nextSample: subtract, sample once on // crossing, redraw. Records hold RAW sampled counts — consumers // (pprof, goroot heapsampling.go) apply the Poisson correction // (scaleHeapSample) themselves, exactly like with gc. - if memProfileRemaining == 0 { - memProfileRemaining = memProfileNextThreshold(rate) + if state.remaining == 0 { + state.remaining = memProfileNextThreshold(state, rate) } - if size < memProfileRemaining { - memProfileRemaining -= size - memProfileInSample = false + if size < state.remaining { + state.remaining -= size return } - memProfileRemaining = memProfileNextThreshold(rate) + state.remaining = memProfileNextThreshold(state, rate) + state.inSample = true sampleMemProfileStack(size) - memProfileInSample = false + state.inSample = false return } sizeClass := memProfileSizeClass(size) @@ -311,12 +320,13 @@ func memProfileStacks(p []MemProfileRecord) (n int, ok bool) { // each individual snapshot must see a valid immutable list. Snapshot // construction may itself allocate, so suppress recursive sampling only // on this physical thread while the table lock is held. - wasInSample := memProfileInSample - memProfileInSample = true + state := &memProfileState + wasInSample := state.inSample + state.inSample = true memStackTabLock.lock() n, ok = memProfileStacksLocked(p) memStackTabLock.unlock() - memProfileInSample = wasInSample + state.inSample = wasInSample return n, ok } diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go index 9e76ffa578..8ff67c6f78 100644 --- a/runtime/internal/runtime/memprofile_atomic.go +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -6,16 +6,12 @@ import "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" type memProfileCounter = uint64 -// Native memory-profile sampling state is per physical thread, matching gc's -// per-M sampling state and keeping recursive allocator entry local to the -// thread that is currently capturing a stack. +// Keep the hot-path fields in one TLS object so one address lookup serves the +// whole allocation decision. The recursion guard remains local to the thread +// that is currently capturing a stack. // //llgo:tls -var ( - memProfileRemaining uintptr - memProfileRandState uint64 - memProfileInSample bool -) +var memProfileState memProfileThreadState func memProfileAddObject(p *memProfileCounter) { atomic.Add(p, memProfileCounter(1)) diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go index 31704ea177..4802dbeb39 100644 --- a/runtime/internal/runtime/memprofile_baremetal.go +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -6,11 +6,7 @@ type memProfileCounter = uintptr // Bare-metal runtimes have a single execution context and must not introduce // native TLS relocations. -var ( - memProfileRemaining uintptr - memProfileRandState uint64 - memProfileInSample bool -) +var memProfileState memProfileThreadState func memProfileAddN(p *memProfileCounter, n uint64) { *p += memProfileCounter(n) From a6f99912cc2b8cf2397f14bb2a8530d0beacf304 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Mon, 17 Aug 2026 19:03:17 +0800 Subject: [PATCH 11/26] test: refresh module path expectations --- cl/_testdata/method/in.go | 2 +- cl/_testmeta/reflect_named/meta-expect.txt | 10 +++++----- cl/_testrt/struct/in.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cl/_testdata/method/in.go b/cl/_testdata/method/in.go index 70e2d04416..d9ae848268 100644 --- a/cl/_testdata/method/in.go +++ b/cl/_testdata/method/in.go @@ -25,7 +25,7 @@ func main() { // CHECK-LABEL: define i64 @"main.(*T).Add"(ptr %0, i64 %1){{.*}} { // CHECK: [[NIL:%[0-9]+]] = icmp eq ptr %0, null -// CHECK-NEXT: call void @"{{.*}}PanicWrapNilPointer"(i1 [[NIL]], %"{{.*}}String" { ptr @{{[0-9]+}}, i64 44 }, %"{{.*}}String" { ptr @{{[0-9]+}}, i64 3 }) +// CHECK-NEXT: call void @"{{.*}}PanicWrapNilPointer"(i1 [[NIL]], %"{{.*}}String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }, %"{{.*}}String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }) // CHECK-NEXT: [[RECEIVER:%[0-9]+]] = load i64, ptr %0 // CHECK-NEXT: [[WRAPPED_SUM:%[0-9]+]] = call i64 @main.T.Add(i64 [[RECEIVER]], i64 %1) // CHECK-NEXT: ret i64 [[WRAPPED_SUM]] diff --git a/cl/_testmeta/reflect_named/meta-expect.txt b/cl/_testmeta/reflect_named/meta-expect.txt index 31e4612586..60d47772f2 100644 --- a/cl/_testmeta/reflect_named/meta-expect.txt +++ b/cl/_testmeta/reflect_named/meta-expect.txt @@ -44,8 +44,8 @@ main.main: [UseIfaceMethod] main.main: - github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E - github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E + github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E + github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY MethodByName _llgo_func$aM2cVUtLQbPq1YHtnabQiM7XJ5Cg5RyV6BIDWrqey7E [UseNamedMethod] main.main: @@ -61,7 +61,7 @@ _llgo_main.T: 1 main.m _llgo_func$2_iS07vIlF2_rZqWB5eU0IvP_9HviM4MYZNkXZDvbac main.(*T).m main.T.m [InterfaceInfo] -github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFpshv9b41oJ45HmjD5KeBlWw: +github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$bReUJH2_12QVkFcde1SpCUgXqmhRT8DRulK5lXtpIkY: Align _llgo_func$ETeB8WwW04JEq0ztcm-XPTJtuYvtpkjIsAc0-2NT9zA AssignableTo _llgo_func$Kxk9fspGkjXcoNWf2ucHG1vOQ5VHxVtYionfm-DnvWE Bits _llgo_func$ETeB8WwW04JEq0ztcm-XPTJtuYvtpkjIsAc0-2NT9zA @@ -101,6 +101,6 @@ github.com/xgo-dev/llgo/runtime/internal/lib/reflect.iface$mD8cc0P9iPHqG1cgYNbFp PkgPath _llgo_func$zNDVRsWTIpUPKouNUS805RGX--IV9qVK8B31IZbg5to Size _llgo_func$1kITCsyu7hFLMxHLR7kDlvu4SOra_HtrtdFUQH9P13s String _llgo_func$zNDVRsWTIpUPKouNUS805RGX--IV9qVK8B31IZbg5to - reflect.common _llgo_func$w6XuV-1SmW103DbauPseXBpW50HpxXAEsUsGFibl0Uw - reflect.uncommon _llgo_func$iG49bujiXjI2lVflYdE0hPXlCAABL-XKRANSNJEKOio + reflect.common _llgo_func$_FtQJl7A5s1VI_ob-KR67FLMXbIG5TjcVaEFuqM5Lo4 + reflect.uncommon _llgo_func$tpkXXN4h9pcUPRsL9rq-8-w7qAx06dW7DgXtepA0U1E diff --git a/cl/_testrt/struct/in.go b/cl/_testrt/struct/in.go index 6fcd3cadd1..97fc4fd738 100644 --- a/cl/_testrt/struct/in.go +++ b/cl/_testrt/struct/in.go @@ -36,7 +36,7 @@ func main() { // CHECK-LABEL: define void @"main.(*Foo).Print"(ptr %0){{.*}} { // CHECK: [[WRAPPER_NIL:%[0-9]+]] = icmp eq ptr %0, null -// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.PanicWrapNilPointer"(i1 [[WRAPPER_NIL]], %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 44 }, %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 5 }) +// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.PanicWrapNilPointer"(i1 [[WRAPPER_NIL]], %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }, %"{{.*}}/runtime/internal/runtime.String" { ptr @{{[0-9]+}}, i64 {{[0-9]+}} }) // CHECK-NEXT: [[WRAPPER_VALUE:%[0-9]+]] = load %main.Foo, ptr %0 // CHECK-NEXT: call void @main.Foo.Print(%main.Foo [[WRAPPER_VALUE]]) From 3859c4d04282fd8e55d124404656a1bc595a79f4 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 00:31:44 +0800 Subject: [PATCH 12/26] runtime: minimize sampled heap profiling overhead --- .../runtime/pprof_memprofile_go123_llgo.go | 12 +- .../lib/runtime/pprof_runtime_stub_llgo.go | 44 ++--- runtime/internal/runtime/memprofile.go | 171 +++--------------- runtime/internal/runtime/memprofile_atomic.go | 18 +- .../internal/runtime/memprofile_baremetal.go | 11 ++ .../runtime/memprofile_record_legacy.go | 27 +++ .../runtime/memprofile_record_native.go | 118 ++++++++++++ test/go/memprofile/memprofile_test.go | 44 +++++ 8 files changed, 277 insertions(+), 168 deletions(-) create mode 100644 runtime/internal/runtime/memprofile_record_legacy.go create mode 100644 runtime/internal/runtime/memprofile_record_native.go diff --git a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go index 0cc01acadc..8a84dfa437 100644 --- a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go +++ b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go @@ -2,7 +2,11 @@ package runtime -import _ "unsafe" +import ( + _ "unsafe" + + llrt "github.com/xgo-dev/llgo/runtime/internal/runtime" +) type pprofMemProfileRecord struct { AllocBytes, FreeBytes int64 @@ -12,6 +16,12 @@ type pprofMemProfileRecord struct { //go:linkname pprof_memProfileInternal runtime.pprof_memProfileInternal func pprof_memProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, ok bool) { + previous := llrt.MemProfilePause() + defer llrt.MemProfileResume(previous) + return pprofMemProfileInternal(p, inuseZero) +} + +func pprofMemProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, ok bool) { n, _ = MemProfile(nil, inuseZero) if len(p) < n { return n, false diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 799824db05..83a0615840 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -29,6 +29,15 @@ type MemProfileRecord struct { Stack0 [32]uintptr } +// MemProfile reuses its caller's record storage for the internal snapshot. +// Keep that zero-copy conversion guarded if either definition changes. +var ( + _ [unsafe.Sizeof(MemProfileRecord{}) - unsafe.Sizeof(llrt.MemProfileRecord{})]byte + _ [unsafe.Sizeof(llrt.MemProfileRecord{}) - unsafe.Sizeof(MemProfileRecord{})]byte + _ [unsafe.Offsetof(MemProfileRecord{}.Stack0) - unsafe.Offsetof(llrt.MemProfileRecord{}.Stack0)]byte + _ [unsafe.Offsetof(llrt.MemProfileRecord{}.Stack0) - unsafe.Offsetof(MemProfileRecord{}.Stack0)]byte +) + func (r *MemProfileRecord) InUseBytes() int64 { return r.AllocBytes - r.FreeBytes } @@ -84,31 +93,22 @@ func isRuntimePlumbingFrame(pc uintptr) bool { } func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { - // Size dynamically with slack and retry: sampling between a sizing - // call and its fill call can grow the bucket set, and a fixed cap - // would make callers that retry-until-ok (pprof) loop forever. - records := make([]llrt.MemProfileRecord, 64) - for attempt := 0; ; attempt++ { - n, ok = llrt.MemProfile(records, inuseZero) - if ok || attempt >= 3 { - break - } - records = make([]llrt.MemProfileRecord, n+n/4+16) - } - if !ok || len(p) < n { + previous := llrt.MemProfilePause() + defer llrt.MemProfileResume(previous) + return memProfile(p, inuseZero) +} + +func memProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + // The public and core records deliberately have the same fixed layout. + // Reuse the caller buffer so reading a rate-1 profile does not recursively + // allocate progressively larger profile buffers and create more buckets. + records := unsafe.Slice((*llrt.MemProfileRecord)(unsafe.Pointer(unsafe.SliceData(p))), len(p)) + n, ok = llrt.MemProfile(records, inuseZero) + if !ok { return n, false } - if n == 0 { - return 0, true - } for i := 0; i < n; i++ { - p[i] = MemProfileRecord{ - AllocBytes: records[i].AllocBytes, - FreeBytes: records[i].FreeBytes, - AllocObjects: records[i].AllocObjects, - FreeObjects: records[i].FreeObjects, - Stack0: trimMemProfileStack(records[i].Stack0), - } + p[i].Stack0 = trimMemProfileStack(p[i].Stack0) } return n, true } diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go index 9b04e52e55..e393927d11 100644 --- a/runtime/internal/runtime/memprofile.go +++ b/runtime/internal/runtime/memprofile.go @@ -2,11 +2,9 @@ package runtime import ( "unsafe" - - "github.com/xgo-dev/llgo/runtime/internal/clite/bitcast" ) -// MemProfileRecord describes allocations aggregated by size class. +// MemProfileRecord describes allocations aggregated into one profile bucket. type MemProfileRecord struct { AllocBytes, FreeBytes int64 AllocObjects, FreeObjects int64 @@ -80,6 +78,7 @@ var MemProfileRatePtr *int type memStackBucket struct { next *memStackBucket hash uintptr + size uintptr allocBytes memProfileCounter allocObjects memProfileCounter nstk int32 @@ -89,118 +88,15 @@ type memStackBucket struct { type memProfileThreadState struct { remaining uintptr rand uint64 - inSample bool } const memStackTabSize = 512 // power of two var ( - memStackTab [memStackTabSize]*memStackBucket + memStackTab [memStackTabSize]memProfileBucketHead memStackTabLock memProfileLock ) -// The sampling state is physical-thread local, LLGo's analogue of gc's per-P -// countdown. Independent allocation streams must not race or phase-lock each -// other. See memprofile_atomic.go for the native TLS declaration. - -func memProfileNextThreshold(state *memProfileThreadState, rate int) uintptr { - // Exponentially distributed with mean rate, like gc's fastexprand: - // the memoryless property is required — with any bounded-support - // distribution a near-periodic allocation pattern phase-locks the - // sampling points onto the large sites and skews per-site estimates - // (observed 1.6x on goroot heapsampling's interleaved sizes). - x := state.rand - if x == 0 { - // Mix the TLS state address into the first state so independently - // initialized threads do not replay the same threshold stream. Using - // the existing state pointer also avoids making a local escape here. - x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(state))) - if x == 0 { - x = 0x9e3779b97f4a7c15 - } - } - x ^= x >> 12 - x ^= x << 25 - x ^= x >> 27 - state.rand = x - r := (x * 0x2545f4914f6cdd1d) >> 11 // 53 random bits - u := float64(r) / (1 << 53) - if u < 1e-12 { - u = 1e-12 - } - t := -lnApprox(u) * float64(rate) - if t < 1 { - t = 1 - } - if max := float64(rate) * 64; t > max { - t = max - } - return uintptr(t) -} - -// lnApprox computes ln(u) for u in (0,1] via exponent split and an -// atanh series on the mantissa — a few 1e-6s of relative error, far -// below sampling noise. -func lnApprox(u float64) float64 { - const ln2 = 0.6931471805599453 - bits := uint64(bitcast.FromFloat64(u)) - e := int((bits>>52)&0x7ff) - 1023 - mbits := (bits &^ (uint64(0x7ff) << 52)) | (uint64(1023) << 52) - m := bitcast.ToFloat64(int64(mbits)) // in [1, 2) - z := (m - 1) / (m + 1) - z2 := z * z - lnm := 2 * z * (1 + z2/3 + z2*z2/5 + z2*z2*z2/7) - return float64(e)*ln2 + lnm -} - -func recordMemProfileAlloc(size uintptr) { - if size == 0 { - return - } - if MemProfileStackCapture != nil && MemProfileRatePtr != nil { - // Check the public rate before touching TLS so disabled profiling has - // no per-thread lookup cost. - rate := *MemProfileRatePtr - if rate <= 0 { - return - } - state := &memProfileState - if state.inSample { - return - } - if rate == 1 { - state.inSample = true - sampleMemProfileStack(size) - state.inSample = false - return - } - // Mirror gc's mcache.nextSample: subtract, sample once on - // crossing, redraw. Records hold RAW sampled counts — consumers - // (pprof, goroot heapsampling.go) apply the Poisson correction - // (scaleHeapSample) themselves, exactly like with gc. - if state.remaining == 0 { - state.remaining = memProfileNextThreshold(state, rate) - } - if size < state.remaining { - state.remaining -= size - return - } - state.remaining = memProfileNextThreshold(state, rate) - state.inSample = true - sampleMemProfileStack(size) - state.inSample = false - return - } - sizeClass := memProfileSizeClass(size) - for i := range memProfileBuckets { - b := &memProfileBuckets[i] - if b.size == sizeClass { - memProfileAddObject(&b.objects) - return - } - } -} - func sampleMemProfileStack(size uintptr) { // Tiny allocations occupy at least one 16-byte granule (bdwgc's // minimum on 64-bit, matching gc's tiny size class): report the @@ -218,14 +114,15 @@ func sampleMemProfileStack(size uintptr) { for i := 0; i < n; i++ { h = h*33 + pcs[i] } + // gc keys heap profile buckets by both stack and allocation size. Besides + // preserving that contract, keeping sizes separate lets the consumer's + // Poisson correction use the actual sample size instead of a mixed mean. + h = h*33 + size slot := h & (memStackTabSize - 1) - memStackTabLock.lock() - if b := findMemStackBucket(slot, h, pcs[:n]); b != nil { + if b := findMemStackBucket(memProfileLoadBucket(&memStackTab[slot]), h, size, pcs[:n]); b != nil { memStackAdd(b, size) - memStackTabLock.unlock() return } - memStackTabLock.unlock() // Allocation re-enters recordMemProfileAlloc, where this physical // thread's recursion guard suppresses the internal allocation. Keep the @@ -233,25 +130,29 @@ func sampleMemProfileStack(size uintptr) { // invert lock order with a concurrent MemProfile call. b := (*memStackBucket)(AllocZ(unsafe.Sizeof(memStackBucket{}))) b.hash = h + b.size = size b.nstk = int32(n) copy(b.stk[:], pcs[:n]) memStackTabLock.lock() - if existing := findMemStackBucket(slot, h, pcs[:n]); existing != nil { - memStackAdd(existing, size) + head := memProfileLoadBucket(&memStackTab[slot]) + if existing := findMemStackBucket(head, h, size, pcs[:n]); existing != nil { memStackTabLock.unlock() + memStackAdd(existing, size) return } memStackAdd(b, size) - b.next = memStackTab[slot] - memStackTab[slot] = b + b.next = head + memProfileStoreBucket(&memStackTab[slot], b) memStackTabLock.unlock() } -// findMemStackBucket is called with memStackTabLock held. -func findMemStackBucket(slot, hash uintptr, pcs []uintptr) *memStackBucket { - for b := memStackTab[slot]; b != nil; b = b.next { - if b.hash == hash && int(b.nstk) == len(pcs) && memStackEqual(b, pcs) { +// Buckets and their next links are immutable after the head is atomically +// published, so lookups need no lock. The insertion lock only resolves the +// rare race between two first samples for the same key. +func findMemStackBucket(head *memStackBucket, hash, size uintptr, pcs []uintptr) *memStackBucket { + for b := head; b != nil; b = b.next { + if b.hash == hash && b.size == size && int(b.nstk) == len(pcs) && memStackEqual(b, pcs) { return b } } @@ -315,24 +216,13 @@ func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { } func memProfileStacks(p []MemProfileRecord) (n int, ok bool) { - // Protect both list publication and enumeration. The public wrappers - // tolerate a bucket appearing between their sizing and fill calls, but - // each individual snapshot must see a valid immutable list. Snapshot - // construction may itself allocate, so suppress recursive sampling only - // on this physical thread while the table lock is held. - state := &memProfileState - wasInSample := state.inSample - state.inSample = true - memStackTabLock.lock() - n, ok = memProfileStacksLocked(p) - memStackTabLock.unlock() - state.inSample = wasInSample - return n, ok -} - -func memProfileStacksLocked(p []MemProfileRecord) (n int, ok bool) { - for i := range memStackTab { - for b := memStackTab[i]; b != nil; b = b.next { + // Save one atomic head per slot. Published chains are immutable, so this + // gives a stable snapshot without blocking allocation samples and without + // touching the caller thread's sampling state. + var heads [memStackTabSize]*memStackBucket + for i := range heads { + heads[i] = memProfileLoadBucket(&memStackTab[i]) + for b := heads[i]; b != nil; b = b.next { n++ } } @@ -340,11 +230,8 @@ func memProfileStacksLocked(p []MemProfileRecord) (n int, ok bool) { return n, false } j := 0 - for i := range memStackTab { - for b := memStackTab[i]; b != nil; b = b.next { - if j >= len(p) { - break - } + for i := range heads { + for b := heads[i]; b != nil; b = b.next { r := MemProfileRecord{ AllocBytes: int64(memProfileLoadObjects(&b.allocBytes)), AllocObjects: int64(memProfileLoadObjects(&b.allocObjects)), diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go index 8ff67c6f78..9315d200e4 100644 --- a/runtime/internal/runtime/memprofile_atomic.go +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -2,13 +2,17 @@ package runtime -import "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" +import ( + "unsafe" + + "github.com/xgo-dev/llgo/runtime/internal/clite/sync/atomic" +) type memProfileCounter = uint64 +type memProfileBucketHead = unsafe.Pointer // Keep the hot-path fields in one TLS object so one address lookup serves the -// whole allocation decision. The recursion guard remains local to the thread -// that is currently capturing a stack. +// whole allocation decision. A sentinel countdown marks recursive sampling. // //llgo:tls var memProfileState memProfileThreadState @@ -24,3 +28,11 @@ func memProfileAddN(p *memProfileCounter, n uint64) { func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { return atomic.Load(p) } + +func memProfileLoadBucket(p *memProfileBucketHead) *memStackBucket { + return (*memStackBucket)(atomic.Load(p)) +} + +func memProfileStoreBucket(p *memProfileBucketHead, b *memStackBucket) { + atomic.Store(p, unsafe.Pointer(b)) +} diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go index 4802dbeb39..26565e81d9 100644 --- a/runtime/internal/runtime/memprofile_baremetal.go +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -2,7 +2,10 @@ package runtime +import "unsafe" + type memProfileCounter = uintptr +type memProfileBucketHead = unsafe.Pointer // Bare-metal runtimes have a single execution context and must not introduce // native TLS relocations. @@ -19,3 +22,11 @@ func memProfileAddObject(p *memProfileCounter) { func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { return *p } + +func memProfileLoadBucket(p *memProfileBucketHead) *memStackBucket { + return (*memStackBucket)(*p) +} + +func memProfileStoreBucket(p *memProfileBucketHead, b *memStackBucket) { + *p = unsafe.Pointer(b) +} diff --git a/runtime/internal/runtime/memprofile_record_legacy.go b/runtime/internal/runtime/memprofile_record_legacy.go new file mode 100644 index 0000000000..b9ec0a4afe --- /dev/null +++ b/runtime/internal/runtime/memprofile_record_legacy.go @@ -0,0 +1,27 @@ +//go:build !llgo || baremetal || wasm + +package runtime + +func MemProfilePause() uintptr { + return 0 +} + +func MemProfileResume(uintptr) { +} + +// Legacy targets do not yet expose the physical native stack walk used by the +// sampled profiler. Keep their existing size-class accounting separate so the +// native allocator hot path has no mode dispatch. +func recordMemProfileAlloc(size uintptr) { + if size == 0 { + return + } + sizeClass := memProfileSizeClass(size) + for i := range memProfileBuckets { + b := &memProfileBuckets[i] + if b.size == sizeClass { + memProfileAddObject(&b.objects) + return + } + } +} diff --git a/runtime/internal/runtime/memprofile_record_native.go b/runtime/internal/runtime/memprofile_record_native.go new file mode 100644 index 0000000000..e5334724ac --- /dev/null +++ b/runtime/internal/runtime/memprofile_record_native.go @@ -0,0 +1,118 @@ +//go:build llgo && !baremetal && !wasm + +package runtime + +import ( + "unsafe" + + "github.com/xgo-dev/llgo/runtime/internal/clite/bitcast" +) + +const memProfileSampling = ^uintptr(0) + +// MemProfilePause suppresses allocations made while materializing a profile. +// Without it, rate-1 profiling would observe its own symbolization and buffer +// work and recursively grow the bucket table. The returned value supports +// nested readers and is restored by MemProfileResume. +func MemProfilePause() uintptr { + state := &memProfileState + previous := state.remaining + state.remaining = memProfileSampling + return previous +} + +func MemProfileResume(previous uintptr) { + memProfileState.remaining = previous +} + +// recordMemProfileAlloc is deliberately small enough to inline into AllocZ/U. +// The usual enabled path does one rate load, one TLS address calculation, and +// one countdown update. Random threshold generation and stack capture stay on +// the sampled slow path. +func recordMemProfileAlloc(size uintptr) { + if size == 0 || MemProfileRatePtr == nil { + return + } + rate := *MemProfileRatePtr + if rate <= 0 { + return + } + state := &memProfileState + remaining := state.remaining + if remaining == memProfileSampling { + return + } + if rate != 1 && size < remaining { + state.remaining = remaining - size + return + } + recordMemProfileAllocSlow(state, size, rate) +} + +//go:noinline +func recordMemProfileAllocSlow(state *memProfileThreadState, size uintptr, rate int) { + if MemProfileStackCapture == nil { + return + } + next := state.remaining + if rate != 1 { + // A zero countdown initializes the stream. A non-zero countdown reached + // here because this allocation crossed the next sampling point. + if next == 0 { + next = memProfileNextThreshold(state, rate) + if size < next { + state.remaining = next - size + return + } + } + next = memProfileNextThreshold(state, rate) + } + state.remaining = memProfileSampling + sampleMemProfileStack(size) + state.remaining = next +} + +func memProfileNextThreshold(state *memProfileThreadState, rate int) uintptr { + // Exponentially distributed with mean rate, like gc's fastexprand: + // the memoryless property is required — with any bounded-support + // distribution a near-periodic allocation pattern phase-locks the + // sampling points onto the large sites and skews per-site estimates. + x := state.rand + if x == 0 { + x = 0x9e3779b97f4a7c15 ^ uint64(uintptr(unsafe.Pointer(state))) + if x == 0 { + x = 0x9e3779b97f4a7c15 + } + } + x ^= x >> 12 + x ^= x << 25 + x ^= x >> 27 + state.rand = x + r := (x * 0x2545f4914f6cdd1d) >> 11 // 53 random bits + u := float64(r) / (1 << 53) + if u < 1e-12 { + u = 1e-12 + } + t := -lnApprox(u) * float64(rate) + if t < 1 { + t = 1 + } + if max := float64(rate) * 64; t > max { + t = max + } + return uintptr(t) +} + +// lnApprox computes ln(u) for u in (0,1] via exponent split and an +// atanh series on the mantissa. Its error is far below sampling noise. +func lnApprox(u float64) float64 { + const ln2 = 0.6931471805599453 + bits := uint64(bitcast.FromFloat64(u)) + e := int((bits>>52)&0x7ff) - 1023 + mbits := (bits &^ (uint64(0x7ff) << 52)) | (uint64(1023) << 52) + m := bitcast.ToFloat64(int64(mbits)) // in [1, 2) + z := (m - 1) / (m + 1) + z2 := z * z + lnm := 2 * z * (1 + z2/3 + z2*z2/5 + z2*z2*z2/7) + return float64(e)*ln2 + lnm +} diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go index 37e499a4e2..eae3e95855 100644 --- a/test/go/memprofile/memprofile_test.go +++ b/test/go/memprofile/memprofile_test.go @@ -6,10 +6,12 @@ import ( "reflect" "runtime" "runtime/pprof" + "strings" "testing" ) var tinySink []*int32 +var mixedSizeSink [][]byte type profiledClosure struct { fn func(int) int @@ -71,6 +73,48 @@ func TestRuntimeMemProfileReportsTinyAllocations(t *testing.T) { t.Fatalf("MemProfile did not report tiny allocations totaling at least %d bytes: %#v", wantBytes, records) } +func TestRuntimeMemProfileSeparatesSizesAtOneStack(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + mixedSizeSink = make([][]byte, 128) + mixedSizeProfileAlloc(mixedSizeSink) + sizes := make(map[int64]bool) + for _, record := range readMemProfile(t) { + if record.AllocObjects == 0 { + continue + } + frames := runtime.CallersFrames(record.Stack()) + for { + frame, more := frames.Next() + if strings.HasSuffix(frame.Function, ".mixedSizeProfileAlloc") { + sizes[record.AllocBytes/record.AllocObjects] = true + break + } + if !more { + break + } + } + } + if !sizes[64] || !sizes[256] { + t.Fatalf("same-stack memory profile sizes = %v, want 64 and 256", sizes) + } +} + +//go:noinline +func mixedSizeProfileAlloc(dst [][]byte) { + for i := range dst { + size := 64 + if i&1 != 0 { + size = 256 + } + dst[i] = make([]byte, size) + } +} + func TestRuntimePprofHeapProfileReportsTinyAllocations(t *testing.T) { oldRate := runtime.MemProfileRate runtime.MemProfileRate = 1 From 1248692a1e136bd55c62f4e8471d00ad3e9a891b Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 00:31:54 +0800 Subject: [PATCH 13/26] ssa,cl: reuse native TLS addresses on hot paths --- cl/_testgo/localitycodegen/in.go | 3 ++- cl/compile.go | 2 +- cl/locality_lower.go | 42 +++++++++++++++++++++++++++++++- cl/locality_test.go | 18 ++++++++++++++ ssa/memory.go | 15 ++++++++++++ 5 files changed, 77 insertions(+), 3 deletions(-) diff --git a/cl/_testgo/localitycodegen/in.go b/cl/_testgo/localitycodegen/in.go index 100ef977ac..86eb136a9a 100644 --- a/cl/_testgo/localitycodegen/in.go +++ b/cl/_testgo/localitycodegen/in.go @@ -31,7 +31,8 @@ package main // CHECK-LABEL: define { i64, ptr, ptr } @main.values() // CHECK: call void @"main.__llgo_tls_init$ensure"() -// CHECK: load i64, ptr @main.scalar +// CHECK-NEXT: [[SCALAR_ADDR:%[0-9]+]] = call ptr @llvm.threadlocal.address{{.*}}(ptr @main.scalar) +// CHECK-NEXT: load i64, ptr [[SCALAR_ADDR]] // CHECK: call ptr @main.__llgo_local_block() // CHECK-LABEL: define ptr @"main._llgo_routine$1"(ptr diff --git a/cl/compile.go b/cl/compile.go index f3c3dad644..7dca9fd07f 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -1948,7 +1948,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr { } if p.options.DebugSymbols && p.localityAllowsGlobalDebug(v) { pos := p.fset.Position(v.Pos()) - b.DIGlobal(val, v.Name(), pos) + b.DIGlobal(p.localityGlobalDebugValue(v, val), v.Name(), pos) } return val case *ssa.Const: diff --git a/cl/locality_lower.go b/cl/locality_lower.go index 1825ec1d11..7e0a2e535d 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -63,6 +63,11 @@ type localEnsureCacheKey struct { kind locality.Kind } +type localTLSCacheKey struct { + block *ssa.BasicBlock + variable *localVariable +} + // localityLowering owns all compiler state for TLS/GLS lowering. Only this // value is embedded in the general compiler context. type localityLowering struct { @@ -75,6 +80,7 @@ type localityFunction struct { block *ssa.BasicBlock packageBases map[localBaseCacheKey]llssa.Expr packageEnsures map[localEnsureCacheKey]bool + tlsAddresses map[localTLSCacheKey]llssa.Expr entry *localEntryContext } @@ -180,6 +186,21 @@ func (p *context) localityAllowsGlobalDebug(global *ssa.Global) bool { return variable == nil || variable.planned.Storage == localitylayout.StorageNativeTLS } +// localityGlobalDebugValue keeps debug metadata on the native TLS global. +// Normal accesses use llvm.threadlocal.address instructions, which are values +// rather than GlobalObjects and therefore cannot carry DIGlobal metadata. +func (p *context) localityGlobalDebugValue(global *ssa.Global, value llssa.Expr) llssa.Expr { + variable := p.locality.variables[global] + if variable == nil || variable.planned.Storage != localitylayout.StorageNativeTLS { + return value + } + direct := variable.owner.direct[variable.planned.Name] + if direct == nil { + panic(fmt.Sprintf("missing native TLS storage for %s", variable.planned.Name)) + } + return direct.Expr +} + func (p *context) localTypesPackage(fullName string) *types.Package { matches := func(pkg *types.Package) bool { if pkg == nil { @@ -345,7 +366,26 @@ func (p *context) localVariableAddr(b llssa.Builder, v *ssa.Global, info llssa.V if direct == nil { panic(fmt.Sprintf("missing native TLS storage for %s", name)) } - return direct.Expr + // Metadata probes have no instruction builder. Real lowering + // materializes and dominance-caches the address so Darwin TLV and ELF + // TLS code generation resolve it only once along a control-flow path. + if b == nil { + return direct.Expr + } + state := &p.locality.function + for block := state.block; block != nil; block = block.Idom() { + if addr, ok := state.tlsAddresses[localTLSCacheKey{block: block, variable: variable}]; ok { + return addr + } + } + addr := b.ThreadLocalAddress(direct) + if state.block != nil { + if state.tlsAddresses == nil { + state.tlsAddresses = make(map[localTLSCacheKey]llssa.Expr) + } + state.tlsAddresses[localTLSCacheKey{block: state.block, variable: variable}] = addr + } + return addr } base := p.localPackageBase(b, variable.owner) return b.FieldAddr(base, variable.planned.Field) diff --git a/cl/locality_test.go b/cl/locality_test.go index f95f765c74..f953f74ed2 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -129,6 +129,24 @@ func value() *int { return pointer } } } +func TestNativeTLSAddressIsReusedWithinDominatingPath(t *testing.T) { + _, ir := compileLocalitySource(t, `package locality + +//llgo:tls +var counter int + +func add() int { + counter++ + counter++ + return counter +} +`) + add := llvmFunction(t, ir, "example.com/locality.add") + if got := strings.Count(add, "@llvm.threadlocal.address"); got != 1 { + t.Fatalf("native TLS address resolutions = %d, want one:\n%s", got, add) + } +} + func llvmFunction(t *testing.T, ir, name string) string { t.Helper() markerAt := strings.Index(ir, `@"`+name+`"(`) diff --git a/ssa/memory.go b/ssa/memory.go index 3241d6cd34..ea90562a47 100644 --- a/ssa/memory.go +++ b/ssa/memory.go @@ -24,6 +24,8 @@ import ( "github.com/xgo-dev/llvm" ) +var threadLocalAddressIntrinsic = llvm.LookupIntrinsicID("llvm.threadlocal.address") + // ----------------------------------------------------------------------------- func (b Builder) aggregateAllocU(t Type, flds ...llvm.Value) llvm.Value { @@ -167,6 +169,19 @@ func (b Builder) AllocaT(t Type) (ret Expr) { return } +// ThreadLocalAddress materializes the address of a native TLS global. Keeping +// this as an SSA value lets a caller reuse one target TLS resolver result for +// all dominated accesses instead of resolving the global at every use. +func (b Builder) ThreadLocalAddress(v Global) Expr { + if threadLocalAddressIntrinsic == 0 { + panic("ssa: llvm.threadlocal.address is unavailable") + } + return Expr{ + b.impl.CreateIntrinsic(v.impl.Type(), threadLocalAddressIntrinsic, []llvm.Value{v.impl}, ""), + v.Type, + } +} + /* TODO(xsw): // AllocaU allocates uninitialized space for n*sizeof(elem) bytes. func (b Builder) AllocaU(elem Type, n ...int64) (ret Expr) { From 26ddb055000fcada55e7ed92998cc9aeb1c2a5dd Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 00:31:59 +0800 Subject: [PATCH 14/26] cl,build: omit unused memory profiling hooks --- cl/caller_frame_test.go | 56 ++++++- cl/caller_tracking_precompute_test.go | 29 ++++ cl/instr.go | 208 +++++++++++++++++++++++--- internal/build/build.go | 22 +++ internal/build/collect.go | 4 + internal/build/fingerprint.go | 15 +- internal/build/fingerprint_test.go | 16 ++ internal/build/module_hook_test.go | 78 ++++++++++ ssa/backend_program_test.go | 3 +- ssa/package.go | 13 ++ 10 files changed, 411 insertions(+), 33 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index c0748c2b39..1ead834efa 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -1200,8 +1200,8 @@ func TestDirectiveFilename(t *testing.T) { } } -// Packages that read the memory profile pin every trackable function: -// per-site heap attribution needs frames and allocation-site anchors. +// Packages that read the memory profile pin allocation paths, not unrelated +// helpers: per-site heap attribution needs only the observable stack. func TestPackageReadsMemProfilePin(t *testing.T) { ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/hp", `package main @@ -1209,22 +1209,27 @@ import "runtime" func allocLeaf() *[64]byte { return new([64]byte) } +func allocWrapper() *[64]byte { return allocLeaf() } + func plainHelper() int { return 1 } func main() { runtime.MemProfileRate = 1 - _ = allocLeaf() + _ = allocWrapper() _ = plainHelper() var r [4]runtime.MemProfileRecord runtime.MemProfile(r[:], true) } `) set := runtimeCallerFuncSet(NewCallerTracking(), ssapkg) - for _, name := range []string{"allocLeaf", "plainHelper", "main"} { + for _, name := range []string{"allocLeaf", "allocWrapper", "main"} { if !set[ssapkg.Func(name)] { t.Fatalf("%s must be pinned in a memprofile-reading package", name) } } + if set[ssapkg.Func("plainHelper")] { + t.Fatal("plainHelper must remain inlineable in a memprofile-reading package") + } quiet, _ := buildCallerFrameSSAPackage(t, "example.com/quiet", `package q func Helper() int { return 2 } @@ -1234,6 +1239,30 @@ func Helper() int { return 2 } } } +func TestMemoryProfileConvertMayAllocate(t *testing.T) { + stringType := types.Typ[types.String] + intType := types.Typ[types.Int] + tests := []struct { + name string + from, to types.Type + want bool + }{ + {"numeric", intType, types.Typ[types.Int64], false}, + {"string to bytes", stringType, types.NewSlice(types.Typ[types.Uint8]), true}, + {"runes to string", types.NewSlice(types.Typ[types.Int32]), stringType, true}, + {"rune to string", types.Typ[types.Int32], stringType, true}, + {"string to ints", stringType, types.NewSlice(intType), false}, + {"pointer", types.NewPointer(intType), types.Typ[types.UnsafePointer], false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := memoryProfileConvertMayAllocate(test.from, test.to); got != test.want { + t.Fatalf("memoryProfileConvertMayAllocate(%v, %v) = %v, want %v", test.from, test.to, got, test.want) + } + }) + } +} + func TestPackageReadsMemProfileDetection(t *testing.T) { tests := []struct { name string @@ -1257,6 +1286,13 @@ import "runtime" func enable() { runtime.MemProfileRate = 1 }`, want: true, }, + { + name: "MemProfile function value", + src: `package p +import "runtime" +var report = runtime.MemProfile`, + want: true, + }, { name: "unrelated runtime use", src: `package p @@ -1275,6 +1311,18 @@ func goos() string { return runtime.GOOS }`, } } +func TestMemProfileConsumer(t *testing.T) { + quiet, _ := buildCallerFrameSSAPackage(t, "example.com/quiet", `package quiet +func Value() int { return 1 }`) + if got := MemProfileConsumer([]*gossa.Package{quiet}); got != "" { + t.Fatal("quiet program unexpectedly enabled memory profiling") + } + pprof, _ := buildCallerFrameSSAPackage(t, "runtime/pprof", `package pprof`) + if got := MemProfileConsumer([]*gossa.Package{quiet, pprof}); got != "runtime/pprof" { + t.Fatal("runtime/pprof did not enable memory profiling") + } +} + func TestPackageReadsMemProfileSkipsFunctionWithoutPackage(t *testing.T) { funcs := map[*gossa.Function]bool{new(gossa.Function): true} if packageReadsMemProfile(funcs) { diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go index 4505862286..38ea8f3146 100644 --- a/cl/caller_tracking_precompute_test.go +++ b/cl/caller_tracking_precompute_test.go @@ -127,6 +127,35 @@ func Plain() { dep.Quiet() } } } +func TestCallerTrackingPrecomputePinsCrossPackageMemoryProfileAllocations(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +func Alloc() *[64]byte { return new([64]byte) } +func Plain() int { return 1 } +`, + "example.com/root", `package root +import ( + "example.com/dep" + "runtime" +) +func UseAlloc() *[64]byte { return dep.Alloc() } +func Plain() int { return dep.Plain() } +func Report(records []runtime.MemProfileRecord) { runtime.MemProfile(records, false) } +`) + tracking := NewCallerTracking() + tracking.SetMemoryProfileAttribution(true) + tracking.Precompute([]*gossa.Package{dep, root}) + if !runtimeCallerFuncSet(tracking, dep)[dep.Func("Alloc")] { + t.Fatal("cross-package allocation leaf was not pinned") + } + if !runtimeCallerFuncSet(tracking, root)[root.Func("UseAlloc")] { + t.Fatal("cross-package allocation wrapper was not pinned") + } + if runtimeCallerFuncSet(tracking, dep)[dep.Func("Plain")] || runtimeCallerFuncSet(tracking, root)[root.Func("Plain")] { + t.Fatal("unrelated cross-package path was pinned") + } +} + func TestCallerTrackingPrecomputeRejectsLatePackages(t *testing.T) { dep, root := buildCallerFrameSSAProgram(t, "example.com/dep", `package dep diff --git a/cl/instr.go b/cl/instr.go index 7006f1b571..6caf4afa19 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -923,8 +923,8 @@ func fnUsesRuntimeCaller(c *CallerTracking, fn *ssa.Function) bool { // 5. the function can run below a defer that consumes panic pcs — recover // exposes the panicked call chain after longjmp has removed those physical // frames, so the compiler must keep and annotate the possible callees. -// 6. the package reads the memory profile — exact per-site allocation -// attribution requires physical frames for all trackable functions. +// 6. the program reads the memory profile — functions on a path to an +// allocation keep physical frames for per-site attribution. // // Criterion 2 tests membership against the callee package's *base* set // (criterion 1 alone), so tracking extends exactly one call level past a @@ -954,7 +954,11 @@ func callerTrackingFuncSetsForPackage(c *CallerTracking, pkg *ssa.Package) calle } base := runtimeCallerBaseSet(c, pkg) funcs, trackable := collectRuntimeCallerFunctions(pkg) - sets := computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkg, funcs, base, trackable, func(dep *ssa.Package) map[*ssa.Function]bool { + var profileFrames map[*ssa.Function]bool + if packageReadsMemProfile(trackable) { + profileFrames = memoryProfileAllocationFrames(trackable) + } + sets := computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkg, funcs, base, trackable, profileFrames, func(dep *ssa.Package) map[*ssa.Function]bool { return runtimeCallerBaseSet(c, dep) }) c.extended[pkg] = sets @@ -966,24 +970,21 @@ type callerTrackingFuncSets struct { recoverPanicSites map[*ssa.Function]bool } -func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs, base, trackable map[*ssa.Function]bool, baseSet func(*ssa.Package) map[*ssa.Function]bool) callerTrackingFuncSets { +func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs, base, trackable, profileFrames map[*ssa.Function]bool, baseSet func(*ssa.Package) map[*ssa.Function]bool) callerTrackingFuncSets { frames := make(map[*ssa.Function]bool, len(base)) for fn := range base { frames[fn] = true } - // Criterion 6: a package that reads the memory profile gets every - // trackable function pinned. Heap records attribute sampled - // allocations to physical frames at exact statement lines; inlining - // any function in such a package would merge its allocation sites - // into the caller and lose per-site attribution (goroot - // heapsampling.go). Profiling packages are rare and accuracy beats - // inlining there — gc keeps per-site attribution via its inline tree. - pinAll := packageReadsMemProfile(trackable) + // Criterion 6: retain only functions that can reach an allocation when the + // program reads the profile. This keeps allocation leaf and wrapper + // identities, including across packages, without disabling inlining for + // unrelated helpers. gc represents the same logical frames in its inline + // tree; LLGo currently keeps these selected physical frames. for fn := range trackable { if frames[fn] { continue } - if pinAll { + if profileFrames[fn] { frames[fn] = true continue } @@ -1023,6 +1024,95 @@ func computeRuntimeCallerFuncSets(recover *recoverFacts, pkg *ssa.Package, funcs return callerTrackingFuncSets{frames: frames, recoverPanicSites: recoverPanicSites} } +// memoryProfileAllocationFrames returns allocation-bearing functions and +// their static callers. Calls whose implementation is outside the analyzed +// function set are conservatively allocation-bearing: the external callee may +// use AllocZ/U, and the local wrapper is part of the exposed profile stack. +func memoryProfileAllocationFrames(funcs map[*ssa.Function]bool) map[*ssa.Function]bool { + frames := make(map[*ssa.Function]bool) + for fn := range funcs { + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if memoryProfileInstructionMayAllocate(instr, funcs) { + frames[fn] = true + break + } + } + if frames[fn] { + break + } + } + } + for changed := true; changed; { + changed = false + for fn := range funcs { + if frames[fn] { + continue + } + forEachCall(fn, func(call *ssa.CallCommon) { + if callee := call.StaticCallee(); callee != nil && frames[callee] { + frames[fn] = true + changed = true + } + }) + } + } + return frames +} + +func memoryProfileInstructionMayAllocate(instr ssa.Instruction, funcs map[*ssa.Function]bool) bool { + switch instr := instr.(type) { + case *ssa.Alloc: + return instr.Heap + case *ssa.MakeChan, *ssa.MakeClosure, *ssa.MakeInterface, *ssa.MakeMap, *ssa.MakeSlice, + *ssa.Defer, *ssa.Go, *ssa.MapUpdate, *ssa.Send: + return true + case *ssa.BinOp: + return instr.Op == token.ADD && types.Identical(instr.Type(), types.Typ[types.String]) + case *ssa.Convert: + return memoryProfileConvertMayAllocate(instr.X.Type(), instr.Type()) + case *ssa.MultiConvert: + // A type-parameter conversion can select an allocating string/slice + // conversion after instantiation. Keep it conservative until its type + // set is lowered to concrete alternatives here. + return true + case ssa.CallInstruction: + call := instr.Common() + if builtin, ok := call.Value.(*ssa.Builtin); ok { + return builtin.Name() == "append" + } + callee := call.StaticCallee() + return callee == nil || !funcs[callee] + } + return false +} + +func memoryProfileConvertMayAllocate(from, to types.Type) bool { + from = types.Unalias(from).Underlying() + to = types.Unalias(to).Underlying() + switch to := to.(type) { + case *types.Basic: + if to.Kind() != types.String { + return false + } + switch from := from.(type) { + case *types.Basic: + return from.Info()&types.IsInteger != 0 + case *types.Slice: + return isByteOrRune(from.Elem()) + } + case *types.Slice: + from, ok := from.(*types.Basic) + return ok && from.Kind() == types.String && isByteOrRune(to.Elem()) + } + return false +} + +func isByteOrRune(typ types.Type) bool { + basic, ok := types.Unalias(typ).Underlying().(*types.Basic) + return ok && (basic.Kind() == types.Uint8 || basic.Kind() == types.Int32) +} + // addRecoverObservableCallees keeps the same-package synchronous call/defer // subtree below a defer that can inspect caller pcs. These frames are no // longer physically live when the deferred function runs after recover; @@ -1215,10 +1305,27 @@ func (a *runtimeCallerAnalysis) callTargets(fn *ssa.Function, call *ssa.CallComm // Precompute before workers start; recover facts also synchronize lazy queries // for nested and synthetic functions that are not package members. type CallerTracking struct { - base map[*ssa.Package]map[*ssa.Function]bool - extended map[*ssa.Package]callerTrackingFuncSets - recover *recoverFacts - precomputed bool + base map[*ssa.Package]map[*ssa.Function]bool + extended map[*ssa.Package]callerTrackingFuncSets + recover *recoverFacts + memoryProfileAttribution bool + memoryProfileConfigured bool + precomputed bool +} + +// SetMemoryProfileAttribution supplies the build-wide profiling decision. +// The build coordinator already computed it for allocator selection, so +// reusing it avoids a second program scan and covers externally callable +// library modes that have no visible profile consumer. +func (c *CallerTracking) SetMemoryProfileAttribution(enable bool) { + if c == nil { + return + } + if c.precomputed { + panic("memory-profile attribution configured after caller tracking") + } + c.memoryProfileAttribution = enable + c.memoryProfileConfigured = true } // Precompute resolves caller-tracking and recover data before package backends @@ -1269,8 +1376,22 @@ func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { analyses[i] = analyzeCallerTrackingPackage(pkgs[i], methods[pkgs[i]]) base[i] = analyses[i].base } + profileEnabled := c.memoryProfileAttribution + if !c.memoryProfileConfigured { + profileEnabled = MemProfileConsumer(pkgs) != "" + } + var profileFrames map[*ssa.Function]bool + if profileEnabled { + allTrackable := make(map[*ssa.Function]bool) + for i := range analyses { + for fn := range analyses[i].trackable { + allTrackable[fn] = true + } + } + profileFrames = memoryProfileAllocationFrames(allTrackable) + } for i := range pkgs { - extended[i] = computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkgs[i], analyses[i].funcs, base[i], analyses[i].trackable, func(dep *ssa.Package) map[*ssa.Function]bool { + extended[i] = computeRuntimeCallerFuncSets(c.recoverAnalysis(), pkgs[i], analyses[i].funcs, base[i], analyses[i].trackable, profileFrames, func(dep *ssa.Package) map[*ssa.Function]bool { j, ok := index[dep] if !ok { panic("caller-tracking dependency was not precomputed") @@ -1414,9 +1535,15 @@ func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { if rand == nil { continue } - if g, ok := (*rand).(*ssa.Global); ok && g.Pkg != nil && - isPublicRuntimePath(g.Pkg.Pkg.Path()) && g.Name() == "MemProfileRate" { - return true + switch value := (*rand).(type) { + case *ssa.Global: + if value.Pkg != nil && isPublicRuntimePath(value.Pkg.Pkg.Path()) && value.Name() == "MemProfileRate" { + return true + } + case *ssa.Function: + if value.Pkg != nil && isPublicRuntimePath(value.Pkg.Pkg.Path()) && value.Name() == "MemProfile" { + return true + } } } } @@ -1425,6 +1552,39 @@ func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { return false } +// MemProfileConsumer returns the package that made whole-program allocation +// recording necessary, or an empty string when it is provably unused. +// It runs after Go SSA construction but before backend compilation, so the +// same decision applies to runtime and every dependency. Loading runtime/pprof +// itself is a consumer even though its runtime call uses go:linkname rather +// than a normal public-runtime reference. +func MemProfileConsumer(pkgs []*ssa.Package) string { + for _, pkg := range pkgs { + if pkg == nil || pkg.Pkg == nil { + continue + } + // The public runtime implementation wires MemProfileRate and the + // capture hook internally; those references exist in every program and + // are providers, not consumers. + if isPublicRuntimePath(pkg.Pkg.Path()) { + continue + } + if pkg.Pkg.Path() == "runtime/pprof" { + return pkg.Pkg.Path() + } + _, funcs := collectRuntimeCallerFunctions(pkg) + if packageReadsMemProfile(funcs) { + return pkg.Pkg.Path() + } + } + return "" +} + +func (p *context) omitMemProfileRecordCall(fn *ssa.Function) bool { + return !p.prog.MemoryProfilingEnabled() && p.pkg != nil && p.pkg.Path() == llssa.PkgRuntime && + fn != nil && fn.Name() == "recordMemProfileAlloc" +} + func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { if fn == nil || fn.Parent() != nil { return false @@ -2514,6 +2674,12 @@ func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallComm args := p.compileValues(b, args, kind) ret = p.emitDo(b, act, ds, false, llssa.Builtin(fn), llssa.Builder.Call, args...) case *ssa.Function: + if p.omitMemProfileRecordCall(cv) { + // The allocator hook has no result. Evaluate arguments for + // completeness, then omit the call itself even at O0. + p.compileValues(b, args, kind) + return + } aFn, pyFn, ftype := p.compileFunction(cv) // TODO(xsw): check ca != llssa.Call switch ftype { diff --git a/internal/build/build.go b/internal/build/build.go index e7ec8f3abf..01792b82c5 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -195,6 +195,10 @@ type Config struct { // for float-to-uint32 conversions. SaturatingFloatToUint32 bool + // memoryProfiling is derived from whole-program SSA before package cache + // lookup. Library build modes set it conservatively. + memoryProfiling bool + // PthreadStackSize sets a custom stack size, in bytes, for pthread-backed // goroutines. A zero value keeps the platform pthread default. PthreadStackSize int64 @@ -679,6 +683,20 @@ func Build(inv Invocation) ([]Package, error) { return nil, err } buildSSAPkgs(ctx, append(append(altEntries, pkgEntries...), depEntries...)) + memProfileConsumer := cl.MemProfileConsumer(progSSA.AllPackages()) + conf.memoryProfiling = enableMemoryProfiling(conf.BuildMode, memProfileConsumer) + prog.EnableMemoryProfiling(conf.memoryProfiling) + if verbose { + reason := memProfileConsumer + if conf.BuildMode != BuildModeExe { + reason = string(conf.BuildMode) + } + if reason == "" { + reason = "not reachable" + } + fmt.Fprintf(os.Stderr, "memory profiling: %t (%s)\n", conf.memoryProfiling, reason) + } + ctx.callerTracking.SetMemoryProfileAttribution(conf.memoryProfiling) callerSpan := buildTrace.startCoordinator("precompute caller tracking", nil) ctx.callerTracking.Precompute(ctx.progSSA.AllPackages()) callerSpan.done() @@ -799,6 +817,10 @@ func Build(inv Invocation) ([]Package, error) { return allPkgs, nil } +func enableMemoryProfiling(mode BuildMode, consumer string) bool { + return mode != BuildModeExe || consumer != "" +} + // cHeaderPackages excludes the patched standard runtime implementation. Its // //export callbacks are linker implementation details and may use internal C // types that are deliberately not representable in a public generated header. diff --git a/internal/build/collect.go b/internal/build/collect.go index ea08d32fea..b212bf1afc 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -29,6 +29,7 @@ import ( "github.com/xgo-dev/llgo/internal/env" "github.com/xgo-dev/llgo/internal/meta" "github.com/xgo-dev/llgo/internal/packages" + llssa "github.com/xgo-dev/llgo/ssa" gopackages "golang.org/x/tools/go/packages" ) @@ -166,6 +167,9 @@ func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error m.pkg.PkgPath = p.PkgPath m.pkg.PkgID = p.ID + if p.PkgPath == llssa.PkgRuntime { + m.pkg.MemoryProfiling = c.buildConf.memoryProfiling + } // Go source files goFilesList, err := digestFilesWithOverlay(p.GoFiles, c.buildConf.Overlay) diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index f6c824ba28..312e39c59f 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -140,16 +140,17 @@ func (s *commonSection) empty() bool { } type packageSection struct { - PkgPath string `yaml:"pkg_path,omitempty"` - PkgID string `yaml:"pkg_id,omitempty"` - GoFiles []fileDigest `yaml:"go_files,omitempty"` - AltGoFiles []fileDigest `yaml:"alt_go_files,omitempty"` - OtherFiles []fileDigest `yaml:"other_files,omitempty"` - RewriteVars orderedStringMap `yaml:"rewrite_vars,omitempty"` + PkgPath string `yaml:"pkg_path,omitempty"` + PkgID string `yaml:"pkg_id,omitempty"` + MemoryProfiling bool `yaml:"memory_profiling,omitempty"` + GoFiles []fileDigest `yaml:"go_files,omitempty"` + AltGoFiles []fileDigest `yaml:"alt_go_files,omitempty"` + OtherFiles []fileDigest `yaml:"other_files,omitempty"` + RewriteVars orderedStringMap `yaml:"rewrite_vars,omitempty"` } func (s *packageSection) empty() bool { - return s.PkgPath == "" && s.PkgID == "" && len(s.GoFiles) == 0 && len(s.AltGoFiles) == 0 && len(s.OtherFiles) == 0 && len(s.RewriteVars) == 0 + return s.PkgPath == "" && s.PkgID == "" && !s.MemoryProfiling && len(s.GoFiles) == 0 && len(s.AltGoFiles) == 0 && len(s.OtherFiles) == 0 && len(s.RewriteVars) == 0 } // manifestBuilder builds manifest text with sorted sections. diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go index 3947925bed..50fbf0890f 100644 --- a/internal/build/fingerprint_test.go +++ b/internal/build/fingerprint_test.go @@ -174,6 +174,22 @@ func TestManifestBuilder_SaturatingFloatToUint32(t *testing.T) { } } +func TestManifestBuilder_MemoryProfiling(t *testing.T) { + plain := newManifestBuilder() + profiled := newManifestBuilder() + profiled.pkg.MemoryProfiling = true + if plain.Fingerprint() == profiled.Fingerprint() { + t.Fatal("memory profiling did not change the package fingerprint") + } + data, err := decodeManifest(profiled.Build()) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Package == nil || !data.Package.MemoryProfiling { + t.Fatalf("memory profiling missing from manifest: %+v", data.Package) + } +} + func TestManifestBuilder_EmptySections(t *testing.T) { m := newManifestBuilder() content := m.Build() diff --git a/internal/build/module_hook_test.go b/internal/build/module_hook_test.go index 99d2bde828..5809475d1b 100644 --- a/internal/build/module_hook_test.go +++ b/internal/build/module_hook_test.go @@ -4,7 +4,12 @@ package build import ( + "os" + "path/filepath" + "strings" "testing" + + llssa "github.com/xgo-dev/llgo/ssa" ) func TestModuleHookReceivesMainPackageModule(t *testing.T) { @@ -35,3 +40,76 @@ func TestModuleHookReceivesMainPackageModule(t *testing.T) { t.Fatalf("expected non-empty module snapshot for %s", mainPkg) } } + +func TestMemoryProfileConsumerSelectsAllocatorInstrumentation(t *testing.T) { + cacheDir := t.TempDir() + oldCacheRoot := cacheRootFunc + cacheRootFunc = func() string { return cacheDir } + defer func() { cacheRootFunc = oldCacheRoot }() + + plain := memoryProfileAllocatorIR(t, `package main +func main() { println("plain") } +`) + if strings.Contains(plain, "recordMemProfileAlloc") { + t.Fatalf("plain executable allocator retained memory profiling:\n%s", plain) + } + + profiled := memoryProfileAllocatorIR(t, `package main +import "runtime" +func main() { runtime.MemProfile(nil, false) } +`) + if !strings.Contains(profiled, "recordMemProfileAlloc") { + t.Fatalf("memory-profile consumer allocator lost recording:\n%s", profiled) + } +} + +func TestMemoryProfileLibraryModeSelection(t *testing.T) { + for _, mode := range []BuildMode{BuildModeCArchive, BuildModeCShared} { + if !enableMemoryProfiling(mode, "") { + t.Errorf("%s build disabled externally callable memory profiling", mode) + } + } + if enableMemoryProfiling(BuildModeExe, "") { + t.Error("plain executable enabled memory profiling") + } + if !enableMemoryProfiling(BuildModeExe, "runtime/pprof") { + t.Error("profile consumer did not enable executable memory profiling") + } +} + +func memoryProfileAllocatorIR(t *testing.T, source string) string { + t.Helper() + dir := t.TempDir() + mainFile := filepath.Join(dir, "main.go") + if err := os.WriteFile(mainFile, []byte(source), 0o644); err != nil { + t.Fatal(err) + } + conf := NewDefaultConf(ModeGen) + var allocator string + conf.ModuleHook = func(pkg Package) { + if pkg.PkgPath != llssa.PkgRuntime { + return + } + ir := pkg.LPkg.String() + marker := `define ptr @"` + llssa.PkgRuntime + `.AllocZ"(` + start := strings.Index(ir, marker) + if start < 0 { + return + } + end := strings.Index(ir[start:], "\n}") + if end >= 0 { + allocator = ir[start : start+end+2] + } + } + pkgs, err := Do([]string{mainFile}, conf) + if err != nil { + t.Fatalf("generate memory-profile allocator IR: %v", err) + } + if len(pkgs) == 1 && pkgs[0].LPkg != nil { + defer pkgs[0].LPkg.Prog.Dispose() + } + if allocator == "" { + t.Fatal("runtime AllocZ module was not observed") + } + return allocator +} diff --git a/ssa/backend_program_test.go b/ssa/backend_program_test.go index 06df50fdc3..dea6fe7216 100644 --- a/ssa/backend_program_test.go +++ b/ssa/backend_program_test.go @@ -28,6 +28,7 @@ func TestNewBackendProgramSharesPreparedGoState(t *testing.T) { coordinator.DisableBoundsChecks(true) coordinator.EnableGoGlobalDCE(true) coordinator.EnableDeadcodeDrop(true) + coordinator.EnableMemoryProfiling(true) coordinator.SetPthreadStackSize(4096) coordinator.EnableLTOPluginMarkers(true) coordinator.EnableFuncInfoMetadata(true) @@ -77,7 +78,7 @@ func TestNewBackendProgramSharesPreparedGoState(t *testing.T) { if backend.python() != nil { t.Fatal("backend Program changed the prepared optional Python package") } - if !backend.disableBoundsChecks || !backend.enableGoGlobalDCE || !backend.enableDeadcodeDrop || + if !backend.disableBoundsChecks || !backend.enableGoGlobalDCE || !backend.enableDeadcodeDrop || !backend.MemoryProfilingEnabled() || backend.pthreadStackSize != 4096 || !backend.enableLTOPluginMarker || !backend.enableFuncInfoMetadata || !backend.enableFuncInfoSites || backend.debugInfoOptimized { t.Fatal("backend Program did not preserve coordinator configuration") diff --git a/ssa/package.go b/ssa/package.go index dda1ec7940..1aff74c816 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -235,6 +235,7 @@ type aProgram struct { enableGoGlobalDCE bool enableDeadcodeDrop bool disableBoundsChecks bool + memoryProfiling bool pthreadStackSize uint64 enableLTOPluginMarker bool @@ -348,6 +349,7 @@ func (p Program) NewBackendProgram() Program { backend.enableGoGlobalDCE = p.enableGoGlobalDCE backend.enableDeadcodeDrop = p.enableDeadcodeDrop backend.disableBoundsChecks = p.disableBoundsChecks + backend.memoryProfiling = p.memoryProfiling backend.pthreadStackSize = p.pthreadStackSize backend.enableLTOPluginMarker = p.enableLTOPluginMarker backend.enableFuncInfoMetadata = p.enableFuncInfoMetadata @@ -391,6 +393,17 @@ func (p Program) EnableGoGlobalDCE(enable bool) { p.enableGoGlobalDCE = enable } +// EnableMemoryProfiling selects whether allocator recording calls are emitted +// for this whole-program build. The build coordinator sets it before package +// backends start. +func (p Program) EnableMemoryProfiling(enable bool) { + p.memoryProfiling = enable +} + +func (p Program) MemoryProfilingEnabled() bool { + return p.memoryProfiling +} + func (p Program) EnableDeadcodeDrop(enable bool) { p.enableDeadcodeDrop = enable } From 08763224e3f896202afe91c31fce71030abfc3dd Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 05:14:29 +0800 Subject: [PATCH 15/26] cl,runtime: omit unused memory profile setup --- cl/caller_frame_test.go | 39 ++++++++++++++-- cl/compile.go | 8 ++-- cl/instr.go | 40 ++++++++++------ internal/build/build_test.go | 5 +- internal/build/module_hook_test.go | 51 ++++++++++++++++----- runtime/internal/lib/runtime/unwind_llgo.go | 7 +++ 6 files changed, 113 insertions(+), 37 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 1ead834efa..4c4e2b98f1 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -4,6 +4,7 @@ package cl import ( + "fmt" "go/ast" "go/importer" "go/parser" @@ -14,6 +15,7 @@ import ( "github.com/goplus/gogen/packages" llssa "github.com/xgo-dev/llgo/ssa" + llabi "github.com/xgo-dev/llgo/ssa/abi" gossa "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" ) @@ -248,7 +250,7 @@ func plain() {} t.Fatal("Version should not be a runtime caller metadata function") } - rtpkg, _ := buildCallerFrameSSAPackage(t, "github.com/xgo-dev/llgo/runtime/internal/lib/runtime", `package runtime + rtpkg, _ := buildCallerFrameSSAPackage(t, llabi.PatchPathPrefix+"runtime", `package runtime func Caller(skip int) (uintptr, string, int, bool) { return 0, "", 0, false } func FuncForPC(pc uintptr) uintptr { return 0 } `) @@ -734,6 +736,35 @@ func leaf() {} } } +func TestCompileMemoryProfileAllocationPCLineMetadata(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/profileline", `package profileline + +var sink *int + +func init() { +//line profile_alloc.go:321 + sink = new(int) +} +`) + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprint(enabled), func(t *testing.T) { + prog := newLLSSAProgForTarget(t, &llssa.Target{GOOS: "linux", GOARCH: "amd64"}) + prog.EnableMemoryProfiling(enabled) + prog.EnableFuncInfoMetadata(true) + prog.EnableFuncInfoSites(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + got := strings.Contains(ir, "!llgo.pcline") && strings.Contains(ir, `!"profile_alloc.go"`) + if got != enabled { + t.Fatalf("allocation pcline present = %v, memory profiling = %v\n%s", got, enabled, ir) + } + }) + } +} + func TestCompileRuntimeCallerPCLineMetadata32Bit(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" @@ -1332,9 +1363,9 @@ func TestPackageReadsMemProfileSkipsFunctionWithoutPackage(t *testing.T) { func TestPublicRuntimePath(t *testing.T) { for path, want := range map[string]bool{ - "runtime": true, - "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": true, - "runtime/debug": false, + "runtime": true, + llabi.PatchPathPrefix + "runtime": true, + "runtime/debug": false, } { if got := isPublicRuntimePath(path); got != want { t.Errorf("isPublicRuntimePath(%q) = %v, want %v", path, got, want) diff --git a/cl/compile.go b/cl/compile.go index 7dca9fd07f..07776b402d 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -755,13 +755,13 @@ func needsRuntimeStackNoInline(pkg *types.Package, f *ssa.Function) bool { if pkg == nil || f == nil || f.Signature.Recv() != nil { return false } - switch pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + path := pkg.Path() + if isPublicRuntimePath(path) { switch f.Name() { case "Caller", "Callers", "callers": return true } - case "github.com/xgo-dev/llgo/runtime/internal/clite/debug": + } else if path == "github.com/xgo-dev/llgo/runtime/internal/clite/debug" { return f.Name() == "StackTrace" } return false @@ -1434,7 +1434,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue return } elem := p.type_(t.Elem(), llssa.InGo) - if v.Heap { + if v.Heap && p.prog.MemoryProfilingEnabled() { // Heap allocations are memory-profile sample sites; give each // one a statement anchor in tracked functions so sampled // records attribute to the allocating line (heapsampling.go diff --git a/cl/instr.go b/cl/instr.go index 6caf4afa19..0c1b313d55 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -31,6 +31,7 @@ import ( "golang.org/x/tools/go/ssa" llssa "github.com/xgo-dev/llgo/ssa" + llabi "github.com/xgo-dev/llgo/ssa/abi" ) var asmRegisterRegex = regexp.MustCompile(`\{[a-zA-Z]+\}`) @@ -1497,7 +1498,7 @@ func NewCallerTracking() *CallerTracking { // it to LLGo's implementation package. func isPublicRuntimePath(path string) bool { return path == "runtime" || - path == "github.com/xgo-dev/llgo/runtime/internal/lib/runtime" + path == llabi.PatchPathPrefix+"runtime" } func packageReadsMemProfile(funcs map[*ssa.Function]bool) bool { @@ -1580,9 +1581,18 @@ func MemProfileConsumer(pkgs []*ssa.Package) string { return "" } -func (p *context) omitMemProfileRecordCall(fn *ssa.Function) bool { - return !p.prog.MemoryProfilingEnabled() && p.pkg != nil && p.pkg.Path() == llssa.PkgRuntime && - fn != nil && fn.Name() == "recordMemProfileAlloc" +func (p *context) omitMemProfileProviderCall(fn *ssa.Function) bool { + if p.prog.MemoryProfilingEnabled() || p.pkg == nil || fn == nil { + return false + } + path := p.pkg.Path() + switch { + case path == llssa.PkgRuntime: + return fn.Name() == "recordMemProfileAlloc" + case isPublicRuntimePath(path): + return fn.Name() == "installMemProfileHooks" + } + return false } func isProgramUniqueFrame(pkg *ssa.Package, fn *ssa.Function) bool { @@ -2051,10 +2061,10 @@ func isRuntimeCallerFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): return isRuntimeCallerName(fn.Name()) - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" default: return false @@ -2065,10 +2075,10 @@ func isRuntimeCallerFrameFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): return isRuntimeCallerFrameName(fn.Name()) - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" default: return false @@ -2079,13 +2089,13 @@ func isRuntimeCallerLookupFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false } - switch fn.Pkg.Pkg.Path() { - case "runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime": + switch path := fn.Pkg.Pkg.Path(); { + case isPublicRuntimePath(path): switch fn.Name() { case "Caller", "Callers", "Stack": return true } - case "runtime/debug": + case path == "runtime/debug": return fn.Name() == "Stack" } return false @@ -2674,8 +2684,8 @@ func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallComm args := p.compileValues(b, args, kind) ret = p.emitDo(b, act, ds, false, llssa.Builtin(fn), llssa.Builder.Call, args...) case *ssa.Function: - if p.omitMemProfileRecordCall(cv) { - // The allocator hook has no result. Evaluate arguments for + if p.omitMemProfileProviderCall(cv) { + // These provider calls have no result. Evaluate arguments for // completeness, then omit the call itself even at O0. p.compileValues(b, args, kind) return diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 06e49164e4..f8c5a412d5 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -1340,12 +1340,13 @@ func TestCHeaderPackagesExcludesStandardRuntime(t *testing.T) { userLPkg := prog.NewPackage("example.com/p", "example.com/p") userLPkg.SetExport("example.com/p.Export", "Export") runtimeLPkg := prog.NewPackage("runtime", "runtime") - llgoRuntimeLPkg := prog.NewPackage("github.com/xgo-dev/llgo/runtime/internal/lib/runtime", "github.com/xgo-dev/llgo/runtime/internal/lib/runtime") + llgoRuntimePath := altPkgPathPrefix + "runtime" + llgoRuntimeLPkg := prog.NewPackage(llgoRuntimePath, llgoRuntimePath) dependencyLPkg := prog.NewPackage("example.com/dep", "example.com/dep") pkgs := []*aPackage{ {Package: &packages.Package{PkgPath: "example.com/p"}, LPkg: userLPkg}, {Package: &packages.Package{PkgPath: "runtime"}, LPkg: runtimeLPkg}, - {Package: &packages.Package{PkgPath: "github.com/xgo-dev/llgo/runtime/internal/lib/runtime"}, LPkg: llgoRuntimeLPkg}, + {Package: &packages.Package{PkgPath: llgoRuntimePath}, LPkg: llgoRuntimeLPkg}, {Package: &packages.Package{PkgPath: "example.com/dep"}, LPkg: dependencyLPkg}, nil, } diff --git a/internal/build/module_hook_test.go b/internal/build/module_hook_test.go index 5809475d1b..628416f4a7 100644 --- a/internal/build/module_hook_test.go +++ b/internal/build/module_hook_test.go @@ -47,19 +47,26 @@ func TestMemoryProfileConsumerSelectsAllocatorInstrumentation(t *testing.T) { cacheRootFunc = func() string { return cacheDir } defer func() { cacheRootFunc = oldCacheRoot }() - plain := memoryProfileAllocatorIR(t, `package main -func main() { println("plain") } + plain := memoryProfileProviderIR(t, `package main +import "runtime" +func main() { println(runtime.GOOS) } `) - if strings.Contains(plain, "recordMemProfileAlloc") { - t.Fatalf("plain executable allocator retained memory profiling:\n%s", plain) + if strings.Contains(plain.allocator, "recordMemProfileAlloc") { + t.Fatalf("plain executable allocator retained memory profiling:\n%s", plain.allocator) + } + if hasMemProfileHookInstall(plain.publicRuntime) { + t.Fatalf("plain executable runtime installed memory-profile hooks:\n%s", plain.publicRuntime) } - profiled := memoryProfileAllocatorIR(t, `package main + profiled := memoryProfileProviderIR(t, `package main import "runtime" func main() { runtime.MemProfile(nil, false) } `) - if !strings.Contains(profiled, "recordMemProfileAlloc") { - t.Fatalf("memory-profile consumer allocator lost recording:\n%s", profiled) + if !strings.Contains(profiled.allocator, "recordMemProfileAlloc") { + t.Fatalf("memory-profile consumer allocator lost recording:\n%s", profiled.allocator) + } + if !hasMemProfileHookInstall(profiled.publicRuntime) { + t.Fatalf("memory-profile consumer runtime lost hook installation:\n%s", profiled.publicRuntime) } } @@ -77,7 +84,12 @@ func TestMemoryProfileLibraryModeSelection(t *testing.T) { } } -func memoryProfileAllocatorIR(t *testing.T, source string) string { +type memoryProfileProviders struct { + allocator string + publicRuntime string +} + +func memoryProfileProviderIR(t *testing.T, source string) memoryProfileProviders { t.Helper() dir := t.TempDir() mainFile := filepath.Join(dir, "main.go") @@ -85,8 +97,11 @@ func memoryProfileAllocatorIR(t *testing.T, source string) string { t.Fatal(err) } conf := NewDefaultConf(ModeGen) - var allocator string + var providers memoryProfileProviders conf.ModuleHook = func(pkg Package) { + if pkg.PkgPath == "runtime" || pkg.PkgPath == altPkgPathPrefix+"runtime" { + providers.publicRuntime = pkg.LPkg.String() + } if pkg.PkgPath != llssa.PkgRuntime { return } @@ -98,7 +113,7 @@ func memoryProfileAllocatorIR(t *testing.T, source string) string { } end := strings.Index(ir[start:], "\n}") if end >= 0 { - allocator = ir[start : start+end+2] + providers.allocator = ir[start : start+end+2] } } pkgs, err := Do([]string{mainFile}, conf) @@ -108,8 +123,20 @@ func memoryProfileAllocatorIR(t *testing.T, source string) string { if len(pkgs) == 1 && pkgs[0].LPkg != nil { defer pkgs[0].LPkg.Prog.Dispose() } - if allocator == "" { + if providers.allocator == "" { t.Fatal("runtime AllocZ module was not observed") } - return allocator + if providers.publicRuntime == "" { + t.Fatal("public runtime module was not observed") + } + return providers +} + +func hasMemProfileHookInstall(ir string) bool { + for line := range strings.SplitSeq(ir, "\n") { + if strings.Contains(line, "call void") && strings.Contains(line, ".installMemProfileHooks(") { + return true + } + } + return false } diff --git a/runtime/internal/lib/runtime/unwind_llgo.go b/runtime/internal/lib/runtime/unwind_llgo.go index 99ffd8c86c..0b24e040fa 100644 --- a/runtime/internal/lib/runtime/unwind_llgo.go +++ b/runtime/internal/lib/runtime/unwind_llgo.go @@ -20,6 +20,13 @@ func init() { rtdebug.PanicRecovered = clearFaultTraceback rtdebug.PanicPCSnapshot = capturePanicPCs rtdebug.RecoverMark = recoverMark + installMemProfileHooks() +} + +// installMemProfileHooks is a separate call so whole-program executable +// builds that do not consume memory profiles can omit the complete setup and +// capture path without changing runtime APIs. +func installMemProfileHooks() { // Table initialization may allocate. Complete it before installing the // allocator hook so the first sample never initializes it from AllocZ/U. initRuntimeFuncPCFrames() From e98d946c1616e5c0d6f4ec12a7aa0aa38280a5aa Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 05:14:35 +0800 Subject: [PATCH 16/26] test: stabilize memory profile rate transition --- test/go/memprofile/memprofile_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go index eae3e95855..d0208da31b 100644 --- a/test/go/memprofile/memprofile_test.go +++ b/test/go/memprofile/memprofile_test.go @@ -12,6 +12,7 @@ import ( var tinySink []*int32 var mixedSizeSink [][]byte +var memProfileWarmup []byte type profiledClosure struct { fn func(int) int @@ -80,6 +81,13 @@ func TestRuntimeMemProfileSeparatesSizesAtOneStack(t *testing.T) { runtime.MemProfileRate = oldRate }() + // gc caches the active rate in each mcache. Allocate enough to force the + // current cache to observe the change before checking the interesting + // allocation stacks (the Go runtime's own profiler tests do the same). + for range 1024 { + memProfileWarmup = make([]byte, 1024) + } + mixedSizeSink = make([][]byte, 128) mixedSizeProfileAlloc(mixedSizeSink) sizes := make(map[int64]bool) From 22e3d20030a94e948a87912011ed8fd5b30efc79 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 06:01:03 +0800 Subject: [PATCH 17/26] test: materialize memory profile size samples --- test/go/memprofile/memprofile_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go index d0208da31b..938419bcb7 100644 --- a/test/go/memprofile/memprofile_test.go +++ b/test/go/memprofile/memprofile_test.go @@ -90,6 +90,10 @@ func TestRuntimeMemProfileSeparatesSizesAtOneStack(t *testing.T) { mixedSizeSink = make([][]byte, 128) mixedSizeProfileAlloc(mixedSizeSink) + // The gc runtime publishes heap profile samples up to two GC cycles + // after allocation. Materialize both sizes before reading them. + runtime.GC() + runtime.GC() sizes := make(map[int64]bool) for _, record := range readMemProfile(t) { if record.AllocObjects == 0 { From 7b1e320e06ec588222c1579179d092943e50d37d Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 08:17:13 +0800 Subject: [PATCH 18/26] benchmark: stabilize and track memory profile paths --- .github/llgo-benchmark.yml | 16 ++++ benchmark/baseline/README.md | 63 ++++++---------- benchmark/baseline/main.go | 9 ++- benchmark/baseline/main_test.go | 1 + benchmark/baseline/run.sh | 74 ++++++++++++++++--- benchmark/memprofile/enabled/main.go | 43 +++++++++++ .../memprofile/internal/runner/runner.go | 40 ++++++++++ benchmark/memprofile/noconsumer/main.go | 35 +++++++++ 8 files changed, 229 insertions(+), 52 deletions(-) create mode 100644 benchmark/memprofile/enabled/main.go create mode 100644 benchmark/memprofile/internal/runner/runner.go create mode 100644 benchmark/memprofile/noconsumer/main.go diff --git a/.github/llgo-benchmark.yml b/.github/llgo-benchmark.yml index 2011ca5f2e..0a59baaadc 100644 --- a/.github/llgo-benchmark.yml +++ b/.github/llgo-benchmark.yml @@ -5,12 +5,14 @@ site-path: benchmark/baseline include: - "^BenchmarkProgram/" - "^Benchmark(MergeCompilerFlags|MergeLinkerFlags|LookupPCRandom)$" + - "^BenchmarkMemProfile(NoConsumer|Rate0|Default)$" - "^Benchmark(RuntimeGetG|Global(Read|Write))$" - "^Benchmark(DirectCall|InterfaceCall|Defer|Goroutine)$" - "^BenchmarkChannel(Buffered|Handoff)$" groups: programs: "^Program/" compiler: "^(MergeCompilerFlags|MergeLinkerFlags|LookupPCRandom)$" + memprofile: "^MemProfile(NoConsumer|Rate0|Default)$" core: match: - "^(RuntimeGetG|Global(Read|Write))$" @@ -41,6 +43,20 @@ views: run-ns: title: Run format: duration-ns + memprofile: + title: Memory profiling allocation benchmarks + select: + groups: "^memprofile$" + metrics: "^ns/op$" + table: + rows: [platform, benchmark] + columns: [metric] + missing: error + empty: error + dimensions: + benchmark: + title: Profile path + trim-prefix: BenchmarkMemProfile core: title: Core language and compiler benchmarks select: diff --git a/benchmark/baseline/README.md b/benchmark/baseline/README.md index 32073268cf..3c67bfb813 100644 --- a/benchmark/baseline/README.md +++ b/benchmark/baseline/README.md @@ -1,7 +1,7 @@ # LLGo baseline benchmarks This suite is the lightweight performance gate for ordinary LLGo changes. It -uses fixed workloads and short calibrated benchmarks on Linux and macOS so it +uses fixed workloads and calibrated benchmarks on Linux and macOS so it can run on every `main` push and pull request. Branch-only series can be run explicitly with `workflow_dispatch`, avoiding duplicate push and pull-request jobs for the same commit. The two native jobs record normalized artifacts; a @@ -15,12 +15,23 @@ The program workloads reuse: - `benchmark/binary_size/fmtprintf`: `fmt.Printf`. For each workload, the collector performs an unmeasured warm build, then records -median build time, median process time, file size, executable-code bytes, -allocated non-executable data, and zero-filled data. On ELF, read-only constants -are included in the data bucket; on Mach-O, `__TEXT` constants are included in -the text bucket. The Go benchmark stream records five samples of selected -compiler helpers and LLGo-generated core-language operations: direct/interface -calls, defer, goroutine creation, channels, `getg`, and global access. +the median of five builds and fifteen process runs, file size, executable-code +bytes, allocated non-executable data, and zero-filled data. On ELF, read-only +constants are included in the data bucket; on Mach-O, `__TEXT` constants are +included in the text bucket. The Go benchmark stream discards the first +one-second sample as warmup, then records seven one-second samples of compiler +helpers and LLGo-generated core-language operations: direct/interface calls, +defer, channels, `getg`, and global access. +Goroutine creation keeps its bounded 100-iteration sample and likewise discards +the first of eight runs. + +The memory-profile allocation group uses standalone LLGo executables so the +whole-program no-consumer path is measurable independently of the retained +profile paths. Every process disables BDWGC, warms with two million allocations, +then measures forty million escaping 16-byte allocations. Seven independent +processes are recorded per mode in rotated order; an additional discarded +process warms loader state. The reported modes are `NoConsumer`, `Rate0`, and +`Default`. For pull requests, each platform job checks out the recorded base and current commits into the same source path, then runs both suites sequentially on the same @@ -48,39 +59,13 @@ updated summary comment linking to their long-term trend page. If no matching `main` history exists yet, the pull-request report is still published and marks every metric as `new`. -Local collection: - -```sh -GOMAXPROCS=2 go build -o .benchmark/llgo ./cmd/llgo -go run ./benchmark/baseline \ - -llgo .benchmark/llgo \ - -out .benchmark/results -``` - -Write the selected Go benchmark output to `.benchmark/results/go.txt`: +Run the complete local collection with the same script as CI: ```sh -results=.benchmark/results/go.txt -GOMAXPROCS=1 go test \ - -run '^$' \ - -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \ - -benchtime=250ms -count=5 -cpu=1 \ - ./internal/clang ./internal/build/funcinfo | tee "$results" -GOMAXPROCS=1 .benchmark/llgo test \ - -run '^$' \ - -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \ - -benchtime=250ms -count=5 \ - ./test/llgoext | tee -a "$results" -GOMAXPROCS=1 .benchmark/llgo test \ - -run '^$' -bench '^BenchmarkGoroutine$' -benchtime=100x -count=5 \ - ./test/llgoext | tee -a "$results" +benchmark/baseline/run.sh \ + "$PWD" \ + "$PWD/.benchmark/llgo" \ + "$PWD/.benchmark/results" ``` -Then validate and export the complete artifact in standard Go benchmark format: - -```sh -go run ./benchmark/baseline \ - -mode export \ - -out .benchmark/results \ - -benchmark-output .benchmark/results/benchmark.txt -``` +The normalized artifact is `.benchmark/results/benchmark.txt`. diff --git a/benchmark/baseline/main.go b/benchmark/baseline/main.go index f9e4cbe19b..5e9281ce01 100644 --- a/benchmark/baseline/main.go +++ b/benchmark/baseline/main.go @@ -69,12 +69,15 @@ var expectedGoBenchmarks = []string{ "BenchmarkGoroutine", "BenchmarkInterfaceCall", "BenchmarkLookupPCRandom", + "BenchmarkMemProfileDefault", + "BenchmarkMemProfileNoConsumer", + "BenchmarkMemProfileRate0", "BenchmarkMergeCompilerFlags", "BenchmarkMergeLinkerFlags", "BenchmarkRuntimeGetG", } -const goBenchmarkSamples = 5 +const goBenchmarkSamples = 7 type footprint struct { file uint64 @@ -99,8 +102,8 @@ func runCLI(ctx context.Context, args []string) error { root := flags.String("root", ".", "LLGo repository root") llgo := flags.String("llgo", "llgo", "LLGo command") out := flags.String("out", filepath.Join("benchmark", "baseline", "out"), "result directory") - buildRuns := flags.Int("build-runs", 3, "build repetitions per workload") - runRuns := flags.Int("run-runs", 7, "process repetitions per workload") + buildRuns := flags.Int("build-runs", 5, "build repetitions per workload") + runRuns := flags.Int("run-runs", 15, "process repetitions per workload") benchmarkOutput := flags.String( "benchmark-output", "", diff --git a/benchmark/baseline/main_test.go b/benchmark/baseline/main_test.go index 2fe56a56e5..fb7379290c 100644 --- a/benchmark/baseline/main_test.go +++ b/benchmark/baseline/main_test.go @@ -97,6 +97,7 @@ func TestExportBenchmarks(t *testing.T) { "Unit file-bytes better=lower assume=exact", "Unit build-ns better=lower", "BenchmarkProgram/cprintf 1 1 file-bytes 1 text-bytes 1 data-bytes 1 bss-bytes 1 build-ns 1 run-ns", + "BenchmarkMemProfileNoConsumer-1 100 12.5 ns/op", "BenchmarkRuntimeGetG-1 100 12.5 ns/op", } { if !strings.Contains(text, want) { diff --git a/benchmark/baseline/run.sh b/benchmark/baseline/run.sh index c0ffdf02ec..9fc08ad857 100755 --- a/benchmark/baseline/run.sh +++ b/benchmark/baseline/run.sh @@ -31,26 +31,44 @@ result_directory="$(cd "$3" && pwd)" go_results="$result_directory/go.txt" : > "$go_results" + +compiler_benchmarks='^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' +core_benchmarks='^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' + +drop_first_benchmark_sample() { + awk ' + /^Benchmark/ { + name = $1 + sub(/-[0-9]+$/, "", name) + if (++seen[name] == 1) next + } + { print } + ' +} + +# The first one-second sample warms each path and is discarded. The following +# seven samples use the same benchmark process and fixed single-CPU conditions. ( cd "$source_root" GOMAXPROCS=1 LLGO_ROOT="$source_root" go test \ -run '^$' \ - -bench '^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' \ - -benchtime=250ms \ - -count=5 \ + -bench "$compiler_benchmarks" \ + -benchtime=1s \ + -count=8 \ -cpu=1 \ ./internal/clang ./internal/build/funcinfo -) | tee -a "$go_results" +) | drop_first_benchmark_sample | tee -a "$go_results" ( cd "$source_root" GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \ -run '^$' \ - -bench '^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' \ - -benchtime=250ms \ - -count=5 \ + -bench "$core_benchmarks" \ + -benchtime=1s \ + -count=8 \ + -cpu=1 \ ./test/llgoext -) | tee -a "$go_results" +) | drop_first_benchmark_sample | tee -a "$go_results" # The current native backend creates one pthread per goroutine and intentionally # has a bounded lifecycle stress limit. Keep creation monitoring deterministic @@ -61,9 +79,45 @@ go_results="$result_directory/go.txt" -run '^$' \ -bench '^BenchmarkGoroutine$' \ -benchtime=100x \ - -count=5 \ + -count=8 \ + -cpu=1 \ ./test/llgoext -) | tee -a "$go_results" +) | drop_first_benchmark_sample | tee -a "$go_results" + +memprofile_noconsumer="$result_directory/bin/memprofile-noconsumer" +memprofile_enabled="$result_directory/bin/memprofile-enabled" +( + cd "$harness_root" + GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true "$llgo_output" build \ + -o "$memprofile_noconsumer" \ + ./benchmark/memprofile/noconsumer + GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true "$llgo_output" build \ + -o "$memprofile_enabled" \ + ./benchmark/memprofile/enabled +) + +# Each executable also warms its allocator internally before timing. These +# discarded processes additionally remove loader and first-execution effects. +GOMAXPROCS=1 "$memprofile_noconsumer" >/dev/null 2>&1 +GOMAXPROCS=1 "$memprofile_enabled" rate0 >/dev/null 2>&1 +GOMAXPROCS=1 "$memprofile_enabled" >/dev/null 2>&1 + +# Rotate the order so every mode occupies each scheduling position across the +# seven independent process samples. +for round in {0..6}; do + case $((round % 3)) in + 0) modes=(noconsumer rate0 default) ;; + 1) modes=(rate0 default noconsumer) ;; + 2) modes=(default noconsumer rate0) ;; + esac + for mode in "${modes[@]}"; do + case "$mode" in + noconsumer) GOMAXPROCS=1 "$memprofile_noconsumer" 2>&1 ;; + rate0) GOMAXPROCS=1 "$memprofile_enabled" rate0 2>&1 ;; + default) GOMAXPROCS=1 "$memprofile_enabled" 2>&1 ;; + esac + done +done | tee -a "$go_results" ( cd "$harness_root" diff --git a/benchmark/memprofile/enabled/main.go b/benchmark/memprofile/enabled/main.go new file mode 100644 index 0000000000..1897133348 --- /dev/null +++ b/benchmark/memprofile/enabled/main.go @@ -0,0 +1,43 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + "runtime" + _ "unsafe" + + "github.com/xgo-dev/llgo/benchmark/memprofile/internal/runner" +) + +//go:linkname disableGC C.GC_disable +func disableGC() + +func main() { + name := "BenchmarkMemProfileDefault-1" + if len(os.Args) == 2 && os.Args[1] == "rate0" { + runtime.MemProfileRate = 0 + name = "BenchmarkMemProfileRate0-1" + } + disableGC() + elapsed := runner.Run() + println(name, runner.Iterations, + float64(elapsed.Nanoseconds())/runner.Iterations, "ns/op") + runtime.MemProfile(nil, false) +} diff --git a/benchmark/memprofile/internal/runner/runner.go b/benchmark/memprofile/internal/runner/runner.go new file mode 100644 index 0000000000..49d5298ec0 --- /dev/null +++ b/benchmark/memprofile/internal/runner/runner.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runner + +import "time" + +const ( + Iterations = 40_000_000 + warmup = 2_000_000 +) + +var sink [4096]*[16]byte + +//go:noinline +func allocate(n int) { + for i := 0; i < n; i++ { + sink[i&(len(sink)-1)] = new([16]byte) + } +} + +func Run() time.Duration { + allocate(warmup) + start := time.Now() + allocate(Iterations) + return time.Since(start) +} diff --git a/benchmark/memprofile/noconsumer/main.go b/benchmark/memprofile/noconsumer/main.go new file mode 100644 index 0000000000..5da377084d --- /dev/null +++ b/benchmark/memprofile/noconsumer/main.go @@ -0,0 +1,35 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + _ "unsafe" + + "github.com/xgo-dev/llgo/benchmark/memprofile/internal/runner" +) + +//go:linkname disableGC C.GC_disable +func disableGC() + +func main() { + disableGC() + elapsed := runner.Run() + println("BenchmarkMemProfileNoConsumer-1", runner.Iterations, + float64(elapsed.Nanoseconds())/runner.Iterations, "ns/op") +} From 2c1865618e0eaa89e577beee62cbbeda3695778d Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 09:10:09 +0800 Subject: [PATCH 19/26] benchmark: publish stable profile workloads --- .github/llgo-benchmark.yml | 16 --- benchmark/baseline/README.md | 24 ++-- benchmark/baseline/main.go | 136 ++++++++++++++---- benchmark/baseline/main_test.go | 30 +++- benchmark/baseline/run.sh | 36 +---- benchmark/memprofile/enabled/main.go | 9 +- .../memprofile/internal/runner/runner.go | 4 +- benchmark/memprofile/noconsumer/main.go | 3 +- 8 files changed, 156 insertions(+), 102 deletions(-) diff --git a/.github/llgo-benchmark.yml b/.github/llgo-benchmark.yml index 0a59baaadc..2011ca5f2e 100644 --- a/.github/llgo-benchmark.yml +++ b/.github/llgo-benchmark.yml @@ -5,14 +5,12 @@ site-path: benchmark/baseline include: - "^BenchmarkProgram/" - "^Benchmark(MergeCompilerFlags|MergeLinkerFlags|LookupPCRandom)$" - - "^BenchmarkMemProfile(NoConsumer|Rate0|Default)$" - "^Benchmark(RuntimeGetG|Global(Read|Write))$" - "^Benchmark(DirectCall|InterfaceCall|Defer|Goroutine)$" - "^BenchmarkChannel(Buffered|Handoff)$" groups: programs: "^Program/" compiler: "^(MergeCompilerFlags|MergeLinkerFlags|LookupPCRandom)$" - memprofile: "^MemProfile(NoConsumer|Rate0|Default)$" core: match: - "^(RuntimeGetG|Global(Read|Write))$" @@ -43,20 +41,6 @@ views: run-ns: title: Run format: duration-ns - memprofile: - title: Memory profiling allocation benchmarks - select: - groups: "^memprofile$" - metrics: "^ns/op$" - table: - rows: [platform, benchmark] - columns: [metric] - missing: error - empty: error - dimensions: - benchmark: - title: Profile path - trim-prefix: BenchmarkMemProfile core: title: Core language and compiler benchmarks select: diff --git a/benchmark/baseline/README.md b/benchmark/baseline/README.md index 3c67bfb813..fc0b687572 100644 --- a/benchmark/baseline/README.md +++ b/benchmark/baseline/README.md @@ -15,23 +15,23 @@ The program workloads reuse: - `benchmark/binary_size/fmtprintf`: `fmt.Printf`. For each workload, the collector performs an unmeasured warm build, then records -the median of five builds and fifteen process runs, file size, executable-code -bytes, allocated non-executable data, and zero-filled data. On ELF, read-only -constants are included in the data bucket; on Mach-O, `__TEXT` constants are -included in the text bucket. The Go benchmark stream discards the first -one-second sample as warmup, then records seven one-second samples of compiler -helpers and LLGo-generated core-language operations: direct/interface calls, -defer, channels, `getg`, and global access. +the median of six builds and eighteen process runs, file size, executable-code +bytes, allocated non-executable data, and zero-filled data. Workload order is +rotated between rounds to balance runner drift and cache position. On ELF, +read-only constants are included in the data bucket; on Mach-O, `__TEXT` +constants are included in the text bucket. The Go benchmark stream discards the +first one-second sample as warmup, then records seven one-second samples of +compiler helpers and LLGo-generated core-language operations: direct/interface +calls, defer, channels, `getg`, and global access. Goroutine creation keeps its bounded 100-iteration sample and likewise discards the first of eight runs. -The memory-profile allocation group uses standalone LLGo executables so the +The program table also includes standalone memory-profile workloads so the whole-program no-consumer path is measurable independently of the retained profile paths. Every process disables BDWGC, warms with two million allocations, -then measures forty million escaping 16-byte allocations. Seven independent -processes are recorded per mode in rotated order; an additional discarded -process warms loader state. The reported modes are `NoConsumer`, `Rate0`, and -`Default`. +then internally times forty million escaping 16-byte allocations. The reported +duration is the median of eighteen processes. The workloads are +`memprofile-no-consumer`, `memprofile-rate0`, and `memprofile-default`. For pull requests, each platform job checks out the recorded base and current commits into the same source path, then runs both suites sequentially on the same diff --git a/benchmark/baseline/main.go b/benchmark/baseline/main.go index 5e9281ce01..074f3cdc1a 100644 --- a/benchmark/baseline/main.go +++ b/benchmark/baseline/main.go @@ -48,15 +48,37 @@ type metric struct { } type workload struct { - name string - source string - output string + name string + source string + output string + args []string + harnessSource bool + internalDuration bool } var workloads = []workload{ {name: "cprintf", source: "benchmark/binary_size/cprintf/main.go", output: "Hello, world\n"}, {name: "println", source: "benchmark/binary_size/println/main.go", output: "Hello, world\n"}, {name: "fmtprintf", source: "benchmark/binary_size/fmtprintf/main.go", output: "Hello, world\n"}, + { + name: "memprofile-no-consumer", + source: "benchmark/memprofile/noconsumer", + harnessSource: true, + internalDuration: true, + }, + { + name: "memprofile-rate0", + source: "benchmark/memprofile/enabled", + args: []string{"rate0"}, + harnessSource: true, + internalDuration: true, + }, + { + name: "memprofile-default", + source: "benchmark/memprofile/enabled", + harnessSource: true, + internalDuration: true, + }, } var expectedGoBenchmarks = []string{ @@ -69,9 +91,6 @@ var expectedGoBenchmarks = []string{ "BenchmarkGoroutine", "BenchmarkInterfaceCall", "BenchmarkLookupPCRandom", - "BenchmarkMemProfileDefault", - "BenchmarkMemProfileNoConsumer", - "BenchmarkMemProfileRate0", "BenchmarkMergeCompilerFlags", "BenchmarkMergeLinkerFlags", "BenchmarkRuntimeGetG", @@ -100,10 +119,11 @@ func runCLI(ctx context.Context, args []string) error { flags.SetOutput(io.Discard) mode := flags.String("mode", "collect", "collect, validate, or export") root := flags.String("root", ".", "LLGo repository root") + harnessRoot := flags.String("harness-root", ".", "benchmark harness repository root") llgo := flags.String("llgo", "llgo", "LLGo command") out := flags.String("out", filepath.Join("benchmark", "baseline", "out"), "result directory") - buildRuns := flags.Int("build-runs", 5, "build repetitions per workload") - runRuns := flags.Int("run-runs", 15, "process repetitions per workload") + buildRuns := flags.Int("build-runs", 6, "build repetitions per workload") + runRuns := flags.Int("run-runs", 18, "process repetitions per workload") benchmarkOutput := flags.String( "benchmark-output", "", @@ -115,7 +135,7 @@ func runCLI(ctx context.Context, args []string) error { switch *mode { case "collect": - return collect(ctx, *root, *llgo, *out, *buildRuns, *runRuns) + return collectWithHarness(ctx, *root, *harnessRoot, *llgo, *out, *buildRuns, *runRuns) case "validate": return validateArtifact(*out) case "export": @@ -199,6 +219,12 @@ func formatMetric(value float64) string { } func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int) error { + return collectWithHarness(ctx, root, root, llgo, out, buildRuns, runRuns) +} + +func collectWithHarness( + ctx context.Context, root, harnessRoot, llgo, out string, buildRuns, runRuns int, +) error { if buildRuns <= 0 || runRuns <= 0 { return errors.New("build and run repetitions must be positive") } @@ -217,30 +243,60 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int if err := os.MkdirAll(binDir, 0o755); err != nil { return err } + harnessRoot, err = filepath.Abs(harnessRoot) + if err != nil { + return err + } env := append(os.Environ(), "GOMAXPROCS=2", "LLGO_ROOT="+root, "LLGO_FULL_RPATH=true", ) - var sizes, timings []metric + type measurement struct { + binary string + buildDurations []time.Duration + runDurations []time.Duration + } + measurements := make([]measurement, len(workloads)) + + // Warm every workload before measuring any of them so the first workload + // does not pay unique toolchain and filesystem cache costs. for _, item := range workloads { binary := filepath.Join(binDir, item.name) - // Keep first-use toolchain and filesystem caches out of the measured - // median so the first revision is not systematically disadvantaged. - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(root, item.source)); err != nil { + sourceRoot := root + if item.harnessSource { + sourceRoot = harnessRoot + } + if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)); err != nil { return fmt.Errorf("warm build %s: %w", item.name, err) } - buildDurations := make([]time.Duration, 0, buildRuns) - for range buildRuns { + } + + // Rotate workload order across rounds to balance runner drift and cache + // position instead of consistently favoring the first or last workload. + for round := range buildRuns { + for offset := range workloads { + index := (round + offset) % len(workloads) + item := workloads[index] + binary := filepath.Join(binDir, item.name) + sourceRoot := root + if item.harnessSource { + sourceRoot = harnessRoot + } start := time.Now() - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(root, item.source)); err != nil { + if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)); err != nil { return fmt.Errorf("build %s: %w", item.name, err) } - buildDurations = append(buildDurations, time.Since(start)) + measurements[index].buildDurations = append(measurements[index].buildDurations, time.Since(start)) } - timings = append(timings, durationMetric("compile/"+item.name, buildDurations)) + } + var sizes, timings []metric + for index, item := range workloads { + binary := filepath.Join(binDir, item.name) + measurements[index].binary = binary + timings = append(timings, durationMetric("compile/"+item.name, measurements[index].buildDurations)) size, err := inspectExecutable(binary) if err != nil { return fmt.Errorf("inspect %s: %w", item.name, err) @@ -253,21 +309,40 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int ) var output bytes.Buffer - if err := run(ctx, env, &output, binary); err != nil { + if err := run(ctx, env, &output, binary, item.args...); err != nil { return fmt.Errorf("execute %s: %w", item.name, err) } - if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { + if item.internalDuration { + if _, err := parseInternalDuration(output.String()); err != nil { + return fmt.Errorf("execute %s: %w", item.name, err) + } + } else if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { return fmt.Errorf("execute %s: output %q, want %q", item.name, got, item.output) } - runDurations := make([]time.Duration, 0, runRuns) - for range runRuns { + } + + for round := range runRuns { + for offset := range workloads { + index := (round + offset) % len(workloads) + item := workloads[index] + measurement := &measurements[index] + var output bytes.Buffer start := time.Now() - if err := run(ctx, env, io.Discard, binary); err != nil { + if err := run(ctx, env, &output, measurement.binary, item.args...); err != nil { return fmt.Errorf("execute %s: %w", item.name, err) } - runDurations = append(runDurations, time.Since(start)) + duration := time.Since(start) + if item.internalDuration { + duration, err = parseInternalDuration(output.String()) + if err != nil { + return fmt.Errorf("execute %s: %w", item.name, err) + } + } + measurement.runDurations = append(measurement.runDurations, duration) } - timings = append(timings, durationMetric("run/"+item.name, runDurations)) + } + for index, item := range workloads { + timings = append(timings, durationMetric("run/"+item.name, measurements[index].runDurations)) } if err := writeMetrics(filepath.Join(out, "size.json"), sizes); err != nil { @@ -276,6 +351,15 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int return writeMetrics(filepath.Join(out, "time.json"), timings) } +func parseInternalDuration(output string) (time.Duration, error) { + value := strings.TrimSpace(output) + nanoseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || nanoseconds < 0 { + return 0, fmt.Errorf("invalid internal duration %q", value) + } + return time.Duration(nanoseconds), nil +} + func run(ctx context.Context, env []string, output io.Writer, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) cmd.Env = env @@ -301,7 +385,7 @@ func durationMetric(name string, values []time.Duration) metric { Value: median, Range: strconv.FormatInt(ordered[0].Nanoseconds(), 10) + ".." + strconv.FormatInt(ordered[len(ordered)-1].Nanoseconds(), 10), - Extra: fmt.Sprintf("median of %d consecutive runs", len(ordered)), + Extra: fmt.Sprintf("median of %d rotated runs", len(ordered)), } } diff --git a/benchmark/baseline/main_test.go b/benchmark/baseline/main_test.go index fb7379290c..fe0306bbcb 100644 --- a/benchmark/baseline/main_test.go +++ b/benchmark/baseline/main_test.go @@ -40,7 +40,7 @@ func TestDurationMetric(t *testing.T) { if got.Name != "compile/test" || got.Unit != "ns" || got.Value != 6 { t.Fatalf("durationMetric = %+v", got) } - if got.Range != "3..9" || got.Extra != "median of 3 consecutive runs" { + if got.Range != "3..9" || got.Extra != "median of 3 rotated runs" { t.Fatalf("duration metadata = %+v", got) } if !slices.Equal(values, []time.Duration{9, 3, 6}) { @@ -53,6 +53,18 @@ func TestDurationMetric(t *testing.T) { } } +func TestParseInternalDuration(t *testing.T) { + got, err := parseInternalDuration(" 123\n") + if err != nil || got != 123*time.Nanosecond { + t.Fatalf("parseInternalDuration = %v, %v", got, err) + } + for _, input := range []string{"", "not-a-duration", "-1"} { + if _, err := parseInternalDuration(input); err == nil { + t.Fatalf("parseInternalDuration(%q) unexpectedly succeeded", input) + } + } +} + func TestWriteAndValidateMetrics(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "metrics.json") @@ -97,7 +109,7 @@ func TestExportBenchmarks(t *testing.T) { "Unit file-bytes better=lower assume=exact", "Unit build-ns better=lower", "BenchmarkProgram/cprintf 1 1 file-bytes 1 text-bytes 1 data-bytes 1 bss-bytes 1 build-ns 1 run-ns", - "BenchmarkMemProfileNoConsumer-1 100 12.5 ns/op", + "BenchmarkProgram/memprofile-no-consumer 1 1 file-bytes 1 text-bytes 1 data-bytes 1 bss-bytes 1 build-ns 1 run-ns", "BenchmarkRuntimeGetG-1 100 12.5 ns/op", } { if !strings.Contains(text, want) { @@ -294,7 +306,7 @@ func TestCollect(t *testing.T) { }) out := filepath.Join(root, "out") - if err := collect(context.Background(), root, fakeLLGo, out, 1, 1); err != nil { + if err := collect(context.Background(), root, fakeLLGo, out, 2, 2); err != nil { t.Fatal(err) } goText := makeGoBenchmarkText() @@ -487,10 +499,20 @@ while [ "$#" -gt 0 ]; do *) shift ;; esac done -cat > "$out" <<'LLGO_BENCH_PROGRAM' +case "$out" in + *memprofile-*) + cat > "$out" <<'LLGO_BENCH_PROGRAM' +#!/bin/sh +printf '1\n' +LLGO_BENCH_PROGRAM + ;; + *) + cat > "$out" <<'LLGO_BENCH_PROGRAM' #!/bin/sh ` + program + ` LLGO_BENCH_PROGRAM + ;; +esac chmod +x "$out" ` return writeScript(t, filepath.Join(root, "llgo"), script) diff --git a/benchmark/baseline/run.sh b/benchmark/baseline/run.sh index 9fc08ad857..44b2a38168 100755 --- a/benchmark/baseline/run.sh +++ b/benchmark/baseline/run.sh @@ -25,6 +25,7 @@ result_directory="$(cd "$3" && pwd)" # suite changes must therefore remain executable against the PR base. LLGO_ROOT="$source_root" go run ./benchmark/baseline \ -root "$source_root" \ + -harness-root "$harness_root" \ -llgo "$llgo_output" \ -out "$result_directory" ) @@ -84,41 +85,6 @@ drop_first_benchmark_sample() { ./test/llgoext ) | drop_first_benchmark_sample | tee -a "$go_results" -memprofile_noconsumer="$result_directory/bin/memprofile-noconsumer" -memprofile_enabled="$result_directory/bin/memprofile-enabled" -( - cd "$harness_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true "$llgo_output" build \ - -o "$memprofile_noconsumer" \ - ./benchmark/memprofile/noconsumer - GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true "$llgo_output" build \ - -o "$memprofile_enabled" \ - ./benchmark/memprofile/enabled -) - -# Each executable also warms its allocator internally before timing. These -# discarded processes additionally remove loader and first-execution effects. -GOMAXPROCS=1 "$memprofile_noconsumer" >/dev/null 2>&1 -GOMAXPROCS=1 "$memprofile_enabled" rate0 >/dev/null 2>&1 -GOMAXPROCS=1 "$memprofile_enabled" >/dev/null 2>&1 - -# Rotate the order so every mode occupies each scheduling position across the -# seven independent process samples. -for round in {0..6}; do - case $((round % 3)) in - 0) modes=(noconsumer rate0 default) ;; - 1) modes=(rate0 default noconsumer) ;; - 2) modes=(default noconsumer rate0) ;; - esac - for mode in "${modes[@]}"; do - case "$mode" in - noconsumer) GOMAXPROCS=1 "$memprofile_noconsumer" 2>&1 ;; - rate0) GOMAXPROCS=1 "$memprofile_enabled" rate0 2>&1 ;; - default) GOMAXPROCS=1 "$memprofile_enabled" 2>&1 ;; - esac - done -done | tee -a "$go_results" - ( cd "$harness_root" go run ./benchmark/baseline \ diff --git a/benchmark/memprofile/enabled/main.go b/benchmark/memprofile/enabled/main.go index 1897133348..7276ca68b5 100644 --- a/benchmark/memprofile/enabled/main.go +++ b/benchmark/memprofile/enabled/main.go @@ -30,14 +30,13 @@ import ( func disableGC() func main() { - name := "BenchmarkMemProfileDefault-1" if len(os.Args) == 2 && os.Args[1] == "rate0" { runtime.MemProfileRate = 0 - name = "BenchmarkMemProfileRate0-1" } + // Keep the profiling consumer in this executable without including profile + // materialization in the timed allocation region. + runtime.MemProfile(nil, false) disableGC() elapsed := runner.Run() - println(name, runner.Iterations, - float64(elapsed.Nanoseconds())/runner.Iterations, "ns/op") - runtime.MemProfile(nil, false) + println(elapsed.Nanoseconds()) } diff --git a/benchmark/memprofile/internal/runner/runner.go b/benchmark/memprofile/internal/runner/runner.go index 49d5298ec0..f9fe3d20aa 100644 --- a/benchmark/memprofile/internal/runner/runner.go +++ b/benchmark/memprofile/internal/runner/runner.go @@ -19,7 +19,7 @@ package runner import "time" const ( - Iterations = 40_000_000 + iterations = 40_000_000 warmup = 2_000_000 ) @@ -35,6 +35,6 @@ func allocate(n int) { func Run() time.Duration { allocate(warmup) start := time.Now() - allocate(Iterations) + allocate(iterations) return time.Since(start) } diff --git a/benchmark/memprofile/noconsumer/main.go b/benchmark/memprofile/noconsumer/main.go index 5da377084d..17cc3d9476 100644 --- a/benchmark/memprofile/noconsumer/main.go +++ b/benchmark/memprofile/noconsumer/main.go @@ -30,6 +30,5 @@ func disableGC() func main() { disableGC() elapsed := runner.Run() - println("BenchmarkMemProfileNoConsumer-1", runner.Iterations, - float64(elapsed.Nanoseconds())/runner.Iterations, "ns/op") + println(elapsed.Nanoseconds()) } From ceb4754ba0ef78c21c29755826def27eba201b99 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 09:28:31 +0800 Subject: [PATCH 20/26] benchmark: interleave pull request comparisons --- .github/workflows/benchmark.yml | 36 ++-- benchmark/baseline/README.md | 29 ++-- benchmark/baseline/main.go | 269 +++++++++++++++++++---------- benchmark/baseline/main_test.go | 54 ++++++ benchmark/baseline/run.sh | 296 +++++++++++++++++++++++--------- 5 files changed, 480 insertions(+), 204 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 68821b81fa..130275b047 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -38,7 +38,16 @@ jobs: with: repository: ${{ github.event.pull_request.base.repo.full_name }} ref: ${{ github.event.pull_request.base.sha }} - path: .benchmark/source + path: .benchmark/base-source + persist-credentials: false + + - name: Check out pull request head benchmark source + if: github.event_name == 'pull_request' + uses: actions/checkout@v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: .benchmark/head-source persist-credentials: false - name: Install dependencies @@ -49,31 +58,22 @@ jobs: - name: Set up Go uses: ./.github/actions/setup-go - - name: Measure pull request base + - name: Measure paired pull request revisions if: github.event_name == 'pull_request' run: | benchmark/baseline/run.sh \ - "$GITHUB_WORKSPACE/.benchmark/source" \ + "$GITHUB_WORKSPACE/.benchmark/base-source" \ "$GITHUB_WORKSPACE/.benchmark/base-llgo" \ - "$GITHUB_WORKSPACE/.benchmark/base-results" - - - name: Check out pull request head benchmark source - if: github.event_name == 'pull_request' - uses: actions/checkout@v7 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - path: .benchmark/source - persist-credentials: false + "$GITHUB_WORKSPACE/.benchmark/base-results" \ + "$GITHUB_WORKSPACE/.benchmark/head-source" \ + "$GITHUB_WORKSPACE/.benchmark/llgo" \ + "$GITHUB_WORKSPACE/.benchmark/results" - name: Measure current revision + if: github.event_name != 'pull_request' run: | - source_root="$GITHUB_WORKSPACE" - if [[ "$GITHUB_EVENT_NAME" == pull_request ]]; then - source_root="$GITHUB_WORKSPACE/.benchmark/source" - fi benchmark/baseline/run.sh \ - "$source_root" \ + "$GITHUB_WORKSPACE" \ "$GITHUB_WORKSPACE/.benchmark/llgo" \ "$GITHUB_WORKSPACE/.benchmark/results" diff --git a/benchmark/baseline/README.md b/benchmark/baseline/README.md index fc0b687572..50c6f2ef8c 100644 --- a/benchmark/baseline/README.md +++ b/benchmark/baseline/README.md @@ -19,12 +19,11 @@ the median of six builds and eighteen process runs, file size, executable-code bytes, allocated non-executable data, and zero-filled data. Workload order is rotated between rounds to balance runner drift and cache position. On ELF, read-only constants are included in the data bucket; on Mach-O, `__TEXT` -constants are included in the text bucket. The Go benchmark stream discards the -first one-second sample as warmup, then records seven one-second samples of -compiler helpers and LLGo-generated core-language operations: direct/interface -calls, defer, channels, `getg`, and global access. -Goroutine creation keeps its bounded 100-iteration sample and likewise discards -the first of eight runs. +constants are included in the text bucket. The Go benchmark stream performs one +unrecorded warmup, then records seven one-second samples of compiler helpers and +LLGo-generated core-language operations: direct/interface calls, defer, +channels, `getg`, and global access. Goroutine creation keeps its bounded +100-iteration samples. The program table also includes standalone memory-profile workloads so the whole-program no-consumer path is measurable independently of the retained @@ -33,15 +32,15 @@ then internally times forty million escaping 16-byte allocations. The reported duration is the median of eighteen processes. The workloads are `memprofile-no-consumer`, `memprofile-rate0`, and `memprofile-default`. -For pull requests, each platform job checks out the recorded base and current -commits into the same source path, then runs both suites sequentially on the same -runner. The pull request comment compares that pair, avoiding differences from -runner machines and embedded source paths. Dependency setup is shared, and Go's -build cache can be reused by unchanged packages; main pushes still run the suite -only once. Very small changes can still be scheduler, frequency, or thermal -noise and should be confirmed by repeated workflow runs. If a workflow does not -provide a paired result, the publisher falls back to the latest matching `main` -data. +For pull requests, each platform job builds the recorded base and current +revisions on one runner, then alternates their measurements within every round. +Compiler and runtime benchmark binaries are built before sampling, so only their +execution is interleaved. This prevents a phase-wide frequency, thermal, or host +load change from being attributed entirely to one revision. Dependency setup is +shared, and Go's build cache can be reused by unchanged packages; main pushes +still run the suite only once. Very small changes can remain scheduler noise and +should be confirmed by repeated workflow runs. If a workflow does not provide a +paired result, the publisher falls back to the latest matching `main` data. The trusted publisher commits the current result history and generated site to the `pages` branch of the configured data repository. Every LLGo repository diff --git a/benchmark/baseline/main.go b/benchmark/baseline/main.go index 074f3cdc1a..21bda52b01 100644 --- a/benchmark/baseline/main.go +++ b/benchmark/baseline/main.go @@ -117,11 +117,14 @@ func main() { func runCLI(ctx context.Context, args []string) error { flags := flag.NewFlagSet("llgo-baseline", flag.ContinueOnError) flags.SetOutput(io.Discard) - mode := flags.String("mode", "collect", "collect, validate, or export") + mode := flags.String("mode", "collect", "collect, collect-paired, validate, or export") root := flags.String("root", ".", "LLGo repository root") harnessRoot := flags.String("harness-root", ".", "benchmark harness repository root") llgo := flags.String("llgo", "llgo", "LLGo command") out := flags.String("out", filepath.Join("benchmark", "baseline", "out"), "result directory") + baseRoot := flags.String("base-root", "", "comparison LLGo repository root") + baseLLGo := flags.String("base-llgo", "", "comparison LLGo command") + baseOut := flags.String("base-out", "", "comparison result directory") buildRuns := flags.Int("build-runs", 6, "build repetitions per workload") runRuns := flags.Int("run-runs", 18, "process repetitions per workload") benchmarkOutput := flags.String( @@ -136,6 +139,17 @@ func runCLI(ctx context.Context, args []string) error { switch *mode { case "collect": return collectWithHarness(ctx, *root, *harnessRoot, *llgo, *out, *buildRuns, *runRuns) + case "collect-paired": + if *baseRoot == "" || *baseLLGo == "" || *baseOut == "" { + return errors.New("collect-paired mode requires base-root, base-llgo, and base-out") + } + return collectPaired( + ctx, + collectionSpec{name: "base", root: *baseRoot, harnessRoot: *harnessRoot, llgo: *baseLLGo, out: *baseOut}, + collectionSpec{name: "current", root: *root, harnessRoot: *harnessRoot, llgo: *llgo, out: *out}, + *buildRuns, + *runRuns, + ) case "validate": return validateArtifact(*out) case "export": @@ -225,99 +239,120 @@ func collect(ctx context.Context, root, llgo, out string, buildRuns, runRuns int func collectWithHarness( ctx context.Context, root, harnessRoot, llgo, out string, buildRuns, runRuns int, ) error { + return collectSpecs(ctx, []collectionSpec{{ + root: root, harnessRoot: harnessRoot, llgo: llgo, out: out, + }}, buildRuns, runRuns) +} + +type collectionSpec struct { + name string + root string + harnessRoot string + llgo string + out string +} + +type workloadMeasurement struct { + binary string + buildDurations []time.Duration + runDurations []time.Duration +} + +type collectionState struct { + collectionSpec + binDir string + env []string + measurements []workloadMeasurement +} + +func collectPaired( + ctx context.Context, base, current collectionSpec, buildRuns, runRuns int, +) error { + return collectSpecs(ctx, []collectionSpec{base, current}, buildRuns, runRuns) +} + +func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRuns int) error { if buildRuns <= 0 || runRuns <= 0 { return errors.New("build and run repetitions must be positive") } - root, err := filepath.Abs(root) - if err != nil { - return err - } - out, err = filepath.Abs(out) - if err != nil { - return err - } - binDir := filepath.Join(out, "bin") - if err := os.RemoveAll(out); err != nil { - return err - } - if err := os.MkdirAll(binDir, 0o755); err != nil { - return err - } - harnessRoot, err = filepath.Abs(harnessRoot) - if err != nil { - return err + if len(specs) == 0 { + return errors.New("at least one collection is required") } - - env := append(os.Environ(), - "GOMAXPROCS=2", - "LLGO_ROOT="+root, - "LLGO_FULL_RPATH=true", - ) - type measurement struct { - binary string - buildDurations []time.Duration - runDurations []time.Duration + states := make([]collectionState, len(specs)) + for index, spec := range specs { + root, err := filepath.Abs(spec.root) + if err != nil { + return err + } + harnessRoot, err := filepath.Abs(spec.harnessRoot) + if err != nil { + return err + } + out, err := filepath.Abs(spec.out) + if err != nil { + return err + } + binDir := filepath.Join(out, "bin") + if err := os.RemoveAll(out); err != nil { + return err + } + if err := os.MkdirAll(binDir, 0o755); err != nil { + return err + } + states[index] = collectionState{ + collectionSpec: collectionSpec{ + name: spec.name, root: root, harnessRoot: harnessRoot, llgo: spec.llgo, out: out, + }, + binDir: binDir, + env: append(os.Environ(), "GOMAXPROCS=2", "LLGO_ROOT="+root, "LLGO_FULL_RPATH=true"), + measurements: make([]workloadMeasurement, len(workloads)), + } } - measurements := make([]measurement, len(workloads)) // Warm every workload before measuring any of them so the first workload - // does not pay unique toolchain and filesystem cache costs. - for _, item := range workloads { - binary := filepath.Join(binDir, item.name) - sourceRoot := root - if item.harnessSource { - sourceRoot = harnessRoot - } - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)); err != nil { - return fmt.Errorf("warm build %s: %w", item.name, err) + // does not pay unique toolchain and filesystem cache costs. Paired + // collections alternate which revision runs first for adjacent workloads. + for workloadIndex, item := range workloads { + for stateOffset := range states { + state := &states[(workloadIndex+stateOffset)%len(states)] + if err := buildWorkload(ctx, state, item); err != nil { + return collectionError(state, "warm build", item.name, err) + } } } - // Rotate workload order across rounds to balance runner drift and cache - // position instead of consistently favoring the first or last workload. + // Rotate both workload and revision order across rounds. In paired mode each + // base/current measurement is adjacent, so runner drift is not attributed to + // one complete revision phase. for round := range buildRuns { for offset := range workloads { index := (round + offset) % len(workloads) item := workloads[index] - binary := filepath.Join(binDir, item.name) - sourceRoot := root - if item.harnessSource { - sourceRoot = harnessRoot - } - start := time.Now() - if err := run(ctx, env, io.Discard, llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)); err != nil { - return fmt.Errorf("build %s: %w", item.name, err) + for stateOffset := range states { + state := &states[(round+offset+stateOffset)%len(states)] + start := time.Now() + if err := buildWorkload(ctx, state, item); err != nil { + return collectionError(state, "build", item.name, err) + } + state.measurements[index].buildDurations = append( + state.measurements[index].buildDurations, time.Since(start), + ) } - measurements[index].buildDurations = append(measurements[index].buildDurations, time.Since(start)) } } - var sizes, timings []metric - for index, item := range workloads { - binary := filepath.Join(binDir, item.name) - measurements[index].binary = binary - timings = append(timings, durationMetric("compile/"+item.name, measurements[index].buildDurations)) - size, err := inspectExecutable(binary) - if err != nil { - return fmt.Errorf("inspect %s: %w", item.name, err) - } - sizes = append(sizes, - byteMetric("binary/"+item.name+"/file", size.file), - byteMetric("binary/"+item.name+"/text", size.text), - byteMetric("binary/"+item.name+"/data", size.data), - byteMetric("binary/"+item.name+"/bss", size.bss), - ) - - var output bytes.Buffer - if err := run(ctx, env, &output, binary, item.args...); err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) - } - if item.internalDuration { - if _, err := parseInternalDuration(output.String()); err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) + // Inspect final binaries and warm every execution path before timing. + for workloadIndex, item := range workloads { + for stateOffset := range states { + state := &states[(workloadIndex+stateOffset)%len(states)] + binary := filepath.Join(state.binDir, item.name) + state.measurements[workloadIndex].binary = binary + if _, err := inspectExecutable(binary); err != nil { + return collectionError(state, "inspect", item.name, err) + } + if _, err := executeWorkload(ctx, state, item, binary); err != nil { + return collectionError(state, "execute", item.name, err) } - } else if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { - return fmt.Errorf("execute %s: output %q, want %q", item.name, got, item.output) } } @@ -325,30 +360,84 @@ func collectWithHarness( for offset := range workloads { index := (round + offset) % len(workloads) item := workloads[index] - measurement := &measurements[index] - var output bytes.Buffer - start := time.Now() - if err := run(ctx, env, &output, measurement.binary, item.args...); err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) - } - duration := time.Since(start) - if item.internalDuration { - duration, err = parseInternalDuration(output.String()) + for stateOffset := range states { + state := &states[(round+offset+stateOffset)%len(states)] + measurement := &state.measurements[index] + duration, err := executeWorkload(ctx, state, item, measurement.binary) if err != nil { - return fmt.Errorf("execute %s: %w", item.name, err) + return collectionError(state, "execute", item.name, err) } + measurement.runDurations = append(measurement.runDurations, duration) } - measurement.runDurations = append(measurement.runDurations, duration) } } - for index, item := range workloads { - timings = append(timings, durationMetric("run/"+item.name, measurements[index].runDurations)) + + for stateIndex := range states { + if err := writeCollection(&states[stateIndex]); err != nil { + return err + } } + return nil +} + +func buildWorkload(ctx context.Context, state *collectionState, item workload) error { + binary := filepath.Join(state.binDir, item.name) + sourceRoot := state.root + if item.harnessSource { + sourceRoot = state.harnessRoot + } + return run(ctx, state.env, io.Discard, state.llgo, "build", "-o", binary, filepath.Join(sourceRoot, item.source)) +} - if err := writeMetrics(filepath.Join(out, "size.json"), sizes); err != nil { +func executeWorkload( + ctx context.Context, state *collectionState, item workload, binary string, +) (time.Duration, error) { + var output bytes.Buffer + start := time.Now() + if err := run(ctx, state.env, &output, binary, item.args...); err != nil { + return 0, err + } + duration := time.Since(start) + if item.internalDuration { + return parseInternalDuration(output.String()) + } + if got := strings.ReplaceAll(output.String(), "\r\n", "\n"); got != item.output { + return 0, fmt.Errorf("output %q, want %q", got, item.output) + } + return duration, nil +} + +func writeCollection(state *collectionState) error { + var sizes, timings []metric + for index, item := range workloads { + measurement := &state.measurements[index] + timings = append(timings, + durationMetric("compile/"+item.name, measurement.buildDurations), + durationMetric("run/"+item.name, measurement.runDurations), + ) + size, err := inspectExecutable(measurement.binary) + if err != nil { + return collectionError(state, "inspect", item.name, err) + } + sizes = append(sizes, + byteMetric("binary/"+item.name+"/file", size.file), + byteMetric("binary/"+item.name+"/text", size.text), + byteMetric("binary/"+item.name+"/data", size.data), + byteMetric("binary/"+item.name+"/bss", size.bss), + ) + } + if err := writeMetrics(filepath.Join(state.out, "size.json"), sizes); err != nil { return err } - return writeMetrics(filepath.Join(out, "time.json"), timings) + return writeMetrics(filepath.Join(state.out, "time.json"), timings) +} + +func collectionError(state *collectionState, operation, workload string, err error) error { + prefix := "" + if state.name != "" { + prefix = state.name + " " + } + return fmt.Errorf("%s%s %s: %w", prefix, operation, workload, err) } func parseInternalDuration(output string) (time.Duration, error) { diff --git a/benchmark/baseline/main_test.go b/benchmark/baseline/main_test.go index fe0306bbcb..22688d3f42 100644 --- a/benchmark/baseline/main_test.go +++ b/benchmark/baseline/main_test.go @@ -318,6 +318,56 @@ func TestCollect(t *testing.T) { } } +func TestCollectPaired(t *testing.T) { + if os.PathSeparator != '/' { + t.Skip("fake compiler uses a POSIX shell") + } + root := t.TempDir() + baseRoot := filepath.Join(root, "base") + currentRoot := filepath.Join(root, "current") + if err := os.MkdirAll(baseRoot, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(currentRoot, 0o755); err != nil { + t.Fatal(err) + } + baseLLGo := writeFakeCompiler(t, baseRoot, "printf 'Hello, world\\n'") + currentLLGo := writeFakeCompiler(t, currentRoot, "printf 'Hello, world\\n'") + + oldInspect := inspectExecutable + inspectExecutable = func(path string) (footprint, error) { + info, err := os.Stat(path) + if err != nil { + return footprint{}, err + } + return footprint{file: uint64(info.Size()), text: 10, data: 2, bss: 1}, nil + } + t.Cleanup(func() { + inspectExecutable = oldInspect + }) + + baseOut := filepath.Join(root, "base-out") + currentOut := filepath.Join(root, "current-out") + err := collectPaired( + context.Background(), + collectionSpec{name: "base", root: baseRoot, harnessRoot: root, llgo: baseLLGo, out: baseOut}, + collectionSpec{name: "current", root: currentRoot, harnessRoot: root, llgo: currentLLGo, out: currentOut}, + 2, + 2, + ) + if err != nil { + t.Fatal(err) + } + for _, out := range []string{baseOut, currentOut} { + if err := os.WriteFile(filepath.Join(out, "go.txt"), []byte(makeGoBenchmarkText()), 0o644); err != nil { + t.Fatal(err) + } + if err := validateArtifact(out); err != nil { + t.Fatal(err) + } + } +} + func TestCollectRejectsInvalidRuns(t *testing.T) { err := collect(context.Background(), ".", "llgo", t.TempDir(), 0, 1) if err == nil || !strings.Contains(err.Error(), "must be positive") { @@ -469,6 +519,10 @@ func TestRunCLI(t *testing.T) { !strings.Contains(err.Error(), "unknown mode") { t.Fatalf("unknown CLI error = %v", err) } + if err := runCLI(context.Background(), []string{"-mode=collect-paired"}); err == nil || + !strings.Contains(err.Error(), "requires base-root") { + t.Fatalf("collect-paired CLI error = %v", err) + } if err := runCLI(context.Background(), []string{"-not-a-flag"}); err == nil { t.Fatal("runCLI unexpectedly accepted an unknown flag") } diff --git a/benchmark/baseline/run.sh b/benchmark/baseline/run.sh index 44b2a38168..2471820a2f 100755 --- a/benchmark/baseline/run.sh +++ b/benchmark/baseline/run.sh @@ -2,93 +2,227 @@ set -euo pipefail -if [[ $# -ne 3 ]]; then - echo "usage: $0 " >&2 +if [[ $# -ne 3 && $# -ne 6 ]]; then + echo "usage: $0 [ ]" >&2 exit 2 fi harness_root="$(cd "$(dirname "$0")/../.." && pwd)" -source_root="$(cd "$1" && pwd)" -mkdir -p "$(dirname "$2")" "$3" -llgo_output="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" -result_directory="$(cd "$3" && pwd)" - -( - cd "$source_root" - LLGO_ROOT="$source_root" go build -p=1 -o "$llgo_output" ./cmd/llgo -) - -( - cd "$harness_root" - # Keep one current-checkout harness for both source revisions. Benchmark - # suite changes must therefore remain executable against the PR base. - LLGO_ROOT="$source_root" go run ./benchmark/baseline \ - -root "$source_root" \ - -harness-root "$harness_root" \ - -llgo "$llgo_output" \ - -out "$result_directory" -) - -go_results="$result_directory/go.txt" -: > "$go_results" - -compiler_benchmarks='^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags|BenchmarkLookupPCRandom)$' +clang_benchmarks='^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags)$' +funcinfo_benchmarks='^BenchmarkLookupPCRandom$' core_benchmarks='^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' -drop_first_benchmark_sample() { - awk ' - /^Benchmark/ { - name = $1 - sub(/-[0-9]+$/, "", name) - if (++seen[name] == 1) next - } - { print } - ' +absolute_output() { + mkdir -p "$(dirname "$1")" + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +absolute_directory() { + mkdir -p "$1" + (cd "$1" && pwd) +} + +build_llgo() { + local source_root="$1" + local llgo_output="$2" + ( + cd "$source_root" + LLGO_ROOT="$source_root" go build -p=1 -o "$llgo_output" ./cmd/llgo + ) +} + +export_result() { + local result_directory="$1" + ( + cd "$harness_root" + go run ./benchmark/baseline \ + -mode export \ + -out "$result_directory" \ + -benchmark-output "$result_directory/benchmark.txt" + ) +} + +run_single() { + local source_root="$1" + local llgo_output="$2" + local result_directory="$3" + + build_llgo "$source_root" "$llgo_output" + ( + cd "$harness_root" + LLGO_ROOT="$source_root" go run ./benchmark/baseline \ + -root "$source_root" \ + -harness-root "$harness_root" \ + -llgo "$llgo_output" \ + -out "$result_directory" + ) + + local go_results="$result_directory/go.txt" + : > "$go_results" + + local binaries="$result_directory/tests" + build_benchmark_binaries "$source_root" "$llgo_output" "$binaries" + run_benchmark_series "$source_root" "$binaries/clang.test" "$go_results" "$clang_benchmarks" 1s + run_benchmark_series "$source_root" "$binaries/funcinfo.test" "$go_results" "$funcinfo_benchmarks" 1s + run_benchmark_series "$source_root" "$binaries/llgoext.test" "$go_results" "$core_benchmarks" 1s + # Goroutine creation is bounded explicitly because LLGo currently maps one + # goroutine to one pthread. + run_benchmark_series "$source_root" "$binaries/llgoext.test" "$go_results" '^BenchmarkGoroutine$' 100x + + export_result "$result_directory" } -# The first one-second sample warms each path and is discarded. The following -# seven samples use the same benchmark process and fixed single-CPU conditions. -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" go test \ - -run '^$' \ - -bench "$compiler_benchmarks" \ - -benchtime=1s \ - -count=8 \ - -cpu=1 \ - ./internal/clang ./internal/build/funcinfo -) | drop_first_benchmark_sample | tee -a "$go_results" - -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \ - -run '^$' \ - -bench "$core_benchmarks" \ - -benchtime=1s \ - -count=8 \ - -cpu=1 \ - ./test/llgoext -) | drop_first_benchmark_sample | tee -a "$go_results" - -# The current native backend creates one pthread per goroutine and intentionally -# has a bounded lifecycle stress limit. Keep creation monitoring deterministic -# instead of letting testing auto-calibrate to millions of host threads. -( - cd "$source_root" - GOMAXPROCS=1 LLGO_ROOT="$source_root" "$llgo_output" test \ - -run '^$' \ - -bench '^BenchmarkGoroutine$' \ - -benchtime=100x \ - -count=8 \ - -cpu=1 \ - ./test/llgoext -) | drop_first_benchmark_sample | tee -a "$go_results" - -( - cd "$harness_root" - go run ./benchmark/baseline \ - -mode export \ - -out "$result_directory" \ - -benchmark-output "$result_directory/benchmark.txt" -) +build_benchmark_binaries() { + local source_root="$1" + local llgo_output="$2" + local binary_directory="$3" + mkdir -p "$binary_directory" + ( + cd "$source_root" + GOMAXPROCS=1 LLGO_ROOT="$source_root" go test -c \ + -o "$binary_directory/clang.test" ./internal/clang + GOMAXPROCS=1 LLGO_ROOT="$source_root" go test -c \ + -o "$binary_directory/funcinfo.test" ./internal/build/funcinfo + GOMAXPROCS=1 LLGO_ROOT="$source_root" LLGO_FULL_RPATH=true \ + "$llgo_output" test -c -o "$binary_directory/llgoext.test" ./test/llgoext + ) +} + +run_benchmark_sample() { + local source_root="$1" + local binary="$2" + local pattern="$3" + local benchtime="$4" + local output="$5" + ( + cd "$source_root" + GOMAXPROCS=1 LLGO_ROOT="$source_root" "$binary" \ + -test.run '^$' \ + -test.bench "$pattern" \ + -test.benchtime "$benchtime" \ + -test.count=1 \ + -test.cpu=1 + ) | tee -a "$output" +} + +run_benchmark_series() { + local source_root="$1" + local binary="$2" + local output="$3" + local pattern="$4" + local benchtime="$5" + + run_benchmark_sample "$source_root" "$binary" "$pattern" "$benchtime" /dev/null + local round + for ((round = 0; round < 7; round++)); do + run_benchmark_sample "$source_root" "$binary" "$pattern" "$benchtime" "$output" + done +} + +run_benchmark_pair() { + local base_root="$1" + local base_binary="$2" + local base_output="$3" + local current_root="$4" + local current_binary="$5" + local current_output="$6" + local pattern="$7" + local benchtime="$8" + local pair_index="$9" + + # Warm both binaries without recording the result. Alternate the leading + # revision between groups, then alternate it again for every measured sample. + if (( pair_index % 2 == 0 )); then + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" /dev/null + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" /dev/null + else + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" /dev/null + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" /dev/null + fi + + local round + for ((round = 0; round < 7; round++)); do + if (( (round + pair_index) % 2 == 0 )); then + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" "$base_output" + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" "$current_output" + else + run_benchmark_sample "$current_root" "$current_binary" "$pattern" "$benchtime" "$current_output" + run_benchmark_sample "$base_root" "$base_binary" "$pattern" "$benchtime" "$base_output" + fi + done +} + +run_paired() { + local base_root="$1" + local base_llgo="$2" + local base_result="$3" + local current_root="$4" + local current_llgo="$5" + local current_result="$6" + + build_llgo "$base_root" "$base_llgo" + build_llgo "$current_root" "$current_llgo" + + # Keep one current-checkout harness for both revisions. Suite changes must + # therefore remain executable against the pull request base. + ( + cd "$harness_root" + go run ./benchmark/baseline \ + -mode collect-paired \ + -base-root "$base_root" \ + -base-llgo "$base_llgo" \ + -base-out "$base_result" \ + -root "$current_root" \ + -harness-root "$harness_root" \ + -llgo "$current_llgo" \ + -out "$current_result" + ) + + local base_binaries="$base_result/tests" + local current_binaries="$current_result/tests" + build_benchmark_binaries "$base_root" "$base_llgo" "$base_binaries" + build_benchmark_binaries "$current_root" "$current_llgo" "$current_binaries" + + local base_go="$base_result/go.txt" + local current_go="$current_result/go.txt" + : > "$base_go" + : > "$current_go" + + run_benchmark_pair \ + "$base_root" "$base_binaries/clang.test" "$base_go" \ + "$current_root" "$current_binaries/clang.test" "$current_go" \ + "$clang_benchmarks" 1s 0 + run_benchmark_pair \ + "$base_root" "$base_binaries/funcinfo.test" "$base_go" \ + "$current_root" "$current_binaries/funcinfo.test" "$current_go" \ + "$funcinfo_benchmarks" 1s 1 + run_benchmark_pair \ + "$base_root" "$base_binaries/llgoext.test" "$base_go" \ + "$current_root" "$current_binaries/llgoext.test" "$current_go" \ + "$core_benchmarks" 1s 2 + # Goroutine creation is bounded explicitly because LLGo currently maps one + # goroutine to one pthread. + run_benchmark_pair \ + "$base_root" "$base_binaries/llgoext.test" "$base_go" \ + "$current_root" "$current_binaries/llgoext.test" "$current_go" \ + '^BenchmarkGoroutine$' 100x 3 + + export_result "$base_result" + export_result "$current_result" +} + +source_root="$(cd "$1" && pwd)" +llgo_output="$(absolute_output "$2")" +result_directory="$(absolute_directory "$3")" + +if [[ $# -eq 3 ]]; then + run_single "$source_root" "$llgo_output" "$result_directory" +else + current_source_root="$(cd "$4" && pwd)" + current_llgo_output="$(absolute_output "$5")" + current_result_directory="$(absolute_directory "$6")" + run_paired \ + "$source_root" "$llgo_output" "$result_directory" \ + "$current_source_root" "$current_llgo_output" "$current_result_directory" +fi From 94289e36421259e9a79600b22a53587a9333144c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 09:48:13 +0800 Subject: [PATCH 21/26] benchmark: pair individual microbenchmarks --- benchmark/baseline/README.md | 13 +++--- benchmark/baseline/run.sh | 76 +++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/benchmark/baseline/README.md b/benchmark/baseline/README.md index 50c6f2ef8c..0d70d3889e 100644 --- a/benchmark/baseline/README.md +++ b/benchmark/baseline/README.md @@ -35,12 +35,13 @@ duration is the median of eighteen processes. The workloads are For pull requests, each platform job builds the recorded base and current revisions on one runner, then alternates their measurements within every round. Compiler and runtime benchmark binaries are built before sampling, so only their -execution is interleaved. This prevents a phase-wide frequency, thermal, or host -load change from being attributed entirely to one revision. Dependency setup is -shared, and Go's build cache can be reused by unchanged packages; main pushes -still run the suite only once. Very small changes can remain scheduler noise and -should be confirmed by repeated workflow runs. If a workflow does not provide a -paired result, the publisher falls back to the latest matching `main` data. +execution is interleaved and each matching base/current sample remains adjacent. +This prevents a phase-wide frequency, thermal, or host load change from being +attributed entirely to one revision. Dependency setup is shared, and Go's build +cache can be reused by unchanged packages; main pushes still run the suite only +once. Very small changes can remain scheduler noise and should be confirmed by +repeated workflow runs. If a workflow does not provide a paired result, the +publisher falls back to the latest matching `main` data. The trusted publisher commits the current result history and generated site to the `pages` branch of the configured data repository. Every LLGo repository diff --git a/benchmark/baseline/run.sh b/benchmark/baseline/run.sh index 2471820a2f..0f7bf48fc9 100755 --- a/benchmark/baseline/run.sh +++ b/benchmark/baseline/run.sh @@ -9,9 +9,34 @@ fi harness_root="$(cd "$(dirname "$0")/../.." && pwd)" -clang_benchmarks='^(BenchmarkMergeCompilerFlags|BenchmarkMergeLinkerFlags)$' -funcinfo_benchmarks='^BenchmarkLookupPCRandom$' -core_benchmarks='^(BenchmarkRuntimeGetG|BenchmarkGlobal(Read|Write)|Benchmark(DirectCall|InterfaceCall|Defer|ChannelBuffered|ChannelHandoff))$' +benchmark_names=( + MergeCompilerFlags + MergeLinkerFlags + LookupPCRandom + RuntimeGetG + GlobalRead + GlobalWrite + DirectCall + InterfaceCall + Defer + ChannelBuffered + ChannelHandoff + Goroutine +) +benchmark_binaries=( + clang + clang + funcinfo + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext + llgoext +) absolute_output() { mkdir -p "$(dirname "$1")" @@ -63,12 +88,17 @@ run_single() { local binaries="$result_directory/tests" build_benchmark_binaries "$source_root" "$llgo_output" "$binaries" - run_benchmark_series "$source_root" "$binaries/clang.test" "$go_results" "$clang_benchmarks" 1s - run_benchmark_series "$source_root" "$binaries/funcinfo.test" "$go_results" "$funcinfo_benchmarks" 1s - run_benchmark_series "$source_root" "$binaries/llgoext.test" "$go_results" "$core_benchmarks" 1s - # Goroutine creation is bounded explicitly because LLGo currently maps one - # goroutine to one pthread. - run_benchmark_series "$source_root" "$binaries/llgoext.test" "$go_results" '^BenchmarkGoroutine$' 100x + local index + for index in "${!benchmark_names[@]}"; do + local benchtime=1s + [[ "${benchmark_names[$index]}" != Goroutine ]] || benchtime=100x + run_benchmark_series \ + "$source_root" \ + "$binaries/${benchmark_binaries[$index]}.test" \ + "$go_results" \ + "^Benchmark${benchmark_names[$index]}$" \ + "$benchtime" + done export_result "$result_directory" } @@ -189,24 +219,16 @@ run_paired() { : > "$base_go" : > "$current_go" - run_benchmark_pair \ - "$base_root" "$base_binaries/clang.test" "$base_go" \ - "$current_root" "$current_binaries/clang.test" "$current_go" \ - "$clang_benchmarks" 1s 0 - run_benchmark_pair \ - "$base_root" "$base_binaries/funcinfo.test" "$base_go" \ - "$current_root" "$current_binaries/funcinfo.test" "$current_go" \ - "$funcinfo_benchmarks" 1s 1 - run_benchmark_pair \ - "$base_root" "$base_binaries/llgoext.test" "$base_go" \ - "$current_root" "$current_binaries/llgoext.test" "$current_go" \ - "$core_benchmarks" 1s 2 - # Goroutine creation is bounded explicitly because LLGo currently maps one - # goroutine to one pthread. - run_benchmark_pair \ - "$base_root" "$base_binaries/llgoext.test" "$base_go" \ - "$current_root" "$current_binaries/llgoext.test" "$current_go" \ - '^BenchmarkGoroutine$' 100x 3 + local index + for index in "${!benchmark_names[@]}"; do + local benchtime=1s + [[ "${benchmark_names[$index]}" != Goroutine ]] || benchtime=100x + local binary="${benchmark_binaries[$index]}.test" + run_benchmark_pair \ + "$base_root" "$base_binaries/$binary" "$base_go" \ + "$current_root" "$current_binaries/$binary" "$current_go" \ + "^Benchmark${benchmark_names[$index]}$" "$benchtime" "$index" + done export_result "$base_result" export_result "$current_result" From d268ae6bcdd0ede3f1b59cdaf958f64ab5f19864 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 09:55:22 +0800 Subject: [PATCH 22/26] benchmark: alternate program revision order --- benchmark/baseline/main.go | 12 ++++++++---- benchmark/baseline/main_test.go | 12 ++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/benchmark/baseline/main.go b/benchmark/baseline/main.go index 21bda52b01..9c131de072 100644 --- a/benchmark/baseline/main.go +++ b/benchmark/baseline/main.go @@ -314,7 +314,7 @@ func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRun // collections alternate which revision runs first for adjacent workloads. for workloadIndex, item := range workloads { for stateOffset := range states { - state := &states[(workloadIndex+stateOffset)%len(states)] + state := &states[collectionStateIndex(0, workloadIndex, stateOffset, len(states))] if err := buildWorkload(ctx, state, item); err != nil { return collectionError(state, "warm build", item.name, err) } @@ -329,7 +329,7 @@ func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRun index := (round + offset) % len(workloads) item := workloads[index] for stateOffset := range states { - state := &states[(round+offset+stateOffset)%len(states)] + state := &states[collectionStateIndex(round, index, stateOffset, len(states))] start := time.Now() if err := buildWorkload(ctx, state, item); err != nil { return collectionError(state, "build", item.name, err) @@ -344,7 +344,7 @@ func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRun // Inspect final binaries and warm every execution path before timing. for workloadIndex, item := range workloads { for stateOffset := range states { - state := &states[(workloadIndex+stateOffset)%len(states)] + state := &states[collectionStateIndex(0, workloadIndex, stateOffset, len(states))] binary := filepath.Join(state.binDir, item.name) state.measurements[workloadIndex].binary = binary if _, err := inspectExecutable(binary); err != nil { @@ -361,7 +361,7 @@ func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRun index := (round + offset) % len(workloads) item := workloads[index] for stateOffset := range states { - state := &states[(round+offset+stateOffset)%len(states)] + state := &states[collectionStateIndex(round, index, stateOffset, len(states))] measurement := &state.measurements[index] duration, err := executeWorkload(ctx, state, item, measurement.binary) if err != nil { @@ -380,6 +380,10 @@ func collectSpecs(ctx context.Context, specs []collectionSpec, buildRuns, runRun return nil } +func collectionStateIndex(round, workloadIndex, stateOffset, stateCount int) int { + return (round + workloadIndex + stateOffset) % stateCount +} + func buildWorkload(ctx context.Context, state *collectionState, item workload) error { binary := filepath.Join(state.binDir, item.name) sourceRoot := state.root diff --git a/benchmark/baseline/main_test.go b/benchmark/baseline/main_test.go index 22688d3f42..de6c5d834f 100644 --- a/benchmark/baseline/main_test.go +++ b/benchmark/baseline/main_test.go @@ -368,6 +368,18 @@ func TestCollectPaired(t *testing.T) { } } +func TestCollectionStateIndexAlternatesEachWorkload(t *testing.T) { + for workloadIndex := range workloads { + first := collectionStateIndex(0, workloadIndex, 0, 2) + if second := collectionStateIndex(1, workloadIndex, 0, 2); second == first { + t.Fatalf("workload %d keeps state %d first across rounds", workloadIndex, first) + } + if peer := collectionStateIndex(0, workloadIndex, 1, 2); peer == first { + t.Fatalf("workload %d state order repeats %d within a round", workloadIndex, first) + } + } +} + func TestCollectRejectsInvalidRuns(t *testing.T) { err := collect(context.Background(), ".", "llgo", t.TempDir(), 0, 1) if err == nil || !strings.Contains(err.Error(), "must be positive") { From 341531caa3e1ddb2c2ae97737fe3ab78c467c34c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 10:00:52 +0800 Subject: [PATCH 23/26] ci: allow benchmark setup variance --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 130275b047..d1ce7a1bc5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,7 +25,7 @@ jobs: id: macos display: macOS runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 env: GOMAXPROCS: "2" LLGO_ROOT: ${{ github.workspace }} From e67b50772a1118e03c0074c9271a8862a4e408e1 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 14:06:30 +0800 Subject: [PATCH 24/26] ci: use setup-benchmark-go-action v1.0.5 --- .github/workflows/benchmark-publish.yml | 2 +- .github/workflows/benchmark.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-publish.yml b/.github/workflows/benchmark-publish.yml index d78b6b0a11..d08e2dffea 100644 --- a/.github/workflows/benchmark-publish.yml +++ b/.github/workflows/benchmark-publish.yml @@ -14,7 +14,7 @@ permissions: jobs: publish: if: github.event.workflow_run.conclusion == 'success' - uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1.0.4 + uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1 with: run_id: ${{ github.event.workflow_run.id }} config_path: .github/llgo-benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d1ce7a1bc5..07446bcc21 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -78,12 +78,14 @@ jobs: "$GITHUB_WORKSPACE/.benchmark/results" - name: Record benchmark result - uses: xgo-dev/setup-benchmark-go-action@v1.0.4 + uses: xgo-dev/setup-benchmark-go-action@v1 with: config: .github/llgo-benchmark.yml benchmark-file: .benchmark/results/benchmark.txt baseline-benchmark-file: >- ${{ github.event_name == 'pull_request' && '.benchmark/base-results/benchmark.txt' || '' }} + sample-pairing: >- + ${{ github.event_name == 'pull_request' && 'index' || '' }} platform-id: ${{ matrix.id }} platform-label: ${{ matrix.display }} From 05367fb72cae6d0c0a665a5829bfcf1b583fdb95 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 18 Aug 2026 14:28:23 +0800 Subject: [PATCH 25/26] ci: pin benchmark action v1.0.5 --- .github/workflows/benchmark-publish.yml | 2 +- .github/workflows/benchmark.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-publish.yml b/.github/workflows/benchmark-publish.yml index d08e2dffea..d120948c4c 100644 --- a/.github/workflows/benchmark-publish.yml +++ b/.github/workflows/benchmark-publish.yml @@ -14,7 +14,7 @@ permissions: jobs: publish: if: github.event.workflow_run.conclusion == 'success' - uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1 + uses: xgo-dev/setup-benchmark-go-action/.github/workflows/publish.yml@v1.0.5 with: run_id: ${{ github.event.workflow_run.id }} config_path: .github/llgo-benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 07446bcc21..b323610e34 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -78,7 +78,7 @@ jobs: "$GITHUB_WORKSPACE/.benchmark/results" - name: Record benchmark result - uses: xgo-dev/setup-benchmark-go-action@v1 + uses: xgo-dev/setup-benchmark-go-action@v1.0.5 with: config: .github/llgo-benchmark.yml benchmark-file: .benchmark/results/benchmark.txt From a7bafd4e07b041ae0807dbca95c85a71be18ab9c Mon Sep 17 00:00:00 2001 From: visualfc Date: Tue, 18 Aug 2026 15:08:36 +0800 Subject: [PATCH 26/26] runtime: fix trace stop deadlock on Go 1.25 --- runtime/internal/lib/runtime/trace_stub_llgo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/internal/lib/runtime/trace_stub_llgo.go b/runtime/internal/lib/runtime/trace_stub_llgo.go index 35f5512b3a..28aa8fed4d 100644 --- a/runtime/internal/lib/runtime/trace_stub_llgo.go +++ b/runtime/internal/lib/runtime/trace_stub_llgo.go @@ -11,7 +11,7 @@ func traceAdvance(stopTrace bool) {} func traceClockNow() uint64 { return 0 } //go:linkname runtime_readTrace runtime/trace.runtime_readTrace -func runtime_readTrace() []byte { return nil } +func runtime_readTrace() []byte { return ReadTrace() } //go:linkname trace_userTaskCreate runtime/trace.userTaskCreate func trace_userTaskCreate(id, parentID uint64, taskType string) {}