From 15b0620a3beea0cb196f8dec4d3aa37df363248c Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 1 Aug 2026 10:06:41 +0800 Subject: [PATCH] cmd/compile: run write barriers before LLVM emission --- src/cmd/compile/internal/ssa/compile.go | 27 ++- src/cmd/compile/internal/ssa/ssa2llvm.go | 93 ++++++--- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 28 +++ src/cmd/internal/testdir/llvm_abi_test.go | 35 +++- src/cmd/internal/testdir/llvm_test.go | 41 ++-- .../testdir/llvm_writebarrier_test.go | 178 ++++++++++++++++++ src/cmd/llvmplugin/CMakeLists.txt | 15 ++ src/cmd/llvmplugin/GoALLCStatepoints.cpp | 20 +- src/cmd/llvmplugin/testdata/statepoint.ll | 16 ++ test/codegen/llvm_memory_order.go | 28 +++ test/codegen/llvm_writebarrier.go | 54 ++++++ test/llvm_tests.json | 3 + test/llvm_writebarrier_gc.go | 102 ++++++++++ 13 files changed, 597 insertions(+), 43 deletions(-) create mode 100644 src/cmd/compile/internal/ssa/ssa2llvm_test.go create mode 100644 src/cmd/internal/testdir/llvm_writebarrier_test.go create mode 100644 test/codegen/llvm_memory_order.go create mode 100644 test/codegen/llvm_writebarrier.go create mode 100644 test/llvm_writebarrier_gc.go diff --git a/src/cmd/compile/internal/ssa/compile.go b/src/cmd/compile/internal/ssa/compile.go index cedea715350f2d..445f9d204c1dc9 100644 --- a/src/cmd/compile/internal/ssa/compile.go +++ b/src/cmd/compile/internal/ssa/compile.go @@ -29,9 +29,6 @@ import ( // - the order of b.Values is the order to emit the Values in each Block // - f has a non-nil regAlloc field func Compile(f *Func) { - if base.Flag.EnableLLVM { - LLVMCompile(f) - } // TODO: debugging - set flags to control verbosity of compiler, // which phases to dump IR before/after, etc. if f.Log() { @@ -168,6 +165,24 @@ func Compile(f *Func) { phaseName = "" } +func llvmWritebarrierPass(f *Func) { + if base.Flag.EnableLLVM { + writebarrier(f) + } +} + +func llvmCompilePass(f *Func) { + if base.Flag.EnableLLVM { + LLVMCompile(f) + } +} + +func nativeWritebarrierPass(f *Func) { + if !base.Flag.EnableLLVM { + writebarrier(f) + } +} + // DumpFileForPhase creates a file from the function name and phase name, // warning and returning nil if this is not possible. func (f *Func) DumpFileForPhase(phaseName string) io.WriteCloser { @@ -460,6 +475,10 @@ commas. For example: // list of passes for the compiler var passes = [...]pass{ {name: "number lines", fn: numberLines, required: true}, + // LLVM expands Go write barriers while calls and logical aggregate values + // still retain their frontend ABI shape, then consumes that generic SSA. + {name: "llvm writebarrier", fn: llvmWritebarrierPass, required: true}, + {name: "llvm", fn: llvmCompilePass, required: true}, {name: "early phielim and copyelim", fn: copyelim}, {name: "early deadcode", fn: deadcode}, // remove generated dead code to avoid doing pointless work during opt {name: "short circuit", fn: shortcircuit}, @@ -490,7 +509,7 @@ var passes = [...]pass{ {name: "check bce", fn: checkbce}, {name: "dse", fn: dse}, {name: "memcombine", fn: memcombine}, - {name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops + {name: "writebarrier", fn: nativeWritebarrierPass, required: true}, // expand write barrier ops {name: "insert resched checks", fn: insertLoopReschedChecks, disabled: !buildcfg.Experiment.PreemptibleLoops}, // insert resched checks in loops. {name: "cpufeatures", fn: cpufeatures, required: buildcfg.Experiment.SIMD, disabled: !buildcfg.Experiment.SIMD}, diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 5275e3dd7d2741..eb8a136c76d9a4 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -50,6 +50,8 @@ const goResultsTupleAttr = "go_results_tuple" const goGCStrategy = "goallc" const goGCLeafFunctionAttr = "gc-leaf-function" const goStackGrowthStatepointAttr = "go-stack-growth-statepoint" +const goAsyncUnsafeAttr = "go-async-unsafe" +const goWriteBarrierIntrinsic = "llvm.go.gc.write.barrier" const llvmFramePointerAttr = "frame-pointer" const llvmFramePointerNonLeaf = "non-leaf" @@ -199,15 +201,11 @@ func llvmMemoryOpInfo(v *Value) (int64, int) { if !ok || t == nil { v.Fatalf("%s has no memory type", v.Op) } - // LLVM lowering runs before the native writebarrier pass. Mirror its - // canonical first safety gate: writes of pointer-containing Zero/Move - // values need no heap write barrier when their destination is provably a - // stack address. This deliberately covers derived stack addresses and - // repeated writes, while heap, global, and argument-derived destinations - // remain fail closed. - if t.HasPointers() && !IsStackAddr(v.Args[0]) { - v.Fatalf("%s of pointer-containing type %v requires write-barrier lowering before LLVM", v.Op, t) - } + // LLVM lowering runs after the complete writebarrier pass. At this point a + // pointer-containing heap Zero/Move is the raw memory operation that follows + // its wbZero/wbMove helper; stack destinations need no helper. The unexpanded + // OpZeroWB/OpMoveWB forms remain unsupported by GenLV and therefore fail + // closed rather than bypassing the write barrier. size := auxIntToInt64(v.AuxInt) if size < 0 { v.Fatalf("%s has negative size %d", v.Op, size) @@ -549,24 +547,43 @@ func (lfc *LLVMFuncContext) aggregate(v *Value, args []*Value) llvm.Value { return result } -// llvmNewprocSignature restores the semantic pointer type of newproc's -// funcval argument. Native ssagen intentionally uses uintptr only to compute -// the raw call's physical ABI assignment; the actual SSA operand is a pointer. -func llvmNewprocSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvmFuncSignature { - if aux == nil || aux.Fn != ir.Syms.Newproc { +// llvmStaticCallSignature restores semantic pointer types for compiler-built +// runtime calls whose AuxCall uses uintptr only to compute physical ABI +// assignments. AuxCall remains the physical ABI authority; the LLVM operands +// and runtime helper parameters are pointers. +func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvmFuncSignature { + if aux == nil || aux.Fn == nil { + return sig + } + wantArgs := int64(0) + switch aux.Fn { + case ir.Syms.Newproc: + wantArgs = 1 + case ir.Syms.WBZero: + wantArgs = 2 + case ir.Syms.WBMove: + wantArgs = 3 + default: return sig } if aux.ABI().Which() != obj.ABIInternal { - v.Fatalf("runtime.newproc uses unsupported ABI %v", aux.ABI().Which()) + v.Fatalf("%s uses unsupported ABI %v", aux.Fn.Name, aux.ABI().Which()) } - if aux.NArgs() != 1 || aux.NResults() != 0 || !aux.TypeOfArg(0).IsUintptr() { - v.Fatalf("runtime.newproc has unexpected raw call signature") + if aux.NArgs() != wantArgs || aux.NResults() != 0 { + v.Fatalf("%s has unexpected raw call signature: %d arguments, %d results", aux.Fn.Name, aux.NArgs(), aux.NResults()) } - if len(v.Args) != 2 || v.Args[0].Type == nil || !v.Args[0].Type.IsPtrShaped() { + if aux.Fn == ir.Syms.Newproc && (len(v.Args) != 2 || v.Args[0].Type == nil || !v.Args[0].Type.IsPtrShaped()) { v.Fatalf("runtime.newproc argument is not pointer-shaped") } + for i := int64(0); i < aux.NArgs(); i++ { + if typ := aux.TypeOfArg(i); typ == nil || !typ.IsUintptr() { + v.Fatalf("argument %d to %s is not raw uintptr", i, aux.Fn.Name) + } + } params := append([]llvm.Type(nil), sig.Type.ParamTypes()...) - params[0] = GlobalCtxt.PointerType(0) + for i := range params { + params[i] = GlobalCtxt.PointerType(0) + } sig.Type = llvm.FunctionType(sig.ReturnType, params, false) return sig } @@ -580,9 +597,10 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { v.Fatalf("static call to %s has %d LLVM arguments, want %d", aux.Fn.Name, got, want) } - sig := llvmNewprocSignature(v, aux, llvmSignature(aux)) + sig := llvmStaticCallSignature(v, aux, llvmSignature(aux)) cc := llvmCallConv(aux.ABI().Which()) fn := getOrInsertLLVMFunction(aux.Fn.Name, sig, cc) + rawWriteBarrier := aux.Fn == ir.Syms.WBZero || aux.Fn == ir.Syms.WBMove args := make([]llvm.Value, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { arg := lfc.GenLV(v.Args[i]) @@ -597,6 +615,9 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { } call := lfc.b.CreateCall(sig.Type, fn, args, name) call.SetInstructionCallConv(cc) + if rawWriteBarrier { + markLLVMGCLeaf(fn, call) + } return call } @@ -723,6 +744,17 @@ func (lfc *LLVMFuncContext) panicBounds(v *Value) llvm.Value { return call } +func (lfc *LLVMFuncContext) llvmWriteBarrier(v *Value) llvm.Value { + entries := auxIntToInt64(v.AuxInt) + if entries < 1 || entries > 8 { + v.Fatalf("write barrier requests %d buffer entries", entries) + } + i32 := GlobalCtxt.Int32Type() + sig := llvm.FunctionType(GlobalCtxt.PointerType(0), []llvm.Type{i32}, false) + fn := getOrInsertLLVMIntrinsic(goWriteBarrierIntrinsic, sig) + return lfc.b.CreateCall(sig, fn, []llvm.Value{llvm.ConstInt(i32, uint64(entries), false)}, v.String()) +} + func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { if lv, ok := lfc.Vs[v.ID]; ok { return lv @@ -740,7 +772,7 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { arg0 := func() llvm.Value { return lfc.GenLV(v.Args[0]) } arg1 := func() llvm.Value { return lfc.GenLV(v.Args[1]) } switch v.Op { - case OpInitMem, OpSP, OpSB, OpInlMark: + case OpInitMem, OpSP, OpSB, OpInlMark, OpWBend: // LLVM models memory ordering through instruction dependencies, not an // explicit SSA memory value. SP/SB are only address-space tokens here. case OpUnknown: @@ -970,6 +1002,8 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { lVal = lfc.b.CreateGEP(getLLVMType(v.Type.Elem()), arg0(), []llvm.Value{arg1()}, v.String()) case OpStaticCall, OpStaticLECall: lVal = lfc.staticCall(v) + case OpWB: + lVal = lfc.llvmWriteBarrier(v) case OpClosureCall, OpClosureLECall: // arg0 is the code pointer loaded from the funcval, arg1 is the // funcval itself. The latter is a hidden REGCTXT input, not an @@ -993,6 +1027,11 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { if sel == 0 { lVal = load } + case OpWB: + call := lfc.GenLV(src) + if sel == 0 { + lVal = call + } default: v.Fatalf("%s selects unsupported tuple source %s", v.Op, src.Op) } @@ -1101,9 +1140,20 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { return lVal } +// llvmValueIsWriteBarrierTombstone recognizes the dead OpNilCheck left in a +// block when writebarrier duplicates that check and resets the original before +// the normal early deadcode pass has run. Reject every other malformed +// OpInvalid through GenLV instead of treating it as generic dead code. +func llvmValueIsWriteBarrierTombstone(v *Value) bool { + return v.Op == OpInvalid && v.Uses == 0 && len(v.Args) == 0 && v.Aux == nil && v.AuxInt == 0 +} + func (lfc *LLVMFuncContext) CompileBlock(BB *Block) { lfc.b.SetInsertPointAtEnd(lfc.BBs[BB.ID]) for _, v := range BB.Values { + if llvmValueIsWriteBarrierTombstone(v) { + continue + } lfc.GenLV(v) } switch BB.Kind { @@ -1171,6 +1221,7 @@ func LLVMCompile(f *Func) { f.fe.Fatalf(f.Entry.Pos, "duplicate LLVM definition for %s", f.OwnAux.Fn.Name) } FCtxt.LF.SetGC(goGCStrategy) + FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goAsyncUnsafeAttr, "")) // TODO(goallc): Once LLVM lowering propagates the compiler's precise // morestack policy, attach this only to functions whose prologue can grow // the Go stack. diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go new file mode 100644 index 00000000000000..6b91191d778048 --- /dev/null +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssa + +import "testing" + +func TestLLVMValueIsWriteBarrierTombstone(t *testing.T) { + tests := []struct { + name string + v *Value + want bool + }{ + {name: "dead invalid", v: &Value{Op: OpInvalid}, want: true}, + {name: "live invalid", v: &Value{Op: OpInvalid, Uses: 1}}, + {name: "invalid with argument", v: &Value{Op: OpInvalid, Args: []*Value{{}}}}, + {name: "invalid with auxiliary value", v: &Value{Op: OpInvalid, Aux: AuxMark}}, + {name: "ordinary dead value", v: &Value{Op: OpConstNil}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := llvmValueIsWriteBarrierTombstone(test.v); got != test.want { + t.Fatalf("llvmValueIsWriteBarrierTombstone(%s, uses=%d) = %t, want %t", test.v.Op, test.v.Uses, got, test.want) + } + }) + } +} diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 00d0114f3e1411..a1f8c7b274129d 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -71,6 +71,8 @@ type llvmABICase struct { goallcArgsMaps [][]int nativeStackMaps []int32 goallcStackMaps []int32 + nativeQueryMaps [][]int + goallcQueryMaps [][]int checkFullMaps bool nativeLocals uint32 goallcLocals uint32 @@ -220,9 +222,10 @@ func runLLVMABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "mixedABI", args: 152, pointerBits: []int{2, 4, 18}, nativeArgsMaps: [][]int{{2, 4, 18}, nil}, - goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, + goallcArgsMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, + goallcStackMaps: []int32{-1, 1, 2, 3, 0}, + goallcQueryMaps: [][]int{{2}, {2}, {2}, {2}, {2, 4, 18}}, }, { name: "liveScalarStackArgument", args: 136, pointerBits: []int{0}, @@ -353,6 +356,14 @@ func runLLVMABIDifferentialTest(t *testing.T, gorootTestDir string) { checkLLVMABIStackMaps(t, "GoALLC", goallcSymbol, tc.goallcArgsMaps, tc.goallcStackMaps) } + if tc.nativeQueryMaps != nil { + checkLLVMABIStackMapQueryBitmaps(t, "native", nativeSymbol, + tc.nativeQueryMaps) + } + if tc.goallcQueryMaps != nil { + checkLLVMABIStackMapQueryBitmaps(t, "GoALLC", goallcSymbol, + tc.goallcQueryMaps) + } }) } @@ -898,6 +909,26 @@ func checkLLVMABIStackMaps(t *testing.T, backend string, symbol llvmABISymbol, w t.Logf("%s ArgsPointerMaps=%v PCDATA=%v", backend, args, pcdata) } +func checkLLVMABIStackMapQueryBitmaps(t *testing.T, backend string, symbol llvmABISymbol, want [][]int) { + t.Helper() + args := llvmABIArgsPointerBitmaps(t, symbol) + got := make([][]int, 0, len(symbol.Function.StackMapQueries)) + for _, query := range symbol.Function.StackMapQueries { + if query.DecodeError != "" { + t.Fatalf("%s stack-map query failed: %s", backend, query.DecodeError) + } + if query.StackMapIndex < 0 || int(query.StackMapIndex) >= len(args) { + t.Fatalf("%s stack-map query index %d is outside %d ArgsPointerMaps", + backend, query.StackMapIndex, len(args)) + } + got = append(got, args[query.StackMapIndex]) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s stack-map query bitmaps=%v, want %v", backend, got, want) + } + t.Logf("%s stack-map query bitmaps=%v", backend, got) +} + func checkLLVMABISourceStackMaps(t *testing.T, backend string, symbol llvmABISymbol, wantLocals uint32, wantArgs, wantMaps [][]int, wantPCData, wantQueries []int32) { t.Helper() if got := symbol.Function.Info.Locals; got != wantLocals { diff --git a/src/cmd/internal/testdir/llvm_test.go b/src/cmd/internal/testdir/llvm_test.go index a67adacf7a25ae..1a23caa70f7c77 100644 --- a/src/cmd/internal/testdir/llvm_test.go +++ b/src/cmd/internal/testdir/llvm_test.go @@ -68,6 +68,8 @@ func runLLVMTests(t *testing.T, common testCommon) { runLLVMAllocaStatepointTest(t, common.gorootTestDir) }) + t.Run("writebarrier-helpers", runLLVMWriteBarrierHelperTest) + t.Run("runtime", func(t *testing.T) { names := sortedLLVMWhitelist(policy.Runtime.Whitelist) for _, name := range names { @@ -87,7 +89,7 @@ func runLLVMTests(t *testing.T, common testCommon) { } }) - t.Run("fail-closed", runLLVMMemoryOpFailClosedTests) + t.Run("writebarrier-ir", runLLVMWriteBarrierIRTests) }) } @@ -298,21 +300,26 @@ func runLLVMCodegenTest(t *testing.T, gorootTestDir, name string) { } } -func runLLVMMemoryOpFailClosedTests(t *testing.T) { +func runLLVMWriteBarrierIRTests(t *testing.T) { tests := []struct { name string source string - want string + want []string }{ { name: "ZeroWithPointers", source: "package p\nfunc zero(dst *[2]*int) { *dst = [2]*int{} }\n", - want: "Zero of pointer-containing type [2]*int requires write-barrier lowering before LLVM", + want: []string{"call goabiinternal void @runtime.wbZero(ptr", "@llvm.memset.inline", `"gc-leaf-function"`}, }, { name: "MoveWithPointers", source: "package p\nfunc move(dst, src *[2]*int) { *dst = *src }\n", - want: "Move of pointer-containing type [2]*int requires write-barrier lowering before LLVM", + want: []string{"call goabiinternal void @runtime.wbMove(ptr", "@llvm.memmove", `"gc-leaf-function"`}, + }, + { + name: "DeletePointer", + source: "package p\nfunc delete(dst **int) { *dst = nil }\n", + want: []string{"@llvm.go.gc.write.barrier"}, }, } for _, tc := range tests { @@ -322,7 +329,7 @@ func runLLVMMemoryOpFailClosedTests(t *testing.T) { if err := os.WriteFile(source, []byte(tc.source), 0o666); err != nil { t.Fatal(err) } - archive := filepath.Join(dir, "fail.a") + archive := filepath.Join(dir, "writebarrier.a") cmd := exec.Command(goTool, "tool", "compile", "-p=p", "-importcfg="+stdlibImportcfgFile(), @@ -333,14 +340,24 @@ func runLLVMMemoryOpFailClosedTests(t *testing.T) { ) cmd.Env = append(os.Environ(), "GOENV=off", "GOFLAGS=") out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("LLVM compilation unexpectedly succeeded") + if err != nil { + t.Fatalf("LLVM compilation failed: %v\n%s", err, out) + } + ir, err := os.ReadFile(archive + ".ll") + if err != nil { + t.Fatal(err) + } + want := append(tc.want, `"go-async-unsafe"`) + for _, want := range want { + if !bytes.Contains(ir, []byte(want)) { + t.Fatalf("LLVM write-barrier IR does not contain %s\n%s", want, ir) + } } - if !bytes.Contains(out, []byte(tc.want)) { - t.Fatalf("LLVM compilation error does not contain %q:\n%s", tc.want, out) + if bytes.Contains(ir, []byte(".rawarg = ptrtoint")) { + t.Fatalf("LLVM write-barrier IR still coerces pointer arguments to raw uintptr\n%s", ir) } - if _, err := os.Stat(archive + ".ll"); !os.IsNotExist(err) { - t.Fatalf("failed LLVM compilation left IR output: %v", err) + if bytes.Contains(ir, []byte("llvm.go.gc.unsafe.point")) { + t.Fatalf("LLVM write-barrier IR still contains unsafe-point marker calls\n%s", ir) } }) } diff --git a/src/cmd/internal/testdir/llvm_writebarrier_test.go b/src/cmd/internal/testdir/llvm_writebarrier_test.go new file mode 100644 index 00000000000000..4b52f5f8aa132e --- /dev/null +++ b/src/cmd/internal/testdir/llvm_writebarrier_test.go @@ -0,0 +1,178 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testdir_test + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "testing" +) + +func runLLVMWriteBarrierHelperTest(t *testing.T) { + t.Helper() + if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { + t.Skip("write-barrier helper IR and RS4GC expectations are qualified on darwin/arm64") + } + + dir := t.TempDir() + source := filepath.Join(dir, "writebarrier.go") + const program = `package p + +type Big [32]*int +type Move8 [8]*int + +func ordinary(*int) + +//go:noinline +func safe(x int) int { return x + 1 } + +//go:noinline +func heapZero(dst *Big, live *int) { + *dst = Big{} + ordinary(live) +} + +//go:noinline +func heapMove(dst, src *Move8) { + *dst = *src +} +` + if err := os.WriteFile(source, []byte(program), 0o666); err != nil { + t.Fatal(err) + } + + archive := filepath.Join(dir, "writebarrier.a") + irPath := archive + ".ll" + runLLVMABICommand(t, nil, goTool, "tool", "compile", + "-l", "-p=p", "-enablellvm", "-llvmironly", "-o", archive, source) + ir, err := os.ReadFile(irPath) + if err != nil { + t.Fatal(err) + } + + for _, check := range []struct { + name string + body []byte + want [][]byte + }{ + { + name: "wbZero", + body: llvmAllocaIRFunction(t, ir, "p.heapZero"), + want: [][]byte{ + []byte("call goabiinternal void @runtime.wbZero(ptr @\"type:p.Big\", ptr %dst)"), + }, + }, + { + name: "wbMove", + body: llvmAllocaIRFunction(t, ir, "p.heapMove"), + want: [][]byte{ + []byte("call goabiinternal void @runtime.wbMove(ptr @\"type:p.Move8\", ptr %dst, ptr %src)"), + }, + }, + } { + t.Run(check.name, func(t *testing.T) { + for _, want := range check.want { + if !bytes.Contains(check.body, want) { + t.Fatalf("input IR does not contain %q\n%s", want, check.body) + } + } + }) + } + for _, want := range [][]byte{ + []byte("declare goabiinternal void @runtime.wbZero(ptr, ptr)"), + []byte("declare goabiinternal void @runtime.wbMove(ptr, ptr, ptr)"), + []byte(`"gc-leaf-function"`), + []byte(`"go-async-unsafe"`), + } { + if !bytes.Contains(ir, want) { + t.Fatalf("input IR does not contain %q", want) + } + } + if bytes.Contains(ir, []byte("llvm.go.gc.unsafe.point")) { + t.Fatalf("input IR still contains unsafe-point marker calls") + } + + opt := llvmToolPath(t, "opt", "GOALLC_OPT") + runLLVMABICommand(t, nil, opt, "-passes=verify", "-disable-output", irPath) + optimized := runLLVMABICommand(t, nil, opt, "-passes=default", "-S", "-o", "-", irPath) + for _, name := range []string{"runtime.wbZero", "runtime.wbMove", "gc-leaf-function", "go-async-unsafe"} { + if !bytes.Contains(optimized, []byte(name)) { + t.Fatalf("optimized IR lost %q", name) + } + } + + llc := llvmToolPath(t, "llc", "GOALLC_LLC") + plugin := llvmABIPassPlugin(t, llc) + rewritten := runLLVMABICommand(t, nil, llc, + "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", + "-filetype=null", "-o", os.DevNull, irPath) + for _, tc := range []struct { + function string + helper string + }{ + {"p.heapZero", "runtime.wbZero"}, + {"p.heapMove", "runtime.wbMove"}, + } { + body := llvmAllocaIRFunction(t, rewritten, tc.function) + if !bytes.Contains(body, []byte("call goabiinternal void @"+tc.helper)) { + t.Fatalf("RS4GC output lost direct leaf call to %s\n%s", tc.helper, body) + } + for _, line := range bytes.Split(body, []byte{'\n'}) { + if bytes.Contains(line, []byte("@llvm.experimental.gc.statepoint")) && bytes.Contains(line, []byte("@"+tc.helper)) { + t.Fatalf("RS4GC statepointized raw helper %s\n%s", tc.helper, body) + } + } + } + zero := llvmAllocaIRFunction(t, rewritten, "p.heapZero") + var ordinaryStatepoints, nilCheckStatepoints int + for _, line := range bytes.Split(zero, []byte{'\n'}) { + if !bytes.Contains(line, []byte("@llvm.experimental.gc.statepoint")) { + continue + } + switch { + case bytes.Contains(line, []byte("@p.ordinary")): + ordinaryStatepoints++ + case bytes.Contains(line, []byte("@runtime.panicmem")): + nilCheckStatepoints++ + default: + t.Fatalf("unexpected heapZero statepoint\n%s", line) + } + } + if ordinaryStatepoints != 1 || nilCheckStatepoints != 2 { + t.Fatalf("heapZero statepoints: ordinary=%d nilcheck=%d, want 1 and 2\n%s", + ordinaryStatepoints, nilCheckStatepoints, zero) + } + runLLVMABICommand(t, rewritten, opt, "-load-pass-plugin="+plugin, + "-passes=verify", "-disable-output", "-") + + object := filepath.Join(dir, "writebarrier.o") + runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, + "-verify-machineinstrs", "-filetype=obj", "-o", object, irPath) + document := readLLVMABIObject(t, object) + for _, tc := range []struct { + name string + want int32 + }{ + {"p.heapZero", -2}, + {"p.heapMove", -2}, + {"p.safe", -2}, + } { + symbol := findLLVMABISymbol(t, document, tc.name) + var values []int32 + for _, pcdata := range symbol.Function.PCData { + if pcdata.Kind != "unsafe_point" { + continue + } + for _, pcRange := range pcdata.Ranges { + values = append(values, pcRange.Value) + } + } + if len(values) != 1 || values[0] != tc.want { + t.Fatalf("%s PCDATA_UnsafePoint values=%v, want whole-function value %d", tc.name, values, tc.want) + } + } +} diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 9c5d7c3572035f..c0901aff884ff7 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -669,6 +669,21 @@ if(BUILD_TESTING) "\"gc-live\"\\(ptr %p\\.relocated2" ) + add_test( + NAME GoALLCStatepoints.OutOfLayoutCallResultRewrite + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/statepoint.ll" + ) + set_tests_properties(GoALLCStatepoints.OutOfLayoutCallResultRewrite PROPERTIES + PASS_REGULAR_EXPRESSION + "define goabiinternal ptr @out_of_layout_call_result" + ) + add_test( NAME GoALLCStatepoints.LoopRelocationRewrite COMMAND diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index fad824330521e4..5572df4e6b773a 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -80,6 +80,7 @@ struct SafepointRecord { uint64_t ID; ValueSet Live; CallInst *Statepoint = nullptr; + CallInst *Result = nullptr; SmallVector Relocates; }; @@ -805,13 +806,23 @@ Error rewriteCall(SafepointRecord &Record, Record.Relocates.push_back(Relocate); } - if (Result) - Call->replaceAllUsesWith(Result); - Call->eraseFromParent(); - Record.Call = nullptr; + Record.Result = Result; return Error::success(); } +void eraseOriginalCalls(ArrayRef Records) { + // Keep every original call and its result alive until all precomputed + // liveness sets have been consumed. LLVM block layout need not follow CFG + // dominance, so a record encountered earlier can legitimately contain the + // result of a call encountered later. Replacing and erasing each call inside + // rewriteCall would leave that record with a dangling Value pointer. + for (const SafepointRecord &Record : Records) { + if (Record.Result) + Record.Call->replaceAllUsesWith(Record.Result); + Record.Call->eraseFromParent(); + } +} + void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { // Re-read gc-live after every ordinary call has been replaced. A @@ -944,6 +955,7 @@ Error rewriteFunction(Function &F) { for (SafepointRecord &Record : llvm::reverse(Records)) if (Error Err = rewriteCall(Record, PointerAllocas)) return Err; + eraseOriginalCalls(Records); repairRelocationSSA(F, DT, Records); return Error::success(); } diff --git a/src/cmd/llvmplugin/testdata/statepoint.ll b/src/cmd/llvmplugin/testdata/statepoint.ll index a6db84ff052891..9178dd6c4f21a7 100644 --- a/src/cmd/llvmplugin/testdata/statepoint.ll +++ b/src/cmd/llvmplugin/testdata/statepoint.ll @@ -59,5 +59,21 @@ entry: ret ptr %pointer } +; Function block layout is intentionally different from CFG dominance. The +; safepoint in %use is visited before the pointer-producing call in %define, +; but its liveness set contains that call result. +define goabiinternal ptr @out_of_layout_call_result() #1 gc "goallc" { +entry: + br label %define + +use: + call goabiinternal void @callee() + ret ptr %pointer + +define: + %pointer = call goabiinternal ptr @make_pointer() + br label %use +} + attributes #0 = { "gc-leaf-function" } attributes #1 = { "go-stack-growth-statepoint" } diff --git a/test/codegen/llvm_memory_order.go b/test/codegen/llvm_memory_order.go new file mode 100644 index 00000000000000..1c5c8a4553ac7d --- /dev/null +++ b/test/codegen/llvm_memory_order.go @@ -0,0 +1,28 @@ +// asmcheck + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +var llvmMemoryOrderRoot *int + +//go:noinline +func llvmMemoryOrderStore(value *int) { + llvmMemoryOrderRoot = value +} + +// LLVM-LABEL: define goabiinternal ptr @codegen.llvmMemoryOrderLoad( +// LLVM: call goabiinternal void @codegen.llvmMemoryOrderStore( +// LLVM: load ptr, ptr @codegen.llvmMemoryOrderRoot +// LLVM: ret ptr +// LLVM-OPT-LABEL: define goabiinternal ptr @codegen.llvmMemoryOrderLoad( +// LLVM-OPT-SAME: ptr{{[^%]*}}%[[VALUE:[a-zA-Z0-9._]+]]) +// LLVM-OPT: call ptr @llvm.go.gc.write.barrier(i32 2) +// LLVM-OPT: store ptr %[[VALUE]], ptr @codegen.llvmMemoryOrderRoot +// LLVM-OPT: ret ptr %[[VALUE]] +func llvmMemoryOrderLoad(value *int) *int { + llvmMemoryOrderStore(value) + return llvmMemoryOrderRoot +} diff --git a/test/codegen/llvm_writebarrier.go b/test/codegen/llvm_writebarrier.go new file mode 100644 index 00000000000000..aed1ed8cd6c352 --- /dev/null +++ b/test/codegen/llvm_writebarrier.go @@ -0,0 +1,54 @@ +// asmcheck + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +// LLVM-LABEL: define goabiinternal ptr @codegen.llvmWriteBarrierStore( +// LLVM-SAME: ptr{{[^%]*}}%dst, ptr{{[^%]*}}%[[VALUE:[a-zA-Z0-9._]+]]) +// LLVM-SAME: #[[WBATTR:[0-9]+]] gc "goallc" { +// LLVM: load i32, ptr @runtime.writeBarrier +// LLVM: br i1 +// LLVM: call ptr @llvm.go.gc.write.barrier(i32 2) +// LLVM: store ptr %[[VALUE]] +// LLVM: store ptr %[[VALUE]], ptr %dst +// LLVM-NOT: call ptr @llvm.go.gc.write.barrier +// LLVM: ret ptr %[[VALUE]] +// LLVM-OPT-LABEL: define goabiinternal ptr @codegen.llvmWriteBarrierStore( +// LLVM-OPT-SAME: ptr{{[^%]*}}%dst, ptr{{[^%]*}}%[[OPT_VALUE:[a-zA-Z0-9._]+]]) +// LLVM-OPT: call ptr @llvm.go.gc.write.barrier(i32 2) +// LLVM-OPT-NOT: call ptr @llvm.go.gc.write.barrier +// LLVM-OPT: ret ptr %[[OPT_VALUE]] +func llvmWriteBarrierStore(dst **int, value *int) *int { + local := value + *dst = local + return local +} + +type llvmWriteBarrierPair struct { + left *int + right *int +} + +// The compiler drains its function queue in reverse source order, so keep the +// checks in emitted IR order rather than beside the two source declarations. +// LLVM-LABEL: define goabiinternal void @codegen.llvmWriteBarrierZero( +// LLVM-SAME: #[[WBATTR]] gc "goallc" { +// LLVM: call ptr @llvm.go.gc.write.barrier(i32 4) +// LLVM: store ptr null +// LLVM: store ptr null +// LLVM-LABEL: define goabiinternal void @codegen.llvmWriteBarrierMove( +// LLVM-SAME: #[[WBATTR]] gc "goallc" { +// LLVM: call ptr @llvm.go.gc.write.barrier(i32 4) +// LLVM: store ptr +// LLVM: store ptr +// LLVM: attributes #[[WBATTR]] = { {{.*}}"go-async-unsafe"{{.*}} } +func llvmWriteBarrierMove(dst, src *llvmWriteBarrierPair) { + *dst = *src +} + +func llvmWriteBarrierZero(dst *llvmWriteBarrierPair) { + *dst = llvmWriteBarrierPair{} +} diff --git a/test/llvm_tests.json b/test/llvm_tests.json index 4e99b30861ca7c..3622fcf7be00ab 100644 --- a/test/llvm_tests.json +++ b/test/llvm_tests.json @@ -34,8 +34,10 @@ "codegen/llvm_data_sections.go": "preserve data, bss, noptrdata, and noptrbss identities in LLVM IR", "codegen/llvm_dereference.go": "address-taken named stack results loaded for return", "codegen/llvm_memops.go": "pointer-free zero and overlap-safe move intrinsics, runtime memequal, and native-int slice masks", + "codegen/llvm_memory_order.go": "preserve Go SSA Memory-token order when LLVM emission precedes native scheduling", "codegen/llvm_newproc.go": "pointer-typed funcval argument for the runtime.newproc raw ABI call", "codegen/llvm_nilcheck.go": "explicit branch to recoverable runtime.panicmem with stable continuation and optimized IR checks", + "codegen/llvm_writebarrier.go": "pre-expand Go write-barrier control flow, whole-function asynchronous-unsafe attribute, and target intrinsic for a heap pointer store", "codegen/memops.go": "typed indexed loads, stores, comparisons, bounds checks, and scalar memory operations", "codegen/memops_bigoffset.go": "typed loads and stores at offsets beyond signed 32-bit byte displacement", "codegen/multiply.go": "integer multiplication across positive, negative, and zero constants", @@ -78,6 +80,7 @@ "llvm_type_descriptor_kinds.go": "all runtime type descriptor kinds, GoObj link, and execution", "llvm_type_descriptor_map.go": "typed map descriptor layout, R_KEEP reachability edge, GoObj link, and execution", "llvm_type_descriptor_methods.go": "uncommon concrete methods, method wrappers, direct calls, GoObj link, and execution", + "llvm_writebarrier_gc.go": "heap/global pointer stores plus aggregate move and zero under concurrent GC and stack growth", "newexpr.go": "new(expr) for scalar, variable, and pointer-free aggregate values", "printbig.go": "signed and unsigned 64-bit boundary constants through runtime print", "sizeof.go": "unsafe Sizeof, Alignof, and deeply embedded Offsetof constants", diff --git a/test/llvm_writebarrier_gc.go b/test/llvm_writebarrier_gc.go new file mode 100644 index 00000000000000..dc9e4ccf6a9046 --- /dev/null +++ b/test/llvm_writebarrier_gc.go @@ -0,0 +1,102 @@ +// run + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "runtime" + +type wbPayload struct { + tag int + pad [32]uintptr +} + +type wbNode struct { + payload *wbPayload + next *wbNode +} + +type wbPair struct { + left *wbPayload + right *wbNode +} + +var wbRoot *wbNode + +//go:noinline +func wbStore(dst **wbNode, value *wbNode) { + *dst = value +} + +//go:noinline +func wbMove(dst *wbPair, value wbPair) { + *dst = value +} + +//go:noinline +func wbZero(dst *wbPair) { + *dst = wbPair{} +} + +//go:noinline +func wbGrow(depth int, value *wbNode) *wbNode { + var pad [4]uintptr + pad[0] = uintptr(depth) + if depth == 0 { + runtime.GC() + runtime.KeepAlive(pad) + return value + } + result := wbGrow(depth-1, value) + runtime.KeepAlive(pad) + return result +} + +func main() { + runtime.GOMAXPROCS(2) + started := make(chan struct{}) + gcDone := make(chan struct{}) + go func() { + close(started) + for i := 0; i < 64; i++ { + garbage := make([]*[256]byte, 64) + for j := 0; j < 64; j++ { + garbage[j] = new([256]byte) + } + runtime.GC() + runtime.KeepAlive(garbage) + } + close(gcDone) + }() + <-started + + pair := new(wbPair) + for i := 1; i <= 8192; i++ { + payload := &wbPayload{tag: i} + node := &wbNode{payload: payload, next: wbRoot} + wbStore(&wbRoot, node) + wbMove(pair, wbPair{left: payload, right: node}) + if wbRoot == nil || wbRoot.payload == nil || wbRoot.payload.tag != i { + panic("write-barrier store lost its pointer") + } + if pair.left == nil || pair.left.tag != i || pair.right != wbRoot { + panic("write-barrier move lost its pointers") + } + wbZero(pair) + if pair.left != nil || pair.right != nil { + panic("write-barrier zero retained pointers") + } + if i%257 == 0 { + if got := wbGrow(256, wbRoot); got == nil || got.payload.tag != i { + panic("stack growth lost a write-barrier result") + } + } + } + <-gcDone + runtime.GC() + if wbRoot == nil || wbRoot.payload == nil || wbRoot.payload.tag != 8192 { + panic("final write-barrier pointer is invalid") + } +}