From 4ded0e60fa1cbf6255626f6933a2fad53a65dec0 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 5 Aug 2026 22:11:46 +0800 Subject: [PATCH] cmd/compile: keep defer results live at safepoints --- src/cmd/compile/internal/ssa/ssa2llvm.go | 18 ++++++++ src/cmd/llvmplugin/CMakeLists.txt | 2 +- src/cmd/llvmplugin/GoALLCStatepoints.cpp | 33 +++++++++++--- .../llvmplugin/testdata/check-defer-edge.py | 44 ++++++++++++++++++- src/cmd/llvmplugin/testdata/defer-edge.ll | 22 ++++++++++ test/codegen/llvm_defer.go | 26 +++++++++-- test/llvm_defer.go | 30 +++++++++++++ test/llvm_tests.json | 2 + 8 files changed, 165 insertions(+), 12 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index a56c978a9cf200..b9d883a6ce3e6f 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -22,6 +22,7 @@ type LLVMFuncContext struct { ResultSlots map[ID]llvm.Value ItabMethods map[ID]bool ClosureCodeLoads map[ID]bool + DeferRecovery map[ID]bool F *Func LF llvm.Value ClosureContext llvm.Value @@ -63,6 +64,7 @@ const goAsyncUnsafeAttr = "go-async-unsafe" const goWriteBarrierIntrinsic = "llvm.go.gc.write.barrier" const goDeferEdgeIntrinsic = "llvm.go.defer.edge" const goSourceAddressTakenMD = "goallc.source_addrtaken" +const goDeferResultMD = "goallc.defer_result" const goObjMarkerRelocMD = "goobj.marker_reloc" const goObjSymbolNameMD = "goobj.symbol.name" const llvmFramePointerAttr = "frame-pointer" @@ -2028,6 +2030,9 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { v.Fatalf("%s address has non-pointer LLVM type", v.Op) } lVal = lfc.b.CreateLoad(typ, addr, v.String()) + if v.Block != nil && lfc.DeferRecovery[v.Block.ID] { + lVal.SetVolatile(true) + } if v.Op == OpDereference { align := v.Type.Alignment() if align <= 0 || align&(align-1) != 0 { @@ -2257,6 +2262,7 @@ func LLVMCompile(f *Func) { ResultSlots: map[ID]llvm.Value{}, ItabMethods: map[ID]bool{}, ClosureCodeLoads: map[ID]bool{}, + DeferRecovery: map[ID]bool{}, F: f, b: GlobalCtxt.NewBuilder(), ReturnType: sig.ReturnType, @@ -2307,6 +2313,11 @@ func LLVMCompile(f *Func) { } FCtxt.BBs[BB.ID] = GlobalCtxt.AddBasicBlock(FCtxt.LF, BB.String()) } + for _, BB := range f.Blocks { + if BB.Kind == BlockDefer && len(BB.Succs) == 2 { + FCtxt.DeferRecovery[BB.Succs[1].Block().ID] = true + } + } for _, BB := range f.Blocks { for _, v := range BB.Values { if (v.Op == OpInterCall || v.Op == OpInterLECall || v.Op == OpTailLECallInter) && len(v.Args) != 0 { @@ -2381,6 +2392,13 @@ func LLVMCompile(f *Func) { slot.SetMetadata(GlobalCtxt.MDKindID(goSourceAddressTakenMD), GlobalCtxt.MDNode([]llvm.Metadata{ sourceAddressTaken.ConstantAsMetadata(), })) + if frontendFunc != nil && frontendFunc.HasDefer() && name.Class == ir.PPARAMOUT { + // Panic recovery resumes at a deferreturn path that is not a + // successor of the suspended call in LLVM's CFG. Keep pointer + // results in every caller stack map; the recovery block's + // volatile reload keeps this alloca as its stable memory home. + slot.SetMetadata(GlobalCtxt.MDKindID(goDeferResultMD), GlobalCtxt.MDNode(nil)) + } } FCtxt.Locals[key] = llvmStackSlot{Value: slot, Type: name.Type()} if name.Class == ir.PPARAM { diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 67ecb66900c365..b1886b5cd726c0 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -96,7 +96,7 @@ if(BUILD_TESTING) PASS_REGULAR_EXPRESSION "callbr void @llvm.go.defer.edge" FAIL_REGULAR_EXPRESSION - "GoALLC statepoints do not yet support invoke or callbr;call goabiinternal void @runtime.deferproc" + "GoALLC statepoints do not yet support invoke or callbr;call goabiinternal void @runtime.deferproc;goallc.defer_result" ) add_test( diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 2570a21e743cc9..e339aba23e81bd 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -40,6 +40,7 @@ constexpr StringLiteral GoALLCGCName = "goallc"; constexpr StringLiteral GCLeafAttr = "gc-leaf-function"; constexpr StringLiteral GoResultsTupleAttr = "go_results_tuple"; constexpr StringLiteral GoSourceAddressTakenMD = "goallc.source_addrtaken"; +constexpr StringLiteral GoDeferResultMD = "goallc.defer_result"; constexpr StringLiteral GoObjMarkerRelocMD = "goobj.marker_reloc"; constexpr StringLiteral StackColoringNoMergeMD = "llvm.stackcoloring.no_merge"; @@ -101,6 +102,7 @@ struct PointerAllocaLeaf { struct PointerAllocaRecord { AllocaInst *Alloca; bool NeedsStackObject; + bool DeferResult; uint64_t ByteSize; uint64_t Alignment; uint64_t BitCount; @@ -665,6 +667,16 @@ Expected sourceMarkedAddressTaken(AllocaInst &Alloca) { return CI->isOne(); } +Expected deferResultAlloca(AllocaInst &Alloca) { + MDNode *MD = Alloca.getMetadata(GoDeferResultMD); + if (!MD) + return false; + if (MD->getNumOperands() != 0) + return createStringError(std::errc::invalid_argument, + "GoALLC defer-result metadata must be empty"); + return true; +} + // Return true when the optimized IR can make the address observable outside // compiler-controlled direct accesses. This is deliberately a structural // post-optimization decision: the frontend Addrtaken bit is provenance, not a @@ -870,12 +882,11 @@ Error computePointerAllocaActivity( void protectStackObjectsFromColoring( MutableArrayRef PointerAllocas) { for (PointerAllocaRecord &Record : PointerAllocas) { - if (!Record.NeedsStackObject) + if (!Record.NeedsStackObject && !Record.DeferResult) continue; - // A Go StackObject has function-wide identity and layout metadata. It - // cannot share storage with another lifetime-disjoint alloca because both - // object records must remain independently addressable at every - // statepoint, including calls outside this object's active lifetime. + // A Go StackObject has function-wide identity and layout metadata. A defer + // result likewise remains a root at every statepoint. Neither can share + // storage with another lifetime-disjoint alloca. Record.Alloca->setMetadata( StackColoringNoMergeMD, MDNode::get(Record.Alloca->getContext(), ArrayRef())); @@ -1000,12 +1011,16 @@ Error collectPointerAllocas( Expected SourceAddressTaken = sourceMarkedAddressTaken(*Alloca); if (!SourceAddressTaken) return SourceAddressTaken.takeError(); + Expected DeferResult = deferResultAlloca(*Alloca); + if (!DeferResult) + return DeferResult.takeError(); // Consume the marker after the optimized use graph has been classified. // In particular, a source Addrtaken alloca can be downgraded when all // observable address uses disappeared during optimization. Alloca->setMetadata(GoSourceAddressTakenMD, nullptr); + Alloca->setMetadata(GoDeferResultMD, nullptr); bool NeedsStackObject = allocaNeedsStackObject(*Alloca); - PointerAllocas.push_back({Alloca, NeedsStackObject, ByteSize, + PointerAllocas.push_back({Alloca, NeedsStackObject, *DeferResult, ByteSize, Alloca->getAlign().value(), BitCount, std::move(BitmapWords), std::move(Leaves)}); } @@ -1381,7 +1396,11 @@ Error rewriteFunction(Function &F) { for (SafepointRecord &Record : llvm::reverse(Records)) { SmallVector AllocaRecords; for (const PointerAllocaRecord &Alloca : PointerAllocas) { - bool IsActive = Record.Live.contains(Alloca.Alloca) || + // A recovered panic resumes outside LLVM's explicit CFG. The frontend + // marks named result homes whose contents must therefore remain visible + // to Go's stack scanner at every possible suspension call. + bool IsActive = Alloca.DeferResult || + Record.Live.contains(Alloca.Alloca) || isPointerAllocaActiveAt(Alloca, *Record.Call); if (IsActive) Record.Live.insert(Alloca.Alloca); diff --git a/src/cmd/llvmplugin/testdata/check-defer-edge.py b/src/cmd/llvmplugin/testdata/check-defer-edge.py index 2fb6c5444c2f22..395512a35abb74 100644 --- a/src/cmd/llvmplugin/testdata/check-defer-edge.py +++ b/src/cmd/llvmplugin/testdata/check-defer-edge.py @@ -7,6 +7,7 @@ FUNCTION = "defer_edge" +RESULT = "defer_result" WRAPPER = "defer_wrapper" @@ -49,6 +50,15 @@ def main(): if metadata is None: fail(f"{FUNCTION} has no function metadata") + result = only( + obj["symbols"], + lambda item: item["name"] == RESULT, + f"{RESULT} symbols", + ) + result_metadata = result.get("function") + if result_metadata is None: + fail(f"{RESULT} has no function metadata") + wrapper = only( obj["symbols"], lambda item: item["name"] == WRAPPER, @@ -85,9 +95,41 @@ def target_name(relocation): if query["stack_map_index"] < 0: fail(f"deferreturn has no valid GC stack map: {query}") + funcdata = { + item["kind"]: item for item in result_metadata["funcdata"] + } + if "stack_objects" in funcdata: + fail(f"{RESULT} emitted StackObjects instead of alloca deopt roots") + locals_maps = funcdata["locals_pointer_maps"]["stack_map"]["bitmaps"] + + def bitmap_has_pointer(index): + return bool(locals_maps[index].get("set_bits")) + + result_calls = { + target_name(relocation): relocation + for relocation in result["relocations"] + if relocation["type"] == "R_CALL" + } + for name in ( + "runtime.deferproc", + "runtime.panicmem", + "runtime.deferreturn", + ): + relocation = result_calls.get(name) + if relocation is None: + fail(f"{RESULT} has no direct {name} relocation") + call_query = only( + result_metadata["stack_map_queries"], + lambda item: item["call_offset"] == relocation["offset"], + f"{name} stack-map queries", + ) + if not bitmap_has_pointer(call_query["stack_map_index"]): + fail(f"{name} does not keep the defer result in locals ptrmap") + print( f"{FUNCTION}: runtime.deferreturn R_CALL at {deferreturn['offset']}, " - f"stack map {query['stack_map_index']}; {WRAPPER}: FuncID 23, flags 0" + f"stack map {query['stack_map_index']}; {RESULT}: result live at all " + f"calls with no StackObjects; {WRAPPER}: FuncID 23, flags 0" ) diff --git a/src/cmd/llvmplugin/testdata/defer-edge.ll b/src/cmd/llvmplugin/testdata/defer-edge.ll index 2b2ab5e30d09ac..bdc665a306419e 100644 --- a/src/cmd/llvmplugin/testdata/defer-edge.ll +++ b/src/cmd/llvmplugin/testdata/defer-edge.ll @@ -2,7 +2,9 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @runtime.deferproc() declare goabiinternal void @runtime.deferreturn() +declare goabiinternal void @runtime.panicmem() declare void @llvm.go.defer.edge() +declare void @llvm.lifetime.start.p0(ptr captures(none)) define goabiinternal void @defer_edge() #0 gc "goallc" { entry: @@ -17,6 +19,25 @@ recover: ret void } +define goabiinternal ptr @defer_result(ptr %pointer) #0 gc "goallc" { +entry: + %result = alloca ptr, align 8, !goallc.defer_result !1 + call void @llvm.lifetime.start.p0(ptr %result) + store ptr null, ptr %result, align 8 + call goabiinternal void @runtime.deferproc() + callbr void @llvm.go.defer.edge() to label %panic [label %recover] + +panic: + store ptr %pointer, ptr %result, align 8 + call goabiinternal void @runtime.panicmem() + unreachable + +recover: + call goabiinternal void @runtime.deferreturn() + %value = load volatile ptr, ptr %result, align 8 + ret ptr %value +} + define goabiinternal void @defer_wrapper() !goobj.func.info !0 { entry: ret void @@ -25,3 +46,4 @@ entry: attributes #0 = { "go-stack-growth-statepoint" } !0 = !{i8 23, i8 0} +!1 = !{} diff --git a/test/codegen/llvm_defer.go b/test/codegen/llvm_defer.go index 6a11cbcc4132f9..dd273cd6b3d647 100644 --- a/test/codegen/llvm_defer.go +++ b/test/codegen/llvm_defer.go @@ -8,21 +8,35 @@ package codegen var llvmDeferSink int +// LLVM-LABEL: define goabiinternal ptr @codegen.llvmDeferPointerResult(ptr{{.*}} %pointer) +// LLVM: [[RESULT:%.*]] = alloca ptr, {{.*}}!goallc.defer_result +// LLVM: callbr void @llvm.go.defer.edge() +// LLVM-NEXT: to label %{{.*}} [label %[[RECOVER:[A-Za-z0-9_.]+]]] +// LLVM: [[RECOVER]]: +// LLVM-NEXT: call goabiinternal void @runtime.deferreturn() +// LLVM-NEXT: {{.*}} = load volatile ptr, ptr [[RESULT]] +// LLVM-OPT-LABEL: define goabiinternal ptr @codegen.llvmDeferPointerResult(ptr{{.*}} %pointer) +// LLVM-OPT: [[RESULT_OPT:%.*]] = alloca ptr, {{.*}}!goallc.defer_result +// LLVM-OPT: callbr void @llvm.go.defer.edge() +// LLVM-OPT-NEXT: to label %{{.*}} [label %[[RECOVER_OPT:[A-Za-z0-9_.]+]]] +// LLVM-OPT: [[RECOVER_OPT]]: +// LLVM-OPT-NEXT: call goabiinternal void @runtime.deferreturn() +// LLVM-OPT-NEXT: {{.*}} = load volatile ptr, ptr [[RESULT_OPT]] + // LLVM-LABEL: define goabiinternal i64 @codegen.llvmDeferStack(i64 %value) // LLVM: call goabiinternal void @runtime.deferprocStack // LLVM: callbr void @llvm.go.defer.edge() // LLVM-NEXT: to label %[[STACK_NORMAL:.*]] [label %[[STACK_RECOVER:.*]]] // LLVM: [[STACK_RECOVER]]: // LLVM: call goabiinternal void @runtime.deferreturn() -// LLVM-NOT: load volatile +// LLVM: load volatile i64 // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmDeferStack(i64 %value) // LLVM-OPT: call goabiinternal void @runtime.deferprocStack // LLVM-OPT: callbr void @llvm.go.defer.edge() // LLVM-OPT-NEXT: to label %{{.*}} [label %[[STACK_OPT_RECOVER:.*]]] // LLVM-OPT: [[STACK_OPT_RECOVER]]: // LLVM-OPT: call goabiinternal void @runtime.deferreturn() -// LLVM-OPT: load i64 -// LLVM-OPT-NOT: load volatile +// LLVM-OPT: load volatile i64 // LLVM-LABEL: define goabiinternal void @codegen.llvmDeferHeap(i64 %count) // LLVM: [[HEAP_NORMAL_RETURN:[A-Za-z0-9_.]+]]: @@ -62,3 +76,9 @@ func llvmDeferStack(value int) (result int) { result = 7 return } + +func llvmDeferPointerResult(pointer *int) (result *int) { + defer func() {}() + result = pointer + panic(llvmDeferSink) +} diff --git a/test/llvm_defer.go b/test/llvm_defer.go index ce3d783989f8d4..e9f52e4172c662 100644 --- a/test/llvm_defer.go +++ b/test/llvm_defer.go @@ -9,6 +9,11 @@ package main import "runtime" var deferTrace int +var deferResultFinalized = make(chan struct{}, 1) + +type deferResultObject struct { + value int +} //go:noinline func normalDefers() (result int) { @@ -73,6 +78,26 @@ func pointerDefer() (result int) { return } +//go:noinline +func namedPointerResultSurvivesPanic() (result *deferResultObject) { + result = &deferResultObject{value: 91} + runtime.SetFinalizer(result, func(*deferResultObject) { + deferResultFinalized <- struct{}{} + }) + defer func() { + runtime.GC() + runtime.GC() + runtime.GC() + select { + case <-deferResultFinalized: + panic("named result was finalized during panic unwinding") + default: + } + recover() + }() + panic("named result liveness") +} + func main() { if got := normalDefers(); got != 17 || deferTrace != 21 { panic("normal defer order or named result is incorrect") @@ -89,4 +114,9 @@ func main() { if pointerDefer() != 73 { panic("defer lost a captured pointer across GC") } + result := namedPointerResultSurvivesPanic() + if result == nil || result.value != 91 { + panic("defer lost a named pointer result during panic unwinding") + } + runtime.KeepAlive(result) } diff --git a/test/llvm_tests.json b/test/llvm_tests.json index ed94a82af126df..7d70aff124c4d5 100644 --- a/test/llvm_tests.json +++ b/test/llvm_tests.json @@ -1084,6 +1084,7 @@ "fixedbugs/issue26094.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "fixedbugs/issue27289.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "fixedbugs/issue27518a.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", + "fixedbugs/issue27518b.go": "defer recovery keeps named pointer results live across panic-time GC", "fixedbugs/issue28748.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "fixedbugs/issue28797.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "fixedbugs/issue29190.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", @@ -1220,6 +1221,7 @@ "fixedbugs/issue26094.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "fixedbugs/issue27289.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "fixedbugs/issue27518a.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "fixedbugs/issue27518b.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "fixedbugs/issue28748.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "fixedbugs/issue28797.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "fixedbugs/issue29190.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",