cmd/compile: keep defer results live at safepoints

src/cmd/compile/internal/ssa/ssa2llvm.go

@@ -22,6 +22,7 @@ type LLVMFuncContext struct {
 	ResultSlots      map[ID]llvm.Value
 	ItabMethods      map[ID]bool
 	ClosureCodeLoads map[ID]bool
+	DeferRecovery    map[ID]bool
 	F                *Func
 	LF               llvm.Value
 	ClosureContext   llvm.Value
@@ -63,6 +64,7 @@ const goAsyncUnsafeAttr = "go-async-unsafe"
 const goWriteBarrierIntrinsic = "llvm.go.gc.write.barrier"
 const goDeferEdgeIntrinsic = "llvm.go.defer.edge"
 const goSourceAddressTakenMD = "goallc.source_addrtaken"
+const goDeferResultMD = "goallc.defer_result"
 const goObjMarkerRelocMD = "goobj.marker_reloc"
 const goObjSymbolNameMD = "goobj.symbol.name"
 const llvmFramePointerAttr = "frame-pointer"
@@ -2028,6 +2030,9 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value {
 			v.Fatalf("%s address has non-pointer LLVM type", v.Op)
 		}
 		lVal = lfc.b.CreateLoad(typ, addr, v.String())
+		if v.Block != nil && lfc.DeferRecovery[v.Block.ID] {
+			lVal.SetVolatile(true)
+		}
 		if v.Op == OpDereference {
 			align := v.Type.Alignment()
 			if align <= 0 || align&(align-1) != 0 {
@@ -2257,6 +2262,7 @@ func LLVMCompile(f *Func) {
 		ResultSlots:      map[ID]llvm.Value{},
 		ItabMethods:      map[ID]bool{},
 		ClosureCodeLoads: map[ID]bool{},
+		DeferRecovery:    map[ID]bool{},
 		F:                f,
 		b:                GlobalCtxt.NewBuilder(),
 		ReturnType:       sig.ReturnType,
@@ -2307,6 +2313,11 @@ func LLVMCompile(f *Func) {
 		}
 		FCtxt.BBs[BB.ID] = GlobalCtxt.AddBasicBlock(FCtxt.LF, BB.String())
 	}
+	for _, BB := range f.Blocks {
+		if BB.Kind == BlockDefer && len(BB.Succs) == 2 {
+			FCtxt.DeferRecovery[BB.Succs[1].Block().ID] = true
+		}
+	}
 	for _, BB := range f.Blocks {
 		for _, v := range BB.Values {
 			if (v.Op == OpInterCall || v.Op == OpInterLECall || v.Op == OpTailLECallInter) && len(v.Args) != 0 {
@@ -2381,6 +2392,13 @@ func LLVMCompile(f *Func) {
 				slot.SetMetadata(GlobalCtxt.MDKindID(goSourceAddressTakenMD), GlobalCtxt.MDNode([]llvm.Metadata{
 					sourceAddressTaken.ConstantAsMetadata(),
 				}))
+				if frontendFunc != nil && frontendFunc.HasDefer() && name.Class == ir.PPARAMOUT {
+					// Panic recovery resumes at a deferreturn path that is not a
+					// successor of the suspended call in LLVM's CFG. Keep pointer
+					// results in every caller stack map; the recovery block's
+					// volatile reload keeps this alloca as its stable memory home.
+					slot.SetMetadata(GlobalCtxt.MDKindID(goDeferResultMD), GlobalCtxt.MDNode(nil))
+				}
 			}
 			FCtxt.Locals[key] = llvmStackSlot{Value: slot, Type: name.Type()}
 			if name.Class == ir.PPARAM {

src/cmd/llvmplugin/CMakeLists.txt

@@ -96,7 +96,7 @@ if(BUILD_TESTING)
     PASS_REGULAR_EXPRESSION
       "callbr void @llvm.go.defer.edge"
     FAIL_REGULAR_EXPRESSION
-      "GoALLC statepoints do not yet support invoke or callbr;call goabiinternal void @runtime.deferproc"
+      "GoALLC statepoints do not yet support invoke or callbr;call goabiinternal void @runtime.deferproc;goallc.defer_result"
   )
 
   add_test(

src/cmd/llvmplugin/GoALLCStatepoints.cpp

@@ -40,6 +40,7 @@ constexpr StringLiteral GoALLCGCName = "goallc";
 constexpr StringLiteral GCLeafAttr = "gc-leaf-function";
 constexpr StringLiteral GoResultsTupleAttr = "go_results_tuple";
 constexpr StringLiteral GoSourceAddressTakenMD = "goallc.source_addrtaken";
+constexpr StringLiteral GoDeferResultMD = "goallc.defer_result";
 constexpr StringLiteral GoObjMarkerRelocMD = "goobj.marker_reloc";
 constexpr StringLiteral StackColoringNoMergeMD = "llvm.stackcoloring.no_merge";
 
@@ -101,6 +102,7 @@ struct PointerAllocaLeaf {
 struct PointerAllocaRecord {
   AllocaInst *Alloca;
   bool NeedsStackObject;
+  bool DeferResult;
   uint64_t ByteSize;
   uint64_t Alignment;
   uint64_t BitCount;
@@ -665,6 +667,16 @@ Expected<bool> sourceMarkedAddressTaken(AllocaInst &Alloca) {
   return CI->isOne();
 }
 
+Expected<bool> deferResultAlloca(AllocaInst &Alloca) {
+  MDNode *MD = Alloca.getMetadata(GoDeferResultMD);
+  if (!MD)
+    return false;
+  if (MD->getNumOperands() != 0)
+    return createStringError(std::errc::invalid_argument,
+                             "GoALLC defer-result metadata must be empty");
+  return true;
+}
+
 // Return true when the optimized IR can make the address observable outside
 // compiler-controlled direct accesses. This is deliberately a structural
 // post-optimization decision: the frontend Addrtaken bit is provenance, not a
@@ -870,12 +882,11 @@ Error computePointerAllocaActivity(
 void protectStackObjectsFromColoring(
     MutableArrayRef<PointerAllocaRecord> PointerAllocas) {
   for (PointerAllocaRecord &Record : PointerAllocas) {
-    if (!Record.NeedsStackObject)
+    if (!Record.NeedsStackObject && !Record.DeferResult)
       continue;
-    // A Go StackObject has function-wide identity and layout metadata.  It
-    // cannot share storage with another lifetime-disjoint alloca because both
-    // object records must remain independently addressable at every
-    // statepoint, including calls outside this object's active lifetime.
+    // A Go StackObject has function-wide identity and layout metadata. A defer
+    // result likewise remains a root at every statepoint. Neither can share
+    // storage with another lifetime-disjoint alloca.
     Record.Alloca->setMetadata(
         StackColoringNoMergeMD,
         MDNode::get(Record.Alloca->getContext(), ArrayRef<Metadata *>()));
@@ -1000,12 +1011,16 @@ Error collectPointerAllocas(
     Expected<bool> SourceAddressTaken = sourceMarkedAddressTaken(*Alloca);
     if (!SourceAddressTaken)
       return SourceAddressTaken.takeError();
+    Expected<bool> DeferResult = deferResultAlloca(*Alloca);
+    if (!DeferResult)
+      return DeferResult.takeError();
     // Consume the marker after the optimized use graph has been classified.
     // In particular, a source Addrtaken alloca can be downgraded when all
     // observable address uses disappeared during optimization.
     Alloca->setMetadata(GoSourceAddressTakenMD, nullptr);
+    Alloca->setMetadata(GoDeferResultMD, nullptr);
     bool NeedsStackObject = allocaNeedsStackObject(*Alloca);
-    PointerAllocas.push_back({Alloca, NeedsStackObject, ByteSize,
+    PointerAllocas.push_back({Alloca, NeedsStackObject, *DeferResult, ByteSize,
                               Alloca->getAlign().value(), BitCount,
                               std::move(BitmapWords), std::move(Leaves)});
   }
@@ -1381,7 +1396,11 @@ Error rewriteFunction(Function &F) {
   for (SafepointRecord &Record : llvm::reverse(Records)) {
     SmallVector<const PointerAllocaRecord *, 8> AllocaRecords;
     for (const PointerAllocaRecord &Alloca : PointerAllocas) {
-      bool IsActive = Record.Live.contains(Alloca.Alloca) ||
+      // A recovered panic resumes outside LLVM's explicit CFG. The frontend
+      // marks named result homes whose contents must therefore remain visible
+      // to Go's stack scanner at every possible suspension call.
+      bool IsActive = Alloca.DeferResult ||
+                      Record.Live.contains(Alloca.Alloca) ||
                       isPointerAllocaActiveAt(Alloca, *Record.Call);
       if (IsActive)
         Record.Live.insert(Alloca.Alloca);

src/cmd/llvmplugin/testdata/check-defer-edge.py

@@ -7,6 +7,7 @@
 
 
 FUNCTION = "defer_edge"
+RESULT = "defer_result"
 WRAPPER = "defer_wrapper"
 
 
@@ -49,6 +50,15 @@ def main():
     if metadata is None:
         fail(f"{FUNCTION} has no function metadata")
 
+    result = only(
+        obj["symbols"],
+        lambda item: item["name"] == RESULT,
+        f"{RESULT} symbols",
+    )
+    result_metadata = result.get("function")
+    if result_metadata is None:
+        fail(f"{RESULT} has no function metadata")
+
     wrapper = only(
         obj["symbols"],
         lambda item: item["name"] == WRAPPER,
@@ -85,9 +95,41 @@ def target_name(relocation):
     if query["stack_map_index"] < 0:
         fail(f"deferreturn has no valid GC stack map: {query}")
 
+    funcdata = {
+        item["kind"]: item for item in result_metadata["funcdata"]
+    }
+    if "stack_objects" in funcdata:
+        fail(f"{RESULT} emitted StackObjects instead of alloca deopt roots")
+    locals_maps = funcdata["locals_pointer_maps"]["stack_map"]["bitmaps"]
+
+    def bitmap_has_pointer(index):
+        return bool(locals_maps[index].get("set_bits"))
+
+    result_calls = {
+        target_name(relocation): relocation
+        for relocation in result["relocations"]
+        if relocation["type"] == "R_CALL"
+    }
+    for name in (
+        "runtime.deferproc",
+        "runtime.panicmem",
+        "runtime.deferreturn",
+    ):
+        relocation = result_calls.get(name)
+        if relocation is None:
+            fail(f"{RESULT} has no direct {name} relocation")
+        call_query = only(
+            result_metadata["stack_map_queries"],
+            lambda item: item["call_offset"] == relocation["offset"],
+            f"{name} stack-map queries",
+        )
+        if not bitmap_has_pointer(call_query["stack_map_index"]):
+            fail(f"{name} does not keep the defer result in locals ptrmap")
+
     print(
         f"{FUNCTION}: runtime.deferreturn R_CALL at {deferreturn['offset']}, "
-        f"stack map {query['stack_map_index']}; {WRAPPER}: FuncID 23, flags 0"
+        f"stack map {query['stack_map_index']}; {RESULT}: result live at all "
+        f"calls with no StackObjects; {WRAPPER}: FuncID 23, flags 0"
     )
 
 

src/cmd/llvmplugin/testdata/defer-edge.ll

@@ -2,7 +2,9 @@ target triple = "x86_64-unknown-linux-goobj"
 
 declare goabiinternal void @runtime.deferproc()
 declare goabiinternal void @runtime.deferreturn()
+declare goabiinternal void @runtime.panicmem()
 declare void @llvm.go.defer.edge()
+declare void @llvm.lifetime.start.p0(ptr captures(none))
 
 define goabiinternal void @defer_edge() #0 gc "goallc" {
 entry:
@@ -17,6 +19,25 @@ recover:
   ret void
 }
 
+define goabiinternal ptr @defer_result(ptr %pointer) #0 gc "goallc" {
+entry:
+  %result = alloca ptr, align 8, !goallc.defer_result !1
+  call void @llvm.lifetime.start.p0(ptr %result)
+  store ptr null, ptr %result, align 8
+  call goabiinternal void @runtime.deferproc()
+  callbr void @llvm.go.defer.edge() to label %panic [label %recover]
+
+panic:
+  store ptr %pointer, ptr %result, align 8
+  call goabiinternal void @runtime.panicmem()
+  unreachable
+
+recover:
+  call goabiinternal void @runtime.deferreturn()
+  %value = load volatile ptr, ptr %result, align 8
+  ret ptr %value
+}
+
 define goabiinternal void @defer_wrapper() !goobj.func.info !0 {
 entry:
   ret void
@@ -25,3 +46,4 @@ entry:
 attributes #0 = { "go-stack-growth-statepoint" }
 
 !0 = !{i8 23, i8 0}
+!1 = !{}

test/codegen/llvm_defer.go

@@ -8,21 +8,35 @@ package codegen
 
 var llvmDeferSink int
 
+// LLVM-LABEL: define goabiinternal ptr @codegen.llvmDeferPointerResult(ptr{{.*}} %pointer)
+// LLVM: [[RESULT:%.*]] = alloca ptr, {{.*}}!goallc.defer_result
+// LLVM: callbr void @llvm.go.defer.edge()
+// LLVM-NEXT: to label %{{.*}} [label %[[RECOVER:[A-Za-z0-9_.]+]]]
+// LLVM: [[RECOVER]]:
+// LLVM-NEXT: call goabiinternal void @runtime.deferreturn()
+// LLVM-NEXT: {{.*}} = load volatile ptr, ptr [[RESULT]]
+// LLVM-OPT-LABEL: define goabiinternal ptr @codegen.llvmDeferPointerResult(ptr{{.*}} %pointer)
+// LLVM-OPT: [[RESULT_OPT:%.*]] = alloca ptr, {{.*}}!goallc.defer_result
+// LLVM-OPT: callbr void @llvm.go.defer.edge()
+// LLVM-OPT-NEXT: to label %{{.*}} [label %[[RECOVER_OPT:[A-Za-z0-9_.]+]]]
+// LLVM-OPT: [[RECOVER_OPT]]:
+// LLVM-OPT-NEXT: call goabiinternal void @runtime.deferreturn()
+// LLVM-OPT-NEXT: {{.*}} = load volatile ptr, ptr [[RESULT_OPT]]
+
 // LLVM-LABEL: define goabiinternal i64 @codegen.llvmDeferStack(i64 %value)
 // LLVM: call goabiinternal void @runtime.deferprocStack
 // LLVM: callbr void @llvm.go.defer.edge()
 // LLVM-NEXT: to label %[[STACK_NORMAL:.*]] [label %[[STACK_RECOVER:.*]]]
 // LLVM: [[STACK_RECOVER]]:
 // LLVM: call goabiinternal void @runtime.deferreturn()
-// LLVM-NOT: load volatile
+// LLVM: load volatile i64
 // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmDeferStack(i64 %value)
 // LLVM-OPT: call goabiinternal void @runtime.deferprocStack
 // LLVM-OPT: callbr void @llvm.go.defer.edge()
 // LLVM-OPT-NEXT: to label %{{.*}} [label %[[STACK_OPT_RECOVER:.*]]]
 // LLVM-OPT: [[STACK_OPT_RECOVER]]:
 // LLVM-OPT: call goabiinternal void @runtime.deferreturn()
-// LLVM-OPT: load i64
-// LLVM-OPT-NOT: load volatile
+// LLVM-OPT: load volatile i64
 
 // LLVM-LABEL: define goabiinternal void @codegen.llvmDeferHeap(i64 %count)
 // LLVM: [[HEAP_NORMAL_RETURN:[A-Za-z0-9_.]+]]:
@@ -62,3 +76,9 @@ func llvmDeferStack(value int) (result int) {
 	result = 7
 	return
 }
+
+func llvmDeferPointerResult(pointer *int) (result *int) {
+	defer func() {}()
+	result = pointer
+	panic(llvmDeferSink)
+}

test/llvm_defer.go

@@ -9,6 +9,11 @@ package main
 import "runtime"
 
 var deferTrace int
+var deferResultFinalized = make(chan struct{}, 1)
+
+type deferResultObject struct {
+	value int
+}
 
 //go:noinline
 func normalDefers() (result int) {
@@ -73,6 +78,26 @@ func pointerDefer() (result int) {
 	return
 }
 
+//go:noinline
+func namedPointerResultSurvivesPanic() (result *deferResultObject) {
+	result = &deferResultObject{value: 91}
+	runtime.SetFinalizer(result, func(*deferResultObject) {
+		deferResultFinalized <- struct{}{}
+	})
+	defer func() {
+		runtime.GC()
+		runtime.GC()
+		runtime.GC()
+		select {
+		case <-deferResultFinalized:
+			panic("named result was finalized during panic unwinding")
+		default:
+		}
+		recover()
+	}()
+	panic("named result liveness")
+}
+
 func main() {
 	if got := normalDefers(); got != 17 || deferTrace != 21 {
 		panic("normal defer order or named result is incorrect")
@@ -89,4 +114,9 @@ func main() {
 	if pointerDefer() != 73 {
 		panic("defer lost a captured pointer across GC")
 	}
+	result := namedPointerResultSurvivesPanic()
+	if result == nil || result.value != 91 {
+		panic("defer lost a named pointer result during panic unwinding")
+	}
+	runtime.KeepAlive(result)
 }

test/llvm_tests.json

@@ -1084,6 +1084,7 @@
       "fixedbugs/issue26094.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "fixedbugs/issue27289.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "fixedbugs/issue27518a.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
+      "fixedbugs/issue27518b.go": "defer recovery keeps named pointer results live across panic-time GC",
       "fixedbugs/issue28748.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "fixedbugs/issue28797.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "fixedbugs/issue29190.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
@@ -1220,6 +1221,7 @@
         "fixedbugs/issue26094.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "fixedbugs/issue27289.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "fixedbugs/issue27518a.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "fixedbugs/issue27518b.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "fixedbugs/issue28748.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "fixedbugs/issue28797.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "fixedbugs/issue29190.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",