cmd/compile: preserve runtime traceback frame semantics

src/cmd/compile/internal/ssa/llvmdebug.go

@@ -257,6 +257,22 @@ func (lfc *LLVMFuncContext) setDebugLocation(xpos src.XPos) {
 	}
 
 	locationScope := llvmDIScopeForPos(scope, pos, 0)
-	lfc.b.SetCurrentDebugLocationMetadata(GlobalCtxt.CreateDebugLocation(
-		pos.RelLine(), pos.RelCol(), locationScope, inlinedAt))
+	location := GlobalCtxt.CreateDebugLocation(
+		pos.RelLine(), pos.RelCol(), locationScope, inlinedAt)
+	lfc.b.SetCurrentDebugLocationMetadata(location)
+
+	// Generic LLVM optimization may combine instructions from different Go
+	// inline frames and keep only one of their DILocations. Record one complete
+	// frontend location for every inline node independently of the instruction
+	// stream. The final machine pass uses this only when an inline edge has
+	// otherwise disappeared, so optimization remains unconstrained while Go's
+	// pcinline tree still has a real final-layout PC for every source edge.
+	if len(chain) != 0 && !lfc.RequiredInlinePos[pos.Base().InliningIndex()] {
+		lfc.RequiredInlinePos[pos.Base().InliningIndex()] = true
+		CurrentModule.AddNamedMetadataOperand(goObjDebugInlineRequiredMD,
+			GlobalCtxt.MDNode([]llvm.Metadata{
+				lfc.LF.ConstantAsMetadata(),
+				location,
+			}))
+	}
 }

src/cmd/compile/internal/ssa/ssa2llvm.go

@@ -25,6 +25,7 @@ type LLVMFuncContext struct {
 	ClosureCodeLoads  map[ID]bool
 	DeferResults      map[llvmLocalKey]bool
 	DeferResultKeys   map[ID]llvmLocalKey
+	RequiredInlinePos map[int]bool
 	OpenDeferBits     llvmLocalKey
 	HasOpenDeferBits  bool
 	OpenDeferSlots    map[llvmLocalKey]int
@@ -79,6 +80,7 @@ const goOpenDeferBitsMD = "goallc.open_defer_bits"
 const goOpenDeferSlotsMD = "goallc.open_defer_slots"
 const goObjMarkerRelocMD = "goobj.marker_reloc"
 const goObjSymbolIndexMD = "goobj.symbol.index"
+const goObjDebugInlineRequiredMD = "goobj.debug.inline.required"
 const llvmFramePointerAttr = "frame-pointer"
 const llvmFramePointerNonLeaf = "non-leaf"
 const llvmTargetCPUAttr = "target-cpu"
@@ -2846,8 +2848,18 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() {
 	deferReturn := getOrInsertLLVMABISymbolRef("runtime.deferreturn", obj.ABIInternal, deferReturnSig, goABIInternalCallConv)
 
 	lfc.b.SetInsertPointAtEnd(lfc.OpenDeferRecovery)
+	frontendFunc := lfc.F.Frontend().Func()
+	if frontendFunc == nil || !frontendFunc.Endlineno.IsKnown() {
+		lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer recovery has no function-end source position")
+	}
+	// Match the native shared deferreturn convention: its synthetic call is
+	// attributed to the function end, after every source-level defer. Besides
+	// giving PCLN a stable line, a call in an LLVM debug-info function must
+	// carry a !dbg location even when it lives in a disconnected recovery block.
+	lfc.setDebugLocation(frontendFunc.Endlineno)
 	call := lfc.b.CreateCall(deferReturnSig.Type, deferReturn, nil, "")
 	call.SetInstructionCallConv(goABIInternalCallConv)
+	lfc.b.ClearCurrentDebugLocation()
 
 	outParams := lfc.F.OwnAux.ABIInfo().OutParams()
 	if len(outParams) != lfc.ResultCount {
@@ -2957,16 +2969,8 @@ func (lfc *LLVMFuncContext) MappingName() {
 	}
 }
 
-func llvmFuncCalls(f *Func, target string) bool {
-	for _, b := range f.Blocks {
-		for _, v := range b.Values {
-			aux, ok := v.Aux.(*AuxCall)
-			if ok && aux.Fn != nil && aux.Fn.Name == target {
-				return true
-			}
-		}
-	}
-	return false
+func llvmIsRuntimeGorecover(f *Func) bool {
+	return f.OwnAux != nil && f.OwnAux.Fn != nil && f.OwnAux.Fn.Name == "runtime.gorecover"
 }
 
 func LLVMCompile(f *Func) {
@@ -2979,20 +2983,21 @@ func LLVMCompile(f *Func) {
 	}
 	cc := llvmCallConv(f.OwnAux.ABI().Which())
 	FCtxt := &LLVMFuncContext{
-		BBs:              map[ID]llvm.BasicBlock{},
-		Vs:               map[ID]llvm.Value{},
-		Locals:           map[llvmLocalKey]llvmStackSlot{},
-		AddressedResults: map[ID][]llvmAddressedResult{},
-		ResultSlots:      map[ID]llvm.Value{},
-		ItabMethods:      map[ID]bool{},
-		ClosureCodeLoads: map[ID]bool{},
-		DeferResults:     map[llvmLocalKey]bool{},
-		DeferResultKeys:  map[ID]llvmLocalKey{},
-		OpenDeferSlots:   map[llvmLocalKey]int{},
-		F:                f,
-		b:                GlobalCtxt.NewBuilder(),
-		ReturnType:       sig.ReturnType,
-		ResultCount:      sig.ResultCount,
+		BBs:               map[ID]llvm.BasicBlock{},
+		Vs:                map[ID]llvm.Value{},
+		Locals:            map[llvmLocalKey]llvmStackSlot{},
+		AddressedResults:  map[ID][]llvmAddressedResult{},
+		ResultSlots:       map[ID]llvm.Value{},
+		ItabMethods:       map[ID]bool{},
+		ClosureCodeLoads:  map[ID]bool{},
+		DeferResults:      map[llvmLocalKey]bool{},
+		DeferResultKeys:   map[ID]llvmLocalKey{},
+		RequiredInlinePos: map[int]bool{},
+		OpenDeferSlots:    map[llvmLocalKey]int{},
+		F:                 f,
+		b:                 GlobalCtxt.NewBuilder(),
+		ReturnType:        sig.ReturnType,
+		ResultCount:       sig.ResultCount,
 	}
 	defer FCtxt.b.Dispose()
 
@@ -3045,8 +3050,17 @@ func LLVMCompile(f *Func) {
 		}
 	}
 	cgoUnsafeArgs := frontendFunc != nil && frontendFunc.Pragma&ir.CgoUnsafeArgs != 0
-	frontendNoInline := frontendFunc != nil && (frontendFunc.Pragma&ir.Noinline != 0 || frontendFunc.HasDefer() || cgoUnsafeArgs)
-	if frontendNoInline || llvmFuncCalls(f, "runtime.gorecover") {
+	// TODO(goallc): Native Go permits nosplit functions to be inlined, and the
+	// linker checks the final GoObj PCSP tables and call relocations against the
+	// nosplit limit. This conservative fence currently keeps LLVM inlining from
+	// increasing runtime's callRet chain beyond that limit. Replace it with a
+	// targeted policy once the specific frame growth is understood.
+	frontendNoInline := f.NoSplit || frontendFunc != nil && (frontendFunc.Pragma&ir.Noinline != 0 || frontendFunc.HasDefer() || cgoUnsafeArgs)
+	// gorecover explicitly advances the physical unwinder past its own frame
+	// before it walks the remaining physical and inline frames. Preserve that
+	// one physical boundary. Direct callers may be represented by Go's inline
+	// tree and are deliberately left available to LLVM's inliner.
+	if frontendNoInline || llvmIsRuntimeGorecover(f) {
 		FCtxt.LF.AddFunctionAttr(llvmNoInlineAttribute())
 	}
 	if f.OpenDeferBits != nil {

src/cmd/compile/internal/ssa/ssa2llvm_test.go

@@ -560,3 +560,31 @@ func TestLLVMTargetCPU(t *testing.T) {
 		})
 	}
 }
+
+func TestLLVMRuntimeGorecoverUsesLinkSymbolName(t *testing.T) {
+	recoverFn := &Func{
+		Name:   "gorecover",
+		OwnAux: &AuxCall{Fn: &obj.LSym{Name: "runtime.gorecover"}},
+	}
+	if !llvmIsRuntimeGorecover(recoverFn) {
+		t.Fatal("gorecover definition was not recognized from its qualified link symbol")
+	}
+
+	unrelated := &Func{
+		Name:   "gorecover",
+		OwnAux: &AuxCall{Fn: &obj.LSym{Name: "other.gorecover"}},
+	}
+	if llvmIsRuntimeGorecover(unrelated) {
+		t.Fatal("unqualified SSA function name incorrectly identified another package's gorecover")
+	}
+
+	caller := &Func{
+		OwnAux: &AuxCall{Fn: &obj.LSym{Name: "runtime.preprintpanics.func1"}},
+		Blocks: []*Block{{
+			Values: []*Value{{Aux: &AuxCall{Fn: &obj.LSym{Name: "runtime.gorecover"}}}},
+		}},
+	}
+	if llvmIsRuntimeGorecover(caller) {
+		t.Fatal("direct gorecover caller was incorrectly identified as gorecover itself")
+	}
+}

src/cmd/llvmplugin/GoALLCInlineAnchors.cpp

@@ -3,14 +3,17 @@
 // license that can be found in the LICENSE file.
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/Module.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -71,6 +74,42 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass {
                            Loc->getAtomRank());
   }
 
+  SmallVector<const DILocation *, 16>
+  requiredInlineLocations(const MachineFunction &MF) const {
+    SmallVector<const DILocation *, 16> Required;
+    const Module *M = MF.getFunction().getParent();
+    const NamedMDNode *Locations =
+        M ? M->getNamedMetadata("goobj.debug.inline.required") : nullptr;
+    if (!Locations)
+      return Required;
+
+    for (const MDNode *Entry : Locations->operands()) {
+      if (Entry->getNumOperands() != 2)
+        report_fatal_error("expected !goobj.debug.inline.required entries to "
+                           "have two operands");
+      const auto *CAM =
+          dyn_cast_or_null<ConstantAsMetadata>(Entry->getOperand(0));
+      const auto *GV = CAM ? dyn_cast<GlobalValue>(CAM->getValue()) : nullptr;
+      const auto *Loc = dyn_cast_or_null<DILocation>(Entry->getOperand(1));
+      if (!GV || !Loc || !Loc->getInlinedAt())
+        report_fatal_error("invalid !goobj.debug.inline.required entry");
+      if (GV == &MF.getFunction())
+        Required.push_back(Loc);
+    }
+
+    llvm::stable_sort(
+        Required, [](const DILocation *LHS, const DILocation *RHS) {
+          auto Depth = [](const DILocation *Loc) {
+            unsigned Result = 0;
+            for (; Loc && Loc->getInlinedAt(); Loc = Loc->getInlinedAt())
+              ++Result;
+            return Result;
+          };
+          return Depth(LHS) > Depth(RHS);
+        });
+    return Required;
+  }
+
 public:
   static char ID;
 
@@ -88,6 +127,51 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass {
     const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
     CanonicalSites.clear();
 
+    // LLVM may merge instructions from distinct frontend inline frames (for
+    // example, SLP-vectorizing adjacent stores) and retain only one debug
+    // location. Materialize a source NOP for each required inline edge that no
+    // longer occurs in the optimized MachineFunction. Deeper chains are
+    // considered first because one such marker also preserves all its parents.
+    DenseSet<const DILocation *> SurvivingCallsites;
+    for (MachineBasicBlock &MBB : MF)
+      for (MachineInstr &MI : MBB)
+        if (!MI.isMetaInstruction())
+          for (const DILocation *Loc = MI.getDebugLoc().get();
+               Loc && Loc->getInlinedAt(); Loc = Loc->getInlinedAt())
+            SurvivingCallsites.insert(Loc->getInlinedAt());
+
+    bool Changed = false;
+    if (!MF.empty()) {
+      MachineBasicBlock *InsertBlock = nullptr;
+      MachineBasicBlock::iterator InsertAt;
+      for (MachineBasicBlock &MBB : MF) {
+        auto It = MBB.begin();
+        while (It != MBB.end() && (It->isMetaInstruction() ||
+                                   It->getFlag(MachineInstr::FrameSetup)))
+          ++It;
+        if (It != MBB.end()) {
+          InsertBlock = &MBB;
+          InsertAt = It;
+          break;
+        }
+      }
+      for (const DILocation *Loc : requiredInlineLocations(MF)) {
+        const DILocation *Innermost = Loc->getInlinedAt();
+        if (SurvivingCallsites.contains(Innermost))
+          continue;
+        if (!InsertBlock)
+          report_fatal_error("GoALLC required inline location has no "
+                             "post-prologue insertion point");
+        TII.insertNoop(*InsertBlock, InsertAt);
+        MachineInstr &Marker = *std::prev(InsertAt);
+        Marker.setDebugLoc(DebugLoc(Loc));
+        for (const DILocation *Site = Loc; Site && Site->getInlinedAt();
+             Site = Site->getInlinedAt())
+          SurvivingCallsites.insert(Site->getInlinedAt());
+        Changed = true;
+      }
+    }
+
     // Normalize complete inline chains before inspecting them. This keeps the
     // anchor pass and the later GoObj debug handler on the same edge identity.
     for (MachineBasicBlock &MBB : MF)
@@ -97,7 +181,6 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass {
               DebugLoc(canonicalizeLocation(MI.getDebugLoc().get())));
 
     DenseSet<const DILocation *> AnchoredCallsites;
-    bool Changed = false;
 
     for (MachineBasicBlock &MBB : MF) {
       for (auto It = MBB.begin(), End = MBB.end(); It != End; ++It) {

src/cmd/llvmplugin/testdata/debug-inline.ll

@@ -115,9 +115,28 @@ entry:
   ret i64 %x, !dbg !55
 }
 
+; The optimized instruction stream has no location for erasedInner. Frontend
+; required-location metadata must make the final machine pass materialize the
+; missing nested inline edge without constraining IR optimization.
+define goabiinternal void @main.erased() !dbg !19 {
+entry:
+  ret void, !dbg !65
+}
+
+define goabiinternal void @main.erasedMid() !dbg !20 {
+entry:
+  ret void, !dbg !66
+}
+
+define goabiinternal void @main.erasedInner() !dbg !21 {
+entry:
+  ret void, !dbg !67
+}
+
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5, !6}
-!goobj.debug.funcs = !{!40, !41, !42, !43, !44, !45, !46, !47, !48}
+!goobj.debug.funcs = !{!40, !41, !42, !43, !44, !45, !46, !47, !48, !49, !56, !57}
+!goobj.debug.inline.required = !{!58}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_Go, file: !1, producer: "goallc-test", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: true, nameTableKind: None)
 !1 = !DIFile(filename: "outer.go", directory: "/tmp/goobj-inline")
@@ -136,6 +155,9 @@ entry:
 !16 = distinct !DISubprogram(name: "main.shared", linkageName: "main.shared", scope: !1, file: !1, line: 70, type: !3, scopeLine: 70, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !17 = distinct !DISubprogram(name: "main.sharedLeft", linkageName: "main.sharedLeft", scope: !1, file: !1, line: 80, type: !3, scopeLine: 80, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 !18 = distinct !DISubprogram(name: "main.sharedRight", linkageName: "main.sharedRight", scope: !1, file: !1, line: 90, type: !3, scopeLine: 90, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!19 = distinct !DISubprogram(name: "main.erased", linkageName: "main.erased", scope: !1, file: !1, line: 100, type: !3, scopeLine: 100, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!20 = distinct !DISubprogram(name: "main.erasedMid", linkageName: "main.erasedMid", scope: !1, file: !1, line: 110, type: !3, scopeLine: 110, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!21 = distinct !DISubprogram(name: "main.erasedInner", linkageName: "main.erasedInner", scope: !1, file: !1, line: 120, type: !3, scopeLine: 120, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
 
 !30 = !DILocation(line: 30, column: 3, scope: !12, inlinedAt: !31)
 !31 = distinct !DILocation(line: 20, column: 3, scope: !11, inlinedAt: !32)
@@ -151,6 +173,12 @@ entry:
 !53 = !DILocation(line: 72, column: 2, scope: !16)
 !54 = !DILocation(line: 81, column: 2, scope: !17)
 !55 = !DILocation(line: 91, column: 2, scope: !18)
+!60 = !DILocation(line: 121, column: 2, scope: !21, inlinedAt: !61)
+!61 = distinct !DILocation(line: 111, column: 2, scope: !20, inlinedAt: !62)
+!62 = distinct !DILocation(line: 101, column: 2, scope: !19)
+!65 = !DILocation(line: 102, column: 2, scope: !19)
+!66 = !DILocation(line: 112, column: 2, scope: !20)
+!67 = !DILocation(line: 122, column: 2, scope: !21)
 
 !40 = !{!10, ptr @main.outer}
 !41 = !{!11, ptr @main.mid}
@@ -161,3 +189,7 @@ entry:
 !46 = !{!16, ptr @main.shared}
 !47 = !{!17, ptr @main.sharedLeft}
 !48 = !{!18, ptr @main.sharedRight}
+!49 = !{!19, ptr @main.erased}
+!56 = !{!20, ptr @main.erasedMid}
+!57 = !{!21, ptr @main.erasedInner}
+!58 = !{ptr @main.erased, !60}

test/codegen/llvm_defer.go

@@ -73,7 +73,7 @@ var llvmDeferSink int
 // LLVM: callbr void @llvm.go.defer.edge()
 // LLVM-NEXT: to label %{{.*}} [label %[[HEAP_RECOVER]]]
 // LLVM: define goabiinternal void @codegen.llvmDeferHeap.deferwrap1({{.*}}) {{.*}}!goobj.func.info ![[WRAPPER_INFO:[0-9]+]]
-// LLVM: define goabiinternal {{.*}} @codegen.llvmRecover(){{.*}} #[[LLVM_NOINLINE]] gc "goallc"
+// LLVM: define goabiinternal {{.*}} @codegen.llvmRecover(){{.*}} #[[LLVM_RECOVER_ATTRS:[0-9]+]] gc "goallc"
 // LLVM: call goabiinternal {{.*}} @"runtime.gorecover<builtin.{{[0-9]+}}>"(
 // LLVM-OPT-LABEL: define goabiinternal void @codegen.llvmDeferHeap(i64 %count)
 // LLVM-OPT: [[HEAP_OPT_RECOVER:common.ret]]:
@@ -82,7 +82,7 @@ var llvmDeferSink int
 // LLVM-OPT: callbr void @llvm.go.defer.edge()
 // LLVM-OPT-NEXT: to label %{{.*}} [label %[[HEAP_OPT_RECOVER]]]
 // LLVM-OPT: define goabiinternal void @codegen.llvmDeferHeap.deferwrap1({{.*}}) {{.*}}!goobj.func.info ![[WRAPPER_OPT_INFO:[0-9]+]]
-// LLVM-OPT: define goabiinternal {{.*}} @codegen.llvmRecover(){{.*}} #[[LLVM_NOINLINE]] gc "goallc"
+// LLVM-OPT: define goabiinternal {{.*}} @codegen.llvmRecover(){{.*}} #[[LLVM_RECOVER_OPT_ATTRS:[0-9]+]] gc "goallc"
 // LLVM-OPT: call goabiinternal {{.*}} @"runtime.gorecover<builtin.{{[0-9]+}}>"(
 
 // An unnamed result still has a recovery-visible home. If evaluating a return
@@ -93,6 +93,7 @@ var llvmDeferSink int
 // LLVM: call goabiinternal void @"runtime.deferreturn<builtin.{{[0-9]+}}>"()
 // LLVM-NEXT: {{.*}} = load volatile i64, ptr [[UNNAMED_RESULT]]
 // LLVM: attributes #[[LLVM_NOINLINE]] = { {{.*}}noinline
+// LLVM-NOT: attributes #[[LLVM_RECOVER_ATTRS]] = { {{.*}}noinline
 // LLVM: ![[WRAPPER_INFO]] = !{i8 23, i8 0}
 // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmDeferUnnamedResult(i64 %value)
 // LLVM-OPT: [[UNNAMED_OPT_RESULT:%.*]] = alloca i64, align 8{{$}}
@@ -103,6 +104,7 @@ var llvmDeferSink int
 // LLVM-OPT-NEXT: call goabiinternal void @"runtime.deferreturn<builtin.{{[0-9]+}}>"()
 // LLVM-OPT-NEXT: br label %[[UNNAMED_OPT_RETURN]]
 // LLVM-OPT: attributes #[[LLVM_NOINLINE]] = { {{.*}}noinline
+// LLVM-OPT-NOT: attributes #[[LLVM_RECOVER_OPT_ATTRS]] = { {{.*}}noinline
 // LLVM-OPT: ![[WRAPPER_OPT_INFO]] = !{i8 23, i8 0}
 
 // A defer in a loop uses runtime.deferproc rather than deferprocStack. Keep this