[Go] Align stack-check elision and entry PCDATA with native Go

.github/workflows/goallc-ci.yml

@@ -106,6 +106,9 @@ jobs:
             "$build/test/CodeGen/Generic/go-abi0-frame.ll"
             "$build/test/CodeGen/Generic/go-argument-homes.ll"
             "$build/test/CodeGen/Generic/goobj-data-kinds.ll"
+            "$build/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll"
+            "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll"
+            "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll"
             "$build/test/CodeGen/X86/go-callconv.ll"
             "$build/test/CodeGen/X86/go-gc-write-barrier.ll"
             "$build/test/CodeGen/X86/go-statepoint-stack-results.ll"

llvm/lib/MC/GoObjObjectWriter.cpp

@@ -975,12 +975,13 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
   Pairs.push_back(BuildPair(*EntryArgsEntry->Entry));
   SmallVector<GoObjPCTabEntry, 16> PCDataEntries;
   // PCSP is derived from the Machine CFG. Whenever control flow returns to
-  // the entry stack depth, only the function-level entry argument map is
-  // valid. This covers the pre-frame morestack slow path without naming the
-  // helper or manufacturing a statepoint for its raw ABI0 call.
+  // the entry stack depth, restore Go's entry value (-1). The runtime
+  // normalizes that value to ArgsPointerMaps bitmap 0. This covers the
+  // pre-frame morestack slow path without naming the helper or manufacturing
+  // a statepoint for its raw ABI0 call.
   for (const GoObjPCTabEntry &Entry : PCSPEntries)
     if (Entry.Value == 0 && Entry.PC < Function.Size)
-      PCDataEntries.push_back({Entry.PC, 0});
+      PCDataEntries.push_back({Entry.PC, -1});
   std::optional<uint64_t> PreviousCallsitePC;
   SmallVector<uint32_t, 4> IndirectCallOffsets;
   for (const ResolvedEntry &Resolved : ResolvedEntries) {
@@ -1087,7 +1088,7 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
   Result.Args = makeStackMap(ArgsNBits, ArgsBitmaps);
   Result.Locals = makeStackMap(NBits, LocalsBitmaps);
   Result.PCData =
-      makePCTab(0, NormalizedPCDataEntries, Function.Size, PCQuantum);
+      makePCTab(-1, NormalizedPCDataEntries, Function.Size, PCQuantum);
   Result.OpenDefer = std::move(OpenDeferData);
   Result.IndirectCallOffsets = std::move(IndirectCallOffsets);
   Result.StackObjects = std::move(FunctionStackObjects);
@@ -2005,7 +2006,7 @@ uint64_t GoObjObjectWriter::writeObject() {
           makePCTab(-1, LineInfo.PCInline, CodeSize, PCQuantum));
       SmallString<0> ArgsMap = makeEmptyStackMap();
       SmallString<0> LocalsMap = makeEmptyStackMap();
-      SmallString<0> StackMapIndex = makeConstantPCTab(0, CodeSize, PCQuantum);
+      SmallString<0> StackMapIndex = makeConstantPCTab(-1, CodeSize, PCQuantum);
       std::optional<uint32_t> StackObjectsSym;
       std::optional<uint32_t> OpenDeferSym;
       if (const auto *Entries = Asm->getContext().getGoObjSymbolStackMapEntries(

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

@@ -1394,6 +1394,12 @@ static void emitAArch64GoStackCheck(MachineFunction &MF,
     report_fatal_error("GoObj stack growth does not support dynamic allocas");
 
   uint64_t StackSize = MFI.getStackSize() + MFI.getUnsafeStackSize();
+  // Match the native Go assembler: a leaf function whose final frame is
+  // smaller than StackSmall is effectively NOSPLIT.  The entry argument map
+  // is still emitted independently, but there is no need for a morestack
+  // edge because the function cannot exhaust the nosplit stack allowance.
+  if (!MFI.hasCalls() && StackSize < GoStackSmall)
+    return;
 
   const DebugLoc DL;
   const AArch64InstrInfo &TII =

llvm/lib/Target/X86/X86FrameLowering.cpp

@@ -230,6 +230,52 @@ static bool shouldEmitGoStackCheck(const MachineFunction &MF) {
          !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr);
 }
 
+static StringRef getGoDirectCalleeName(const MachineInstr &MI,
+                                       const X86InstrInfo &TII) {
+  const MachineOperand &Callee = MI.getOpcode() == TargetOpcode::STATEPOINT
+                                     ? StatepointOpers(&MI).getCallTarget()
+                                     : TII.getCalleeOperand(MI);
+  if (Callee.isGlobal())
+    return Callee.getGlobal()->getName();
+  if (Callee.isSymbol())
+    return Callee.getSymbolName();
+  if (Callee.isMCSymbol())
+    return Callee.getMCSymbol()->getName();
+  return {};
+}
+
+static bool isGoLeafLikeRuntimeCall(const MachineInstr &MI,
+                                    const X86InstrInfo &TII) {
+  StringRef Name = getGoDirectCalleeName(MI, TII);
+  return Name == "runtime.panicdivide" || Name == "runtime.panicwrap" ||
+         Name == "runtime.panicshift" || Name == "runtime.panicBounds" ||
+         Name == "runtime.panicExtend";
+}
+
+static bool isGoSmallLeafLikeFunction(const MachineFunction &MF,
+                                      const X86InstrInfo &TII,
+                                      uint64_t StackSize) {
+  bool HasLeafLikeCall = false;
+  for (const MachineBasicBlock &MBB : MF)
+    for (const MachineInstr &MI : MBB) {
+      if (!MI.isCall())
+        continue;
+      // The entry argument STACKMAP is a zero-byte data marker. LLVM models
+      // the pseudo as call-like for register-mask purposes, but it is not a
+      // machine call and must not turn every Go function into a non-leaf.
+      if (MI.getOpcode() == TargetOpcode::STACKMAP)
+        continue;
+      if (!isGoLeafLikeRuntimeCall(MI, TII))
+        return false;
+      HasLeafLikeCall = true;
+    }
+
+  // CALL itself pushes an eight-byte return PC. Native x86 permits these
+  // leaf-like runtime calls only while frame+return-PC remains StackSmall.
+  uint64_t CallDepth = HasLeafLikeCall ? 8 : 0;
+  return StackSize < GoStackSmall - CallDepth;
+}
+
 static bool hasGoClosureContext(const Function &F) {
   for (const Argument &Arg : F.args())
     if (Arg.hasNestAttr())
@@ -304,8 +350,15 @@ static void emitGoStackCheck(MachineFunction &MF,
   if (!MF.getSubtarget<X86Subtarget>().getFrameLowering()->hasReservedCallFrame(
           MF))
     StackSize += MFI.getMaxCallFrameSize();
-  const DebugLoc DL;
   const X86InstrInfo &TII = *MF.getSubtarget<X86Subtarget>().getInstrInfo();
+  // Match the native Go assembler: a leaf function whose final frame is
+  // smaller than StackSmall is effectively NOSPLIT. X86 also treats its
+  // zero-argument panic helpers as leaf-like when their CALL return PC still
+  // fits in StackSmall. Keep entry argument-map emission independent from this
+  // decision.
+  if (isGoSmallLeafLikeFunction(MF, TII, StackSize))
+    return;
+  const DebugLoc DL;
   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   ArrayRef<X86MachineFunctionInfo::GoArgHome> Homes = X86FI->getGoArgHomes();
   MachineBasicBlock &EntryMBB = getGoStackCheckEntryMBB(MF, PrologueMBB);

llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll

@@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" }
 
 ; OBJ: symdef 0: main.safe
 ; OBJ: symdef 1: main.async_unsafe
-; OBJ: aux 0.6: type=pcdata target= pc=[0-10:-1]
-; OBJ: aux 1.14: type=pcdata target= pc=[0-10:-2]
+; OBJ: aux 0.6: type=pcdata target= pc=[0-2:-1]
+; OBJ: aux 1.14: type=pcdata target= pc=[0-2:-2]

llvm/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll

@@ -8,9 +8,11 @@
 
 ; A normal statepoint selects a live frame map. When physical block placement
 ; later reaches a CFG path at the entry SP depth, PCDATA_StackMapIndex must
-; return to the function-level entry map without identifying morestack by name.
+; return to Go's entry value (-1), which runtime getStackMap normalizes to
+; function-level ArgsPointerMaps bitmap 0, without identifying morestack by
+; name.
 
-; CHECK: type=pcdata target= pc=[0-{{[0-9]+}}:0,{{[0-9]+}}-{{[0-9]+}}:1,{{[0-9]+}}-{{[0-9]+}}:0]
+; CHECK: type=pcdata target= pc=[0-{{[0-9]+}}:-1,{{[0-9]+}}-{{[0-9]+}}:1,{{[0-9]+}}-{{[0-9]+}}:-1]
 
 declare goabiinternal void @callee()
 

llvm/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll

@@ -0,0 +1,18 @@
+; REQUIRES: aarch64-registered-target, x86-registered-target
+; RUN: llc -mtriple=aarch64-apple-darwin-goobj -filetype=obj %s -o %t.a64.o
+; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.a64.o | \
+; RUN:   FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj %s -o %t.x86.o
+; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.x86.o | \
+; RUN:   FileCheck %s
+
+; EntryArgsStackMapID still produces ArgsPointerMaps bitmap 0, while native Go
+; encodes PCDATA_StackMapIndex=-1 at entry. runtime getStackMap maps -1 to 0.
+;
+; CHECK: type=funcdata target= data=01000000
+; CHECK: type=pcdata target= pc=[0-{{[0-9]+}}:-1]
+
+define goabiinternal ptr @entry_stackmap_sentinel(ptr %pointer) {
+entry:
+  ret ptr %pointer
+}

llvm/test/CodeGen/Generic/goobj-stack-check-policy.ll

@@ -0,0 +1,128 @@
+; REQUIRES: aarch64-registered-target, x86-registered-target
+; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \
+; RUN:   -stop-after=prolog-epilog < %s | FileCheck %s --check-prefix=MIR
+; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \
+; RUN:   -stop-after=prolog-epilog < %s | FileCheck %s --check-prefix=MIR
+; RUN: llc -mtriple=aarch64-apple-darwin-goobj < %s | \
+; RUN:   FileCheck %s --check-prefix=A64-ASM
+; RUN: llc -mtriple=x86_64-unknown-linux-goobj < %s | \
+; RUN:   FileCheck %s --check-prefix=X86-ASM
+
+declare goabiinternal void @callee()
+declare goabiinternal void @runtime.panicdivide()
+
+define goabiinternal void @zero_frame_leaf() "frame-pointer"="non-leaf" {
+entry:
+  ret void
+}
+
+define goabiinternal void @small_frame_leaf() "frame-pointer"="non-leaf" {
+entry:
+  %buf = alloca [64 x i8], align 8
+  %slot = getelementptr inbounds [64 x i8], ptr %buf, i64 0, i64 63
+  store volatile i8 1, ptr %slot, align 1
+  ret void
+}
+
+define goabiinternal void @large_frame_leaf() "frame-pointer"="non-leaf" {
+entry:
+  %buf = alloca [256 x i8], align 8
+  %slot = getelementptr inbounds [256 x i8], ptr %buf, i64 0, i64 255
+  store volatile i8 1, ptr %slot, align 1
+  ret void
+}
+
+define goabiinternal void @small_frame_non_leaf() "frame-pointer"="non-leaf" {
+entry:
+  call goabiinternal void @callee()
+  ret void
+}
+
+define goabiinternal void @small_frame_x86_leaf_like_runtime_call()
+    "frame-pointer"="non-leaf" {
+entry:
+  call goabiinternal void @runtime.panicdivide()
+  ret void
+}
+
+define goabiinternal void @x86_leaf_like_runtime_call_at_limit()
+    "frame-pointer"="non-leaf" {
+entry:
+  %buf = alloca [112 x i8], align 8
+  %slot = getelementptr inbounds [112 x i8], ptr %buf, i64 0, i64 111
+  store volatile i8 1, ptr %slot, align 1
+  call goabiinternal void @runtime.panicdivide()
+  ret void
+}
+
+define goabiinternal void @nosplit_large_non_leaf() "frame-pointer"="non-leaf"
+    "go-nosplit" {
+entry:
+  %buf = alloca [256 x i8], align 8
+  %slot = getelementptr inbounds [256 x i8], ptr %buf, i64 0, i64 255
+  store volatile i8 1, ptr %slot, align 1
+  call goabiinternal void @callee()
+  ret void
+}
+
+; Entry maps remain present even when the native-Go leaf policy elides the
+; stack-growth prologue.
+; MIR-LABEL: name: zero_frame_leaf
+; MIR: STACKMAP 5147419139155979380, 0
+; MIR-NOT: runtime.morestack
+; MIR-LABEL: name: small_frame_leaf
+; MIR: STACKMAP 5147419139155979380, 0
+; MIR-NOT: runtime.morestack
+
+; A leaf frame at or above StackSmall and any non-leaf function still get a
+; stack-growth edge by default.
+; MIR-LABEL: name: large_frame_leaf
+; MIR: runtime.morestack_noctxt
+; MIR: STACKMAP 5147419139155979380, 0
+; MIR-LABEL: name: small_frame_non_leaf
+; MIR: runtime.morestack_noctxt
+; MIR: STACKMAP 5147419139155979380, 0
+
+; The explicit source policy remains the unconditional opt-out.
+; MIR-LABEL: name: nosplit_large_non_leaf
+; MIR: STACKMAP 5147419139155979380, 0
+; MIR-NOT: runtime.morestack
+
+; On AArch64 a zero-frame leaf starts directly with its original RET: no
+; eight-instruction (32-byte) stack-growth sequence precedes it.
+; A64-ASM-LABEL: zero_frame_leaf:
+; A64-ASM-NOT: runtime.morestack
+; A64-ASM: ret
+; A64-ASM-LABEL: small_frame_leaf:
+; A64-ASM-NOT: runtime.morestack
+; A64-ASM: ret
+; A64-ASM-LABEL: large_frame_leaf:
+; A64-ASM: bl "runtime.morestack_noctxt<ABI0>"
+; A64-ASM-LABEL: small_frame_non_leaf:
+; A64-ASM: bl "runtime.morestack_noctxt<ABI0>"
+; A64-ASM-LABEL: small_frame_x86_leaf_like_runtime_call:
+; A64-ASM: bl "runtime.morestack_noctxt<ABI0>"
+; A64-ASM-LABEL: x86_leaf_like_runtime_call_at_limit:
+; A64-ASM: bl "runtime.morestack_noctxt<ABI0>"
+; A64-ASM-LABEL: nosplit_large_non_leaf:
+; A64-ASM-NOT: runtime.morestack
+; A64-ASM: bl callee
+
+; X86-ASM-LABEL: zero_frame_leaf:
+; X86-ASM-NOT: runtime.morestack
+; X86-ASM: retq
+; X86-ASM-LABEL: small_frame_leaf:
+; X86-ASM-NOT: runtime.morestack
+; X86-ASM: retq
+; X86-ASM-LABEL: large_frame_leaf:
+; X86-ASM: callq "runtime.morestack_noctxt<ABI0>"
+; X86-ASM-LABEL: small_frame_non_leaf:
+; X86-ASM: callq "runtime.morestack_noctxt<ABI0>"
+; X86-ASM-LABEL: small_frame_x86_leaf_like_runtime_call:
+; X86-ASM-NOT: runtime.morestack
+; X86-ASM: callq runtime.panicdivide
+; X86-ASM-LABEL: x86_leaf_like_runtime_call_at_limit:
+; X86-ASM: callq "runtime.morestack_noctxt<ABI0>"
+; X86-ASM-LABEL: nosplit_large_non_leaf:
+; X86-ASM-NOT: runtime.morestack
+; X86-ASM: callq callee

llvm/test/CodeGen/X86/goobj-function-abi.ll

@@ -16,4 +16,6 @@ entry:
 ; CHECK-NEXT: symdef 1: internal_func abi=1 type=1 size={{[0-9]+}}
 ; CHECK-NEXT: symdef 2:  abi=0 type=3 size={{[0-9]+}}
 ; CHECK-NEXT: symdef 3:  abi=0 type=3 size={{[0-9]+}}
-; CHECK: nonpkgref {{[0-9]+}}: runtime.morestack_noctxt abi=0 type=0 size=0
+; Both definitions are zero-frame leaves, so native-Go stack-check policy does
+; not introduce an otherwise unrelated runtime reference.
+; CHECK-NOT: runtime.morestack

llvm/test/CodeGen/X86/goobj-function-unsafe.ll

@@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" }
 
 ; OBJ: symdef 0: main.safe
 ; OBJ: symdef 1: main.async_unsafe
-; OBJ: aux 0.6: type=pcdata target= pc=[0-48:-1]
-; OBJ: aux 1.14: type=pcdata target= pc=[0-33:-2]
+; OBJ: aux 0.6: type=pcdata target= pc=[0-4:-1]
+; OBJ: aux 1.14: type=pcdata target= pc=[0-4:-2]

llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll

@@ -57,12 +57,12 @@ done:
 ; COMMON-NEXT: file 0: {{.*}}cdebug.c
 ; COMMON: hasheddef-count: 5
 ; COMMON: nonpkgdef-count: 0
-; X86: hash 0: b3ea8d63c732dc89ddc1accf4206a7bb
-; X86-NEXT: hash 1: 2f05713252bb1b9673be7cf4f639e025
-; X86-NEXT: hash 2: 2d9a46e763fb2afa2f6bf504aef97634
-; ARM64: hash 0: f933ff4cb32d167004d69c3934a4e464
-; ARM64-NEXT: hash 1: cbf5923fb1da07cca3254ffa87009bcc
-; ARM64-NEXT: hash 2: ba4d2125291d2de2a0592dd4bdc679e7
+; X86: hash 0: 76ebad1f8f49595ad91b08e48dc9dd56
+; X86-NEXT: hash 1: 2e8b983aec877a1c003f94b814d56e69
+; X86-NEXT: hash 2: 1689233d6b8d24e3422d5ebae6f408ee
+; ARM64: hash 0: 105a58e8d53963b571ff833d8449eeda
+; ARM64-NEXT: hash 1: 3bef6118e9cf260f78533cdf1a6375ec
+; ARM64-NEXT: hash 2: 3a577c7591ae76fd8b51f8e7ea4ac9d8
 ; COMMON-NEXT: hash 3: 90eb4206729e98003d69bb53d855ecb0
 ; COMMON-NEXT: hash 4: 4b0e7a681c0340c9a97ef4802a3af2f8
 ; COMMON: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=010000000100000000 pkg=hashed sym=3
@@ -75,4 +75,4 @@ done:
 ; COMMON-SAME: :5
 ; COMMON-SAME: pkg=hashed sym=1
 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=2
-; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}}:0{{.*}} pkg=hashed sym=0
+; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}}:-1{{.*}} pkg=hashed sym=2

llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll

@@ -69,7 +69,7 @@ join:
 ; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}}
 ; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}}
 ; CHECK: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1]
-; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:0]
+; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1]
 ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack_noctxt
 ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack
 ; The outgoing frame is 72 bytes of alignment/register-argument space plus 23