From f931e51484bdf844c99c647aae64784fb2907f67 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 16:40:34 +0800 Subject: [PATCH 1/2] [Go] Model runtime stack semantics --- llvm/include/llvm/CodeGen/GoCallingConv.h | 11 +++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 38 +++++----- .../Target/AArch64/AArch64FrameLowering.cpp | 69 ++++++++++++++++--- .../Target/AArch64/AArch64ISelLowering.cpp | 8 ++- llvm/lib/Target/X86/X86FrameLowering.cpp | 61 ++++++++++++++-- llvm/test/CodeGen/AArch64/go-callconv.ll | 3 +- .../AArch64/goobj-stack-growth-statepoint.ll | 24 +++++++ llvm/test/CodeGen/Generic/goobj-nosplit.ll | 48 +++++++++++++ .../X86/goobj-stack-growth-statepoint.ll | 19 +++++ 9 files changed, 248 insertions(+), 33 deletions(-) create mode 100644 llvm/test/CodeGen/Generic/goobj-nosplit.ll diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index 8666893bfd616..a3dc20ea9521f 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -36,8 +36,19 @@ inline constexpr StringLiteral PadTypeName = "go.abi.pad"; // lowering represent the late morestack call with a root-free STATEPOINT. inline constexpr StringLiteral StackGrowthStatepointAttr = "go-stack-growth-statepoint"; +// Target frame lowering must not synthesize a morestack edge for such a +// function, even though StackGrowthStatepointAttr remains present to describe +// the entry-argument map and statepoint form used when growth is permitted. +inline constexpr StringLiteral NoSplitAttr = "go-nosplit"; +// A //go:systemstack function checks g.stackguard1 and traps through +// runtime.morestackc if it is entered on an ordinary goroutine stack. +inline constexpr StringLiteral SystemStackAttr = "go-systemstack"; inline constexpr uint64_t StackGrowthStatepointID = GoObj::StackGrowthStatepointID; +// A nosplit function has no morestack call at which to attach the entry +// argument map. Targets use a zero-byte STACKMAP with this internal ID at PC +// zero; the GoALLC stack-map bridge translates it to the ordinary entry map. +inline constexpr uint64_t NoSplitEntryStackMapID = 0x476f4e6f53706c74ULL; inline bool isGoABIInternalCallingConv(CallingConv::ID CC) { return CC == CallingConv::GoABIInternal; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 7dbd2c879e3f3..1f4ce20a20da9 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -698,7 +698,13 @@ bool AsmPrinter::doInitialization(Module &M) { if ((Target.isOSWindows() || (Target.isUEFI() && EmitCodeView)) && M.getNamedMetadata("llvm.dbg.cu")) Handlers.push_back(std::make_unique(this)); - if (!EmitCodeView || M.getDwarfVersion()) { + // GoObj consumes source locations through GoObjDebug to build pcfile, + // pcline, and pcinline after final machine layout. Generic monolithic + // DWARF sections are not Go linker carriers: their relocations can point + // at functions removed by Go dead-code elimination. Keep target-object + // DWARF disabled until it is represented by Go's per-function carriers. + if (!Target.isOSBinFormatGoObj() && + (!EmitCodeView || M.getDwarfVersion())) { if (hasDebugInfo()) { DD = createDwarfDebug(); Handlers.push_back(std::unique_ptr(DD)); @@ -1081,12 +1087,12 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { const MDNode *MD = GO.getMetadata("goobj.symbol.nonpackage"); if (!MD) continue; - const auto *Marker = MD->getNumOperands() == 1 - ? mdconst::dyn_extract( - MD->getOperand(0)) - : nullptr; - if (!Marker || !Marker->getType()->isIntegerTy(1) || - !Marker->isOne() || GO.isDeclaration()) + const auto *Marker = + MD->getNumOperands() == 1 + ? mdconst::dyn_extract(MD->getOperand(0)) + : nullptr; + if (!Marker || !Marker->getType()->isIntegerTy(1) || !Marker->isOne() || + GO.isDeclaration()) report_fatal_error("invalid !goobj.symbol.nonpackage attachment"); AP.OutContext.setGoObjSymbolNonPackage(AP.getSymbol(&GO)); } @@ -1165,8 +1171,7 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { } } - if (const NamedMDNode *Markers = - M.getNamedMetadata("goobj.marker_relocs")) { + if (const NamedMDNode *Markers = M.getNamedMetadata("goobj.marker_relocs")) { DenseMap> Relocs; for (const MDNode *Entry : Markers->operands()) { @@ -1174,11 +1179,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { report_fatal_error( "expected !goobj.marker_relocs entries to have four operands"); const GlobalValue *Source = - getGoObjMetadataGlobal(Entry->getOperand(0), - "goobj.marker_relocs"); + getGoObjMetadataGlobal(Entry->getOperand(0), "goobj.marker_relocs"); const GlobalValue *Target = - getGoObjMetadataGlobal(Entry->getOperand(1), - "goobj.marker_relocs"); + getGoObjMetadataGlobal(Entry->getOperand(1), "goobj.marker_relocs"); const auto *Type = mdconst::dyn_extract(Entry->getOperand(2)); const auto *Addend = @@ -1195,9 +1198,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { default: report_fatal_error("unsupported !goobj.marker_relocs type"); } - Relocs[Source].push_back( - {AP.getSymbol(Target), static_cast(Type->getZExtValue()), - Addend->getSExtValue()}); + Relocs[Source].push_back({AP.getSymbol(Target), + static_cast(Type->getZExtValue()), + Addend->getSExtValue()}); } for (auto &[Source, SourceRelocs] : Relocs) AP.OutContext.setGoObjMarkerRelocs(AP.getSymbol(Source), @@ -3746,8 +3749,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { if (TM.getTargetTriple().isOSBinFormatGoObj()) { if (std::optional> Flags = getGoObjSymbolFlags(&F)) - OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, - Flags->second); + OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, Flags->second); if (std::optional> Info = getGoObjFunctionInfo(F)) OutContext.setGoObjFunctionInfo(CurrentFnSym, Info->first, Info->second); diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 6e555ce21954a..c3bb495185790 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1248,9 +1248,16 @@ namespace { constexpr uint64_t GoStackSmall = 128; constexpr uint64_t GoStackBig = 4096; constexpr int64_t GoGStackGuard0Offset = 16; +constexpr int64_t GoGStackGuard1Offset = 24; static bool shouldEmitAArch64GoStackCheck(const MachineFunction &MF) { - return AArch64FrameLowering::usesGoFrameLayout(MF); + return AArch64FrameLowering::usesGoFrameLayout(MF) && + !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); +} + +static bool isAArch64GoNoSplitFunction(const MachineFunction &MF) { + return AArch64FrameLowering::usesGoFrameLayout(MF) && + MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); } static bool hasAArch64GoClosureContext(const Function &F) { @@ -1336,6 +1343,47 @@ getAArch64GoStackCheckEntryMBB(MachineFunction &MF, return FallbackMBB; } +static void emitAArch64GoNoSplitEntryStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!isAArch64GoNoSplitFunction(MF) || + !MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) + return; + for (const MachineBasicBlock &MBB : MF) + for (const MachineInstr &MI : MBB) + if (MI.getOpcode() == TargetOpcode::STACKMAP && + MI.getOperand(0).isImm() && + static_cast(MI.getOperand(0).getImm()) == + goabi::NoSplitEntryStackMapID) + return; + + MachineBasicBlock &EntryMBB = getAArch64GoStackCheckEntryMBB(MF, FallbackMBB); + const AArch64InstrInfo &TII = + *MF.getSubtarget().getInstrInfo(); + MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), + TII.get(TargetOpcode::STACKMAP)) + .addImm(goabi::NoSplitEntryStackMapID) + .addImm(0); + const MachineFrameInfo &MFI = MF.getFrameInfo(); + uint64_t PointerSize = MF.getDataLayout().getPointerSize(); + for (const AArch64FunctionInfo::GoArgPointerSlot &Slot : + MF.getInfo()->getGoArgPointerSlots()) { + if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) + report_fatal_error( + "AArch64 Go entry argument pointer slot is not a fixed object"); + int64_t Offset = MFI.getObjectOffset(Slot.FrameIndex) + + static_cast(Slot.OffsetWithinObject); + int64_t ExpectedOffset = static_cast(PointerSize) + + static_cast(Slot.ArgWord) * PointerSize; + if (PointerSize == 0 || Offset != ExpectedOffset || !isInt<32>(Offset)) + report_fatal_error( + "AArch64 Go entry argument pointer slot has invalid SP offset"); + StackMap.addImm(StackMaps::IndirectMemRefOp) + .addImm(PointerSize) + .addReg(AArch64::SP) + .addImm(Offset); + } +} + static unsigned getAArch64GoSpillOpcode(unsigned Size, bool IsFP, bool Reload) { if (IsFP) { switch (Size) { @@ -1475,6 +1523,7 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, bool UseStackGrowthStatepoint = MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); + bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); Register ScratchReg = AArch64::SP; if (StackSize > GoStackBig) { @@ -1502,7 +1551,8 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, BuildMI(CompareMBB, DL, TII.get(AArch64::LDRXui), AArch64::X17) .addReg(AArch64::X28) - .addImm(GoGStackGuard0Offset / 8); + .addImm((IsSystemStack ? GoGStackGuard1Offset : GoGStackGuard0Offset) / + 8); BuildMI(CompareMBB, DL, TII.get(AArch64::SUBSXrx64), AArch64::XZR) .addReg(ScratchReg) .addReg(AArch64::X17) @@ -1516,16 +1566,18 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, BuildMI(MorestackMBB, DL, TII.get(TargetOpcode::COPY), AArch64::X3) .addReg(AArch64::LR); bool HasClosureContext = hasAArch64GoClosureContext(MF.getFunction()); - const char *MorestackName = - HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; + const char *MorestackName = IsSystemStack ? "runtime.morestackc" + : HasClosureContext ? "runtime.morestack" + : "runtime.morestack_noctxt"; MachineInstrBuilder Morestack = - UseStackGrowthStatepoint ? buildAArch64GoStackGrowthStatepoint( - MF, *MorestackMBB, DL, TII, MorestackName) - : BuildMI(MorestackMBB, DL, TII.get(AArch64::BL)); + UseStackGrowthStatepoint + ? buildAArch64GoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, + MorestackName) + : BuildMI(MorestackMBB, DL, TII.get(AArch64::BL)); if (!UseStackGrowthStatepoint) goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); Morestack.addReg(AArch64::X3, RegState::Implicit); - if (HasClosureContext) + if (HasClosureContext && !IsSystemStack) Morestack.addReg(AArch64::X26, RegState::Implicit); emitAArch64GoRegSpills(MF, *MorestackMBB, AFI->getGoArgHomes(), /*Reload=*/true); @@ -1559,6 +1611,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, report_fatal_error("GoObj stack growth does not support dynamic allocas"); AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this); PrologueEmitter.emitPrologue(); + emitAArch64GoNoSplitEntryStackMap(MF, MBB); emitAArch64GoStackCheck(MF, MBB); } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d053a411fd991..2fb204536838e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9337,7 +9337,13 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); - unsigned NumBytes = Layout.TotalStackSize + StackBias; + // Layout.TotalStackSize rounds the logical Go argument area to the target + // stack alignment before the physical entry-SP bias is applied. Adding the + // bias after that rounding reserves an extra word for one-word calls (the + // common ABI0 funcval case) and inflates every containing nosplit frame. + // The caller frame itself remains stack-aligned; reserve only the bytes + // through the last physical argument home here. + unsigned NumBytes = Layout.ArgSize + StackBias; Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index bc779ead6c3f3..ef219c34666e1 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -146,6 +146,7 @@ namespace { constexpr uint64_t GoStackSmall = 128; constexpr uint64_t GoStackBig = 4096; constexpr int64_t GoGStackGuard0Offset = 16; +constexpr int64_t GoGStackGuard1Offset = 24; static unsigned getIntegerStoreOpcode(unsigned Size) { switch (Size) { @@ -221,7 +222,16 @@ static bool shouldEmitGoStackCheck(const MachineFunction &MF) { const Function &F = MF.getFunction(); return MF.getTarget().getTargetTriple().isOSBinFormatGoObj() && MF.getTarget().getTargetTriple().getArch() == Triple::x86_64 && - goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg(); + goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg() && + !F.hasFnAttribute(goabi::NoSplitAttr); +} + +static bool isGoNoSplitFunction(const MachineFunction &MF) { + const Function &F = MF.getFunction(); + return MF.getTarget().getTargetTriple().isOSBinFormatGoObj() && + MF.getTarget().getTargetTriple().getArch() == Triple::x86_64 && + goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg() && + F.hasFnAttribute(goabi::NoSplitAttr); } static void checkGoStackGrowthStatepointContract(const MachineFunction &MF) { @@ -310,6 +320,44 @@ getGoStackCheckEntryMBB(MachineFunction &MF, MachineBasicBlock &FallbackMBB) { return FallbackMBB; } +static void emitGoNoSplitEntryStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!isGoNoSplitFunction(MF) || + !MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) + return; + for (const MachineBasicBlock &MBB : MF) + for (const MachineInstr &MI : MBB) + if (MI.getOpcode() == TargetOpcode::STACKMAP && + MI.getOperand(0).isImm() && + static_cast(MI.getOperand(0).getImm()) == + goabi::NoSplitEntryStackMapID) + return; + + MachineBasicBlock &EntryMBB = getGoStackCheckEntryMBB(MF, FallbackMBB); + const X86InstrInfo &TII = *MF.getSubtarget().getInstrInfo(); + MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), + TII.get(TargetOpcode::STACKMAP)) + .addImm(goabi::NoSplitEntryStackMapID) + .addImm(0); + const MachineFrameInfo &MFI = MF.getFrameInfo(); + uint64_t PointerSize = MF.getDataLayout().getPointerSize(); + for (const X86MachineFunctionInfo::GoArgPointerSlot &Slot : + MF.getInfo()->getGoArgPointerSlots()) { + if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) + report_fatal_error( + "X86 Go entry argument pointer slot is not a fixed object"); + int64_t ExpectedOffset = static_cast(PointerSize) + + static_cast(Slot.ArgWord) * PointerSize; + if (PointerSize == 0 || Slot.EntryOffset != ExpectedOffset) + report_fatal_error( + "X86 Go entry argument pointer slot has invalid RSP offset"); + StackMap.addImm(StackMaps::IndirectMemRefOp) + .addImm(PointerSize) + .addReg(X86::RSP) + .addImm(Slot.EntryOffset); + } +} + static void emitGoStackCheck(MachineFunction &MF, MachineBasicBlock &PrologueMBB) { if (!shouldEmitGoStackCheck(MF)) @@ -380,6 +428,7 @@ static void emitGoStackCheck(MachineFunction &MF, bool UseStackGrowthStatepoint = MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); + bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); checkGoStackGrowthStatepointContract(MF); unsigned ScratchReg = X86::R12; @@ -407,7 +456,7 @@ static void emitGoStackCheck(MachineFunction &MF, .addReg(X86::R14) .addImm(1) .addReg(X86::NoRegister) - .addImm(GoGStackGuard0Offset) + .addImm(IsSystemStack ? GoGStackGuard1Offset : GoGStackGuard0Offset) .addReg(X86::NoRegister); BuildMI(CompareMBB, DL, TII.get(X86::JCC_1)) .addMBB(&EntryMBB) @@ -415,8 +464,9 @@ static void emitGoStackCheck(MachineFunction &MF, emitGoRegSpills(MF, *MorestackMBB, Homes, /*Reload=*/false); bool HasClosureContext = hasGoClosureContext(MF.getFunction()); - const char *MorestackName = - HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; + const char *MorestackName = IsSystemStack ? "runtime.morestackc" + : HasClosureContext ? "runtime.morestack" + : "runtime.morestack_noctxt"; MachineInstrBuilder Morestack = UseStackGrowthStatepoint ? buildGoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, @@ -424,7 +474,7 @@ static void emitGoStackCheck(MachineFunction &MF, : BuildMI(MorestackMBB, DL, TII.get(X86::CALL64pcrel32)); if (!UseStackGrowthStatepoint) goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); - if (HasClosureContext) + if (HasClosureContext && !IsSystemStack) Morestack.addReg(X86::RDX, RegState::Implicit); emitGoRegSpills(MF, *MorestackMBB, Homes, /*Reload=*/true); BuildMI(MorestackMBB, DL, TII.get(X86::JMP_1)).addMBB(CheckMBB); @@ -1967,6 +2017,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, DebugLoc DL; Register ArgBaseReg; + emitGoNoSplitEntryStackMap(MF, MBB); emitGoStackCheck(MF, MBB); // Emit extra prolog for argument stack slot reference. diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index c7544538eba30..d41426c27aa01 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -71,7 +71,8 @@ define goabi0 i64 @abi0_call_second_int() { ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] -; A64: str x[[RET]], [sp, #72] +; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. +; A64: str x[[RET]], [sp, #56] entry: %ret = call goabi0 i64 @abi0_second_int(i64 11, i64 22) ret i64 %ret diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll index afb9de6d20cbc..6ac0a3a2df8e4 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll @@ -2,6 +2,10 @@ ; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \ ; RUN: -stop-after=prolog-epilog < %s | FileCheck %s +declare !goobj.symbol.name !0 goabi0 void @runtime.morestack.goallc.abi0() +declare !goobj.symbol.name !1 goabi0 void @runtime.morestack_noctxt.goallc.abi0() +declare !goobj.symbol.name !2 goabi0 void @runtime.morestackc.goallc.abi0() + define goabiinternal i64 @closure_morestack_statepoint( i64 %value, ptr nest %ctxt) "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { @@ -37,6 +41,15 @@ entry: ret ptr %pointer } +define goabiinternal void @systemstack_growth() "frame-pointer"="non-leaf" + "go-stack-growth-statepoint" "go-systemstack" { +entry: + %buf = alloca [8192 x i8], align 16 + %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 + store volatile i8 1, ptr %slot, align 1 + ret void +} + ; CHECK-LABEL: name: closure_morestack_statepoint ; CHECK-NOT: ANNOTATION_LABEL ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack", @@ -65,3 +78,14 @@ entry: ; CHECK-SAME: csr_aarch64_go, implicit-def $sp, ; CHECK-SAME: implicit-def dead early-clobber $lr, ; CHECK-SAME: implicit $x3 + +; CHECK-LABEL: name: systemstack_growth +; CHECK: $x17 = LDRXui $x28, 3 +; CHECK: STATEPOINT 5147424658422983495, 0, 0, @runtime.morestackc.goallc.abi0, +; CHECK-SAME: csr_aarch64_go, implicit-def $sp, +; CHECK-SAME: implicit-def dead early-clobber $lr, +; CHECK-SAME: implicit $x3 + +!0 = !{!"runtime.morestack"} +!1 = !{!"runtime.morestack_noctxt"} +!2 = !{!"runtime.morestackc"} diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll new file mode 100644 index 0000000000000..9726ef2a491ba --- /dev/null +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -0,0 +1,48 @@ +; REQUIRES: aarch64-registered-target, x86-registered-target +; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog < %s | FileCheck %s --check-prefixes=CHECK,A64 +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog < %s | FileCheck %s + +declare goabiinternal void @callee(ptr) +declare goabi0 void @callee.abi0(ptr) + +define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" + "go-stack-growth-statepoint" { +entry: + %slot = alloca ptr, align 8 + store volatile ptr %pointer, ptr %slot, align 8 + call goabiinternal void @callee(ptr %pointer) + ret void +} + +define goabiinternal void @split(ptr %pointer) + "go-stack-growth-statepoint" { +entry: + %slot = alloca ptr, align 8 + store volatile ptr %pointer, ptr %slot, align 8 + call goabiinternal void @callee(ptr %pointer) + ret void +} + +define goabiinternal void @nosplit_abi0_call(ptr %pointer) "go-nosplit" + "go-stack-growth-statepoint" { +entry: + %closure = alloca [3 x ptr], align 8 + %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 + %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 + store volatile ptr @callee, ptr %code, align 8 + store volatile ptr %pointer, ptr %context, align 8 + call goabi0 void @callee.abi0(ptr %closure) + ret void +} + +; CHECK-LABEL: name: nosplit +; CHECK: STACKMAP +; CHECK-NOT: runtime.morestack +; CHECK-LABEL: name: split +; CHECK: runtime.morestack_noctxt + +; A64-LABEL: name: nosplit_abi0_call +; A64: stackSize: 48 +; A64: maxCallFrameSize: 16 diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll index 005b774550eb6..08ccc82a53fa9 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll @@ -12,6 +12,8 @@ } declare goabiinternal void @use_three_pointers(ptr, ptr, ptr) +declare !goobj.symbol.name !0 goabi0 void @runtime.morestack_noctxt.goallc.abi0() +declare !goobj.symbol.name !1 goabi0 void @runtime.morestackc.goallc.abi0() define goabiinternal i64 @morestack_statepoint(i64 %value) "go-stack-growth-statepoint" { entry: @@ -76,6 +78,15 @@ entry: ret { ptr, ptr } %r1 } +define goabiinternal void @systemstack_growth() + "go-stack-growth-statepoint" "go-systemstack" { +entry: + %buf = alloca [5000 x i8], align 8 + %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 + store volatile i8 1, ptr %slot, align 1 + ret void +} + ; CHECK-LABEL: name: morestack_statepoint ; CHECK-NOT: ANNOTATION_LABEL ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", @@ -134,3 +145,11 @@ entry: ; CHECK-SAME: 1, 8, $rsp, 48, 1, 8, $rsp, 64, ; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp + +; CHECK-LABEL: name: systemstack_growth +; CHECK: CMP64rm $r12, $r14, 1, $noreg, 24, $noreg +; CHECK: STATEPOINT 5147424658422983495, 0, 0, @runtime.morestackc.goallc.abi0, +; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp + +!0 = !{!"runtime.morestack_noctxt"} +!1 = !{!"runtime.morestackc"} From c2800d8975e995b867e44680e558fa41f5977a79 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 00:38:33 +0800 Subject: [PATCH 2/2] [Go] Make stack growth the default GoObj policy --- llvm/include/llvm/BinaryFormat/GoObj.h | 8 +- llvm/include/llvm/CodeGen/GoCallingConv.h | 15 +-- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 103 ++++++++++++++++++ llvm/lib/MC/GoObjObjectWriter.cpp | 58 +++++++--- .../Target/AArch64/AArch64FrameLowering.cpp | 77 +++---------- .../Target/AArch64/AArch64ISelLowering.cpp | 26 ++--- llvm/lib/Target/X86/X86FrameLowering.cpp | 91 ++++------------ llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 26 ++--- .../CodeGen/AArch64/goobj-function-unsafe.ll | 4 +- llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll | 12 +- .../AArch64/goobj-register-argument-homes.ll | 6 +- .../AArch64/goobj-stack-growth-statepoint.ll | 33 +++--- .../Generic/go-statepoint-alloca-address.ll | 2 +- llvm/test/CodeGen/Generic/goobj-nosplit.ll | 22 ++-- .../goobj-statepoint-default-stack-growth.ll | 31 ++++++ ...tepoint-requires-stack-growth-attribute.ll | 27 ----- llvm/test/CodeGen/X86/go-stack-alignment.ll | 2 +- .../X86/goobj-alloca-argument-homes.ll | 7 +- .../test/CodeGen/X86/goobj-function-unsafe.ll | 4 +- llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll | 25 +++-- llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll | 2 +- .../X86/goobj-stack-growth-metadata.ll | 11 +- .../X86/goobj-stack-growth-statepoint.ll | 54 +++++---- 23 files changed, 330 insertions(+), 316 deletions(-) create mode 100644 llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll delete mode 100644 llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll diff --git a/llvm/include/llvm/BinaryFormat/GoObj.h b/llvm/include/llvm/BinaryFormat/GoObj.h index c4a370aba1b00..f63ae9cae61d4 100644 --- a/llvm/include/llvm/BinaryFormat/GoObj.h +++ b/llvm/include/llvm/BinaryFormat/GoObj.h @@ -23,8 +23,14 @@ namespace GoObj { // Go function. GoObj serialization strips it and records ABI0 separately. inline constexpr char ABI0SymbolSuffix[] = ""; +// "GoNoSplt" encoded as the stable STACKMAP identifier for the function-level +// entry argument pointer map. This record is metadata-only: it is present for +// both split and nosplit functions and never denotes a callsite. +inline constexpr uint64_t EntryArgsStackMapID = 0x476f4e6f53706c74ULL; + // "GoStackG" encoded as the stable statepoint identifier for the pre-frame -// runtime.morestack slow path. +// runtime.morestack slow path. Unlike EntryArgsStackMapID, this record denotes +// a real callsite and is absent from nosplit functions. inline constexpr uint64_t StackGrowthStatepointID = 0x476f537461636b47ULL; // GoALLC encodes pointer maps for fixed allocas as a self-describing suffix of diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index a3dc20ea9521f..e2d399f216121 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -32,23 +32,18 @@ namespace goabi { inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple"; inline constexpr StringLiteral PadTypeName = "go.abi.pad"; -// The Go statepoint pass uses this attribute to request that target frame -// lowering represent the late morestack call with a root-free STATEPOINT. -inline constexpr StringLiteral StackGrowthStatepointAttr = - "go-stack-growth-statepoint"; // Target frame lowering must not synthesize a morestack edge for such a -// function, even though StackGrowthStatepointAttr remains present to describe -// the entry-argument map and statepoint form used when growth is permitted. +// function. GoObj Go functions otherwise use the native Go default: emit a +// stack check and represent its late morestack call as a root-free statepoint. inline constexpr StringLiteral NoSplitAttr = "go-nosplit"; // A //go:systemstack function checks g.stackguard1 and traps through // runtime.morestackc if it is entered on an ordinary goroutine stack. inline constexpr StringLiteral SystemStackAttr = "go-systemstack"; inline constexpr uint64_t StackGrowthStatepointID = GoObj::StackGrowthStatepointID; -// A nosplit function has no morestack call at which to attach the entry -// argument map. Targets use a zero-byte STACKMAP with this internal ID at PC -// zero; the GoALLC stack-map bridge translates it to the ordinary entry map. -inline constexpr uint64_t NoSplitEntryStackMapID = 0x476f4e6f53706c74ULL; +// Every GoObj Go function carries its entry argument pointer map in a +// zero-byte STACKMAP. It is function metadata, not a stack-growth callsite. +inline constexpr uint64_t EntryArgsStackMapID = GoObj::EntryArgsStackMapID; inline bool isGoABIInternalCallingConv(CallingConv::ID CC) { return CC == CallingConv::GoABIInternal; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 1f4ce20a20da9..637367158b2fe 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -464,6 +464,102 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL, return Alignment; } +static uint64_t +getGoObjStackMapNonnegativeConstant(const StackMaps::Location &Location, + StringRef Description) { + if (Location.Type != StackMaps::Location::Constant || Location.Offset < 0) + report_fatal_error("malformed GoObj stackmap " + Description); + return static_cast(Location.Offset); +} + +static MCContext::GoObjStackMapLocation::LocationType +convertGoObjStackMapLocationType(StackMaps::Location::LocationType Type) { + using GoLocation = MCContext::GoObjStackMapLocation; + switch (Type) { + case StackMaps::Location::Unprocessed: + return GoLocation::Unprocessed; + case StackMaps::Location::Register: + return GoLocation::Register; + case StackMaps::Location::Direct: + return GoLocation::Direct; + case StackMaps::Location::Indirect: + return GoLocation::Indirect; + case StackMaps::Location::Constant: + return GoLocation::Constant; + case StackMaps::Location::ConstantIndex: + return GoLocation::ConstantIndex; + } + llvm_unreachable("unknown StackMaps location type"); +} + +static void emitGoObjStackMaps(StackMaps &SM, AsmPrinter &AP) { + auto &Callsites = SM.getCSInfos(); + auto Callsite = Callsites.begin(); + uint32_t PointerSize = AP.getPointerSize(); + if (!PointerSize) + report_fatal_error("GoObj statepoint target has no pointer size"); + + for (const auto &[Function, Info] : SM.getFnInfos()) { + for (uint64_t I = 0; I != Info.RecordCount; ++I) { + if (Callsite == Callsites.end()) + report_fatal_error( + "GoObj stackmap function record count exceeds callsites"); + + const StackMaps::CallsiteInfo &CSI = *Callsite++; + bool IsEntryArgs = CSI.ID == GoObj::EntryArgsStackMapID; + uint64_t NumDeopts = 0; + ArrayRef Locations = CSI.Locations; + if (!IsEntryArgs) { + // LLVM's statepoint parser prefixes locations with the calling + // convention, flags, deopt count, and then the deopt operands. These + // entries are not GC roots. EntryArgsStackMapID is a plain STACKMAP + // and contains only function-level argument pointer homes. + if (Locations.size() < 3) + report_fatal_error("malformed GoObj statepoint location list"); + (void)getGoObjStackMapNonnegativeConstant(Locations[0], + "calling convention"); + (void)getGoObjStackMapNonnegativeConstant(Locations[1], "flags"); + NumDeopts = + getGoObjStackMapNonnegativeConstant(Locations[2], "deopt count"); + if (NumDeopts > Locations.size() - 3) + report_fatal_error("malformed GoObj statepoint deopt operands"); + if (NumDeopts > std::numeric_limits::max()) + report_fatal_error("GoObj statepoint has too many deopt operands"); + Locations = Locations.drop_front(3); + } + + MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, + CSI.ID, + CSI.IsIndirectCall, + Info.StackSize, + PointerSize, + static_cast(NumDeopts), + {}}; + Entry.Locations.reserve(Locations.size()); + for (const StackMaps::Location &Location : Locations) { + auto Type = convertGoObjStackMapLocationType(Location.Type); + int64_t Offset = Location.Offset; + if (Location.Type == StackMaps::Location::Constant || + Location.Type == StackMaps::Location::ConstantIndex) { + std::optional Constant = SM.getConstantValue(Location); + if (!Constant) + report_fatal_error( + "GoObj statepoint contains an invalid constant-pool index"); + Type = MCContext::GoObjStackMapLocation::Constant; + Offset = *Constant; + } + Entry.Locations.push_back({Type, Location.Size, Location.Reg, Offset}); + } + AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry)); + } + } + + if (Callsite != Callsites.end()) + report_fatal_error( + "GoObj stackmap callsites exceed function record counts"); + SM.reset(); +} + AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer, char &ID) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), @@ -503,6 +599,10 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer, MP->finishAssembly(M, *MI, *this); }; EmitStackMaps = [this](Module &M) { + if (OutContext.isGoObj()) { + emitGoObjStackMaps(SM, *this); + return; + } GCModuleInfo *MI = getAnalysisIfAvailable(); assert(MI && "AsmPrinter didn't require GCModuleInfo?"); bool NeedsDefault = false; @@ -878,6 +978,9 @@ getGoObjSymbolFlags(const GlobalObject *GO) { Flag2 |= GoObj::SymFlagItab; } } + if (const auto *F = dyn_cast(GO); + F && F->hasFnAttribute(goabi::NoSplitAttr)) + Flag |= GoObj::SymFlagNoSplit; if (const MDNode *MD = GO->getMetadata("goobj.symbol.flags")) { if (MD->getNumOperands() != 2) diff --git a/llvm/lib/MC/GoObjObjectWriter.cpp b/llvm/lib/MC/GoObjObjectWriter.cpp index f1252339a1abc..68b22ec4d4406 100644 --- a/llvm/lib/MC/GoObjObjectWriter.cpp +++ b/llvm/lib/MC/GoObjObjectWriter.cpp @@ -691,17 +691,18 @@ GoObjStatepointStackMaps makeStatepointStackMaps( SmallVector FunctionAllocaRecords; uint64_t OrdinaryEntryCount = llvm::count_if(ResolvedEntries, [](const ResolvedEntry &Resolved) { - return Resolved.Entry->ID != GoObj::StackGrowthStatepointID; + return Resolved.Entry->ID != GoObj::EntryArgsStackMapID && + Resolved.Entry->ID != GoObj::StackGrowthStatepointID; }); std::optional FunctionOpenDefer; uint64_t OpenDeferEntryCount = 0; for (const ResolvedEntry &Resolved : ResolvedEntries) { std::optional Record = parseOpenDeferRecord(*Resolved.Entry); - if (Resolved.Entry->ID == GoObj::StackGrowthStatepointID) { + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID || + Resolved.Entry->ID == GoObj::StackGrowthStatepointID) { if (Record) - report_fatal_error( - "GoObj stack-growth statepoint contains open-defer state"); + report_fatal_error("GoObj entry metadata contains open-defer state"); continue; } if (!Record) @@ -757,6 +758,7 @@ GoObjStatepointStackMaps makeStatepointStackMaps( } auto BuildPair = [&](const MCContext::GoObjStackMapEntry &Entry) { + bool IsEntryArgs = Entry.ID == GoObj::EntryArgsStackMapID; bool IsStackGrowth = Entry.ID == GoObj::StackGrowthStatepointID; GoObjStackMapPair Pair{SmallVector(ArgsBytesPerBitmap, 0), SmallVector(LocalsBytesPerBitmap, 0)}; @@ -769,9 +771,8 @@ GoObjStatepointStackMaps makeStatepointStackMaps( ArrayRef(Entry.Locations).drop_front(Entry.NumDeoptLocations); for (const GoObjAllocaPtrMapRecord &Record : parseAllocaPtrMapRecords(Entry)) { - if (IsStackGrowth) - report_fatal_error( - "GoObj stack-growth statepoint contains an alloca ptrmap"); + if (IsEntryArgs || IsStackGrowth) + report_fatal_error("GoObj entry metadata contains an alloca ptrmap"); MCContext::GoObjStackMapLocation RecordBase = NormalizeFrameLocation(Record.Base); if (RecordBase.Size != PointerSize || @@ -914,16 +915,18 @@ GoObjStatepointStackMaps makeStatepointStackMaps( ", offset=" + Twine(Loc.Offset)); for (int64_t WordOffset : *PointerWordOffsets) { - if (IsStackGrowth) { + if (IsEntryArgs) { std::optional Bit = goobj::classifyStackGrowthStackMapSlot( WordOffset, PointerSize, FrameLayout.EntryArgsStart, ArgSize); if (Loc.Type != MCContext::GoObjStackMapLocation::Indirect || !Bit) report_fatal_error( - "GoObj stack-growth statepoint contains an invalid argument " + "GoObj entry argument stack map contains an invalid argument " "pointer slot"); Pair.Args[*Bit / 8] |= uint8_t(1u << (*Bit % 8)); continue; } + if (IsStackGrowth) + report_fatal_error("GoObj stack-growth statepoint is not root-free"); goobj::StackMapSlot Slot = goobj::classifyOrdinaryStackMapSlot( WordOffset, Loc.Type == MCContext::GoObjStackMapLocation::Indirect, @@ -958,24 +961,45 @@ GoObjStatepointStackMaps makeStatepointStackMaps( return Pair; }; + const ResolvedEntry *EntryArgsEntry = nullptr; const ResolvedEntry *StackGrowthEntry = nullptr; for (const ResolvedEntry &Resolved : ResolvedEntries) { - if (Resolved.Entry->ID != GoObj::StackGrowthStatepointID) + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) { + if (EntryArgsEntry) + report_fatal_error( + "GoObj function contains multiple entry argument stack maps"); + EntryArgsEntry = &Resolved; continue; - if (StackGrowthEntry) - report_fatal_error( - "GoObj function contains multiple stack-growth statepoints"); - StackGrowthEntry = &Resolved; + } + if (Resolved.Entry->ID == GoObj::StackGrowthStatepointID) { + if (StackGrowthEntry) + report_fatal_error( + "GoObj function contains multiple stack-growth statepoints"); + StackGrowthEntry = &Resolved; + } } - if (!StackGrowthEntry) - report_fatal_error("GoObj function has no stack-growth statepoint"); + if (!EntryArgsEntry) + report_fatal_error("GoObj function has no entry argument stack map"); + if (EntryArgsEntry->Entry->IsIndirectCall) + report_fatal_error("GoObj entry argument stack map is a callsite"); + + bool IsNoSplit = (Function.Flag & GoObj::SymFlagNoSplit) != 0; + if (IsNoSplit && StackGrowthEntry) + report_fatal_error("GoObj nosplit function has a stack-growth statepoint"); + if (!IsNoSplit && !StackGrowthEntry) + report_fatal_error("GoObj split function has no stack-growth statepoint"); + if (StackGrowthEntry && (StackGrowthEntry->Entry->NumDeoptLocations != 0 || + !StackGrowthEntry->Entry->Locations.empty())) + report_fatal_error("GoObj stack-growth statepoint is not root-free"); SmallVector Pairs; - Pairs.push_back(BuildPair(*StackGrowthEntry->Entry)); + Pairs.push_back(BuildPair(*EntryArgsEntry->Entry)); SmallVector PCDataEntries; std::optional PreviousCallsitePC; SmallVector IndirectCallOffsets; for (const ResolvedEntry &Resolved : ResolvedEntries) { + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) + continue; if (PreviousCallsitePC && *PreviousCallsitePC == Resolved.CallsitePC) report_fatal_error("GoObj statepoint callsites have duplicate PCs"); PreviousCallsitePC = Resolved.CallsitePC; diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index c3bb495185790..02c2a5eca423b 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1255,11 +1255,6 @@ static bool shouldEmitAArch64GoStackCheck(const MachineFunction &MF) { !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); } -static bool isAArch64GoNoSplitFunction(const MachineFunction &MF) { - return AArch64FrameLowering::usesGoFrameLayout(MF) && - MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); -} - static bool hasAArch64GoClosureContext(const Function &F) { for (const Argument &Arg : F.args()) if (Arg.hasNestAttr()) @@ -1267,19 +1262,6 @@ static bool hasAArch64GoClosureContext(const Function &F) { return false; } -static void -checkAArch64GoStackGrowthStatepointContract(const MachineFunction &MF) { - if (!AArch64FrameLowering::usesGoFrameLayout(MF) || - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) - return; - for (const MachineBasicBlock &MBB : MF) - for (const MachineInstr &MI : MBB) - if (MI.getOpcode() == TargetOpcode::STATEPOINT) - report_fatal_error( - "GoObj statepoints require the go-stack-growth-statepoint " - "function attribute"); -} - static MachineInstrBuilder buildAArch64GoStackGrowthStatepoint( MachineFunction &MF, MachineBasicBlock &MBB, const DebugLoc &DL, const AArch64InstrInfo &TII, StringRef CalleeName) { @@ -1292,34 +1274,12 @@ static MachineInstrBuilder buildAArch64GoStackGrowthStatepoint( auto AddConstant = [&](uint64_t Value) { Statepoint.addImm(StackMaps::ConstantOp).addImm(Value); }; - const MachineFrameInfo &MFI = MF.getFrameInfo(); - ArrayRef PointerSlots = - MF.getInfo()->getGoArgPointerSlots(); - uint64_t PointerSize = MF.getDataLayout().getPointerSize(); AddConstant(MF.getFunction().getCallingConv()); - AddConstant(0); // Statepoint flags. - AddConstant(0); // Deopt arguments. - AddConstant(PointerSlots.size()); // GC pointers. - for (const AArch64FunctionInfo::GoArgPointerSlot &Slot : PointerSlots) { - if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) - report_fatal_error( - "AArch64 Go entry argument pointer slot is not a fixed object"); - int64_t Offset = MFI.getObjectOffset(Slot.FrameIndex) + - static_cast(Slot.OffsetWithinObject); - int64_t ExpectedOffset = static_cast(PointerSize) + - static_cast(Slot.ArgWord) * PointerSize; - if (PointerSize == 0 || Offset != ExpectedOffset || !isInt<32>(Offset)) - report_fatal_error( - "AArch64 Go entry argument pointer slot has invalid SP offset"); - Statepoint.addImm(StackMaps::IndirectMemRefOp) - .addImm(PointerSize) - .addReg(AArch64::SP) - .addImm(Offset); - } - AddConstant(0); // GC allocas. - AddConstant(PointerSlots.size()); // GC base/derived map entries. - for (uint64_t I = 0; I != PointerSlots.size(); ++I) - Statepoint.addImm(I).addImm(I); + AddConstant(0); // Statepoint flags. + AddConstant(0); // Deopt arguments. + AddConstant(0); // GC pointers. Entry args live in EntryArgsStackMapID. + AddConstant(0); // GC allocas. + AddConstant(0); // GC base/derived map entries. Statepoint .addRegMask( MF.getSubtarget() @@ -1343,17 +1303,16 @@ getAArch64GoStackCheckEntryMBB(MachineFunction &MF, return FallbackMBB; } -static void emitAArch64GoNoSplitEntryStackMap(MachineFunction &MF, - MachineBasicBlock &FallbackMBB) { - if (!isAArch64GoNoSplitFunction(MF) || - !MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) +static void emitAArch64GoEntryArgsStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!AArch64FrameLowering::usesGoFrameLayout(MF)) return; for (const MachineBasicBlock &MBB : MF) for (const MachineInstr &MI : MBB) if (MI.getOpcode() == TargetOpcode::STACKMAP && MI.getOperand(0).isImm() && static_cast(MI.getOperand(0).getImm()) == - goabi::NoSplitEntryStackMapID) + goabi::EntryArgsStackMapID) return; MachineBasicBlock &EntryMBB = getAArch64GoStackCheckEntryMBB(MF, FallbackMBB); @@ -1361,7 +1320,7 @@ static void emitAArch64GoNoSplitEntryStackMap(MachineFunction &MF, *MF.getSubtarget().getInstrInfo(); MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), TII.get(TargetOpcode::STACKMAP)) - .addImm(goabi::NoSplitEntryStackMapID) + .addImm(goabi::EntryArgsStackMapID) .addImm(0); const MachineFrameInfo &MFI = MF.getFrameInfo(); uint64_t PointerSize = MF.getDataLayout().getPointerSize(); @@ -1464,8 +1423,6 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, report_fatal_error("GoObj stack growth does not support dynamic allocas"); uint64_t StackSize = MFI.getStackSize() + MFI.getUnsafeStackSize(); - if (StackSize == 0 && !MFI.hasCalls()) - return; const DebugLoc DL; const AArch64InstrInfo &TII = @@ -1521,8 +1478,6 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, MF.push_front(CheckMBB); MF.push_front(StartMBB); - bool UseStackGrowthStatepoint = - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); Register ScratchReg = AArch64::SP; @@ -1569,13 +1524,8 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, const char *MorestackName = IsSystemStack ? "runtime.morestackc" : HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; - MachineInstrBuilder Morestack = - UseStackGrowthStatepoint - ? buildAArch64GoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, - MorestackName) - : BuildMI(MorestackMBB, DL, TII.get(AArch64::BL)); - if (!UseStackGrowthStatepoint) - goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); + MachineInstrBuilder Morestack = buildAArch64GoStackGrowthStatepoint( + MF, *MorestackMBB, DL, TII, MorestackName); Morestack.addReg(AArch64::X3, RegState::Implicit); if (HasClosureContext && !IsSystemStack) Morestack.addReg(AArch64::X26, RegState::Implicit); @@ -1606,12 +1556,11 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { - checkAArch64GoStackGrowthStatepointContract(MF); if (usesGoFrameLayout(MF) && MF.getFrameInfo().hasVarSizedObjects()) report_fatal_error("GoObj stack growth does not support dynamic allocas"); AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this); PrologueEmitter.emitPrologue(); - emitAArch64GoNoSplitEntryStackMap(MF, MBB); + emitAArch64GoEntryArgsStackMap(MF, MBB); emitAArch64GoStackCheck(MF, MBB); } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 2fb204536838e..095b899328ead 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9105,13 +9105,9 @@ static SDValue lowerAArch64GoFormalArguments( ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), DAG.getDataLayout(), ABIConfig); - std::optional EntryArgs; - SmallBitVector MatchedEntryArgWords; - if (F.hasFnAttribute(goabi::StackGrowthStatepointAttr)) { - EntryArgs = goabi::computeEntryArgsInfo(ArgTys, Layout, DAG.getDataLayout(), - ABIConfig); - MatchedEntryArgWords.resize(EntryArgs->NumBits); - } + goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( + ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); SmallVector ArgSpillOffsets(ArgTys.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; @@ -9130,10 +9126,8 @@ static SDValue lowerAArch64GoFormalArguments( unsigned StackBias = getAArch64GoStackBias(F.getCallingConv()); auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size) { - if (!EntryArgs) - return; - uint64_t PointerSize = EntryArgs->PointerSize; - for (uint32_t Word : EntryArgs->PointerWords) { + uint64_t PointerSize = EntryArgs.PointerSize; + for (uint32_t Word : EntryArgs.PointerWords) { uint64_t PointerOffset = static_cast(Word) * PointerSize; if (PointerOffset < ArgOffset || PointerOffset + PointerSize > ArgOffset + Size) @@ -9222,12 +9216,10 @@ static SDValue lowerAArch64GoFormalArguments( } } - if (EntryArgs) { - for (uint32_t Word : EntryArgs->PointerWords) - if (!MatchedEntryArgWords.test(Word)) - report_fatal_error( - "Go entry argument pointer word has no AArch64 fixed object"); - } + for (uint32_t Word : EntryArgs.PointerWords) + if (!MatchedEntryArgWords.test(Word)) + report_fatal_error( + "Go entry argument pointer word has no AArch64 fixed object"); return Chain; } diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index ef219c34666e1..812173a95af4b 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -218,32 +218,16 @@ static void emitGoRegSpills(MachineFunction &MF, MachineBasicBlock &MBB, } } -static bool shouldEmitGoStackCheck(const MachineFunction &MF) { - const Function &F = MF.getFunction(); - return MF.getTarget().getTargetTriple().isOSBinFormatGoObj() && - MF.getTarget().getTargetTriple().getArch() == Triple::x86_64 && - goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg() && - !F.hasFnAttribute(goabi::NoSplitAttr); -} - -static bool isGoNoSplitFunction(const MachineFunction &MF) { +static bool isGoObjGoFunction(const MachineFunction &MF) { const Function &F = MF.getFunction(); return MF.getTarget().getTargetTriple().isOSBinFormatGoObj() && MF.getTarget().getTargetTriple().getArch() == Triple::x86_64 && - goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg() && - F.hasFnAttribute(goabi::NoSplitAttr); + goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg(); } -static void checkGoStackGrowthStatepointContract(const MachineFunction &MF) { - if (!shouldEmitGoStackCheck(MF) || - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) - return; - for (const MachineBasicBlock &MBB : MF) - for (const MachineInstr &MI : MBB) - if (MI.getOpcode() == TargetOpcode::STATEPOINT) - report_fatal_error( - "GoObj statepoints require the go-stack-growth-statepoint " - "function attribute"); +static bool shouldEmitGoStackCheck(const MachineFunction &MF) { + return isGoObjGoFunction(MF) && + !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); } static bool hasGoClosureContext(const Function &F) { @@ -267,37 +251,12 @@ static MachineInstrBuilder buildGoStackGrowthStatepoint(MachineFunction &MF, auto AddConstant = [&](uint64_t Value) { Statepoint.addImm(StackMaps::ConstantOp).addImm(Value); }; - const MachineFrameInfo &MFI = MF.getFrameInfo(); - ArrayRef PointerSlots = - MF.getInfo()->getGoArgPointerSlots(); - uint64_t PointerSize = MF.getDataLayout().getPointerSize(); - // X86 entry RSP points at the return address for both Go calling - // conventions. GoArgPointerSlot::EntryOffset is a physical stack-map - // location even though the corresponding fixed home uses a logical Go - // argument-area offset. - int64_t StackMapBias = static_cast(PointerSize); AddConstant(MF.getFunction().getCallingConv()); - AddConstant(0); // Statepoint flags. - AddConstant(0); // Deopt arguments. - AddConstant(PointerSlots.size()); // GC pointers. - for (const X86MachineFunctionInfo::GoArgPointerSlot &Slot : PointerSlots) { - if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) - report_fatal_error( - "X86 Go entry argument pointer slot is not a fixed object"); - int64_t ExpectedOffset = - StackMapBias + static_cast(Slot.ArgWord) * PointerSize; - if (PointerSize == 0 || Slot.EntryOffset != ExpectedOffset) - report_fatal_error( - "X86 Go entry argument pointer slot has invalid RSP offset"); - Statepoint.addImm(StackMaps::IndirectMemRefOp) - .addImm(PointerSize) - .addReg(X86::RSP) - .addImm(Slot.EntryOffset); - } - AddConstant(0); // GC allocas. - AddConstant(PointerSlots.size()); // GC base/derived map entries. - for (uint64_t I = 0; I != PointerSlots.size(); ++I) - Statepoint.addImm(I).addImm(I); + AddConstant(0); // Statepoint flags. + AddConstant(0); // Deopt arguments. + AddConstant(0); // GC pointers. Entry args live in EntryArgsStackMapID. + AddConstant(0); // GC allocas. + AddConstant(0); // GC base/derived map entries. Statepoint .addRegMask( MF.getSubtarget() @@ -320,24 +279,23 @@ getGoStackCheckEntryMBB(MachineFunction &MF, MachineBasicBlock &FallbackMBB) { return FallbackMBB; } -static void emitGoNoSplitEntryStackMap(MachineFunction &MF, - MachineBasicBlock &FallbackMBB) { - if (!isGoNoSplitFunction(MF) || - !MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) +static void emitGoEntryArgsStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!isGoObjGoFunction(MF)) return; for (const MachineBasicBlock &MBB : MF) for (const MachineInstr &MI : MBB) if (MI.getOpcode() == TargetOpcode::STACKMAP && MI.getOperand(0).isImm() && static_cast(MI.getOperand(0).getImm()) == - goabi::NoSplitEntryStackMapID) + goabi::EntryArgsStackMapID) return; MachineBasicBlock &EntryMBB = getGoStackCheckEntryMBB(MF, FallbackMBB); const X86InstrInfo &TII = *MF.getSubtarget().getInstrInfo(); MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), TII.get(TargetOpcode::STACKMAP)) - .addImm(goabi::NoSplitEntryStackMapID) + .addImm(goabi::EntryArgsStackMapID) .addImm(0); const MachineFrameInfo &MFI = MF.getFrameInfo(); uint64_t PointerSize = MF.getDataLayout().getPointerSize(); @@ -426,10 +384,7 @@ static void emitGoStackCheck(MachineFunction &MF, MF.push_front(CheckMBB); MF.push_front(StartMBB); - bool UseStackGrowthStatepoint = - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); - checkGoStackGrowthStatepointContract(MF); unsigned ScratchReg = X86::R12; if (StackSize <= GoStackSmall) { @@ -468,12 +423,7 @@ static void emitGoStackCheck(MachineFunction &MF, : HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; MachineInstrBuilder Morestack = - UseStackGrowthStatepoint - ? buildGoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, - MorestackName) - : BuildMI(MorestackMBB, DL, TII.get(X86::CALL64pcrel32)); - if (!UseStackGrowthStatepoint) - goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); + buildGoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, MorestackName); if (HasClosureContext && !IsSystemStack) Morestack.addReg(X86::RDX, RegState::Implicit); emitGoRegSpills(MF, *MorestackMBB, Homes, /*Reload=*/true); @@ -2017,7 +1967,9 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, DebugLoc DL; Register ArgBaseReg; - emitGoNoSplitEntryStackMap(MF, MBB); + if (isGoObjGoFunction(MF) && MF.getFrameInfo().hasVarSizedObjects()) + report_fatal_error("GoObj stack growth does not support dynamic allocas"); + emitGoEntryArgsStackMap(MF, MBB); emitGoStackCheck(MF, MBB); // Emit extra prolog for argument stack slot reference. @@ -3718,11 +3670,6 @@ bool X86FrameLowering::restoreCalleeSavedRegisters( void X86FrameLowering::determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const { - // Reject an incomplete Go statepoint contract before callee-save spill - // placement. On X86, statepoints can otherwise require an aligned vector - // spill and trip MachineFrameInfo's non-realignment assertion before the - // morestack prologue gets a chance to report the actionable error. - checkGoStackGrowthStatepointContract(MF); TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); // Spill the BasePtr if it's used. diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 04b7bda0828bd..28d6c4b741c0d 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -205,13 +205,9 @@ static SDValue lowerX86GoFormalArguments( ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), DAG.getDataLayout(), ABIConfig); - std::optional EntryArgs; - SmallBitVector MatchedEntryArgWords; - if (F.hasFnAttribute(goabi::StackGrowthStatepointAttr)) { - EntryArgs = goabi::computeEntryArgsInfo(ArgTys, Layout, DAG.getDataLayout(), - ABIConfig); - MatchedEntryArgWords.resize(EntryArgs->NumBits); - } + goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( + ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); SmallVector ArgSpillOffsets(ArgTys.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; @@ -231,10 +227,8 @@ static SDValue lowerX86GoFormalArguments( FuncInfo->clearGoArgPointerSlots(); auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size) { - if (!EntryArgs) - return; - uint64_t PointerSize = EntryArgs->PointerSize; - for (uint32_t Word : EntryArgs->PointerWords) { + uint64_t PointerSize = EntryArgs.PointerSize; + for (uint32_t Word : EntryArgs.PointerWords) { uint64_t PointerOffset = static_cast(Word) * PointerSize; if (PointerOffset < ArgOffset || PointerOffset + PointerSize > ArgOffset + Size) @@ -335,12 +329,10 @@ static SDValue lowerX86GoFormalArguments( } } - if (EntryArgs) { - for (uint32_t Word : EntryArgs->PointerWords) - if (!MatchedEntryArgWords.test(Word)) - report_fatal_error( - "Go entry argument pointer word has no X86 fixed object"); - } + for (uint32_t Word : EntryArgs.PointerWords) + if (!MatchedEntryArgWords.test(Word)) + report_fatal_error( + "Go entry argument pointer word has no X86 fixed object"); return Chain; } diff --git a/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll b/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll index 8ba776b9c5379..ee5d4d530bbe8 100644 --- a/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll +++ b/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll @@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" } ; OBJ: symdef 0: main.safe ; OBJ: symdef 1: main.async_unsafe -; OBJ: aux 0.6: type=pcdata target= pc=[0-2:-1] -; OBJ: aux 1.14: type=pcdata target= pc=[0-2:-2] +; OBJ: aux 0.6: type=pcdata target= pc=[0-10:-1] +; OBJ: aux 1.14: type=pcdata target= pc=[0-10:-2] diff --git a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll index 90094e13195eb..ec681c7ae5e11 100644 --- a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll +++ b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll @@ -34,17 +34,17 @@ attributes #0 = { "frame-pointer"="non-leaf" } !0 = !{!"branch_weights", i32 1, i32 1000} ; MachineBlockPlacement puts the cold then block after the epilogue and return, -; but the block is reached while the frame is still active. The later +; but the block is reached while the frame is still active. The pre-frame ; morestack block has no frame. ; ; ASM-LABEL: pcsp_cfg: +; ASM: bl "runtime.morestack_noctxt" ; ASM: bl runtime.panicmem ; ASM: ldr x30, [sp], #32 ; ASM: ret ; ASM: bl runtime.GC -; ASM: bl "runtime.morestack_noctxt" -; The return occupies PC quanta 18-19. The out-of-line then block at 19-23 -; restores the 32-byte frame depth before morestack restores the entry depth. -; OBJ: aux 0.3: type=pcsp target= pc=[0-4:0,4-18:32,18-19:0,19-23:32,23-30:0] -; OBJ: reloc 0.2: off=76 size=4 type=9 add=0 target=runtime.GC +; The stack check and morestack path occupy PC quanta 0-11. The return occupies +; 25-26; the out-of-line then block at 26-30 executes with the frame active. +; OBJ: aux 0.3: type=pcsp target= pc=[0-11:0,11-25:32,25-26:0,26-30:32] +; OBJ: reloc 0.3: off=104 size=4 type=9 add=0 target=runtime.GC diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index e337f0fd2a41c..e82475912a23f 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -5,7 +5,7 @@ declare goabiinternal void @"runtime.GC"() define goabiinternal void @subword_homes(i8 %a, i16 %b) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret void @@ -13,7 +13,7 @@ entry: define goabiinternal i64 @large_home_offset([4096 x i64] %stackarg, i64 %regarg) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg @@ -21,7 +21,7 @@ entry: define goabiinternal i64 @large_home_boundary([4094 x i64] %stackarg, i64 %regarg) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll index 6ac0a3a2df8e4..174837180c92b 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll @@ -2,13 +2,13 @@ ; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \ ; RUN: -stop-after=prolog-epilog < %s | FileCheck %s -declare !goobj.symbol.name !0 goabi0 void @runtime.morestack.goallc.abi0() -declare !goobj.symbol.name !1 goabi0 void @runtime.morestack_noctxt.goallc.abi0() -declare !goobj.symbol.name !2 goabi0 void @runtime.morestackc.goallc.abi0() +declare goabi0 void @"runtime.morestack"() +declare goabi0 void @"runtime.morestack_noctxt"() +declare goabi0 void @"runtime.morestackc"() define goabiinternal i64 @closure_morestack_statepoint( i64 %value, ptr nest %ctxt) "frame-pointer"="non-leaf" - "go-stack-growth-statepoint" { + { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -19,7 +19,7 @@ entry: } define goabiinternal ptr @pointer_morestack_statepoint(ptr %pointer) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -32,7 +32,7 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %p16) "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + ptr %p16) "frame-pointer"="non-leaf" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -42,7 +42,7 @@ entry: } define goabiinternal void @systemstack_growth() "frame-pointer"="non-leaf" - "go-stack-growth-statepoint" "go-systemstack" { + "go-systemstack" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -57,35 +57,34 @@ entry: ; CHECK-SAME: csr_aarch64_go, implicit-def $sp, ; CHECK-SAME: implicit-def dead early-clobber $lr, ; CHECK-SAME: implicit $x3, implicit $x26 +; CHECK: STACKMAP 5147419139155979380, 0 ; CHECK-NOT: BL ; CHECK-LABEL: name: pointer_morestack_statepoint ; CHECK: STRXui $x0, $sp, 1 ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, 1, 8, $sp, 8, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_aarch64_go, implicit-def $sp, ; CHECK-SAME: implicit-def dead early-clobber $lr, ; CHECK-SAME: implicit $x3 ; CHECK: $x0 = LDRXui $sp, 1 +; CHECK: STACKMAP 5147419139155979380, 0, 1, 8, $sp, 8 ; CHECK-NOT: BL ; CHECK-LABEL: name: mixed_register_and_stack_pointer_args ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $sp, 16, 1, 8, $sp, 8, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_aarch64_go, implicit-def $sp, ; CHECK-SAME: implicit-def dead early-clobber $lr, ; CHECK-SAME: implicit $x3 +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $sp, 16, 1, 8, $sp, 8 ; CHECK-LABEL: name: systemstack_growth ; CHECK: $x17 = LDRXui $x28, 3 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, @runtime.morestackc.goallc.abi0, +; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestackc", +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_aarch64_go, implicit-def $sp, ; CHECK-SAME: implicit-def dead early-clobber $lr, ; CHECK-SAME: implicit $x3 - -!0 = !{!"runtime.morestack"} -!1 = !{!"runtime.morestack_noctxt"} -!2 = !{!"runtime.morestackc"} +; CHECK: STACKMAP 5147419139155979380, 0 diff --git a/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll b/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll index c9bdc70acc821..410112ab3c0db 100644 --- a/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll +++ b/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll @@ -11,7 +11,7 @@ declare goabiinternal void @safepoint() declare goabiinternal void @observe(ptr addrspace(1)) define goabiinternal void @first_class_alloca_address() - "go-stack-growth-statepoint" gc "statepoint-example" { + gc "statepoint-example" { ; X86-LABEL: name: first_class_alloca_address ; X86: STATEPOINT 1, ; X86-NEXT: $rax = LEA64r diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll index 9726ef2a491ba..d0472728415b9 100644 --- a/llvm/test/CodeGen/Generic/goobj-nosplit.ll +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -3,12 +3,13 @@ ; RUN: -stop-after=prolog-epilog < %s | FileCheck %s --check-prefixes=CHECK,A64 ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ ; RUN: -stop-after=prolog-epilog < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-apple-darwin-goobj -filetype=obj -o %t.a64.o %s +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s declare goabiinternal void @callee(ptr) -declare goabi0 void @callee.abi0(ptr) +declare goabi0 void @"callee.abi0"(ptr) -define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" - "go-stack-growth-statepoint" { +define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { entry: %slot = alloca ptr, align 8 store volatile ptr %pointer, ptr %slot, align 8 @@ -16,8 +17,7 @@ entry: ret void } -define goabiinternal void @split(ptr %pointer) - "go-stack-growth-statepoint" { +define goabiinternal void @split(ptr %pointer) { entry: %slot = alloca ptr, align 8 store volatile ptr %pointer, ptr %slot, align 8 @@ -25,23 +25,25 @@ entry: ret void } -define goabiinternal void @nosplit_abi0_call(ptr %pointer) "go-nosplit" - "go-stack-growth-statepoint" { +define goabiinternal void @nosplit_abi0_call(ptr %pointer) "go-nosplit" { entry: %closure = alloca [3 x ptr], align 8 %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 store volatile ptr @callee, ptr %code, align 8 store volatile ptr %pointer, ptr %context, align 8 - call goabi0 void @callee.abi0(ptr %closure) + call goabi0 void @"callee.abi0"(ptr %closure) ret void } ; CHECK-LABEL: name: nosplit -; CHECK: STACKMAP +; CHECK: STACKMAP 5147419139155979380, 0 +; CHECK-NOT: STATEPOINT 5147424658422983495 ; CHECK-NOT: runtime.morestack ; CHECK-LABEL: name: split -; CHECK: runtime.morestack_noctxt +; CHECK-DAG: STACKMAP 5147419139155979380, 0 +; CHECK-DAG: STATEPOINT 5147424658422983495, 0, 0, +; CHECK-DAG: runtime.morestack_noctxt ; A64-LABEL: name: nosplit_abi0_call ; A64: stackSize: 48 diff --git a/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll b/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll new file mode 100644 index 0000000000000..df7ece08f4759 --- /dev/null +++ b/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll @@ -0,0 +1,31 @@ +; REQUIRES: aarch64-registered-target, x86-registered-target +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog -o - %s | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog -o - %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s +; RUN: llc -mtriple=aarch64-unknown-linux-goobj -filetype=obj -o %t.a64.o %s + +; GoObj Go functions use stack growth by default. An ordinary statepoint does +; not need a second frontend attribute to make the target synthesize the late +; root-free morestack statepoint and the function-level entry-args stack map. + +declare goabiinternal void @callee() + +define goabiinternal void @default_stack_growth() "frame-pointer"="non-leaf" + gc "statepoint-example" { +entry: + call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 0, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, + i32 0, i32 0) + ret void +} + +declare token @llvm.experimental.gc.statepoint.p0( + i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) + +; CHECK-LABEL: name: default_stack_growth +; CHECK-DAG: STACKMAP 5147419139155979380, 0 +; CHECK-DAG: STATEPOINT 5147424658422983495, 0, 0, +; CHECK-DAG: STATEPOINT 0, 0, 0, @callee diff --git a/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll b/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll deleted file mode 100644 index 9df6561709bb7..0000000000000 --- a/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll +++ /dev/null @@ -1,27 +0,0 @@ -; RUN: not --crash llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj \ -; RUN: -o /dev/null %s 2>&1 | FileCheck %s -; RUN: not --crash llc -mtriple=aarch64-unknown-linux-goobj -filetype=obj \ -; RUN: -o /dev/null %s 2>&1 | FileCheck %s - -; A GoObj function with Machine StackMaps must also describe its late -; stack-growth call as a statepoint. Without the frontend attribute, reject the -; function instead of allowing the preceding call's map to cover the raw -; runtime.morestack call. - -declare goabiinternal void @callee() - -define goabiinternal void @missing_stack_growth_attribute() - gc "statepoint-example" { -entry: - call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 0, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, - i32 0, i32 0) - ret void -} - -declare token @llvm.experimental.gc.statepoint.p0( - i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) - -; CHECK: LLVM ERROR: GoObj statepoints require the -; CHECK-SAME: go-stack-growth-statepoint function attribute diff --git a/llvm/test/CodeGen/X86/go-stack-alignment.ll b/llvm/test/CodeGen/X86/go-stack-alignment.ll index f54f5cdcc6761..c1a0034cfe6dc 100644 --- a/llvm/test/CodeGen/X86/go-stack-alignment.ll +++ b/llvm/test/CodeGen/X86/go-stack-alignment.ll @@ -30,4 +30,4 @@ entry: declare void @llvm.memset.inline.p0.i64(ptr, i8, i64, i1 immarg) attributes #0 = { "frame-pointer"="non-leaf" "go-async-unsafe" - "go-stack-growth-statepoint" } + } diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index e33917ed9c72f..0547d1a782189 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -13,7 +13,7 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) ; The alloca is the canonical home of a register argument. Its direct gc-live ; base makes its pointer contents active at the callsite, so bit zero belongs ; to ArgsPointerMaps rather than LocalsPointerMaps. -define goabiinternal ptr @active_scalar(ptr %value) #0 gc "statepoint-example" { +define goabiinternal ptr @active_scalar(ptr %value) gc "statepoint-example" { entry: %home = alloca ptr, align 8 call void @llvm.lifetime.start.p0(i64 8, ptr %home) @@ -46,7 +46,7 @@ entry: ; Its alloca base is not directly live at the ordinary statepoint, so the ; function needs one StackObject at non-negative offset zero relative to argp. define goabiinternal void @inactive_aggregate(%aggregate %value) - #0 gc "statepoint-example" { + gc "statepoint-example" { entry: %home = alloca %aggregate, align 8 call void @llvm.lifetime.start.p0(i64 24, ptr %home) @@ -74,7 +74,7 @@ entry: ; An aggregate assigned wholly to the stack reuses its caller-populated slot; ; the lifetime marker must not make SelectionDAG allocate and copy a local. define goabiinternal void @inactive_stack_aggregate(%stack_aggregate %value) - #0 gc "statepoint-example" { + gc "statepoint-example" { entry: %home = alloca %stack_aggregate, align 8 call void @llvm.lifetime.start.p0(i64 16, ptr %home) @@ -103,4 +103,3 @@ entry: declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/llvm/test/CodeGen/X86/goobj-function-unsafe.ll b/llvm/test/CodeGen/X86/goobj-function-unsafe.ll index a635d4348d54f..1b008df6f71a9 100644 --- a/llvm/test/CodeGen/X86/goobj-function-unsafe.ll +++ b/llvm/test/CodeGen/X86/goobj-function-unsafe.ll @@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" } ; OBJ: symdef 0: main.safe ; OBJ: symdef 1: main.async_unsafe -; OBJ: aux 0.6: type=pcdata target= pc=[0-32:-1] -; OBJ: aux 1.14: type=pcdata target= pc=[0-29:-2] +; OBJ: aux 0.6: type=pcdata target= pc=[0-48:-1] +; OBJ: aux 1.14: type=pcdata target= pc=[0-33:-2] diff --git a/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll b/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll index 75fe4cefb738c..24d6697765bb8 100644 --- a/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll +++ b/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll @@ -55,17 +55,20 @@ done: ; COMMON: file-count: 1 ; COMMON-NEXT: file 0: {{.*}}cdebug.c -; COMMON: hasheddef-count: 4 +; COMMON: hasheddef-count: 6 ; COMMON: nonpkgdef-count: 0 -; X86: hash 0: a21821a940f91b1a89b53461b092269e -; X86-NEXT: hash 1: f9ced9dca799cb1833bd530443fd1f9e -; X86-NEXT: hash 2: 2677aa574f61b902c15de55332c2c2ea -; ARM64: hash 0: 105a58e8d53963b571ff833d8449eeda -; ARM64-NEXT: hash 1: 3bef6118e9cf260f78533cdf1a6375ec -; ARM64-NEXT: hash 2: 3a577c7591ae76fd8b51f8e7ea4ac9d8 -; COMMON-NEXT: hash 3: 4b0e7a681c0340c9a97ef4802a3af2f8 -; COMMON: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=3 -; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=3 +; X86: hash 0: b3ea8d63c732dc89ddc1accf4206a7bb +; X86-NEXT: hash 1: 2f05713252bb1b9673be7cf4f639e025 +; X86-NEXT: hash 2: 2d9a46e763fb2afa2f6bf504aef97634 +; ARM64: hash 0: f933ff4cb32d167004d69c3934a4e464 +; ARM64-NEXT: hash 1: cbf5923fb1da07cca3254ffa87009bcc +; ARM64-NEXT: hash 2: ba4d2125291d2de2a0592dd4bdc679e7 +; COMMON-NEXT: hash 3: 90eb4206729e98003d69bb53d855ecb0 +; COMMON-NEXT: hash 4: 4b0e7a681c0340c9a97ef4802a3af2f8 +; X86-NEXT: hash 5: e612ab846d7727f8f5dc337465ae0fd2 +; ARM64-NEXT: hash 5: b0262f452ad49f58880d4f82680e3f08 +; COMMON: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=010000000100000000 pkg=hashed sym=3 +; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=4 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcsp target= pc={{.*}} pkg=hashed sym=0 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcfile target= pc={{.*}}:0{{.*}} pkg=hashed sym=0 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcline target= pc=[ @@ -74,4 +77,4 @@ done: ; COMMON-SAME: :5 ; COMMON-SAME: pkg=hashed sym=1 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=2 -; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=0 +; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=5 diff --git a/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll b/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll index 69795833b09c3..63ac8207196cb 100644 --- a/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll +++ b/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll @@ -45,5 +45,5 @@ attributes #0 = { "frame-pointer"="non-leaf" } ; The return occupies PC quanta 57-58. The out-of-line then block at 59-66 ; restores the 24-byte frame depth before morestack restores the entry depth. -; OBJ: aux 0.3: type=pcsp target= pc=[0-7:0,7-14:8,14-57:24,57-58:8,58-59:0,59-66:24,66-92:0] +; OBJ: aux 0.3: type=pcsp target= pc=[0-7:0,7-14:8,14-57:24,57-58:8,58-59:0,59-66:24,66-98:0] ; OBJ: reloc 0.2: off=60 size=4 type=7 add=0 target=runtime.GC diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index 6b46c8b4a98b3..cb035c7bb4405 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -66,10 +66,10 @@ join: ; CHECK: nonpkgref {{[0-9]+}}: runtime.morestack_noctxt abi=0 type=0 size=0 ; CHECK: nonpkgref {{[0-9]+}}: runtime.morestack abi=0 type=0 size=0 ; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcinfo target= args={{[1-9][0-9]*}} locals={{[1-9][0-9][0-9][0-9][0-9]*}} -; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 -; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 +; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}} +; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}} ; CHECK: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1] -; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:0] +; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1,{{[0-9]+}}-{{[0-9]+}}:0] ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack_noctxt ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack ; The outgoing frame is 72 bytes of alignment/register-argument space plus 23 @@ -112,11 +112,12 @@ join: ; PEI: fixedStack: ; PEI-NEXT: - { id: 0, type: spill-slot, offset: 0, size: 8 ; PEI: MOV64mr {{.*rsp}}, 1, {{.*noreg}}, 8, {{.*noreg}}, {{.*rax}} -; PEI: CALL64pcrel32 &"runtime.morestack_noctxt" +; PEI: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt" ; PEI: {{.*rax}} = MOV64rm {{.*rsp}}, 1, {{.*noreg}}, 8, {{.*noreg}} ; PEI-LABEL: name: big_closure_frame -; PEI: CALL64pcrel32 &"runtime.morestack", {{.*}}implicit $rdx +; PEI: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack", +; PEI-SAME: implicit $rdx ; PEI-LABEL: name: large_outgoing_frame ; PEI: stackSize: 0 diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll index 08ccc82a53fa9..d3b08034be0d3 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll @@ -12,10 +12,10 @@ } declare goabiinternal void @use_three_pointers(ptr, ptr, ptr) -declare !goobj.symbol.name !0 goabi0 void @runtime.morestack_noctxt.goallc.abi0() -declare !goobj.symbol.name !1 goabi0 void @runtime.morestackc.goallc.abi0() +declare goabi0 void @"runtime.morestack_noctxt"() +declare goabi0 void @"runtime.morestackc"() -define goabiinternal i64 @morestack_statepoint(i64 %value) "go-stack-growth-statepoint" { +define goabiinternal i64 @morestack_statepoint(i64 %value) { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -24,7 +24,7 @@ entry: } define goabi0 void @"abi0_pointer_arguments"(ptr %first, ptr %second, ptr %third) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @use_three_pointers( ptr %first, ptr %second, ptr %third) @@ -32,7 +32,7 @@ entry: } define goabiinternal %many.results @initialized_pointer_result(ptr %pointer) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -42,7 +42,7 @@ entry: define goabiinternal %partial.results @partial_aggregate_result( ptr %first, ptr %second) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -54,7 +54,7 @@ define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) "go-stack-growth-statepoint" { + ptr %pointer) { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -66,7 +66,7 @@ define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, %pointer.aggregate %value) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -79,7 +79,7 @@ entry: } define goabiinternal void @systemstack_growth() - "go-stack-growth-statepoint" "go-systemstack" { + "go-systemstack" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -92,6 +92,7 @@ entry: ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", ; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0 ; CHECK-NOT: CALL64pcrel32 ; The RSP offsets below include the 8-byte amd64 return address. The argument @@ -107,10 +108,10 @@ entry: ; CHECK: offset: 8, size: 8 ; CHECK: offset: 0, size: 8 ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 23, 2, 0, 2, 0, 2, 3, -; CHECK-SAME: 1, 8, $rsp, 8, 1, 8, $rsp, 16, 1, 8, $rsp, 24, -; CHECK-SAME: 2, 0, 2, 3, 0, 0, 1, 1, 2, 2, +; CHECK-SAME: 2, 23, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $rsp, 8, 1, 8, $rsp, 16, 1, 8, $rsp, 24 ; CHECK: renamable $rax = MOV64rm $rbp, 1, $noreg, 16, $noreg ; CHECK: renamable $rbx = MOV64rm $rbp, 1, $noreg, 24, $noreg ; CHECK: renamable $rcx = MOV64rm $rbp, 1, $noreg, 32, $noreg @@ -118,38 +119,35 @@ entry: ; CHECK-LABEL: name: initialized_pointer_result ; CHECK: MOV64mr $rsp, 1, $noreg, 72, $noreg, $rax ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, -; CHECK-SAME: 1, 8, $rsp, 72, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0, 1, 8, $rsp, 72 ; CHECK-LABEL: name: partial_aggregate_result ; CHECK: MOV64mr $rsp, 1, $noreg, 80, $noreg, $rax ; CHECK: MOV64mr $rsp, 1, $noreg, 88, $noreg, $rbx ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $rsp, 80, 1, 8, $rsp, 88, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $rsp, 80, 1, 8, $rsp, 88 ; CHECK-LABEL: name: scalar_stack_argument ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, -; CHECK-SAME: 1, 8, $rsp, 64, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0, 1, 8, $rsp, 64 ; CHECK-LABEL: name: aggregate_stack_argument ; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $rsp, 48, 1, 8, $rsp, 64, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $rsp, 48, 1, 8, $rsp, 64 ; CHECK-LABEL: name: systemstack_growth ; CHECK: CMP64rm $r12, $r14, 1, $noreg, 24, $noreg -; CHECK: STATEPOINT 5147424658422983495, 0, 0, @runtime.morestackc.goallc.abi0, +; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestackc", +; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, ; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp - -!0 = !{!"runtime.morestack_noctxt"} -!1 = !{!"runtime.morestackc"} +; CHECK: STACKMAP 5147419139155979380, 0