diff --git a/llvm/include/llvm/BinaryFormat/GoObj.h b/llvm/include/llvm/BinaryFormat/GoObj.h index c4a370aba1b00..8e3f749a07c9e 100644 --- a/llvm/include/llvm/BinaryFormat/GoObj.h +++ b/llvm/include/llvm/BinaryFormat/GoObj.h @@ -23,9 +23,10 @@ namespace GoObj { // Go function. GoObj serialization strips it and records ABI0 separately. inline constexpr char ABI0SymbolSuffix[] = ""; -// "GoStackG" encoded as the stable statepoint identifier for the pre-frame -// runtime.morestack slow path. -inline constexpr uint64_t StackGrowthStatepointID = 0x476f537461636b47ULL; +// "GoNoSplt" encoded as the stable STACKMAP identifier for the function-level +// entry argument pointer map. This record is metadata-only: it is present for +// both split and nosplit functions and never denotes a callsite. +inline constexpr uint64_t EntryArgsStackMapID = 0x476f4e6f53706c74ULL; // GoALLC encodes pointer maps for fixed allocas as a self-describing suffix of // statepoint deopt locations: diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index 8666893bfd616..0e1fa62d30e7c 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -32,12 +32,16 @@ namespace goabi { inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple"; inline constexpr StringLiteral PadTypeName = "go.abi.pad"; -// The Go statepoint pass uses this attribute to request that target frame -// lowering represent the late morestack call with a root-free STATEPOINT. -inline constexpr StringLiteral StackGrowthStatepointAttr = - "go-stack-growth-statepoint"; -inline constexpr uint64_t StackGrowthStatepointID = - GoObj::StackGrowthStatepointID; +// Target frame lowering must not synthesize a morestack edge for such a +// function. GoObj Go functions otherwise use the native Go default: emit a +// stack check and call the ABI0 runtime.morestack helper on the slow path. +inline constexpr StringLiteral NoSplitAttr = "go-nosplit"; +// A //go:systemstack function checks g.stackguard1 and traps through +// runtime.morestackc if it is entered on an ordinary goroutine stack. +inline constexpr StringLiteral SystemStackAttr = "go-systemstack"; +// Every GoObj Go function carries its entry argument pointer map in a +// zero-byte STACKMAP. It is function metadata, not a stack-growth callsite. +inline constexpr uint64_t EntryArgsStackMapID = GoObj::EntryArgsStackMapID; inline bool isGoABIInternalCallingConv(CallingConv::ID CC) { return CC == CallingConv::GoABIInternal; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 125b73e59b218..f0967c4904ea6 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -5138,6 +5138,15 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { llvm_unreachable("Not Implemented"); } + /// Let a target append mandatory call-boundary work after a statepoint has + /// replaced the temporary target call node. + virtual SDValue finalizeStatepointCallChain(SDValue Chain, + CallingConv::ID CalleeCC, + const SDLoc &DL, + SelectionDAG &DAG) const { + return Chain; + } + /// Target-specific cleanup for formal ByVal parameters. virtual void HandleByVal(CCState *, unsigned &, Align) const {} diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 7dbd2c879e3f3..637367158b2fe 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -464,6 +464,102 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL, return Alignment; } +static uint64_t +getGoObjStackMapNonnegativeConstant(const StackMaps::Location &Location, + StringRef Description) { + if (Location.Type != StackMaps::Location::Constant || Location.Offset < 0) + report_fatal_error("malformed GoObj stackmap " + Description); + return static_cast(Location.Offset); +} + +static MCContext::GoObjStackMapLocation::LocationType +convertGoObjStackMapLocationType(StackMaps::Location::LocationType Type) { + using GoLocation = MCContext::GoObjStackMapLocation; + switch (Type) { + case StackMaps::Location::Unprocessed: + return GoLocation::Unprocessed; + case StackMaps::Location::Register: + return GoLocation::Register; + case StackMaps::Location::Direct: + return GoLocation::Direct; + case StackMaps::Location::Indirect: + return GoLocation::Indirect; + case StackMaps::Location::Constant: + return GoLocation::Constant; + case StackMaps::Location::ConstantIndex: + return GoLocation::ConstantIndex; + } + llvm_unreachable("unknown StackMaps location type"); +} + +static void emitGoObjStackMaps(StackMaps &SM, AsmPrinter &AP) { + auto &Callsites = SM.getCSInfos(); + auto Callsite = Callsites.begin(); + uint32_t PointerSize = AP.getPointerSize(); + if (!PointerSize) + report_fatal_error("GoObj statepoint target has no pointer size"); + + for (const auto &[Function, Info] : SM.getFnInfos()) { + for (uint64_t I = 0; I != Info.RecordCount; ++I) { + if (Callsite == Callsites.end()) + report_fatal_error( + "GoObj stackmap function record count exceeds callsites"); + + const StackMaps::CallsiteInfo &CSI = *Callsite++; + bool IsEntryArgs = CSI.ID == GoObj::EntryArgsStackMapID; + uint64_t NumDeopts = 0; + ArrayRef Locations = CSI.Locations; + if (!IsEntryArgs) { + // LLVM's statepoint parser prefixes locations with the calling + // convention, flags, deopt count, and then the deopt operands. These + // entries are not GC roots. EntryArgsStackMapID is a plain STACKMAP + // and contains only function-level argument pointer homes. + if (Locations.size() < 3) + report_fatal_error("malformed GoObj statepoint location list"); + (void)getGoObjStackMapNonnegativeConstant(Locations[0], + "calling convention"); + (void)getGoObjStackMapNonnegativeConstant(Locations[1], "flags"); + NumDeopts = + getGoObjStackMapNonnegativeConstant(Locations[2], "deopt count"); + if (NumDeopts > Locations.size() - 3) + report_fatal_error("malformed GoObj statepoint deopt operands"); + if (NumDeopts > std::numeric_limits::max()) + report_fatal_error("GoObj statepoint has too many deopt operands"); + Locations = Locations.drop_front(3); + } + + MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, + CSI.ID, + CSI.IsIndirectCall, + Info.StackSize, + PointerSize, + static_cast(NumDeopts), + {}}; + Entry.Locations.reserve(Locations.size()); + for (const StackMaps::Location &Location : Locations) { + auto Type = convertGoObjStackMapLocationType(Location.Type); + int64_t Offset = Location.Offset; + if (Location.Type == StackMaps::Location::Constant || + Location.Type == StackMaps::Location::ConstantIndex) { + std::optional Constant = SM.getConstantValue(Location); + if (!Constant) + report_fatal_error( + "GoObj statepoint contains an invalid constant-pool index"); + Type = MCContext::GoObjStackMapLocation::Constant; + Offset = *Constant; + } + Entry.Locations.push_back({Type, Location.Size, Location.Reg, Offset}); + } + AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry)); + } + } + + if (Callsite != Callsites.end()) + report_fatal_error( + "GoObj stackmap callsites exceed function record counts"); + SM.reset(); +} + AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer, char &ID) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), @@ -503,6 +599,10 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer, MP->finishAssembly(M, *MI, *this); }; EmitStackMaps = [this](Module &M) { + if (OutContext.isGoObj()) { + emitGoObjStackMaps(SM, *this); + return; + } GCModuleInfo *MI = getAnalysisIfAvailable(); assert(MI && "AsmPrinter didn't require GCModuleInfo?"); bool NeedsDefault = false; @@ -698,7 +798,13 @@ bool AsmPrinter::doInitialization(Module &M) { if ((Target.isOSWindows() || (Target.isUEFI() && EmitCodeView)) && M.getNamedMetadata("llvm.dbg.cu")) Handlers.push_back(std::make_unique(this)); - if (!EmitCodeView || M.getDwarfVersion()) { + // GoObj consumes source locations through GoObjDebug to build pcfile, + // pcline, and pcinline after final machine layout. Generic monolithic + // DWARF sections are not Go linker carriers: their relocations can point + // at functions removed by Go dead-code elimination. Keep target-object + // DWARF disabled until it is represented by Go's per-function carriers. + if (!Target.isOSBinFormatGoObj() && + (!EmitCodeView || M.getDwarfVersion())) { if (hasDebugInfo()) { DD = createDwarfDebug(); Handlers.push_back(std::unique_ptr(DD)); @@ -872,6 +978,9 @@ getGoObjSymbolFlags(const GlobalObject *GO) { Flag2 |= GoObj::SymFlagItab; } } + if (const auto *F = dyn_cast(GO); + F && F->hasFnAttribute(goabi::NoSplitAttr)) + Flag |= GoObj::SymFlagNoSplit; if (const MDNode *MD = GO->getMetadata("goobj.symbol.flags")) { if (MD->getNumOperands() != 2) @@ -1081,12 +1190,12 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { const MDNode *MD = GO.getMetadata("goobj.symbol.nonpackage"); if (!MD) continue; - const auto *Marker = MD->getNumOperands() == 1 - ? mdconst::dyn_extract( - MD->getOperand(0)) - : nullptr; - if (!Marker || !Marker->getType()->isIntegerTy(1) || - !Marker->isOne() || GO.isDeclaration()) + const auto *Marker = + MD->getNumOperands() == 1 + ? mdconst::dyn_extract(MD->getOperand(0)) + : nullptr; + if (!Marker || !Marker->getType()->isIntegerTy(1) || !Marker->isOne() || + GO.isDeclaration()) report_fatal_error("invalid !goobj.symbol.nonpackage attachment"); AP.OutContext.setGoObjSymbolNonPackage(AP.getSymbol(&GO)); } @@ -1165,8 +1274,7 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { } } - if (const NamedMDNode *Markers = - M.getNamedMetadata("goobj.marker_relocs")) { + if (const NamedMDNode *Markers = M.getNamedMetadata("goobj.marker_relocs")) { DenseMap> Relocs; for (const MDNode *Entry : Markers->operands()) { @@ -1174,11 +1282,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { report_fatal_error( "expected !goobj.marker_relocs entries to have four operands"); const GlobalValue *Source = - getGoObjMetadataGlobal(Entry->getOperand(0), - "goobj.marker_relocs"); + getGoObjMetadataGlobal(Entry->getOperand(0), "goobj.marker_relocs"); const GlobalValue *Target = - getGoObjMetadataGlobal(Entry->getOperand(1), - "goobj.marker_relocs"); + getGoObjMetadataGlobal(Entry->getOperand(1), "goobj.marker_relocs"); const auto *Type = mdconst::dyn_extract(Entry->getOperand(2)); const auto *Addend = @@ -1195,9 +1301,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { default: report_fatal_error("unsupported !goobj.marker_relocs type"); } - Relocs[Source].push_back( - {AP.getSymbol(Target), static_cast(Type->getZExtValue()), - Addend->getSExtValue()}); + Relocs[Source].push_back({AP.getSymbol(Target), + static_cast(Type->getZExtValue()), + Addend->getSExtValue()}); } for (auto &[Source, SourceRelocs] : Relocs) AP.OutContext.setGoObjMarkerRelocs(AP.getSymbol(Source), @@ -3746,8 +3852,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { if (TM.getTargetTriple().isOSBinFormatGoObj()) { if (std::optional> Flags = getGoObjSymbolFlags(&F)) - OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, - Flags->second); + OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, Flags->second); if (std::optional> Info = getGoObjFunctionInfo(F)) OutContext.setGoObjFunctionInfo(CurrentFnSym, Info->first, Info->second); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 259fa480ec1e5..4a16380483f01 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -1070,7 +1070,10 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Since we always emit CopyToRegs (even for local relocates), we must // update root, so that they are emitted before any local uses. - (void)getControlRoot(); + SDValue FinalChain = getControlRoot(); + FinalChain = DAG.getTargetLoweringInfo().finalizeStatepointCallChain( + FinalChain, SI.CLI.CallConv, getCurSDLoc(), DAG); + DAG.setRoot(FinalChain); // TODO: A better future implementation would be to emit a single variable // argument, variable return value STATEPOINT node here and then hookup the diff --git a/llvm/lib/MC/GoObjObjectWriter.cpp b/llvm/lib/MC/GoObjObjectWriter.cpp index f1252339a1abc..fabc317c6d273 100644 --- a/llvm/lib/MC/GoObjObjectWriter.cpp +++ b/llvm/lib/MC/GoObjObjectWriter.cpp @@ -355,9 +355,9 @@ GoObjGCFrameLayout getGoObjGCFrameLayout(const Triple &TT, uint32_t StackSize, uint32_t PointerSize, bool HasFramePointer) { // At an amd64 function entry, RSP points at the return address. Entry - // argument pointer locations in the stack-growth statepoint are relative to - // that pre-frame RSP, so the caller's argument area begins one word above it. - // The same bias applies after subtracting StackSize for ordinary statepoints. + // argument pointer locations in the entry stack map are relative to that + // pre-frame RSP, so the caller's argument area begins one word above it. The + // same bias applies after subtracting StackSize for ordinary statepoints. if (TT.getArch() == Triple::x86_64) { if (StackSize == 0) return {0, 0, 0, 0, 0, PointerSize}; @@ -612,7 +612,8 @@ parseAllocaPtrMapRecords(const MCContext::GoObjStackMapEntry &Entry) { GoObjStatepointStackMaps makeStatepointStackMaps( const MCAssembler &Asm, const GoObjSymbol &Function, uint32_t StackSize, uint32_t ArgSize, uint32_t PCQuantum, - ArrayRef StackMapEntries) { + ArrayRef StackMapEntries, + ArrayRef PCSPEntries) { struct ResolvedEntry { uint64_t CallsitePC; const MCContext::GoObjStackMapEntry *Entry; @@ -691,17 +692,16 @@ GoObjStatepointStackMaps makeStatepointStackMaps( SmallVector FunctionAllocaRecords; uint64_t OrdinaryEntryCount = llvm::count_if(ResolvedEntries, [](const ResolvedEntry &Resolved) { - return Resolved.Entry->ID != GoObj::StackGrowthStatepointID; + return Resolved.Entry->ID != GoObj::EntryArgsStackMapID; }); std::optional FunctionOpenDefer; uint64_t OpenDeferEntryCount = 0; for (const ResolvedEntry &Resolved : ResolvedEntries) { std::optional Record = parseOpenDeferRecord(*Resolved.Entry); - if (Resolved.Entry->ID == GoObj::StackGrowthStatepointID) { + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) { if (Record) - report_fatal_error( - "GoObj stack-growth statepoint contains open-defer state"); + report_fatal_error("GoObj entry metadata contains open-defer state"); continue; } if (!Record) @@ -757,7 +757,7 @@ GoObjStatepointStackMaps makeStatepointStackMaps( } auto BuildPair = [&](const MCContext::GoObjStackMapEntry &Entry) { - bool IsStackGrowth = Entry.ID == GoObj::StackGrowthStatepointID; + bool IsEntryArgs = Entry.ID == GoObj::EntryArgsStackMapID; GoObjStackMapPair Pair{SmallVector(ArgsBytesPerBitmap, 0), SmallVector(LocalsBytesPerBitmap, 0)}; SmallVector, 4> AllocaRanges; @@ -769,9 +769,8 @@ GoObjStatepointStackMaps makeStatepointStackMaps( ArrayRef(Entry.Locations).drop_front(Entry.NumDeoptLocations); for (const GoObjAllocaPtrMapRecord &Record : parseAllocaPtrMapRecords(Entry)) { - if (IsStackGrowth) - report_fatal_error( - "GoObj stack-growth statepoint contains an alloca ptrmap"); + if (IsEntryArgs) + report_fatal_error("GoObj entry metadata contains an alloca ptrmap"); MCContext::GoObjStackMapLocation RecordBase = NormalizeFrameLocation(Record.Base); if (RecordBase.Size != PointerSize || @@ -914,17 +913,16 @@ GoObjStatepointStackMaps makeStatepointStackMaps( ", offset=" + Twine(Loc.Offset)); for (int64_t WordOffset : *PointerWordOffsets) { - if (IsStackGrowth) { - std::optional Bit = goobj::classifyStackGrowthStackMapSlot( + if (IsEntryArgs) { + std::optional Bit = goobj::classifyEntryArgsStackMapSlot( WordOffset, PointerSize, FrameLayout.EntryArgsStart, ArgSize); if (Loc.Type != MCContext::GoObjStackMapLocation::Indirect || !Bit) report_fatal_error( - "GoObj stack-growth statepoint contains an invalid argument " + "GoObj entry argument stack map contains an invalid argument " "pointer slot"); Pair.Args[*Bit / 8] |= uint8_t(1u << (*Bit % 8)); continue; } - goobj::StackMapSlot Slot = goobj::classifyOrdinaryStackMapSlot( WordOffset, Loc.Type == MCContext::GoObjStackMapLocation::Indirect, PointerSize, FrameLayout.GCLocalsStart, FrameLayout.GCLocalsSize, @@ -958,30 +956,40 @@ GoObjStatepointStackMaps makeStatepointStackMaps( return Pair; }; - const ResolvedEntry *StackGrowthEntry = nullptr; + const ResolvedEntry *EntryArgsEntry = nullptr; for (const ResolvedEntry &Resolved : ResolvedEntries) { - if (Resolved.Entry->ID != GoObj::StackGrowthStatepointID) + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) { + if (EntryArgsEntry) + report_fatal_error( + "GoObj function contains multiple entry argument stack maps"); + EntryArgsEntry = &Resolved; continue; - if (StackGrowthEntry) - report_fatal_error( - "GoObj function contains multiple stack-growth statepoints"); - StackGrowthEntry = &Resolved; + } } - if (!StackGrowthEntry) - report_fatal_error("GoObj function has no stack-growth statepoint"); + if (!EntryArgsEntry) + report_fatal_error("GoObj function has no entry argument stack map"); + if (EntryArgsEntry->Entry->IsIndirectCall) + report_fatal_error("GoObj entry argument stack map is a callsite"); SmallVector Pairs; - Pairs.push_back(BuildPair(*StackGrowthEntry->Entry)); + Pairs.push_back(BuildPair(*EntryArgsEntry->Entry)); SmallVector PCDataEntries; + // PCSP is derived from the Machine CFG. Whenever control flow returns to + // the entry stack depth, only the function-level entry argument map is + // valid. This covers the pre-frame morestack slow path without naming the + // helper or manufacturing a statepoint for its raw ABI0 call. + for (const GoObjPCTabEntry &Entry : PCSPEntries) + if (Entry.Value == 0 && Entry.PC < Function.Size) + PCDataEntries.push_back({Entry.PC, 0}); std::optional PreviousCallsitePC; SmallVector IndirectCallOffsets; for (const ResolvedEntry &Resolved : ResolvedEntries) { + if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) + continue; if (PreviousCallsitePC && *PreviousCallsitePC == Resolved.CallsitePC) report_fatal_error("GoObj statepoint callsites have duplicate PCs"); PreviousCallsitePC = Resolved.CallsitePC; const MCContext::GoObjStackMapEntry &Entry = *Resolved.Entry; - if (Entry.ID == GoObj::StackGrowthStatepointID && Entry.IsIndirectCall) - report_fatal_error("GoObj stack-growth statepoint call is indirect"); if (Entry.IsIndirectCall) { if (Resolved.CallsitePC >= Function.Size) report_fatal_error( @@ -992,23 +1000,21 @@ GoObjStatepointStackMaps makeStatepointStackMaps( if (Entry.PointerSize != PointerSize) report_fatal_error( "GoObj statepoint pointer size changes within a function"); - uint32_t MapIndex = 0; - if (Entry.ID != GoObj::StackGrowthStatepointID) { - GoObjStackMapPair Pair = BuildPair(Entry); - auto It = llvm::find(Pairs, Pair); - if (It == Pairs.end()) { - MapIndex = checkedUint32(Pairs.size(), "GoObj stack map index"); - Pairs.push_back(std::move(Pair)); - } else { - MapIndex = checkedUint32(It - Pairs.begin(), "GoObj stack map index"); - } + GoObjStackMapPair Pair = BuildPair(Entry); + auto It = llvm::find(Pairs, Pair); + uint32_t MapIndex; + if (It == Pairs.end()) { + MapIndex = checkedUint32(Pairs.size(), "GoObj stack map index"); + Pairs.push_back(std::move(Pair)); + } else { + MapIndex = checkedUint32(It - Pairs.begin(), "GoObj stack map index"); } if (MapIndex > static_cast(std::numeric_limits::max())) report_fatal_error("GoObj stack map index exceeds int32 limit"); // GoObj records statepoint callsites at the beginning of the CALL. The - // live-out map remains in effect until another statepoint, including the - // stack-growth call, changes it. + // live-out map remains in effect until another statepoint or a CFG-derived + // return to the entry stack depth changes it. PCDataEntries.push_back( {Resolved.CallsitePC, static_cast(MapIndex)}); } @@ -1081,7 +1087,7 @@ GoObjStatepointStackMaps makeStatepointStackMaps( Result.Args = makeStackMap(ArgsNBits, ArgsBitmaps); Result.Locals = makeStackMap(NBits, LocalsBitmaps); Result.PCData = - makePCTab(-1, NormalizedPCDataEntries, Function.Size, PCQuantum); + makePCTab(0, NormalizedPCDataEntries, Function.Size, PCQuantum); Result.OpenDefer = std::move(OpenDeferData); Result.IndirectCallOffsets = std::move(IndirectCallOffsets); Result.StackObjects = std::move(FunctionStackObjects); @@ -2006,7 +2012,8 @@ uint64_t GoObjObjectWriter::writeObject() { Symbols[I].Symbol)) { if (!Entries->empty()) { GoObjStatepointStackMaps Maps = makeStatepointStackMaps( - *Asm, Symbols[I], StackSize, ArgSize, PCQuantum, *Entries); + *Asm, Symbols[I], StackSize, ArgSize, PCQuantum, *Entries, + PCSPEntries); for (uint32_t Offset : Maps.IndirectCallOffsets) Symbols[I].Relocations.push_back({Offset, 0, GoObj::R_CALLIND, 0, GoObj::PkgIdxInvalid, 0, @@ -2362,8 +2369,6 @@ uint64_t GoObjObjectWriter::writeObject() { "GoObj private constants with relocations are not supported"); int64_t Addend = getGoObjRelocAddend(Reloc); - GoObjSymRef TargetSymRef = GetTargetSymRef(Reloc, Addend); - uint16_t RelocType = checkedUint16(Reloc.Type, "relocation type"); if (Source.Symbol) { if (const auto *Overrides = @@ -2385,6 +2390,16 @@ uint64_t GoObjObjectWriter::writeObject() { RelocType |= GoObj::R_WEAK; } + // Native x86 Go objects intentionally leave the internal-linking TLS + // relocation target empty. The linker resolves R_TLS_LE against its + // synthetic runtime.tlsg symbol and supplies that symbol itself when it + // translates the relocation for external ELF linking. + const Triple::ArchType Arch = Asm->getContext().getTargetTriple().getArch(); + const bool IsX86TLSLE = (Arch == Triple::x86 || Arch == Triple::x86_64) && + (RelocType & ~GoObj::R_WEAK) == GoObj::R_TLS_LE; + GoObjSymRef TargetSymRef = + IsX86TLSLE ? GoObjSymRef{} : GetTargetSymRef(Reloc, Addend); + Source.Relocations.push_back( {static_cast(LocalOffset), Reloc.Size, RelocType, Addend, TargetSymRef.PkgIdx, TargetSymRef.SymIdx, std::nullopt}); diff --git a/llvm/lib/MC/GoObjStackMapUtils.h b/llvm/lib/MC/GoObjStackMapUtils.h index 6978c731126d6..5045c0eab29fb 100644 --- a/llvm/lib/MC/GoObjStackMapUtils.h +++ b/llvm/lib/MC/GoObjStackMapUtils.h @@ -60,8 +60,8 @@ expandStackMapPointerWords(int64_t Offset, uint16_t Size, bool IsIndirect, } inline std::optional -classifyStackGrowthStackMapSlot(int64_t Offset, uint32_t PointerSize, - uint64_t ArgsStart, uint64_t ArgsSize) { +classifyEntryArgsStackMapSlot(int64_t Offset, uint32_t PointerSize, + uint64_t ArgsStart, uint64_t ArgsSize) { if (Offset < 0 || !PointerSize) return std::nullopt; uint64_t UOffset = static_cast(Offset); diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 6e555ce21954a..72345349c5484 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -1248,9 +1248,11 @@ namespace { constexpr uint64_t GoStackSmall = 128; constexpr uint64_t GoStackBig = 4096; constexpr int64_t GoGStackGuard0Offset = 16; +constexpr int64_t GoGStackGuard1Offset = 24; static bool shouldEmitAArch64GoStackCheck(const MachineFunction &MF) { - return AArch64FrameLowering::usesGoFrameLayout(MF); + return AArch64FrameLowering::usesGoFrameLayout(MF) && + !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); } static bool hasAArch64GoClosureContext(const Function &F) { @@ -1260,40 +1262,41 @@ static bool hasAArch64GoClosureContext(const Function &F) { return false; } -static void -checkAArch64GoStackGrowthStatepointContract(const MachineFunction &MF) { - if (!AArch64FrameLowering::usesGoFrameLayout(MF) || - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) +static MachineBasicBlock & +getAArch64GoStackCheckEntryMBB(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + const BasicBlock &EntryBB = MF.getFunction().getEntryBlock(); + for (MachineBasicBlock &MBB : MF) + if (MBB.getBasicBlock() == &EntryBB) + return MBB; + if (MachineBasicBlock *MBB = MF.getBlockNumbered(0)) + return *MBB; + return FallbackMBB; +} + +static void emitAArch64GoEntryArgsStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!AArch64FrameLowering::usesGoFrameLayout(MF)) return; for (const MachineBasicBlock &MBB : MF) for (const MachineInstr &MI : MBB) - if (MI.getOpcode() == TargetOpcode::STATEPOINT) - report_fatal_error( - "GoObj statepoints require the go-stack-growth-statepoint " - "function attribute"); -} + if (MI.getOpcode() == TargetOpcode::STACKMAP && + MI.getOperand(0).isImm() && + static_cast(MI.getOperand(0).getImm()) == + goabi::EntryArgsStackMapID) + return; -static MachineInstrBuilder buildAArch64GoStackGrowthStatepoint( - MachineFunction &MF, MachineBasicBlock &MBB, const DebugLoc &DL, - const AArch64InstrInfo &TII, StringRef CalleeName) { - MachineInstrBuilder Statepoint = - BuildMI(&MBB, DL, TII.get(TargetOpcode::STATEPOINT)) - .addImm(goabi::StackGrowthStatepointID) - .addImm(0) - .addImm(0); - goabi::addGoObjABI0Callee(Statepoint, MF, CalleeName); - auto AddConstant = [&](uint64_t Value) { - Statepoint.addImm(StackMaps::ConstantOp).addImm(Value); - }; + MachineBasicBlock &EntryMBB = getAArch64GoStackCheckEntryMBB(MF, FallbackMBB); + const AArch64InstrInfo &TII = + *MF.getSubtarget().getInstrInfo(); + MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), + TII.get(TargetOpcode::STACKMAP)) + .addImm(goabi::EntryArgsStackMapID) + .addImm(0); const MachineFrameInfo &MFI = MF.getFrameInfo(); - ArrayRef PointerSlots = - MF.getInfo()->getGoArgPointerSlots(); uint64_t PointerSize = MF.getDataLayout().getPointerSize(); - AddConstant(MF.getFunction().getCallingConv()); - AddConstant(0); // Statepoint flags. - AddConstant(0); // Deopt arguments. - AddConstant(PointerSlots.size()); // GC pointers. - for (const AArch64FunctionInfo::GoArgPointerSlot &Slot : PointerSlots) { + for (const AArch64FunctionInfo::GoArgPointerSlot &Slot : + MF.getInfo()->getGoArgPointerSlots()) { if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) report_fatal_error( "AArch64 Go entry argument pointer slot is not a fixed object"); @@ -1304,36 +1307,11 @@ static MachineInstrBuilder buildAArch64GoStackGrowthStatepoint( if (PointerSize == 0 || Offset != ExpectedOffset || !isInt<32>(Offset)) report_fatal_error( "AArch64 Go entry argument pointer slot has invalid SP offset"); - Statepoint.addImm(StackMaps::IndirectMemRefOp) + StackMap.addImm(StackMaps::IndirectMemRefOp) .addImm(PointerSize) .addReg(AArch64::SP) .addImm(Offset); } - AddConstant(0); // GC allocas. - AddConstant(PointerSlots.size()); // GC base/derived map entries. - for (uint64_t I = 0; I != PointerSlots.size(); ++I) - Statepoint.addImm(I).addImm(I); - Statepoint - .addRegMask( - MF.getSubtarget() - .getRegisterInfo() - ->getCallPreservedMask(MF, MF.getFunction().getCallingConv())) - .addReg(AArch64::SP, RegState::ImplicitDefine) - .addReg(AArch64::LR, RegState::ImplicitDefine | RegState::Dead | - RegState::EarlyClobber); - return Statepoint; -} - -static MachineBasicBlock & -getAArch64GoStackCheckEntryMBB(MachineFunction &MF, - MachineBasicBlock &FallbackMBB) { - const BasicBlock &EntryBB = MF.getFunction().getEntryBlock(); - for (MachineBasicBlock &MBB : MF) - if (MBB.getBasicBlock() == &EntryBB) - return MBB; - if (MachineBasicBlock *MBB = MF.getBlockNumbered(0)) - return *MBB; - return FallbackMBB; } static unsigned getAArch64GoSpillOpcode(unsigned Size, bool IsFP, bool Reload) { @@ -1416,8 +1394,6 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, report_fatal_error("GoObj stack growth does not support dynamic allocas"); uint64_t StackSize = MFI.getStackSize() + MFI.getUnsafeStackSize(); - if (StackSize == 0 && !MFI.hasCalls()) - return; const DebugLoc DL; const AArch64InstrInfo &TII = @@ -1473,8 +1449,7 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, MF.push_front(CheckMBB); MF.push_front(StartMBB); - bool UseStackGrowthStatepoint = - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); + bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); Register ScratchReg = AArch64::SP; if (StackSize > GoStackBig) { @@ -1502,7 +1477,8 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, BuildMI(CompareMBB, DL, TII.get(AArch64::LDRXui), AArch64::X17) .addReg(AArch64::X28) - .addImm(GoGStackGuard0Offset / 8); + .addImm((IsSystemStack ? GoGStackGuard1Offset : GoGStackGuard0Offset) / + 8); BuildMI(CompareMBB, DL, TII.get(AArch64::SUBSXrx64), AArch64::XZR) .addReg(ScratchReg) .addReg(AArch64::X17) @@ -1516,16 +1492,14 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, BuildMI(MorestackMBB, DL, TII.get(TargetOpcode::COPY), AArch64::X3) .addReg(AArch64::LR); bool HasClosureContext = hasAArch64GoClosureContext(MF.getFunction()); - const char *MorestackName = - HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; + const char *MorestackName = IsSystemStack ? "runtime.morestackc" + : HasClosureContext ? "runtime.morestack" + : "runtime.morestack_noctxt"; MachineInstrBuilder Morestack = - UseStackGrowthStatepoint ? buildAArch64GoStackGrowthStatepoint( - MF, *MorestackMBB, DL, TII, MorestackName) - : BuildMI(MorestackMBB, DL, TII.get(AArch64::BL)); - if (!UseStackGrowthStatepoint) - goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); + BuildMI(MorestackMBB, DL, TII.get(AArch64::BL)); + goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); Morestack.addReg(AArch64::X3, RegState::Implicit); - if (HasClosureContext) + if (HasClosureContext && !IsSystemStack) Morestack.addReg(AArch64::X26, RegState::Implicit); emitAArch64GoRegSpills(MF, *MorestackMBB, AFI->getGoArgHomes(), /*Reload=*/true); @@ -1554,11 +1528,11 @@ static void emitAArch64GoStackCheck(MachineFunction &MF, void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { - checkAArch64GoStackGrowthStatepointContract(MF); if (usesGoFrameLayout(MF) && MF.getFrameInfo().hasVarSizedObjects()) report_fatal_error("GoObj stack growth does not support dynamic allocas"); AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this); PrologueEmitter.emitPrologue(); + emitAArch64GoEntryArgsStackMap(MF, MBB); emitAArch64GoStackCheck(MF, MBB); } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d053a411fd991..b3a69f3134f28 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9105,13 +9105,9 @@ static SDValue lowerAArch64GoFormalArguments( ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), DAG.getDataLayout(), ABIConfig); - std::optional EntryArgs; - SmallBitVector MatchedEntryArgWords; - if (F.hasFnAttribute(goabi::StackGrowthStatepointAttr)) { - EntryArgs = goabi::computeEntryArgsInfo(ArgTys, Layout, DAG.getDataLayout(), - ABIConfig); - MatchedEntryArgWords.resize(EntryArgs->NumBits); - } + goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( + ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); SmallVector ArgSpillOffsets(ArgTys.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; @@ -9129,11 +9125,10 @@ static SDValue lowerAArch64GoFormalArguments( FuncInfo->clearGoArgPointerSlots(); unsigned StackBias = getAArch64GoStackBias(F.getCallingConv()); - auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size) { - if (!EntryArgs) - return; - uint64_t PointerSize = EntryArgs->PointerSize; - for (uint32_t Word : EntryArgs->PointerWords) { + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, + bool IsLiveAtEntry) { + uint64_t PointerSize = EntryArgs.PointerSize; + for (uint32_t Word : EntryArgs.PointerWords) { uint64_t PointerOffset = static_cast(Word) * PointerSize; if (PointerOffset < ArgOffset || PointerOffset + PointerSize > ArgOffset + Size) @@ -9146,8 +9141,9 @@ static SDValue lowerAArch64GoFormalArguments( if (WithinObject > UINT32_MAX) report_fatal_error( "Go entry argument pointer offset exceeds AArch64 metadata range"); - FuncInfo->addGoArgPointerSlot(FI, static_cast(WithinObject), - Word); + if (IsLiveAtEntry) + FuncInfo->addGoArgPointerSlot(FI, static_cast(WithinObject), + Word); MatchedEntryArgWords.set(Word); } }; @@ -9181,7 +9177,14 @@ static SDValue lowerAArch64GoFormalArguments( /*IsImmutable=*/true); AArch64FunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); - RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size); + // LLVM may replace an unused incoming pointer with poison at every call + // edge. Keep its ABI home so morestack can preserve the complete register + // assignment, but do not expose that uninitialized word as a GC root. + bool IsLiveAtEntry = llvm::any_of( + ArrayRef(Ins).slice(Group.Start, Group.End - Group.Start), + [](const ISD::InputArg &In) { return In.Used; }); + RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, + IsLiveAtEntry); unsigned IntPiece = 0; unsigned FPPiece = 0; @@ -9222,12 +9225,10 @@ static SDValue lowerAArch64GoFormalArguments( } } - if (EntryArgs) { - for (uint32_t Word : EntryArgs->PointerWords) - if (!MatchedEntryArgWords.test(Word)) - report_fatal_error( - "Go entry argument pointer word has no AArch64 fixed object"); - } + for (uint32_t Word : EntryArgs.PointerWords) + if (!MatchedEntryArgWords.test(Word)) + report_fatal_error( + "Go entry argument pointer word has no AArch64 fixed object"); return Chain; } @@ -9337,7 +9338,13 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); - unsigned NumBytes = Layout.TotalStackSize + StackBias; + // Layout.TotalStackSize rounds the logical Go argument area to the target + // stack alignment before the physical entry-SP bias is applied. Adding the + // bias after that rounding reserves an extra word for one-word calls (the + // common ABI0 funcval case) and inflates every containing nosplit frame. + // The caller frame itself remains stack-aligned; reserve only the bytes + // through the last physical argument home here. + unsigned NumBytes = Layout.ArgSize + StackBias; Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index cc73cc14a4abd..5f7af5c5bab40 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -1199,6 +1199,7 @@ def CSR_64_AllRegs_AVX512 : CalleeSavedRegs<(sub (add CSR_64_MostRegs, RAX, (sequence "K%u", 0, 7)), (sequence "XMM%u", 0, 15))>; def CSR_64_NoneRegs : CalleeSavedRegs<(add RBP)>; +def CSR_64_GoABI0 : CalleeSavedRegs<(add RBP)>; def CSR_64_Go : CalleeSavedRegs<(add R14, RBP, XMM15)>; // Standard C + YMM6-15 diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index bb2294b7de622..39ae8aae41ad8 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -315,6 +315,23 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB, MI.eraseFromParent(); return true; } + case X86::GO_REPAIR_ABI_INTERNAL_REGS: { + // The native Go compiler emits exactly this sequence when crossing from + // ABI0 into ABIInternal, and again after an ABIInternal caller returns + // from ABI0. R_TLS_LE deliberately has no symbol in internal Go linking; + // the linker resolves it to runtime.tlsg. + BuildMI(MBB, MBBI, DL, TII->get(X86::XORPSrr), X86::XMM15) + .addReg(X86::XMM15, RegState::Undef) + .addReg(X86::XMM15, RegState::Undef); + BuildMI(MBB, MBBI, DL, TII->get(X86::MOV64rm), X86::R14) + .addReg(X86::NoRegister) + .addImm(1) + .addReg(X86::NoRegister) + .addExternalSymbol("runtime.tlsg", X86II::MO_TPOFF) + .addReg(X86::FS); + MI.eraseFromParent(); + return true; + } case X86::TCRETURNdi: case X86::TCRETURNdicc: case X86::TCRETURNri: diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index bc779ead6c3f3..2286709c65b8e 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -146,6 +146,7 @@ namespace { constexpr uint64_t GoStackSmall = 128; constexpr uint64_t GoStackBig = 4096; constexpr int64_t GoGStackGuard0Offset = 16; +constexpr int64_t GoGStackGuard1Offset = 24; static unsigned getIntegerStoreOpcode(unsigned Size) { switch (Size) { @@ -217,23 +218,16 @@ static void emitGoRegSpills(MachineFunction &MF, MachineBasicBlock &MBB, } } -static bool shouldEmitGoStackCheck(const MachineFunction &MF) { +static bool isGoObjGoFunction(const MachineFunction &MF) { const Function &F = MF.getFunction(); return MF.getTarget().getTargetTriple().isOSBinFormatGoObj() && MF.getTarget().getTargetTriple().getArch() == Triple::x86_64 && goabi::isGoCallingConv(F.getCallingConv()) && !F.isVarArg(); } -static void checkGoStackGrowthStatepointContract(const MachineFunction &MF) { - if (!shouldEmitGoStackCheck(MF) || - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr)) - return; - for (const MachineBasicBlock &MBB : MF) - for (const MachineInstr &MI : MBB) - if (MI.getOpcode() == TargetOpcode::STATEPOINT) - report_fatal_error( - "GoObj statepoints require the go-stack-growth-statepoint " - "function attribute"); +static bool shouldEmitGoStackCheck(const MachineFunction &MF) { + return isGoObjGoFunction(MF) && + !MF.getFunction().hasFnAttribute(goabi::NoSplitAttr); } static bool hasGoClosureContext(const Function &F) { @@ -243,71 +237,53 @@ static bool hasGoClosureContext(const Function &F) { return false; } -static MachineInstrBuilder buildGoStackGrowthStatepoint(MachineFunction &MF, - MachineBasicBlock &MBB, - const DebugLoc &DL, - const X86InstrInfo &TII, - StringRef CalleeName) { - MachineInstrBuilder Statepoint = - BuildMI(&MBB, DL, TII.get(TargetOpcode::STATEPOINT)) - .addImm(goabi::StackGrowthStatepointID) - .addImm(0) - .addImm(0); - goabi::addGoObjABI0Callee(Statepoint, MF, CalleeName); - auto AddConstant = [&](uint64_t Value) { - Statepoint.addImm(StackMaps::ConstantOp).addImm(Value); - }; +static MachineBasicBlock & +getGoStackCheckEntryMBB(MachineFunction &MF, MachineBasicBlock &FallbackMBB) { + const BasicBlock &EntryBB = MF.getFunction().getEntryBlock(); + for (MachineBasicBlock &MBB : MF) + if (MBB.getBasicBlock() == &EntryBB) + return MBB; + + if (MachineBasicBlock *MBB = MF.getBlockNumbered(0)) + return *MBB; + return FallbackMBB; +} + +static void emitGoEntryArgsStackMap(MachineFunction &MF, + MachineBasicBlock &FallbackMBB) { + if (!isGoObjGoFunction(MF)) + return; + for (const MachineBasicBlock &MBB : MF) + for (const MachineInstr &MI : MBB) + if (MI.getOpcode() == TargetOpcode::STACKMAP && + MI.getOperand(0).isImm() && + static_cast(MI.getOperand(0).getImm()) == + goabi::EntryArgsStackMapID) + return; + + MachineBasicBlock &EntryMBB = getGoStackCheckEntryMBB(MF, FallbackMBB); + const X86InstrInfo &TII = *MF.getSubtarget().getInstrInfo(); + MachineInstrBuilder StackMap = BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), + TII.get(TargetOpcode::STACKMAP)) + .addImm(goabi::EntryArgsStackMapID) + .addImm(0); const MachineFrameInfo &MFI = MF.getFrameInfo(); - ArrayRef PointerSlots = - MF.getInfo()->getGoArgPointerSlots(); uint64_t PointerSize = MF.getDataLayout().getPointerSize(); - // X86 entry RSP points at the return address for both Go calling - // conventions. GoArgPointerSlot::EntryOffset is a physical stack-map - // location even though the corresponding fixed home uses a logical Go - // argument-area offset. - int64_t StackMapBias = static_cast(PointerSize); - AddConstant(MF.getFunction().getCallingConv()); - AddConstant(0); // Statepoint flags. - AddConstant(0); // Deopt arguments. - AddConstant(PointerSlots.size()); // GC pointers. - for (const X86MachineFunctionInfo::GoArgPointerSlot &Slot : PointerSlots) { + for (const X86MachineFunctionInfo::GoArgPointerSlot &Slot : + MF.getInfo()->getGoArgPointerSlots()) { if (!MFI.isFixedObjectIndex(Slot.FrameIndex)) report_fatal_error( "X86 Go entry argument pointer slot is not a fixed object"); - int64_t ExpectedOffset = - StackMapBias + static_cast(Slot.ArgWord) * PointerSize; + int64_t ExpectedOffset = static_cast(PointerSize) + + static_cast(Slot.ArgWord) * PointerSize; if (PointerSize == 0 || Slot.EntryOffset != ExpectedOffset) report_fatal_error( "X86 Go entry argument pointer slot has invalid RSP offset"); - Statepoint.addImm(StackMaps::IndirectMemRefOp) + StackMap.addImm(StackMaps::IndirectMemRefOp) .addImm(PointerSize) .addReg(X86::RSP) .addImm(Slot.EntryOffset); } - AddConstant(0); // GC allocas. - AddConstant(PointerSlots.size()); // GC base/derived map entries. - for (uint64_t I = 0; I != PointerSlots.size(); ++I) - Statepoint.addImm(I).addImm(I); - Statepoint - .addRegMask( - MF.getSubtarget() - .getRegisterInfo() - ->getCallPreservedMask(MF, MF.getFunction().getCallingConv())) - .addReg(X86::RSP, RegState::ImplicitDefine) - .addReg(X86::SSP, RegState::ImplicitDefine); - return Statepoint; -} - -static MachineBasicBlock & -getGoStackCheckEntryMBB(MachineFunction &MF, MachineBasicBlock &FallbackMBB) { - const BasicBlock &EntryBB = MF.getFunction().getEntryBlock(); - for (MachineBasicBlock &MBB : MF) - if (MBB.getBasicBlock() == &EntryBB) - return MBB; - - if (MachineBasicBlock *MBB = MF.getBlockNumbered(0)) - return *MBB; - return FallbackMBB; } static void emitGoStackCheck(MachineFunction &MF, @@ -378,9 +354,7 @@ static void emitGoStackCheck(MachineFunction &MF, MF.push_front(CheckMBB); MF.push_front(StartMBB); - bool UseStackGrowthStatepoint = - MF.getFunction().hasFnAttribute(goabi::StackGrowthStatepointAttr); - checkGoStackGrowthStatepointContract(MF); + bool IsSystemStack = MF.getFunction().hasFnAttribute(goabi::SystemStackAttr); unsigned ScratchReg = X86::R12; if (StackSize <= GoStackSmall) { @@ -407,7 +381,7 @@ static void emitGoStackCheck(MachineFunction &MF, .addReg(X86::R14) .addImm(1) .addReg(X86::NoRegister) - .addImm(GoGStackGuard0Offset) + .addImm(IsSystemStack ? GoGStackGuard1Offset : GoGStackGuard0Offset) .addReg(X86::NoRegister); BuildMI(CompareMBB, DL, TII.get(X86::JCC_1)) .addMBB(&EntryMBB) @@ -415,16 +389,13 @@ static void emitGoStackCheck(MachineFunction &MF, emitGoRegSpills(MF, *MorestackMBB, Homes, /*Reload=*/false); bool HasClosureContext = hasGoClosureContext(MF.getFunction()); - const char *MorestackName = - HasClosureContext ? "runtime.morestack" : "runtime.morestack_noctxt"; + const char *MorestackName = IsSystemStack ? "runtime.morestackc" + : HasClosureContext ? "runtime.morestack" + : "runtime.morestack_noctxt"; MachineInstrBuilder Morestack = - UseStackGrowthStatepoint - ? buildGoStackGrowthStatepoint(MF, *MorestackMBB, DL, TII, - MorestackName) - : BuildMI(MorestackMBB, DL, TII.get(X86::CALL64pcrel32)); - if (!UseStackGrowthStatepoint) - goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); - if (HasClosureContext) + BuildMI(MorestackMBB, DL, TII.get(X86::CALL64pcrel32)); + goabi::addGoObjABI0Callee(Morestack, MF, MorestackName); + if (HasClosureContext && !IsSystemStack) Morestack.addReg(X86::RDX, RegState::Implicit); emitGoRegSpills(MF, *MorestackMBB, Homes, /*Reload=*/true); BuildMI(MorestackMBB, DL, TII.get(X86::JMP_1)).addMBB(CheckMBB); @@ -1967,6 +1938,9 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, DebugLoc DL; Register ArgBaseReg; + if (isGoObjGoFunction(MF) && MF.getFrameInfo().hasVarSizedObjects()) + report_fatal_error("GoObj stack growth does not support dynamic allocas"); + emitGoEntryArgsStackMap(MF, MBB); emitGoStackCheck(MF, MBB); // Emit extra prolog for argument stack slot reference. @@ -3147,6 +3121,19 @@ StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, bool IsWin64Prologue = MF.getTarget().getMCAsmInfo().usesWindowsCFI(); int64_t FPDelta = 0; + // Go keeps a frame-pointer chain for profiling and traceback, but its local + // frame slots are addressed from SP. This is more than a code-generation + // preference: runtime.gogo can resume a suspended frame after restoring SP + // while deliberately clearing BP. A BP-relative local would then become + // inaccessible even though the Go frame is otherwise valid. Go frames have + // a reserved call frame, so SP remains a stable base for ordinary locals. + if (goabi::isGoCallingConv(MF.getFunction().getCallingConv()) && !IsFixed && + !TRI->hasStackRealignment(MF) && !TRI->hasBasePointer(MF) && + hasReservedCallFrame(MF)) { + FrameReg = TRI->getStackRegister(); + return StackOffset::getFixed(Offset + StackSize); + } + // In an x86 interrupt, remove the offset we added to account for the return // address from any stack object allocated in the caller's frame. Interrupts // do not have a standard return address. Fixed objects in the current frame, @@ -3667,11 +3654,6 @@ bool X86FrameLowering::restoreCalleeSavedRegisters( void X86FrameLowering::determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const { - // Reject an incomplete Go statepoint contract before callee-save spill - // placement. On X86, statepoints can otherwise require an aligned vector - // spill and trip MachineFrameInfo's non-realignment assertion before the - // morestack prologue gets a chance to report the actionable error. - checkGoStackGrowthStatepointContract(MF); TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); // Spill the BasePtr if it's used. diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 99dfaa7bf81e3..fef56314d9d21 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -867,6 +867,9 @@ namespace llvm { int getGoABI0FrameIndex(MachineFunction &MF) const override; SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl &InVals) const override; + SDValue finalizeStatepointCallChain(SDValue Chain, CallingConv::ID CalleeCC, + const SDLoc &DL, + SelectionDAG &DAG) const override; SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Outs, diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 04b7bda0828bd..498e7323664b5 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -30,6 +30,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Statepoint.h" #include "llvm/Transforms/CFGuard.h" #define DEBUG_TYPE "x86-isel" @@ -205,13 +206,9 @@ static SDValue lowerX86GoFormalArguments( ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), DAG.getDataLayout(), ABIConfig); - std::optional EntryArgs; - SmallBitVector MatchedEntryArgWords; - if (F.hasFnAttribute(goabi::StackGrowthStatepointAttr)) { - EntryArgs = goabi::computeEntryArgsInfo(ArgTys, Layout, DAG.getDataLayout(), - ABIConfig); - MatchedEntryArgWords.resize(EntryArgs->NumBits); - } + goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( + ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); SmallVector ArgSpillOffsets(ArgTys.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; @@ -230,11 +227,10 @@ static SDValue lowerX86GoFormalArguments( FuncInfo->clearGoArgHomes(); FuncInfo->clearGoArgPointerSlots(); - auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size) { - if (!EntryArgs) - return; - uint64_t PointerSize = EntryArgs->PointerSize; - for (uint32_t Word : EntryArgs->PointerWords) { + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, + bool IsLiveAtEntry) { + uint64_t PointerSize = EntryArgs.PointerSize; + for (uint32_t Word : EntryArgs.PointerWords) { uint64_t PointerOffset = static_cast(Word) * PointerSize; if (PointerOffset < ArgOffset || PointerOffset + PointerSize > ArgOffset + Size) @@ -256,9 +252,10 @@ static SDValue lowerX86GoFormalArguments( !isInt<32>(EntryOffset)) report_fatal_error( "Go entry argument pointer word has an invalid X86 fixed object"); - FuncInfo->addGoArgPointerSlot( - FI, static_cast(WithinObject), - static_cast(EntryOffset), Word); + if (IsLiveAtEntry) + FuncInfo->addGoArgPointerSlot( + FI, static_cast(WithinObject), + static_cast(EntryOffset), Word); MatchedEntryArgWords.set(Word); } }; @@ -294,7 +291,14 @@ static SDValue lowerX86GoFormalArguments( /*IsImmutable=*/true); X86MachineFunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); - RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size); + // LLVM may replace an unused incoming pointer with poison at every call + // edge. Keep its ABI home so morestack can preserve the complete register + // assignment, but do not expose that uninitialized word as a GC root. + bool IsLiveAtEntry = llvm::any_of( + ArrayRef(Ins).slice(Group.Start, Group.End - Group.Start), + [](const ISD::InputArg &In) { return In.Used; }); + RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, + IsLiveAtEntry); unsigned IntPiece = 0; unsigned FPPiece = 0; @@ -335,12 +339,10 @@ static SDValue lowerX86GoFormalArguments( } } - if (EntryArgs) { - for (uint32_t Word : EntryArgs->PointerWords) - if (!MatchedEntryArgWords.test(Word)) - report_fatal_error( - "Go entry argument pointer word has no X86 fixed object"); - } + for (uint32_t Word : EntryArgs.PointerWords) + if (!MatchedEntryArgWords.test(Word)) + report_fatal_error( + "Go entry argument pointer word has no X86 fixed object"); return Chain; } @@ -433,6 +435,12 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, const X86Subtarget &Subtarget = MF.getSubtarget(); const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); + CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); + bool RepairBeforeCall = goabi::isGoABI0CallingConv(CallerCC) && + goabi::isGoABIInternalCallingConv(CLI.CallConv); + bool RepairAfterCall = goabi::isGoABIInternalCallingConv(CallerCC) && + goabi::isGoABI0CallingConv(CLI.CallConv); + bool IsStatepoint = CLI.CB && isa(CLI.CB); CLI.IsTailCall = false; if (CLI.IsVarArg) @@ -518,6 +526,11 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, Callee = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Callee); SDValue InGlue; + if (RepairBeforeCall) { + Chain = SDValue(DAG.getMachineNode(X86::GO_REPAIR_ABI_INTERNAL_REGS, DL, + MVT::Other, Chain), + 0); + } for (const auto &[Reg, Val] : RegsToPass) { Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue); InGlue = Chain.getValue(1); @@ -528,7 +541,14 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, Ops.push_back(Callee); for (const auto &[Reg, Val] : RegsToPass) Ops.push_back(DAG.getRegister(Reg, Val.getValueType())); - Ops.push_back(DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); + // An ABI0 callee may clobber R14/XMM15, but the mandatory adjacent repair + // restores their ABIInternal values before any following instruction can + // observe them. Model that compound boundary as preserving the registers so + // PEI does not spill and then restore stale copies around the whole function. + CallingConv::ID EffectiveCallCC = + RepairAfterCall ? CallingConv::GoABIInternal : CLI.CallConv; + Ops.push_back( + DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, EffectiveCallCC))); if (InGlue.getNode()) Ops.push_back(InGlue); @@ -538,6 +558,13 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); InGlue = Chain.getValue(1); + if (RepairAfterCall && !IsStatepoint) { + Chain = SDValue(DAG.getMachineNode(X86::GO_REPAIR_ABI_INTERNAL_REGS, DL, + MVT::Other, Chain), + 0); + InGlue = SDValue(); + } + SmallVector ResultVals(Ins.size()); for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; @@ -613,6 +640,19 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, } // namespace +SDValue X86TargetLowering::finalizeStatepointCallChain( + SDValue Chain, CallingConv::ID CalleeCC, const SDLoc &DL, + SelectionDAG &DAG) const { + CallingConv::ID CallerCC = + DAG.getMachineFunction().getFunction().getCallingConv(); + if (!goabi::isGoABIInternalCallingConv(CallerCC) || + !goabi::isGoABI0CallingConv(CalleeCC)) + return Chain; + return SDValue(DAG.getMachineNode(X86::GO_REPAIR_ABI_INTERNAL_REGS, DL, + MVT::Other, Chain), + 0); +} + std::optional X86TargetLowering::getArgumentCopyElisionFrameInfo(const Argument &Arg, MachineFunction &MF) const { diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td index 98752c1f43ab8..1fb6b277d29d2 100644 --- a/llvm/lib/Target/X86/X86InstrControl.td +++ b/llvm/lib/Target/X86/X86InstrControl.td @@ -465,6 +465,15 @@ def GO_GC_WRITE_BARRIER (int_go_gc_write_barrier (i32 timm:$entries)))]>, Requires<[In64BitMode]>; +// Go ABI0 neither supplies nor preserves ABIInternal's dedicated g and zero +// registers. Keep the repair sequence as a call-adjacent pseudo until after +// register allocation so it cannot be treated as an ordinary callee-saved +// register definition before ABI0's reduced callee-save policy is applied. +let isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1, + hasNoSchedulingInfo = 1 in +def GO_REPAIR_ABI_INTERNAL_REGS + : PseudoI<(outs), (ins), []>, Requires<[In64BitMode]>; + // Conditional tail calls are similar to the above, but they are branches // rather than barriers, and they use EFLAGS. let isCall = 1, isTerminator = 1, isReturn = 1, isBranch = 1, diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp index 072ec85719767..4a1fdbe3ecb24 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.cpp +++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp @@ -284,8 +284,9 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { return CSR_64_RT_AllRegs_SaveList; case CallingConv::PreserveNone: return CSR_64_NoneRegs_SaveList; - case CallingConv::GoABIInternal: case CallingConv::GoABI0: + return CSR_64_GoABI0_SaveList; + case CallingConv::GoABIInternal: return CSR_64_Go_SaveList; case CallingConv::CXX_FAST_TLS: if (Is64Bit) @@ -428,8 +429,9 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF, return CSR_64_RT_AllRegs_RegMask; case CallingConv::PreserveNone: return CSR_64_NoneRegs_RegMask; - case CallingConv::GoABIInternal: case CallingConv::GoABI0: + return CSR_64_GoABI0_RegMask; + case CallingConv::GoABIInternal: return CSR_64_Go_RegMask; case CallingConv::CXX_FAST_TLS: if (Is64Bit) diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index c7544538eba30..d41426c27aa01 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -71,7 +71,8 @@ define goabi0 i64 @abi0_call_second_int() { ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] -; A64: str x[[RET]], [sp, #72] +; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. +; A64: str x[[RET]], [sp, #56] entry: %ret = call goabi0 i64 @abi0_second_int(i64 11, i64 22) ret i64 %ret diff --git a/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll b/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll index 8ba776b9c5379..ee5d4d530bbe8 100644 --- a/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll +++ b/llvm/test/CodeGen/AArch64/goobj-function-unsafe.ll @@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" } ; OBJ: symdef 0: main.safe ; OBJ: symdef 1: main.async_unsafe -; OBJ: aux 0.6: type=pcdata target= pc=[0-2:-1] -; OBJ: aux 1.14: type=pcdata target= pc=[0-2:-2] +; OBJ: aux 0.6: type=pcdata target= pc=[0-10:-1] +; OBJ: aux 1.14: type=pcdata target= pc=[0-10:-2] diff --git a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll index 90094e13195eb..ec681c7ae5e11 100644 --- a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll +++ b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg.ll @@ -34,17 +34,17 @@ attributes #0 = { "frame-pointer"="non-leaf" } !0 = !{!"branch_weights", i32 1, i32 1000} ; MachineBlockPlacement puts the cold then block after the epilogue and return, -; but the block is reached while the frame is still active. The later +; but the block is reached while the frame is still active. The pre-frame ; morestack block has no frame. ; ; ASM-LABEL: pcsp_cfg: +; ASM: bl "runtime.morestack_noctxt" ; ASM: bl runtime.panicmem ; ASM: ldr x30, [sp], #32 ; ASM: ret ; ASM: bl runtime.GC -; ASM: bl "runtime.morestack_noctxt" -; The return occupies PC quanta 18-19. The out-of-line then block at 19-23 -; restores the 32-byte frame depth before morestack restores the entry depth. -; OBJ: aux 0.3: type=pcsp target= pc=[0-4:0,4-18:32,18-19:0,19-23:32,23-30:0] -; OBJ: reloc 0.2: off=76 size=4 type=9 add=0 target=runtime.GC +; The stack check and morestack path occupy PC quanta 0-11. The return occupies +; 25-26; the out-of-line then block at 26-30 executes with the frame active. +; OBJ: aux 0.3: type=pcsp target= pc=[0-11:0,11-25:32,25-26:0,26-30:32] +; OBJ: reloc 0.3: off=104 size=4 type=9 add=0 target=runtime.GC diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index e337f0fd2a41c..50ee4a27d6226 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -5,7 +5,7 @@ declare goabiinternal void @"runtime.GC"() define goabiinternal void @subword_homes(i8 %a, i16 %b) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret void @@ -13,7 +13,7 @@ entry: define goabiinternal i64 @large_home_offset([4096 x i64] %stackarg, i64 %regarg) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg @@ -21,7 +21,7 @@ entry: define goabiinternal i64 @large_home_boundary([4094 x i64] %stackarg, i64 %regarg) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg @@ -33,7 +33,7 @@ entry: ; CHECK-DAG: offset: 8, size: 1 ; CHECK: STRBBui $w0, $sp, 8 ; CHECK-NEXT: STRHHui $w1, $sp, 5 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt" +; CHECK: BL &"runtime.morestack_noctxt" ; CHECK: $w0 = LDRBBui $sp, 8 ; CHECK-NEXT: $w1 = LDRHHui $sp, 5 @@ -41,12 +41,12 @@ entry: ; CHECK: offset: 32776, size: 8 ; CHECK: $x27 = ADDXri $sp, 16, 0 ; CHECK-NEXT: STRXui $x0, $x27, 4095 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt" +; CHECK: BL &"runtime.morestack_noctxt" ; CHECK: $x27 = ADDXri $sp, 16, 0 ; CHECK-NEXT: $x0 = LDRXui $x27, 4095 ; CHECK-LABEL: name: large_home_boundary ; CHECK: offset: 32760, size: 8 ; CHECK: STRXui $x0, $sp, 4095 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt" +; CHECK: BL &"runtime.morestack_noctxt" ; CHECK: $x0 = LDRXui $sp, 4095 diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll similarity index 50% rename from llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll rename to llvm/test/CodeGen/AArch64/goobj-stack-growth.ll index afb9de6d20cbc..7ffb1f52cf568 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll @@ -2,9 +2,13 @@ ; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \ ; RUN: -stop-after=prolog-epilog < %s | FileCheck %s -define goabiinternal i64 @closure_morestack_statepoint( +declare goabi0 void @"runtime.morestack"() +declare goabi0 void @"runtime.morestack_noctxt"() +declare goabi0 void @"runtime.morestackc"() + +define goabiinternal i64 @closure_morestack_call( i64 %value, ptr nest %ctxt) "frame-pointer"="non-leaf" - "go-stack-growth-statepoint" { + { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -14,8 +18,8 @@ entry: ret i64 %sum } -define goabiinternal ptr @pointer_morestack_statepoint(ptr %pointer) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { +define goabiinternal ptr @pointer_morestack_call(ptr %pointer) + "frame-pointer"="non-leaf" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -28,7 +32,7 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %p16) "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + ptr %p16) "frame-pointer"="non-leaf" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 @@ -37,31 +41,38 @@ entry: ret ptr %pointer } -; CHECK-LABEL: name: closure_morestack_statepoint +define goabiinternal void @systemstack_growth() "frame-pointer"="non-leaf" + "go-systemstack" { +entry: + %buf = alloca [8192 x i8], align 16 + %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 + store volatile i8 1, ptr %slot, align 1 + ret void +} + +; CHECK-LABEL: name: closure_morestack_call ; CHECK-NOT: ANNOTATION_LABEL -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, -; CHECK-SAME: csr_aarch64_go, implicit-def $sp, -; CHECK-SAME: implicit-def dead early-clobber $lr, +; CHECK: BL &"runtime.morestack", implicit-def $lr, implicit $sp, ; CHECK-SAME: implicit $x3, implicit $x26 -; CHECK-NOT: BL +; CHECK: STACKMAP 5147419139155979380, 0 +; CHECK-NOT: STATEPOINT -; CHECK-LABEL: name: pointer_morestack_statepoint +; CHECK-LABEL: name: pointer_morestack_call ; CHECK: STRXui $x0, $sp, 1 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, 1, 8, $sp, 8, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, -; CHECK-SAME: csr_aarch64_go, implicit-def $sp, -; CHECK-SAME: implicit-def dead early-clobber $lr, +; CHECK: BL &"runtime.morestack_noctxt", implicit-def $lr, implicit $sp, ; CHECK-SAME: implicit $x3 ; CHECK: $x0 = LDRXui $sp, 1 +; CHECK: STACKMAP 5147419139155979380, 0, 1, 8, $sp, 8 ; CHECK-NOT: BL ; CHECK-LABEL: name: mixed_register_and_stack_pointer_args -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $sp, 16, 1, 8, $sp, 8, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, -; CHECK-SAME: csr_aarch64_go, implicit-def $sp, -; CHECK-SAME: implicit-def dead early-clobber $lr, +; CHECK: BL &"runtime.morestack_noctxt", implicit-def $lr, implicit $sp, +; CHECK-SAME: implicit $x3 +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $sp, 16, 1, 8, $sp, 8 + +; CHECK-LABEL: name: systemstack_growth +; CHECK: $x17 = LDRXui $x28, 3 +; CHECK: BL &"runtime.morestackc", implicit-def $lr, implicit $sp, ; CHECK-SAME: implicit $x3 +; CHECK: STACKMAP 5147419139155979380, 0 diff --git a/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll b/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll index c9bdc70acc821..410112ab3c0db 100644 --- a/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll +++ b/llvm/test/CodeGen/Generic/go-statepoint-alloca-address.ll @@ -11,7 +11,7 @@ declare goabiinternal void @safepoint() declare goabiinternal void @observe(ptr addrspace(1)) define goabiinternal void @first_class_alloca_address() - "go-stack-growth-statepoint" gc "statepoint-example" { + gc "statepoint-example" { ; X86-LABEL: name: first_class_alloca_address ; X86: STATEPOINT 1, ; X86-NEXT: $rax = LEA64r diff --git a/llvm/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll b/llvm/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll new file mode 100644 index 0000000000000..9ac318dd7a7c5 --- /dev/null +++ b/llvm/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll @@ -0,0 +1,34 @@ +; REQUIRES: aarch64-registered-target, x86-registered-target +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -goobj-package-path=main -filetype=obj %s -o %t.x86.o +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.x86.o | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -goobj-package-path=main -filetype=obj %s -o %t.a64.o +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.a64.o | FileCheck %s + +; A normal statepoint selects a live frame map. When physical block placement +; later reaches a CFG path at the entry SP depth, PCDATA_StackMapIndex must +; return to the function-level entry map without identifying morestack by name. + +; CHECK: type=pcdata target= pc=[0-{{[0-9]+}}:0,{{[0-9]+}}-{{[0-9]+}}:1,{{[0-9]+}}-{{[0-9]+}}:0] + +declare goabiinternal void @callee() + +define goabiinternal ptr addrspace(1) + @entry_map_cfg(ptr addrspace(1) %pointer) "frame-pointer"="non-leaf" + gc "statepoint-example" { +entry: + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, ptr elementtype(void ()) @callee, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %pointer) ] + %relocated = call coldcc ptr addrspace(1) + @llvm.experimental.gc.relocate.p1(token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + +declare token @llvm.experimental.gc.statepoint.p0( + i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) +declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token, i32 immarg, i32 immarg) diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll new file mode 100644 index 0000000000000..428cdd6055b65 --- /dev/null +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -0,0 +1,49 @@ +; REQUIRES: aarch64-registered-target, x86-registered-target +; RUN: llc -mtriple=aarch64-apple-darwin-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog < %s | FileCheck %s --check-prefixes=CHECK,A64 +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-apple-darwin-goobj -filetype=obj -o %t.a64.o %s +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s + +declare goabiinternal void @callee(ptr) +declare goabi0 void @"callee.abi0"(ptr) + +define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { +entry: + %slot = alloca ptr, align 8 + store volatile ptr %pointer, ptr %slot, align 8 + call goabiinternal void @callee(ptr %pointer) + ret void +} + +define goabiinternal void @split(ptr %pointer) { +entry: + %slot = alloca ptr, align 8 + store volatile ptr %pointer, ptr %slot, align 8 + call goabiinternal void @callee(ptr %pointer) + ret void +} + +define goabiinternal void @nosplit_abi0_call(ptr %pointer) "go-nosplit" { +entry: + %closure = alloca [3 x ptr], align 8 + %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 + %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 + store volatile ptr @callee, ptr %code, align 8 + store volatile ptr %pointer, ptr %context, align 8 + call goabi0 void @"callee.abi0"(ptr %closure) + ret void +} + +; CHECK-LABEL: name: nosplit +; CHECK: STACKMAP 5147419139155979380, 0 +; CHECK-NOT: STATEPOINT +; CHECK-NOT: runtime.morestack +; CHECK-LABEL: name: split +; CHECK-DAG: STACKMAP 5147419139155979380, 0 +; CHECK-DAG: runtime.morestack_noctxt + +; A64-LABEL: name: nosplit_abi0_call +; A64: stackSize: 48 +; A64: maxCallFrameSize: 16 diff --git a/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll b/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll new file mode 100644 index 0000000000000..7a300e74ce406 --- /dev/null +++ b/llvm/test/CodeGen/Generic/goobj-statepoint-default-stack-growth.ll @@ -0,0 +1,31 @@ +; REQUIRES: aarch64-registered-target, x86-registered-target +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog -o - %s | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -stop-after=prolog-epilog -o - %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s +; RUN: llc -mtriple=aarch64-unknown-linux-goobj -filetype=obj -o %t.a64.o %s + +; GoObj Go functions use stack growth by default. An ordinary statepoint does +; not need a second frontend attribute to make the target synthesize the late +; ABI0 morestack call and the function-level entry-args stack map. + +declare goabiinternal void @callee() + +define goabiinternal void @default_stack_growth() "frame-pointer"="non-leaf" + gc "statepoint-example" { +entry: + call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 0, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, + i32 0, i32 0) + ret void +} + +declare token @llvm.experimental.gc.statepoint.p0( + i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) + +; CHECK-LABEL: name: default_stack_growth +; CHECK-DAG: STACKMAP 5147419139155979380, 0 +; CHECK-DAG: runtime.morestack_noctxt +; CHECK-DAG: STATEPOINT 0, 0, 0, @callee diff --git a/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll b/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll deleted file mode 100644 index 9df6561709bb7..0000000000000 --- a/llvm/test/CodeGen/Generic/goobj-statepoint-requires-stack-growth-attribute.ll +++ /dev/null @@ -1,27 +0,0 @@ -; RUN: not --crash llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj \ -; RUN: -o /dev/null %s 2>&1 | FileCheck %s -; RUN: not --crash llc -mtriple=aarch64-unknown-linux-goobj -filetype=obj \ -; RUN: -o /dev/null %s 2>&1 | FileCheck %s - -; A GoObj function with Machine StackMaps must also describe its late -; stack-growth call as a statepoint. Without the frontend attribute, reject the -; function instead of allowing the preceding call's map to cover the raw -; runtime.morestack call. - -declare goabiinternal void @callee() - -define goabiinternal void @missing_stack_growth_attribute() - gc "statepoint-example" { -entry: - call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 0, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, - i32 0, i32 0) - ret void -} - -declare token @llvm.experimental.gc.statepoint.p0( - i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) - -; CHECK: LLVM ERROR: GoObj statepoints require the -; CHECK-SAME: go-stack-growth-statepoint function attribute diff --git a/llvm/test/CodeGen/X86/go-abi-transition.ll b/llvm/test/CodeGen/X86/go-abi-transition.ll new file mode 100644 index 0000000000000..6acf6c8e0ed9e --- /dev/null +++ b/llvm/test/CodeGen/X86/go-abi-transition.ll @@ -0,0 +1,111 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=ASM,ASM-O0 +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=ASM,ASM-O2 +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -goobj-package-path=main \ +; RUN: -verify-machineinstrs -filetype=obj -o %t.o %s +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.o | \ +; RUN: FileCheck %s --check-prefix=OBJ + +declare goabiinternal void @internal_callee() +declare goabi0 void @"abi0_callee"() +declare goabi0 i64 @"abi0_result"() + +define goabi0 void @"abi0_to_internal"() "go-nosplit" { +; ASM-LABEL: "abi0_to_internal": +; ASM-NOT: pushq %r14 +; ASM-NOT: movaps %xmm15 +; ASM: xorps %xmm15, %xmm15 +; ASM-NEXT: movq %fs:runtime.tlsg@TPOFF, %r14 +; ASM-NEXT: callq internal_callee +; ASM-NOT: movaps {{.*}}, %xmm15 +; ASM-NOT: popq %r14 +; ASM: retq +entry: + call goabiinternal void @internal_callee() + ret void +} + +define goabiinternal void @internal_to_abi0() "go-nosplit" { +; ASM-LABEL: internal_to_abi0: +; ASM: callq "abi0_callee" +; ASM-NEXT: xorps %xmm15, %xmm15 +; ASM-NEXT: movq %fs:runtime.tlsg@TPOFF, %r14 +; ASM-NOT: movaps {{.*}}, %xmm15 +; ASM-NOT: popq %r14 +; ASM: retq +entry: + call goabi0 void @"abi0_callee"() + ret void +} + +define goabiinternal void @internal_statepoint_to_abi0() + "go-nosplit" gc "statepoint-example" { +; ASM-LABEL: internal_statepoint_to_abi0: +; ASM: callq "abi0_callee" +; A statepoint label records the return PC between the call and the repair. +; ASM: xorps %xmm15, %xmm15 +; ASM-NEXT: movq %fs:runtime.tlsg@TPOFF, %r14 +; ASM: retq +entry: + call goabi0 token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 0, i32 0, ptr elementtype(void ()) @"abi0_callee", + i32 0, i32 0, i32 0, i32 0) + ret void +} + +define goabiinternal i64 @internal_statepoint_result_from_abi0() + "go-nosplit" gc "statepoint-example" { +; ABI0 returns through the caller frame. Complete that result load before the +; statepoint's final ABIInternal repair, without creating a chain/glue cycle. +; ASM-LABEL: internal_statepoint_result_from_abi0: +; ASM: callq "abi0_result" +; ASM-O0: movq %rsp, [[RESULT_BASE:%r[a-z0-9]+]] +; ASM-O0-NEXT: movq ([[RESULT_BASE]]), %rax +; ASM-O2: movq (%rsp), %rax +; ASM: xorps %xmm15, %xmm15 +; ASM-NEXT: movq %fs:runtime.tlsg@TPOFF, %r14 +; ASM: retq +entry: + %token = call goabi0 token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 0, i32 0, ptr elementtype(i64 ()) @"abi0_result", + i32 0, i32 0, i32 0, i32 0) + %result = call i64 @llvm.experimental.gc.result.i64(token %token) + ret i64 %result +} + +define goabiinternal i8 @go_local_is_sp_relative() "go-nosplit" + "frame-pointer"="non-leaf" { +; Go context restoration can restore SP while clearing BP. Keep local slots +; usable across that non-local resume while retaining the frame-pointer chain. +; ASM-LABEL: go_local_is_sp_relative: +; ASM: pushq %rbp +; ASM: movq %rsp, %rbp +; ASM: movb $7, {{[0-9]+}}(%rsp) +; ASM: callq internal_callee +; ASM: mov{{(b|zbl)}} {{[0-9]+}}(%rsp), %{{(al|eax)}} +; ASM: popq %rbp +; ASM: retq +entry: + %local = alloca i8, align 1 + store volatile i8 7, ptr %local, align 1 + call goabiinternal void @internal_callee() + %value = load volatile i8, ptr %local, align 1 + ret i8 %value +} + +declare token @llvm.experimental.gc.statepoint.p0( + i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) +declare i64 @llvm.experimental.gc.result.i64(token) + +; Each repair has one symbol-free R_TLS_LE relocation, matching native x86 Go +; objects. Calls retain their ABI-specific named targets. +; OBJ-NOT: nonpkgref {{[0-9]+}}: runtime.tlsg +; OBJ: reloc {{.*}} type=15 {{.*}} kind=unknown pkg=0 sym=0 +; OBJ: reloc {{.*}} kind=R_CALL +; OBJ: reloc {{.*}} kind=R_CALL +; OBJ: reloc {{.*}} type=15 {{.*}} kind=unknown pkg=0 sym=0 +; OBJ: reloc {{.*}} kind=R_CALL +; OBJ: reloc {{.*}} type=15 {{.*}} kind=unknown pkg=0 sym=0 +; OBJ: reloc {{.*}} kind=R_CALL +; OBJ: reloc {{.*}} type=15 {{.*}} kind=unknown pkg=0 sym=0 diff --git a/llvm/test/CodeGen/X86/go-stack-alignment.ll b/llvm/test/CodeGen/X86/go-stack-alignment.ll index f54f5cdcc6761..89c2fe02f0dbd 100644 --- a/llvm/test/CodeGen/X86/go-stack-alignment.ll +++ b/llvm/test/CodeGen/X86/go-stack-alignment.ll @@ -12,9 +12,10 @@ define goabiinternal void @stack_slot(ptr %value) #0 { ; CHECK-NOT: AND64 ; CHECK: $rsp = frame-setup SUB64ri32 $rsp, 40 ; CHECK-NOT: MOVAPSmr -; CHECK: MOVUPSmr $rbp, 1, $noreg, -16, $noreg +; Go locals stay SP-relative because runtime context restoration may clear BP. +; CHECK: MOVUPSmr $rsp, 1, $noreg, 24, $noreg ; CHECK-SAME: align 8 -; CHECK: MOVUPSmr $rbp, 1, $noreg, -32, $noreg +; CHECK: MOVUPSmr $rsp, 1, $noreg, 8, $noreg ; CHECK-SAME: align 8 ; CHECK: CALL64pcrel32 @sink entry: @@ -30,4 +31,4 @@ entry: declare void @llvm.memset.inline.p0.i64(ptr, i8, i64, i1 immarg) attributes #0 = { "frame-pointer"="non-leaf" "go-async-unsafe" - "go-stack-growth-statepoint" } + } diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index e33917ed9c72f..0547d1a782189 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -13,7 +13,7 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) ; The alloca is the canonical home of a register argument. Its direct gc-live ; base makes its pointer contents active at the callsite, so bit zero belongs ; to ArgsPointerMaps rather than LocalsPointerMaps. -define goabiinternal ptr @active_scalar(ptr %value) #0 gc "statepoint-example" { +define goabiinternal ptr @active_scalar(ptr %value) gc "statepoint-example" { entry: %home = alloca ptr, align 8 call void @llvm.lifetime.start.p0(i64 8, ptr %home) @@ -46,7 +46,7 @@ entry: ; Its alloca base is not directly live at the ordinary statepoint, so the ; function needs one StackObject at non-negative offset zero relative to argp. define goabiinternal void @inactive_aggregate(%aggregate %value) - #0 gc "statepoint-example" { + gc "statepoint-example" { entry: %home = alloca %aggregate, align 8 call void @llvm.lifetime.start.p0(i64 24, ptr %home) @@ -74,7 +74,7 @@ entry: ; An aggregate assigned wholly to the stack reuses its caller-populated slot; ; the lifetime marker must not make SelectionDAG allocate and copy a local. define goabiinternal void @inactive_stack_aggregate(%stack_aggregate %value) - #0 gc "statepoint-example" { + gc "statepoint-example" { entry: %home = alloca %stack_aggregate, align 8 call void @llvm.lifetime.start.p0(i64 16, ptr %home) @@ -103,4 +103,3 @@ entry: declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/llvm/test/CodeGen/X86/goobj-function-unsafe.ll b/llvm/test/CodeGen/X86/goobj-function-unsafe.ll index a635d4348d54f..1b008df6f71a9 100644 --- a/llvm/test/CodeGen/X86/goobj-function-unsafe.ll +++ b/llvm/test/CodeGen/X86/goobj-function-unsafe.ll @@ -25,5 +25,5 @@ attributes #0 = { "go-async-unsafe" } ; OBJ: symdef 0: main.safe ; OBJ: symdef 1: main.async_unsafe -; OBJ: aux 0.6: type=pcdata target= pc=[0-32:-1] -; OBJ: aux 1.14: type=pcdata target= pc=[0-29:-2] +; OBJ: aux 0.6: type=pcdata target= pc=[0-48:-1] +; OBJ: aux 1.14: type=pcdata target= pc=[0-33:-2] diff --git a/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll b/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll index 75fe4cefb738c..9740733b3e0a0 100644 --- a/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll +++ b/llvm/test/CodeGen/X86/goobj-pcfile-pcline.ll @@ -55,17 +55,18 @@ done: ; COMMON: file-count: 1 ; COMMON-NEXT: file 0: {{.*}}cdebug.c -; COMMON: hasheddef-count: 4 +; COMMON: hasheddef-count: 5 ; COMMON: nonpkgdef-count: 0 -; X86: hash 0: a21821a940f91b1a89b53461b092269e -; X86-NEXT: hash 1: f9ced9dca799cb1833bd530443fd1f9e -; X86-NEXT: hash 2: 2677aa574f61b902c15de55332c2c2ea -; ARM64: hash 0: 105a58e8d53963b571ff833d8449eeda -; ARM64-NEXT: hash 1: 3bef6118e9cf260f78533cdf1a6375ec -; ARM64-NEXT: hash 2: 3a577c7591ae76fd8b51f8e7ea4ac9d8 -; COMMON-NEXT: hash 3: 4b0e7a681c0340c9a97ef4802a3af2f8 -; COMMON: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=3 -; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=3 +; X86: hash 0: b3ea8d63c732dc89ddc1accf4206a7bb +; X86-NEXT: hash 1: 2f05713252bb1b9673be7cf4f639e025 +; X86-NEXT: hash 2: 2d9a46e763fb2afa2f6bf504aef97634 +; ARM64: hash 0: f933ff4cb32d167004d69c3934a4e464 +; ARM64-NEXT: hash 1: cbf5923fb1da07cca3254ffa87009bcc +; ARM64-NEXT: hash 2: ba4d2125291d2de2a0592dd4bdc679e7 +; COMMON-NEXT: hash 3: 90eb4206729e98003d69bb53d855ecb0 +; COMMON-NEXT: hash 4: 4b0e7a681c0340c9a97ef4802a3af2f8 +; COMMON: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=010000000100000000 pkg=hashed sym=3 +; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 pkg=hashed sym=4 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcsp target= pc={{.*}} pkg=hashed sym=0 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcfile target= pc={{.*}}:0{{.*}} pkg=hashed sym=0 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcline target= pc=[ @@ -74,4 +75,4 @@ done: ; COMMON-SAME: :5 ; COMMON-SAME: pkg=hashed sym=1 ; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=2 -; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}} pkg=hashed sym=0 +; COMMON-NEXT: aux {{[0-9]+}}.{{[0-9]+}}: type=pcdata target= pc={{.*}}:0{{.*}} pkg=hashed sym=0 diff --git a/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll b/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll index 69795833b09c3..86f136c5eab90 100644 --- a/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll +++ b/llvm/test/CodeGen/X86/goobj-pcsp-cfg.ll @@ -43,7 +43,7 @@ attributes #0 = { "frame-pointer"="non-leaf" } ; ASM: callq runtime.GC ; ASM: callq "runtime.morestack_noctxt" -; The return occupies PC quanta 57-58. The out-of-line then block at 59-66 +; The return occupies PC quanta 59-60. The out-of-line then block at 61-68 ; restores the 24-byte frame depth before morestack restores the entry depth. -; OBJ: aux 0.3: type=pcsp target= pc=[0-7:0,7-14:8,14-57:24,57-58:8,58-59:0,59-66:24,66-92:0] -; OBJ: reloc 0.2: off=60 size=4 type=7 add=0 target=runtime.GC +; OBJ: aux 0.3: type=pcsp target= pc=[0-7:0,7-14:8,14-59:24,59-60:8,60-61:0,61-68:24,68-100:0] +; OBJ: reloc 0.2: off=62 size=4 type=7 add=0 target=runtime.GC diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index 6b46c8b4a98b3..34cdba82af48b 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -66,8 +66,8 @@ join: ; CHECK: nonpkgref {{[0-9]+}}: runtime.morestack_noctxt abi=0 type=0 size=0 ; CHECK: nonpkgref {{[0-9]+}}: runtime.morestack abi=0 type=0 size=0 ; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcinfo target= args={{[1-9][0-9]*}} locals={{[1-9][0-9][0-9][0-9][0-9]*}} -; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 -; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data=0100000000000000 +; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}} +; CHECK: aux {{[0-9]+}}.{{[0-9]+}}: type=funcdata target= data={{[0-9a-f]+}} ; CHECK: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1] ; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:0] ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack_noctxt @@ -116,7 +116,8 @@ join: ; PEI: {{.*rax}} = MOV64rm {{.*rsp}}, 1, {{.*noreg}}, 8, {{.*noreg}} ; PEI-LABEL: name: big_closure_frame -; PEI: CALL64pcrel32 &"runtime.morestack", {{.*}}implicit $rdx +; PEI: CALL64pcrel32 &"runtime.morestack" +; PEI-SAME: implicit $rsp, implicit $ssp, implicit $rdx ; PEI-LABEL: name: large_outgoing_frame ; PEI: stackSize: 0 diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll b/llvm/test/CodeGen/X86/goobj-stack-growth.ll similarity index 64% rename from llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll rename to llvm/test/CodeGen/X86/goobj-stack-growth.ll index 005b774550eb6..5e583ecff8139 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-statepoint.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth.ll @@ -12,8 +12,10 @@ } declare goabiinternal void @use_three_pointers(ptr, ptr, ptr) +declare goabi0 void @"runtime.morestack_noctxt"() +declare goabi0 void @"runtime.morestackc"() -define goabiinternal i64 @morestack_statepoint(i64 %value) "go-stack-growth-statepoint" { +define goabiinternal i64 @morestack_call(i64 %value) { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -22,7 +24,7 @@ entry: } define goabi0 void @"abi0_pointer_arguments"(ptr %first, ptr %second, ptr %third) - "frame-pointer"="non-leaf" "go-stack-growth-statepoint" { + "frame-pointer"="non-leaf" { entry: call goabiinternal void @use_three_pointers( ptr %first, ptr %second, ptr %third) @@ -30,7 +32,7 @@ entry: } define goabiinternal %many.results @initialized_pointer_result(ptr %pointer) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -40,7 +42,7 @@ entry: define goabiinternal %partial.results @partial_aggregate_result( ptr %first, ptr %second) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -52,7 +54,7 @@ define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) "go-stack-growth-statepoint" { + ptr %pointer) { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -64,7 +66,7 @@ define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, %pointer.aggregate %value) - "go-stack-growth-statepoint" "go_results_tuple" { + "go_results_tuple" { entry: %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 @@ -76,12 +78,20 @@ entry: ret { ptr, ptr } %r1 } -; CHECK-LABEL: name: morestack_statepoint +define goabiinternal void @systemstack_growth() + "go-systemstack" { +entry: + %buf = alloca [5000 x i8], align 8 + %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 + store volatile i8 1, ptr %slot, align 1 + ret void +} + +; CHECK-LABEL: name: morestack_call ; CHECK-NOT: ANNOTATION_LABEL -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp -; CHECK-NOT: CALL64pcrel32 +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0 +; CHECK-NOT: STATEPOINT ; The RSP offsets below include the 8-byte amd64 return address. The argument ; words themselves are numbered from the start of the Go ABI arg/result/home @@ -95,42 +105,36 @@ entry: ; CHECK: offset: 16, size: 8 ; CHECK: offset: 8, size: 8 ; CHECK: offset: 0, size: 8 -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 23, 2, 0, 2, 0, 2, 3, -; CHECK-SAME: 1, 8, $rsp, 8, 1, 8, $rsp, 16, 1, 8, $rsp, 24, -; CHECK-SAME: 2, 0, 2, 3, 0, 0, 1, 1, 2, 2, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $rsp, 8, 1, 8, $rsp, 16, 1, 8, $rsp, 24 ; CHECK: renamable $rax = MOV64rm $rbp, 1, $noreg, 16, $noreg ; CHECK: renamable $rbx = MOV64rm $rbp, 1, $noreg, 24, $noreg ; CHECK: renamable $rcx = MOV64rm $rbp, 1, $noreg, 32, $noreg ; CHECK-LABEL: name: initialized_pointer_result ; CHECK: MOV64mr $rsp, 1, $noreg, 72, $noreg, $rax -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, -; CHECK-SAME: 1, 8, $rsp, 72, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; The unused pointer parameter may arrive as poison. Its home is preserved for +; the ABI retry path but must not be scanned. +; CHECK: STACKMAP 5147419139155979380, 0{{$}} ; CHECK-LABEL: name: partial_aggregate_result ; CHECK: MOV64mr $rsp, 1, $noreg, 80, $noreg, $rax ; CHECK: MOV64mr $rsp, 1, $noreg, 88, $noreg, $rbx -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $rsp, 80, 1, 8, $rsp, 88, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0{{$}} ; CHECK-LABEL: name: scalar_stack_argument -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 1, -; CHECK-SAME: 1, 8, $rsp, 64, -; CHECK-SAME: 2, 0, 2, 1, 0, 0, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0, 1, 8, $rsp, 64 ; CHECK-LABEL: name: aggregate_stack_argument -; CHECK: STATEPOINT 5147424658422983495, 0, 0, &"runtime.morestack_noctxt", -; CHECK-SAME: 2, 22, 2, 0, 2, 0, 2, 2, -; CHECK-SAME: 1, 8, $rsp, 48, 1, 8, $rsp, 64, -; CHECK-SAME: 2, 0, 2, 2, 0, 0, 1, 1, -; CHECK-SAME: csr_64_go, implicit-def $rsp, implicit-def $ssp +; CHECK: CALL64pcrel32 &"runtime.morestack_noctxt", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0, +; CHECK-SAME: 1, 8, $rsp, 48, 1, 8, $rsp, 64 + +; CHECK-LABEL: name: systemstack_growth +; CHECK: CMP64rm $r12, $r14, 1, $noreg, 24, $noreg +; CHECK: CALL64pcrel32 &"runtime.morestackc", implicit $rsp, implicit $ssp +; CHECK: STACKMAP 5147419139155979380, 0 diff --git a/llvm/unittests/MC/GoObjStackMapUtilsTest.cpp b/llvm/unittests/MC/GoObjStackMapUtilsTest.cpp index 3669ad869b81e..4787a636c10bd 100644 --- a/llvm/unittests/MC/GoObjStackMapUtilsTest.cpp +++ b/llvm/unittests/MC/GoObjStackMapUtilsTest.cpp @@ -97,21 +97,21 @@ TEST(GoObjStackMapUtilsTest, RejectsInvalidVectorWordPlacement) { goobj::StackMapSlotKind::Invalid); } -TEST(GoObjStackMapUtilsTest, ClassifiesStackGrowthPointerVectorWords) { +TEST(GoObjStackMapUtilsTest, ClassifiesEntryArgsPointerVectorWords) { auto Words = goobj::expandStackMapPointerWords( /*Offset=*/64, /*Size=*/16, /*IsIndirect=*/true, /*PointerSize=*/8); ASSERT_TRUE(Words); - EXPECT_EQ(goobj::classifyStackGrowthStackMapSlot( + EXPECT_EQ(goobj::classifyEntryArgsStackMapSlot( (*Words)[0], /*PointerSize=*/8, /*ArgsStart=*/64, /*ArgsSize=*/16), 0u); - EXPECT_EQ(goobj::classifyStackGrowthStackMapSlot( + EXPECT_EQ(goobj::classifyEntryArgsStackMapSlot( (*Words)[1], /*PointerSize=*/8, /*ArgsStart=*/64, /*ArgsSize=*/16), 1u); - EXPECT_FALSE(goobj::classifyStackGrowthStackMapSlot( + EXPECT_FALSE(goobj::classifyEntryArgsStackMapSlot( /*Offset=*/68, /*PointerSize=*/8, /*ArgsStart=*/64, /*ArgsSize=*/16)); - EXPECT_FALSE(goobj::classifyStackGrowthStackMapSlot( + EXPECT_FALSE(goobj::classifyEntryArgsStackMapSlot( /*Offset=*/80, /*PointerSize=*/8, /*ArgsStart=*/64, /*ArgsSize=*/16)); }