diff --git a/llvm/include/llvm/BinaryFormat/GoObj.h b/llvm/include/llvm/BinaryFormat/GoObj.h index bcb11d1b817aa..b9c7fcc7f5476 100644 --- a/llvm/include/llvm/BinaryFormat/GoObj.h +++ b/llvm/include/llvm/BinaryFormat/GoObj.h @@ -23,6 +23,26 @@ namespace GoObj { // runtime.morestack slow path. inline constexpr uint64_t StackGrowthStatepointID = 0x476f537461636b47ULL; +// GoALLC encodes pointer maps for fixed allocas as a self-describing suffix of +// statepoint deopt locations: +// +// ordinary-deopt*, BEGIN, protocol-length, record-count, +// (TAG, record-length, direct-base, byte-offset, byte-size, alignment, +// pointer-size, bit-count, word-bits, word-count, bitmap-word*)*, +// END, protocol-length +// +// Protocol length counts BEGIN through END and excludes its trailing duplicate; +// record length counts TAG through the final bitmap word. The first contract +// has no version, requires a whole alloca at byte offset zero, and uses 64-bit +// bitmap words. Bit N, stored low-bit first, describes the pointer-sized slot at +// direct-base + byte-offset + N * pointer-size. Padding bits must be zero. +// These tags are intentionally small enough to remain inline StackMaps +// constants; bitmap payload words may use the StackMaps constant pool. +inline constexpr int64_t AllocaPtrMapBeginMagic = 0x47414c41; // "GALA" +inline constexpr int64_t AllocaPtrMapEndMagic = 0x414c4c43; // "ALLC" +inline constexpr int64_t AllocaPtrMapRecordTag = 0x5054524d; // "PTRM" +inline constexpr uint32_t AllocaPtrMapBitmapWordBits = 64; + inline constexpr char Magic[] = {'\0', 'g', 'o', '1', '2', '0', 'l', 'd'}; inline constexpr uint32_t MagicSize = sizeof(Magic); inline constexpr uint32_t FingerprintSize = 8; diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index ab6eb01e17b22..c6ed06e89296a 100644 --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -111,19 +111,11 @@ class FunctionLoweringInfo { // relocates only. SDValueNode, } type = NoRelocate; - struct SpillLocation { - int FI; - int64_t Offset; - uint64_t Size; - uint64_t Alignment; - }; - - // Payload contains either the exact stack subslot in which the value was - // spilled/already homed, or the virtual register which contains the - // re-definition. + // Payload contains either frame index of the stack slot in which the value + // was spilled, or virtual register which contains the re-definition. union payload_t { - payload_t() : Spill({-1, 0, 0, 1}) {} - SpillLocation Spill; + payload_t() : FI(-1) {} + int FI; Register Reg; } payload; }; diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h index 255e509dfccc1..1bf6be06b2e0d 100644 --- a/llvm/include/llvm/CodeGen/StackMaps.h +++ b/llvm/include/llvm/CodeGen/StackMaps.h @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace llvm { @@ -357,6 +358,11 @@ class StackMaps { /// Get function info. FnInfoMap &getFnInfos() { return FnInfos; } + /// Resolve an inline or constant-pool stackmap location to its signed value. + /// Returns std::nullopt for non-constant locations or an invalid pool index. + LLVM_ABI std::optional + getConstantValue(const Location &Loc) const; + private: static const char *WSMP; diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 0e0e9f27b3324..dc795d53b2d8d 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -119,7 +119,7 @@ class MCContext { LocationType Type; uint16_t Size; uint16_t DwarfRegNum; - int32_t Offset; + int64_t Offset; }; struct GoObjStackMapEntry { @@ -128,6 +128,7 @@ class MCContext { bool IsIndirectCall; uint64_t StackSize; uint32_t PointerSize; + uint32_t NumDeoptLocations; std::vector Locations; }; diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index b8ac5355e4c7e..effdbfda95df4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -44,7 +44,6 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CheckedArithmetic.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include @@ -77,10 +76,6 @@ static cl::opt MaxRegistersForGCPointers( cl::desc("Max number of VRegs allowed to pass GC pointer meta args in")); typedef FunctionLoweringInfo::StatepointRelocationRecord RecordType; -typedef StatepointLoweringState::FixedStackHome FixedStackHome; - -static constexpr StringLiteral FixedStackHomeMD = - "llvm.statepoint.fixed_stack_home"; static std::optional> getArgumentValueOffset(const Value *V, const DataLayout &DL) { @@ -109,68 +104,6 @@ getArgumentValueOffset(const Value *V, const DataLayout &DL) { return std::pair(Arg, Offset); } -static std::optional -getMarkedFixedStackHome(const Value *V, SelectionDAGBuilder &Builder) { - const auto *Load = dyn_cast(V); - if (!Load) - return std::nullopt; - const MDNode *Marker = Load->getMetadata(FixedStackHomeMD); - if (!Marker) - return std::nullopt; - - auto Invalid = [](const Twine &Reason) -> void { - report_fatal_error(Twine("invalid ") + FixedStackHomeMD + - " load: " + Reason); - }; - - if (Marker->getNumOperands() != 0) - Invalid("metadata must be an empty node"); - if (!Load->getType()->isPointerTy()) - Invalid("result is not a scalar pointer"); - // Canonical producers make these loads volatile so SelectionDAG cannot - // forward or CSE distinct logical roots before their fixed homes are - // recorded. Volatile is valid here and adds no extra stack object; atomic - // accesses are outside this internal marker's contract. - if (Load->isAtomic()) - Invalid("load is atomic"); - - const DataLayout &DL = Builder.DAG.getDataLayout(); - const Value *Pointer = Load->getPointerOperand(); - APInt Offset(DL.getIndexTypeSizeInBits(Pointer->getType()), 0, - /*isSigned=*/true); - const Value *Base = Pointer->stripAndAccumulateConstantOffsets( - DL, Offset, /*AllowNonInbounds=*/true); - const auto *Alloca = dyn_cast(Base); - if (!Alloca || !Alloca->isStaticAlloca()) - Invalid("address is not a static alloca plus a constant offset"); - if (Offset.isNegative() || Offset.getActiveBits() > 64) - Invalid("byte offset is negative or does not fit in uint64"); - - auto AllocaIt = Builder.FuncInfo.StaticAllocaMap.find(Alloca); - if (AllocaIt == Builder.FuncInfo.StaticAllocaMap.end()) - Invalid("static alloca has no fixed frame index"); - - TypeSize StoreSize = DL.getTypeStoreSize(Load->getType()); - if (StoreSize.isScalable()) - Invalid("pointer store size is scalable"); - uint64_t Size = StoreSize.getFixedValue(); - uint64_t ByteOffset = Offset.getZExtValue(); - auto End = checkedAddUnsigned(ByteOffset, Size); - if (!End) - Invalid("stack subslot end offset overflows"); - - MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo(); - int FI = AllocaIt->second; - int64_t ObjectSize = MFI.getObjectSize(FI); - if (ObjectSize < 0 || *End > uint64_t(ObjectSize)) - Invalid("stack subslot is outside the alloca frame object"); - if (MFI.getStackID(FI) != TargetStackID::Default) - Invalid("alloca uses a non-default target stack"); - - return FixedStackHome{FI, int64_t(ByteOffset), Size, - commonAlignment(MFI.getObjectAlign(FI), ByteOffset)}; -} - static void pushStackMapConstant(SmallVectorImpl& Ops, SelectionDAGBuilder &Builder, uint64_t Value) { SDLoc L = Builder.getCurSDLoc(); @@ -179,24 +112,11 @@ static void pushStackMapConstant(SmallVectorImpl& Ops, Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64)); } -static void pushStackMapIndirectLocation(SmallVectorImpl &Ops, - SelectionDAGBuilder &Builder, - const FixedStackHome &Home) { - SDLoc L = Builder.getCurSDLoc(); - Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::IndirectMemRefOp, L, - MVT::i64)); - Ops.push_back(Builder.DAG.getTargetConstant(Home.Size, L, MVT::i64)); - Ops.push_back( - Builder.DAG.getTargetFrameIndex(Home.FI, Builder.getFrameIndexTy())); - Ops.push_back(Builder.DAG.getTargetConstant(Home.Offset, L, MVT::i64)); -} - void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) { // Consistency check assert(PendingGCRelocateCalls.empty() && "Trying to visit statepoint before finished processing previous one"); Locations.clear(); - FixedStackHomes.clear(); NextSlotToAllocate = 0; // Need to resize this on each safepoint - we need the two to stay in sync and // the clear patterns of a SelectionDAGBuilder have no relation to @@ -207,7 +127,6 @@ void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) { void StatepointLoweringState::clear() { Locations.clear(); - FixedStackHomes.clear(); AllocatedStackSlots.clear(); assert(PendingGCRelocateCalls.empty() && "cleared before statepoint sequence completed"); @@ -293,10 +212,7 @@ static std::optional findPreviousSpillSlot(const Value *Val, if (Record.type != RecordType::Spill) return std::nullopt; - int FI = Record.payload.Spill.FI; - if (!is_contained(Builder.FuncInfo.StatepointStackSlots, FI)) - return std::nullopt; - return FI; + return Record.payload.FI; } // Look through bitcast instructions. @@ -494,15 +410,6 @@ static MachineMemOperand* getMachineMemOperand(MachineFunction &MF, MFI.getObjectAlign(FI.getIndex())); } -static MachineMemOperand * -getMachineMemOperand(MachineFunction &MF, const FixedStackHome &Home) { - auto PtrInfo = - MachinePointerInfo::getFixedStack(MF, Home.FI, Home.Offset); - auto MMOFlags = MachineMemOperand::MOStore | MachineMemOperand::MOLoad | - MachineMemOperand::MOVolatile; - return MF.getMachineMemOperand(PtrInfo, MMOFlags, Home.Size, Home.Alignment); -} - /// Spill a value incoming to the statepoint. It might be either part of /// vmstate /// or gcstate. In both cases unconditionally spill it on the stack unless it @@ -565,13 +472,6 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl &Ops, SmallVectorImpl &MemRefs, SelectionDAGBuilder &Builder) { - if (const FixedStackHome *Home = - Builder.StatepointLowering.getFixedStackHome(Incoming)) { - pushStackMapIndirectLocation(Ops, Builder, *Home); - MemRefs.push_back( - getMachineMemOperand(Builder.DAG.getMachineFunction(), *Home)); - return; - } if (willLowerDirectly(Incoming)) { if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { @@ -720,65 +620,12 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, return !willLowerDirectly(SD); }; - // Validate and collect marked homes before deduplicating pointer SDValues or - // deciding which values may use vregs. Two conflicting marked homes for the - // same lowered value are not safe to choose between. Neither is a marked IR - // value colliding with a distinct unmarked GC IR value: pre-call DAG CSE does - // not imply that the two values have the same post-statepoint semantics. - DenseMap MarkedHomes; - DenseMap MarkedSources; - DenseMap UnmarkedSources; - auto collectRootSource = [&](const Value *V) { - std::optional Home = getMarkedFixedStackHome(V, Builder); - SDValue PtrSD = Builder.getValue(V); - if (!Home) { - UnmarkedSources.try_emplace(PtrSD, V); - return; - } - MarkedSources.try_emplace(PtrSD, V); - auto [It, Inserted] = MarkedHomes.try_emplace(PtrSD, *Home); - if (!Inserted && - (It->second.FI != Home->FI || It->second.Offset != Home->Offset || - It->second.Size != Home->Size || - It->second.Alignment != Home->Alignment)) - report_fatal_error( - "conflicting llvm.statepoint.fixed_stack_home locations for one " - "lowered GC pointer"); - }; - // SI.Ptrs has already been deduplicated by lowered SDValue. Inspect the - // original IR carrier lists as well so that deduplication cannot hide a - // marked/unmarked identity collision from this check. - for (const Use &U : SI.GCLives) - collectRootSource(U.get()); - for (const GCRelocateInst *Relocate : SI.GCRelocates) { - collectRootSource(Relocate->getDerivedPtr()); - collectRootSource(Relocate->getBasePtr()); - } - for (const Value *V : SI.Ptrs) - collectRootSource(V); - for (const Value *V : SI.Bases) - collectRootSource(V); - for (const auto &[PtrSD, Home] : MarkedHomes) { - auto Unmarked = UnmarkedSources.find(PtrSD); - assert(MarkedSources.count(PtrSD) && "marked home has no IR source"); - if (Unmarked != UnmarkedSources.end() && - Unmarked->second != MarkedSources.find(PtrSD)->second) - report_fatal_error( - "llvm.statepoint.fixed_stack_home value and a distinct unmarked GC " - "value lower to the same pre-call value"); - } - auto processGCPtr = [&](const Value *V) { SDValue PtrSD = Builder.getValue(V); if (!LoweredGCPtrs.insert(PtrSD)) return; // skip duplicates GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1; - if (auto Home = MarkedHomes.find(PtrSD); Home != MarkedHomes.end()) { - Builder.StatepointLowering.setFixedStackHome(PtrSD, Home->second); - return; - } - if (auto ArgValue = getArgumentValueOffset(V, Builder.DAG.getDataLayout())) { uint64_t Size = PtrSD.getValueType().getStoreSize().getKnownMinValue(); @@ -1130,8 +977,6 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( SDValue Loc = StatepointLowering.getLocation(SDV); bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent()); - const FixedStackHome *Home = - StatepointLowering.getFixedStackHome(SDV); RecordType Record; if (LowerAsVReg.count(SDV)) { @@ -1144,17 +989,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( assert(It != VirtRegs.end()); Record.payload.Reg = It->second; } - } else if (Home) { - Record.type = RecordType::Spill; - Record.payload.Spill = {Home->FI, Home->Offset, Home->Size, - Home->Alignment.value()}; } else if (Loc.getNode()) { Record.type = RecordType::Spill; - int FI = cast(Loc)->getIndex(); - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - Record.payload.Spill = { - FI, 0, uint64_t(MFI.getObjectSize(FI)), - MFI.getObjectAlign(FI).value()}; + Record.payload.FI = cast(Loc)->getIndex(); } else { Record.type = RecordType::NoRelocate; // If we didn't relocate a value, we'll essentialy end up inserting an @@ -1486,12 +1323,8 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { } if (Record.type == RecordType::Spill) { - int Index = Record.payload.Spill.FI; + unsigned Index = Record.payload.FI; SDValue SpillSlot = DAG.getFrameIndex(Index, getFrameIndexTy()); - int64_t Offset = Record.payload.Spill.Offset; - if (Offset != 0) - SpillSlot = DAG.getObjectPtrOffset(getCurSDLoc(), SpillSlot, - TypeSize::getFixed(Offset)); // All the reloads are independent and are reading memory only modified by // statepoints (i.e. no other aliasing stores); informing SelectionDAG of @@ -1503,16 +1336,14 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { const SDValue Chain = DAG.getRoot(); // != Builder.getRoot() auto &MF = DAG.getMachineFunction(); - auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index, Offset); - auto *LoadMMO = MF.getMachineMemOperand( - PtrInfo, MachineMemOperand::MOLoad, Record.payload.Spill.Size, - Align(Record.payload.Spill.Alignment)); + auto &MFI = MF.getFrameInfo(); + auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index); + auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, + MFI.getObjectSize(Index), + MFI.getObjectAlign(Index)); auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), Relocate.getType()); - assert(LoadVT.getStoreSize().getKnownMinValue() == - Record.payload.Spill.Size && - "relocated value size does not match its fixed stack home"); SDValue SpillLoad = DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h index 82bef59f8c320..addc0a7eef3a4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h @@ -19,9 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/Support/Alignment.h" #include -#include namespace llvm { @@ -36,13 +34,6 @@ class StatepointLoweringState { public: StatepointLoweringState() = default; - struct FixedStackHome { - int FI; - int64_t Offset; - uint64_t Size; - Align Alignment; - }; - /// Reset all state tracking for a newly encountered safepoint. Also /// performs some consistency checking. void startNewStatepoint(SelectionDAGBuilder &Builder); @@ -69,17 +60,6 @@ class StatepointLoweringState { Locations[Val] = Location; } - const FixedStackHome *getFixedStackHome(SDValue Val) const { - auto I = FixedStackHomes.find(Val); - return I == FixedStackHomes.end() ? nullptr : &I->second; - } - - void setFixedStackHome(SDValue Val, FixedStackHome Home) { - assert(!FixedStackHomes.count(Val) && - "Trying to set an already assigned fixed stack home"); - FixedStackHomes.try_emplace(Val, Home); - } - /// Record the fact that we expect to encounter a given gc_relocate /// before the next statepoint. If we don't see it, we'll report /// an assertion. @@ -128,11 +108,6 @@ class StatepointLoweringState { /// into it's location (currently only stack slots) DenseMap Locations; - /// Exact stack subslots which already contain incoming GC pointer values. - /// Unlike Locations, these homes may have a non-zero offset within an - /// existing frame object and do not require an additional spill store. - DenseMap FixedStackHomes; - /// A boolean indicator for each slot listed in the FunctionInfo as to /// whether it has been used in the current statepoint. Since we try to /// preserve stack slots across safepoints, there can be gaps in which diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp index 13cd61bbb4be6..65234cd14a80f 100644 --- a/llvm/lib/CodeGen/StackMaps.cpp +++ b/llvm/lib/CodeGen/StackMaps.cpp @@ -44,6 +44,16 @@ static cl::opt StackMapVersion( const char *StackMaps::WSMP = "Stack Maps: "; +std::optional +StackMaps::getConstantValue(const StackMaps::Location &Loc) const { + if (Loc.Type == Location::Constant) + return Loc.Offset; + if (Loc.Type != Location::ConstantIndex || Loc.Offset < 0 || + static_cast(Loc.Offset) >= ConstPool.size()) + return std::nullopt; + return std::next(ConstPool.begin(), Loc.Offset)->second; +} + static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) { assert(MI.getOperand(Idx).isImm() && MI.getOperand(Idx).getImm() == StackMaps::ConstantOp); diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 39a533043339b..71333af36e879 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -1626,37 +1626,6 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, [](MachineOperand &Operand) { return Operand.isFI(); })) return MBB; - BitVector PreEncodedMemRefFIs(MI->getNumOperands()); - if (MI->getOpcode() == TargetOpcode::STATEPOINT) { - StatepointOpers Opers(MI); - auto CollectSection = [&](unsigned CountIdx) { - uint64_t Count = MI->getOperand(CountIdx).getImm(); - unsigned CurIdx = CountIdx + 1; - while (Count--) { - const MachineOperand &Tag = MI->getOperand(CurIdx); - if (Tag.isImm() && Tag.getImm() == StackMaps::DirectMemRefOp) { - assert(CurIdx + 2 < MI->getNumOperands() && - MI->getOperand(CurIdx + 1).isFI() && - MI->getOperand(CurIdx + 2).isImm() && - "malformed pre-encoded direct statepoint location"); - PreEncodedMemRefFIs.set(CurIdx + 1); - } else if (Tag.isImm() && - Tag.getImm() == StackMaps::IndirectMemRefOp) { - assert(CurIdx + 3 < MI->getNumOperands() && - MI->getOperand(CurIdx + 1).isImm() && - MI->getOperand(CurIdx + 2).isFI() && - MI->getOperand(CurIdx + 3).isImm() && - "malformed pre-encoded indirect statepoint location"); - PreEncodedMemRefFIs.set(CurIdx + 2); - } - CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); - } - }; - CollectSection(Opers.getNumDeoptArgsIdx()); - CollectSection(Opers.getNumGCPtrIdx()); - CollectSection(Opers.getNumAllocaIdx()); - } - MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc()); // Inherit previous memory operands. @@ -1682,15 +1651,6 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, // with the canonical set of five x86 addressing-mode operands. int FI = MO.getIndex(); - // Statepoint lowering can already encode an exact fixed-stack subslot as a - // DirectMemRefOp/IndirectMemRefOp tuple. Preserve that FI here; PEI will - // replace it with the frame register and add the final frame offset to the - // tuple's existing byte offset. - if (PreEncodedMemRefFIs.test(i)) { - MIB.add(MO); - continue; - } - // Add frame index operands recognized by stackmaps.cpp if (MFI.isStatepointSpillSlotObjectIndex(FI)) { // indirect-mem-ref tag, size, #FI, offset. diff --git a/llvm/lib/MC/GoObjObjectWriter.cpp b/llvm/lib/MC/GoObjObjectWriter.cpp index 4600f90179e8e..95e3a14317f24 100644 --- a/llvm/lib/MC/GoObjObjectWriter.cpp +++ b/llvm/lib/MC/GoObjObjectWriter.cpp @@ -362,6 +362,120 @@ struct GoObjStackMapPair { } }; +struct GoObjAllocaPtrMapRecord { + MCContext::GoObjStackMapLocation Base; + uint64_t ByteOffset; + uint64_t ByteSize; + uint64_t Alignment; + uint64_t PointerSize; + uint64_t BitCount; + SmallVector BitmapWords; +}; + +int64_t getAllocaPtrMapConstant( + const MCContext::GoObjStackMapLocation &Location, StringRef Description) { + if (Location.Type != MCContext::GoObjStackMapLocation::Constant) + report_fatal_error("GoObj alloca ptrmap " + Description + + " is not a constant"); + return Location.Offset; +} + +uint64_t getNonnegativeAllocaPtrMapConstant( + const MCContext::GoObjStackMapLocation &Location, StringRef Description) { + int64_t Value = getAllocaPtrMapConstant(Location, Description); + if (Value < 0) + report_fatal_error("GoObj alloca ptrmap " + Description + + " is negative"); + return static_cast(Value); +} + +SmallVector +parseAllocaPtrMapRecords(const MCContext::GoObjStackMapEntry &Entry) { + if (Entry.NumDeoptLocations > Entry.Locations.size()) + report_fatal_error("GoObj statepoint deopt location count is invalid"); + ArrayRef Deopts = + ArrayRef(Entry.Locations).take_front(Entry.NumDeoptLocations); + + auto IsConstant = [](const auto &Location, int64_t Value) { + return Location.Type == MCContext::GoObjStackMapLocation::Constant && + Location.Offset == Value; + }; + bool HasProtocolMarker = llvm::any_of(Deopts, [&](const auto &Location) { + return IsConstant(Location, GoObj::AllocaPtrMapBeginMagic) || + IsConstant(Location, GoObj::AllocaPtrMapEndMagic); + }); + if (!HasProtocolMarker) + return {}; + if (Deopts.size() < 6 || + !IsConstant(Deopts[Deopts.size() - 2], + GoObj::AllocaPtrMapEndMagic)) + report_fatal_error("GoObj alloca ptrmap protocol is truncated"); + + uint64_t ProtocolLength = getNonnegativeAllocaPtrMapConstant( + Deopts.back(), "trailing protocol length"); + if (ProtocolLength < 4 || ProtocolLength >= Deopts.size()) + report_fatal_error("GoObj alloca ptrmap protocol length is invalid"); + size_t ProtocolStart = Deopts.size() - ProtocolLength - 1; + if (!IsConstant(Deopts[ProtocolStart], GoObj::AllocaPtrMapBeginMagic) || + getNonnegativeAllocaPtrMapConstant(Deopts[ProtocolStart + 1], + "leading protocol length") != + ProtocolLength || + !IsConstant(Deopts[ProtocolStart + ProtocolLength - 1], + GoObj::AllocaPtrMapEndMagic)) + report_fatal_error("GoObj alloca ptrmap protocol envelope is malformed"); + + uint64_t RecordCount = getNonnegativeAllocaPtrMapConstant( + Deopts[ProtocolStart + 2], "record count"); + size_t Cursor = ProtocolStart + 3; + size_t RecordsEnd = ProtocolStart + ProtocolLength - 1; + if (RecordCount > (RecordsEnd - Cursor) / 10) + report_fatal_error("GoObj alloca ptrmap record count is invalid"); + SmallVector Records; + Records.reserve(static_cast(RecordCount)); + for (uint64_t RecordIndex = 0; RecordIndex != RecordCount; ++RecordIndex) { + if (Cursor > RecordsEnd || RecordsEnd - Cursor < 10 || + !IsConstant(Deopts[Cursor], GoObj::AllocaPtrMapRecordTag)) + report_fatal_error("GoObj alloca ptrmap record header is malformed"); + uint64_t RecordLength = getNonnegativeAllocaPtrMapConstant( + Deopts[Cursor + 1], "record length"); + uint64_t WordCount = getNonnegativeAllocaPtrMapConstant( + Deopts[Cursor + 9], "bitmap word count"); + if (WordCount > RecordsEnd - Cursor - 10 || + RecordLength != 10 + WordCount || + RecordLength > RecordsEnd - Cursor) + report_fatal_error("GoObj alloca ptrmap record length is invalid"); + + const auto &Base = Deopts[Cursor + 2]; + if (Base.Type != MCContext::GoObjStackMapLocation::Direct) + report_fatal_error( + "GoObj alloca ptrmap base is not a direct frame location"); + uint64_t WordBits = getNonnegativeAllocaPtrMapConstant( + Deopts[Cursor + 8], "bitmap word width"); + if (WordBits != GoObj::AllocaPtrMapBitmapWordBits) + report_fatal_error("GoObj alloca ptrmap bitmap word width is invalid"); + + GoObjAllocaPtrMapRecord Record{ + Base, + getNonnegativeAllocaPtrMapConstant(Deopts[Cursor + 3], "byte offset"), + getNonnegativeAllocaPtrMapConstant(Deopts[Cursor + 4], "byte size"), + getNonnegativeAllocaPtrMapConstant(Deopts[Cursor + 5], "alignment"), + getNonnegativeAllocaPtrMapConstant(Deopts[Cursor + 6], "pointer size"), + getNonnegativeAllocaPtrMapConstant(Deopts[Cursor + 7], "bit count"), + {}}; + Record.BitmapWords.reserve(WordCount); + for (uint64_t Word = 0; Word != WordCount; ++Word) + Record.BitmapWords.push_back(static_cast( + getAllocaPtrMapConstant(Deopts[Cursor + 10 + Word], + "bitmap word"))); + Records.push_back(std::move(Record)); + Cursor += RecordLength; + } + if (Cursor != RecordsEnd) + report_fatal_error( + "GoObj alloca ptrmap record count does not cover protocol payload"); + return Records; +} + GoObjStatepointStackMaps makeStatepointStackMaps( const MCAssembler &Asm, const GoObjSymbol &Function, uint32_t StackSize, uint32_t ArgSize, uint32_t PCQuantum, @@ -428,7 +542,78 @@ GoObjStatepointStackMaps makeStatepointStackMaps( bool IsStackGrowth = Entry.ID == GoObj::StackGrowthStatepointID; GoObjStackMapPair Pair{SmallVector(ArgsBytesPerBitmap, 0), SmallVector(LocalsBytesPerBitmap, 0)}; - for (const MCContext::GoObjStackMapLocation &Loc : Entry.Locations) { + SmallVector, 4> AllocaRanges; + DenseSet AllocaPointerBits; + for (const GoObjAllocaPtrMapRecord &Record : + parseAllocaPtrMapRecords(Entry)) { + if (IsStackGrowth) + report_fatal_error( + "GoObj stack-growth statepoint contains an alloca ptrmap"); + if (Record.Base.Size != PointerSize || + Record.Base.DwarfRegNum != StackPointerDwarfRegNum) + report_fatal_error( + "GoObj alloca ptrmap base is not a pointer-sized SP location"); + if (Record.ByteOffset != 0) + report_fatal_error( + "GoObj alloca ptrmap first version requires zero byte offset"); + if (!Record.ByteSize || Record.ByteSize % PointerSize != 0 || + Record.PointerSize != PointerSize || + Record.BitCount != Record.ByteSize / PointerSize || + Record.BitmapWords.size() != divideCeil(Record.BitCount, 64u)) + report_fatal_error("GoObj alloca ptrmap layout is inconsistent"); + if (Record.Alignment < PointerSize || + !isPowerOf2_64(Record.Alignment) || Record.Base.Offset < 0 || + static_cast(Record.Base.Offset) % Record.Alignment != 0) + report_fatal_error("GoObj alloca ptrmap alignment is invalid"); + if (Record.ByteSize > + static_cast(std::numeric_limits::max()) - + static_cast(Record.Base.Offset)) + report_fatal_error("GoObj alloca ptrmap frame range overflows"); + int64_t RangeStart = Record.Base.Offset; + int64_t RangeEnd = RangeStart + static_cast(Record.ByteSize); + for (const auto &[ExistingStart, ExistingEnd] : AllocaRanges) { + if (RangeStart == ExistingStart && RangeEnd == ExistingEnd) + report_fatal_error( + "GoObj alloca ptrmap contains a duplicate frame record"); + if (RangeStart < ExistingEnd && ExistingStart < RangeEnd) + report_fatal_error( + "GoObj alloca ptrmap records overlap in one callsite"); + } + AllocaRanges.push_back({RangeStart, RangeEnd}); + + uint64_t PaddingBits = Record.BitmapWords.size() * 64 - Record.BitCount; + if (PaddingBits && + (Record.BitmapWords.back() >> (64 - PaddingBits)) != 0) + report_fatal_error( + "GoObj alloca ptrmap bitmap padding bits are nonzero"); + bool HasPointer = false; + for (uint64_t Bit = 0; Bit != Record.BitCount; ++Bit) { + int64_t SlotOffset = + RangeStart + static_cast(Bit * PointerSize); + goobj::StackMapSlot Slot = goobj::classifyOrdinaryStackMapSlot( + SlotOffset, /*IsIndirect=*/true, PointerSize, + FrameLayout.GCLocalsStart, FrameLayout.GCLocalsSize, + FrameLayout.GCLocalsBitOffset, OrdinaryArgsStart, ArgSize); + if (Slot.Kind != goobj::StackMapSlotKind::Locals) + report_fatal_error( + "GoObj alloca ptrmap range is not entirely in locals"); + if ((Record.BitmapWords[Bit / 64] & (uint64_t(1) << (Bit % 64))) == + 0) + continue; + HasPointer = true; + if (!AllocaPointerBits.insert(Slot.Bit).second) + report_fatal_error( + "GoObj alloca ptrmap contains a duplicate pointer slot"); + Pair.Locals[Slot.Bit / 8] |= uint8_t(1u << (Slot.Bit % 8)); + } + if (!HasPointer) + report_fatal_error("GoObj alloca ptrmap contains no pointer slots"); + } + + if (Entry.NumDeoptLocations > Entry.Locations.size()) + report_fatal_error("GoObj statepoint deopt location count is invalid"); + for (const MCContext::GoObjStackMapLocation &Loc : + ArrayRef(Entry.Locations).drop_front(Entry.NumDeoptLocations)) { switch (Loc.Type) { case MCContext::GoObjStackMapLocation::Direct: case MCContext::GoObjStackMapLocation::Indirect: @@ -482,6 +667,9 @@ GoObjStatepointStackMaps makeStatepointStackMaps( Pair.Args[Slot.Bit / 8] |= uint8_t(1u << (Slot.Bit % 8)); break; case goobj::StackMapSlotKind::Locals: + if (AllocaPointerBits.contains(Slot.Bit)) + report_fatal_error( + "GoObj alloca ptrmap overlaps an ordinary GC root slot"); Pair.Locals[Slot.Bit / 8] |= uint8_t(1u << (Slot.Bit % 8)); break; } diff --git a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-collision.ll b/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-collision.ll deleted file mode 100644 index 452a893844013..0000000000000 --- a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-collision.ll +++ /dev/null @@ -1,40 +0,0 @@ -; REQUIRES: x86-registered-target -; RUN: not --crash llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 \ -; RUN: | FileCheck %s - -declare void @mutate(ptr) - -define ptr addrspace(1) @marked_unmarked_collision(ptr addrspace(1) %p) - gc "statepoint-example" { -entry: - %home = alloca ptr addrspace(1), align 8 - store ptr addrspace(1) %p, ptr %home, align 8 - %marked = load ptr addrspace(1), ptr %home, align 8, - !llvm.statepoint.fixed_stack_home !0 - %unmarked = select i1 true, ptr addrspace(1) %marked, - ptr addrspace(1) %marked - %token = call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 1, i32 0, ptr elementtype(void (ptr)) @mutate, - i32 1, i32 0, ptr %home, i32 0, i32 0) - [ "gc-live"(ptr addrspace(1) %marked, - ptr addrspace(1) %unmarked) ] - %marked.relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 0, i32 0) - %unmarked.relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 1, i32 1) - %same = icmp eq ptr addrspace(1) %marked.relocated, %unmarked.relocated - %result = select i1 %same, ptr addrspace(1) %marked.relocated, - ptr addrspace(1) %unmarked.relocated - ret ptr addrspace(1) %result -} - -declare token @llvm.experimental.gc.statepoint.p0( - i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( - token, i32 immarg, i32 immarg) - -!0 = !{} - -; CHECK: LLVM ERROR: llvm.statepoint.fixed_stack_home value and a distinct -; CHECK-SAME: unmarked GC value lower to the same pre-call value diff --git a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-invalid.ll b/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-invalid.ll deleted file mode 100644 index 0471b977e1995..0000000000000 --- a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home-invalid.ll +++ /dev/null @@ -1,31 +0,0 @@ -; REQUIRES: x86-registered-target -; RUN: not --crash llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 \ -; RUN: | FileCheck %s - -@root = global ptr addrspace(1) null - -declare void @safepoint() - -define ptr addrspace(1) @not_an_alloca() gc "statepoint-example" { -entry: - %live = load ptr addrspace(1), ptr @root, align 8, - !llvm.statepoint.fixed_stack_home !0 - %token = call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 1, i32 0, ptr elementtype(void ()) @safepoint, - i32 0, i32 0, i32 0, i32 0) - [ "gc-live"(ptr addrspace(1) %live) ] - %relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 0, i32 0) - ret ptr addrspace(1) %relocated -} - -declare token @llvm.experimental.gc.statepoint.p0( - i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( - token, i32 immarg, i32 immarg) - -!0 = !{} - -; CHECK: LLVM ERROR: invalid llvm.statepoint.fixed_stack_home load: -; CHECK-SAME: address is not a static alloca plus a constant offset diff --git a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home.ll b/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home.ll deleted file mode 100644 index 4c0a40f09cd35..0000000000000 --- a/llvm/test/CodeGen/Generic/statepoint-fixed-stack-home.ll +++ /dev/null @@ -1,107 +0,0 @@ -; REQUIRES: aarch64-registered-target -; REQUIRES: x86-registered-target -; RUN: llc -mtriple=aarch64-unknown-linux-gnu -verify-machineinstrs \ -; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=AARCH64-MIR -; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs \ -; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=X86-MIR -; RUN: llc -mtriple=aarch64-unknown-linux-gnu -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefix=AARCH64-ASM -; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefix=X86-ASM - -%roots = type { i64, ptr addrspace(1), ptr addrspace(1) } -%pair = type { ptr addrspace(1), ptr addrspace(1) } - -declare void @safepoint() -declare void @mutate(ptr) - -define %pair @two_subslots(ptr addrspace(1) %p, ptr addrspace(1) %q) - gc "statepoint-example" { -entry: - %roots = alloca %roots, align 8 - %p.addr = getelementptr inbounds %roots, ptr %roots, i32 0, i32 1 - %q.addr = getelementptr inbounds %roots, ptr %roots, i32 0, i32 2 - store ptr addrspace(1) %p, ptr %p.addr, align 8 - store ptr addrspace(1) %q, ptr %q.addr, align 8 - %p.live = load volatile ptr addrspace(1), ptr %p.addr, align 8, - !llvm.statepoint.fixed_stack_home !0 - %q.live = load volatile ptr addrspace(1), ptr %q.addr, align 8, - !llvm.statepoint.fixed_stack_home !0 - %token = call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 1, i32 0, ptr elementtype(void ()) @safepoint, - i32 0, i32 0, i32 0, i32 0) - [ "gc-live"(ptr addrspace(1) %p.live, - ptr addrspace(1) %q.live) ] - %p.relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 0, i32 0) - %q.relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 1, i32 1) - store ptr addrspace(1) %p.relocated, ptr %p.addr, align 8 - store ptr addrspace(1) %q.relocated, ptr %q.addr, align 8 - %result.0 = insertvalue %pair poison, ptr addrspace(1) %p.relocated, 0 - %result.1 = insertvalue %pair %result.0, ptr addrspace(1) %q.relocated, 1 - ret %pair %result.1 -} - -; The call receives the field address and may replace its pointer value. The -; relocate must reload that post-call value from the exact home; lowering must -; not spill the stale pre-call load to another slot or back over the field. -define ptr addrspace(1) @callee_mutates_home(ptr addrspace(1) %p) - gc "statepoint-example" { -entry: - %roots = alloca %roots, align 8 - %p.addr = getelementptr inbounds %roots, ptr %roots, i32 0, i32 1 - store ptr addrspace(1) %p, ptr %p.addr, align 8 - %p.live = load volatile ptr addrspace(1), ptr %p.addr, align 8, - !llvm.statepoint.fixed_stack_home !0 - %token = call token (i64, i32, ptr, i32, i32, ...) - @llvm.experimental.gc.statepoint.p0( - i64 2, i32 0, ptr elementtype(void (ptr)) @mutate, - i32 1, i32 0, ptr %p.addr, i32 0, i32 0) - [ "gc-live"(ptr addrspace(1) %p.live) ] - %p.relocated = call ptr addrspace(1) - @llvm.experimental.gc.relocate.p1(token %token, i32 0, i32 0) - ret ptr addrspace(1) %p.relocated -} - -declare token @llvm.experimental.gc.statepoint.p0( - i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( - token, i32 immarg, i32 immarg) - -!0 = !{} - -; AARCH64-MIR-LABEL: name: two_subslots -; AARCH64-MIR: stack: -; AARCH64-MIR: - { id: 0, name: roots, type: default, offset: 0, size: 24, -; AARCH64-MIR-NOT: id: 1 -; AARCH64-MIR: STATEPOINT 1, -; AARCH64-MIR-SAME: 2, 2, 1, 8, %stack.0{{[^, ]*}}, 16, -; AARCH64-MIR-SAME: 1, 8, %stack.0{{[^, ]*}}, 8, -; AARCH64-MIR-SAME: (volatile load store (s64) on %stack.0{{[^ ]*}} + 16), -; AARCH64-MIR-SAME: (volatile load store (s64) on %stack.0{{[^ ]*}} + 8) -; AARCH64-MIR-DAG: LDRXui %stack.0{{[^, ]*}}, 1 -; AARCH64-MIR-DAG: LDRXui %stack.0{{[^, ]*}}, 2 - -; X86-MIR-LABEL: name: two_subslots -; X86-MIR: stack: -; X86-MIR: - { id: 0, name: roots, type: default, offset: 0, size: 24, -; X86-MIR-NOT: id: 1 -; X86-MIR: STATEPOINT 1, -; X86-MIR-SAME: 2, 2, 1, 8, %stack.0{{[^, ]*}}, 16, -; X86-MIR-SAME: 1, 8, %stack.0{{[^, ]*}}, 8, -; X86-MIR-SAME: (volatile load store (s64) on %stack.0{{[^ ]*}} + 16), -; X86-MIR-SAME: (volatile load store (s64) on %stack.0{{[^ ]*}} + 8) -; X86-MIR-DAG: MOV64rm %stack.0{{[^, ]*}}, 1, $noreg, 8, -; X86-MIR-DAG: MOV64rm %stack.0{{[^, ]*}}, 1, $noreg, 16, - -; AARCH64-ASM-LABEL: callee_mutates_home: -; AARCH64-ASM: str x0, [sp, #[[AARCH64_HOME:[0-9]+]]] -; AARCH64-ASM: bl mutate -; AARCH64-ASM: ldr x0, [sp, #[[AARCH64_HOME]]] - -; X86-ASM-LABEL: callee_mutates_home: -; X86-ASM: movq %rdi, [[X86_HOME:[0-9]+]](%rsp) -; X86-ASM: callq mutate@PLT -; X86-ASM: movq [[X86_HOME]](%rsp), %rax