[Integration][Go] Runtime LLVM support

llvm/include/llvm/BinaryFormat/GoObj.h

@@ -23,9 +23,10 @@ namespace GoObj {
 // Go function. GoObj serialization strips it and records ABI0 separately.
 inline constexpr char ABI0SymbolSuffix[] = "<ABI0>";
 
-// "GoStackG" encoded as the stable statepoint identifier for the pre-frame
-// runtime.morestack slow path.
-inline constexpr uint64_t StackGrowthStatepointID = 0x476f537461636b47ULL;
+// "GoNoSplt" encoded as the stable STACKMAP identifier for the function-level
+// entry argument pointer map. This record is metadata-only: it is present for
+// both split and nosplit functions and never denotes a callsite.
+inline constexpr uint64_t EntryArgsStackMapID = 0x476f4e6f53706c74ULL;
 
 // GoALLC encodes pointer maps for fixed allocas as a self-describing suffix of
 // statepoint deopt locations:

llvm/include/llvm/CodeGen/GoCallingConv.h

@@ -32,12 +32,16 @@ namespace goabi {
 
 inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple";
 inline constexpr StringLiteral PadTypeName = "go.abi.pad";
-// The Go statepoint pass uses this attribute to request that target frame
-// lowering represent the late morestack call with a root-free STATEPOINT.
-inline constexpr StringLiteral StackGrowthStatepointAttr =
-    "go-stack-growth-statepoint";
-inline constexpr uint64_t StackGrowthStatepointID =
-    GoObj::StackGrowthStatepointID;
+// Target frame lowering must not synthesize a morestack edge for such a
+// function. GoObj Go functions otherwise use the native Go default: emit a
+// stack check and call the ABI0 runtime.morestack helper on the slow path.
+inline constexpr StringLiteral NoSplitAttr = "go-nosplit";
+// A //go:systemstack function checks g.stackguard1 and traps through
+// runtime.morestackc if it is entered on an ordinary goroutine stack.
+inline constexpr StringLiteral SystemStackAttr = "go-systemstack";
+// Every GoObj Go function carries its entry argument pointer map in a
+// zero-byte STACKMAP. It is function metadata, not a stack-growth callsite.
+inline constexpr uint64_t EntryArgsStackMapID = GoObj::EntryArgsStackMapID;
 
 inline bool isGoABIInternalCallingConv(CallingConv::ID CC) {
   return CC == CallingConv::GoABIInternal;

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -5138,6 +5138,15 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     llvm_unreachable("Not Implemented");
   }
 
+  /// Let a target append mandatory call-boundary work after a statepoint has
+  /// replaced the temporary target call node.
+  virtual SDValue finalizeStatepointCallChain(SDValue Chain,
+                                              CallingConv::ID CalleeCC,
+                                              const SDLoc &DL,
+                                              SelectionDAG &DAG) const {
+    return Chain;
+  }
+
   /// Target-specific cleanup for formal ByVal parameters.
   virtual void HandleByVal(CCState *, unsigned &, Align) const {}
 

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -464,6 +464,102 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
   return Alignment;
 }
 
+static uint64_t
+getGoObjStackMapNonnegativeConstant(const StackMaps::Location &Location,
+                                    StringRef Description) {
+  if (Location.Type != StackMaps::Location::Constant || Location.Offset < 0)
+    report_fatal_error("malformed GoObj stackmap " + Description);
+  return static_cast<uint64_t>(Location.Offset);
+}
+
+static MCContext::GoObjStackMapLocation::LocationType
+convertGoObjStackMapLocationType(StackMaps::Location::LocationType Type) {
+  using GoLocation = MCContext::GoObjStackMapLocation;
+  switch (Type) {
+  case StackMaps::Location::Unprocessed:
+    return GoLocation::Unprocessed;
+  case StackMaps::Location::Register:
+    return GoLocation::Register;
+  case StackMaps::Location::Direct:
+    return GoLocation::Direct;
+  case StackMaps::Location::Indirect:
+    return GoLocation::Indirect;
+  case StackMaps::Location::Constant:
+    return GoLocation::Constant;
+  case StackMaps::Location::ConstantIndex:
+    return GoLocation::ConstantIndex;
+  }
+  llvm_unreachable("unknown StackMaps location type");
+}
+
+static void emitGoObjStackMaps(StackMaps &SM, AsmPrinter &AP) {
+  auto &Callsites = SM.getCSInfos();
+  auto Callsite = Callsites.begin();
+  uint32_t PointerSize = AP.getPointerSize();
+  if (!PointerSize)
+    report_fatal_error("GoObj statepoint target has no pointer size");
+
+  for (const auto &[Function, Info] : SM.getFnInfos()) {
+    for (uint64_t I = 0; I != Info.RecordCount; ++I) {
+      if (Callsite == Callsites.end())
+        report_fatal_error(
+            "GoObj stackmap function record count exceeds callsites");
+
+      const StackMaps::CallsiteInfo &CSI = *Callsite++;
+      bool IsEntryArgs = CSI.ID == GoObj::EntryArgsStackMapID;
+      uint64_t NumDeopts = 0;
+      ArrayRef<StackMaps::Location> Locations = CSI.Locations;
+      if (!IsEntryArgs) {
+        // LLVM's statepoint parser prefixes locations with the calling
+        // convention, flags, deopt count, and then the deopt operands. These
+        // entries are not GC roots. EntryArgsStackMapID is a plain STACKMAP
+        // and contains only function-level argument pointer homes.
+        if (Locations.size() < 3)
+          report_fatal_error("malformed GoObj statepoint location list");
+        (void)getGoObjStackMapNonnegativeConstant(Locations[0],
+                                                  "calling convention");
+        (void)getGoObjStackMapNonnegativeConstant(Locations[1], "flags");
+        NumDeopts =
+            getGoObjStackMapNonnegativeConstant(Locations[2], "deopt count");
+        if (NumDeopts > Locations.size() - 3)
+          report_fatal_error("malformed GoObj statepoint deopt operands");
+        if (NumDeopts > std::numeric_limits<uint32_t>::max())
+          report_fatal_error("GoObj statepoint has too many deopt operands");
+        Locations = Locations.drop_front(3);
+      }
+
+      MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr,
+                                          CSI.ID,
+                                          CSI.IsIndirectCall,
+                                          Info.StackSize,
+                                          PointerSize,
+                                          static_cast<uint32_t>(NumDeopts),
+                                          {}};
+      Entry.Locations.reserve(Locations.size());
+      for (const StackMaps::Location &Location : Locations) {
+        auto Type = convertGoObjStackMapLocationType(Location.Type);
+        int64_t Offset = Location.Offset;
+        if (Location.Type == StackMaps::Location::Constant ||
+            Location.Type == StackMaps::Location::ConstantIndex) {
+          std::optional<int64_t> Constant = SM.getConstantValue(Location);
+          if (!Constant)
+            report_fatal_error(
+                "GoObj statepoint contains an invalid constant-pool index");
+          Type = MCContext::GoObjStackMapLocation::Constant;
+          Offset = *Constant;
+        }
+        Entry.Locations.push_back({Type, Location.Size, Location.Reg, Offset});
+      }
+      AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry));
+    }
+  }
+
+  if (Callsite != Callsites.end())
+    report_fatal_error(
+        "GoObj stackmap callsites exceed function record counts");
+  SM.reset();
+}
+
 AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer,
                        char &ID)
     : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()),
@@ -503,6 +599,10 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer,
         MP->finishAssembly(M, *MI, *this);
   };
   EmitStackMaps = [this](Module &M) {
+    if (OutContext.isGoObj()) {
+      emitGoObjStackMaps(SM, *this);
+      return;
+    }
     GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>();
     assert(MI && "AsmPrinter didn't require GCModuleInfo?");
     bool NeedsDefault = false;
@@ -698,7 +798,13 @@ bool AsmPrinter::doInitialization(Module &M) {
     if ((Target.isOSWindows() || (Target.isUEFI() && EmitCodeView)) &&
         M.getNamedMetadata("llvm.dbg.cu"))
       Handlers.push_back(std::make_unique<CodeViewDebug>(this));
-    if (!EmitCodeView || M.getDwarfVersion()) {
+    // GoObj consumes source locations through GoObjDebug to build pcfile,
+    // pcline, and pcinline after final machine layout. Generic monolithic
+    // DWARF sections are not Go linker carriers: their relocations can point
+    // at functions removed by Go dead-code elimination. Keep target-object
+    // DWARF disabled until it is represented by Go's per-function carriers.
+    if (!Target.isOSBinFormatGoObj() &&
+        (!EmitCodeView || M.getDwarfVersion())) {
       if (hasDebugInfo()) {
         DD = createDwarfDebug();
         Handlers.push_back(std::unique_ptr<DwarfDebug>(DD));
@@ -872,6 +978,9 @@ getGoObjSymbolFlags(const GlobalObject *GO) {
         Flag2 |= GoObj::SymFlagItab;
     }
   }
+  if (const auto *F = dyn_cast<Function>(GO);
+      F && F->hasFnAttribute(goabi::NoSplitAttr))
+    Flag |= GoObj::SymFlagNoSplit;
 
   if (const MDNode *MD = GO->getMetadata("goobj.symbol.flags")) {
     if (MD->getNumOperands() != 2)
@@ -1081,12 +1190,12 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
     const MDNode *MD = GO.getMetadata("goobj.symbol.nonpackage");
     if (!MD)
       continue;
-    const auto *Marker = MD->getNumOperands() == 1
-                             ? mdconst::dyn_extract<ConstantInt>(
-                                   MD->getOperand(0))
-                             : nullptr;
-    if (!Marker || !Marker->getType()->isIntegerTy(1) ||
-        !Marker->isOne() || GO.isDeclaration())
+    const auto *Marker =
+        MD->getNumOperands() == 1
+            ? mdconst::dyn_extract<ConstantInt>(MD->getOperand(0))
+            : nullptr;
+    if (!Marker || !Marker->getType()->isIntegerTy(1) || !Marker->isOne() ||
+        GO.isDeclaration())
       report_fatal_error("invalid !goobj.symbol.nonpackage attachment");
     AP.OutContext.setGoObjSymbolNonPackage(AP.getSymbol(&GO));
   }
@@ -1165,20 +1274,17 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
     }
   }
 
-  if (const NamedMDNode *Markers =
-          M.getNamedMetadata("goobj.marker_relocs")) {
+  if (const NamedMDNode *Markers = M.getNamedMetadata("goobj.marker_relocs")) {
     DenseMap<const GlobalValue *, std::vector<MCContext::GoObjMarkerReloc>>
         Relocs;
     for (const MDNode *Entry : Markers->operands()) {
       if (Entry->getNumOperands() != 4)
         report_fatal_error(
             "expected !goobj.marker_relocs entries to have four operands");
       const GlobalValue *Source =
-          getGoObjMetadataGlobal(Entry->getOperand(0),
-                                 "goobj.marker_relocs");
+          getGoObjMetadataGlobal(Entry->getOperand(0), "goobj.marker_relocs");
       const GlobalValue *Target =
-          getGoObjMetadataGlobal(Entry->getOperand(1),
-                                 "goobj.marker_relocs");
+          getGoObjMetadataGlobal(Entry->getOperand(1), "goobj.marker_relocs");
       const auto *Type =
           mdconst::dyn_extract<ConstantInt>(Entry->getOperand(2));
       const auto *Addend =
@@ -1195,9 +1301,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
       default:
         report_fatal_error("unsupported !goobj.marker_relocs type");
       }
-      Relocs[Source].push_back(
-          {AP.getSymbol(Target), static_cast<uint16_t>(Type->getZExtValue()),
-           Addend->getSExtValue()});
+      Relocs[Source].push_back({AP.getSymbol(Target),
+                                static_cast<uint16_t>(Type->getZExtValue()),
+                                Addend->getSExtValue()});
     }
     for (auto &[Source, SourceRelocs] : Relocs)
       AP.OutContext.setGoObjMarkerRelocs(AP.getSymbol(Source),
@@ -3746,8 +3852,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   if (TM.getTargetTriple().isOSBinFormatGoObj()) {
     if (std::optional<std::pair<uint8_t, uint8_t>> Flags =
             getGoObjSymbolFlags(&F))
-      OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first,
-                                     Flags->second);
+      OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, Flags->second);
     if (std::optional<std::pair<uint8_t, uint8_t>> Info =
             getGoObjFunctionInfo(F))
       OutContext.setGoObjFunctionInfo(CurrentFnSym, Info->first, Info->second);

llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

@@ -1070,7 +1070,10 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
 
   // Since we always emit CopyToRegs (even for local relocates), we must
   // update root, so that they are emitted before any local uses.
-  (void)getControlRoot();
+  SDValue FinalChain = getControlRoot();
+  FinalChain = DAG.getTargetLoweringInfo().finalizeStatepointCallChain(
+      FinalChain, SI.CLI.CallConv, getCurSDLoc(), DAG);
+  DAG.setRoot(FinalChain);
 
   // TODO: A better future implementation would be to emit a single variable
   // argument, variable return value STATEPOINT node here and then hookup the