[Go] Emit GoObj entry and stack-growth maps

llvm/include/llvm/BinaryFormat/GoObj.h

@@ -23,8 +23,14 @@ namespace GoObj {
 // Go function. GoObj serialization strips it and records ABI0 separately.
 inline constexpr char ABI0SymbolSuffix[] = "<ABI0>";
 
+// "GoNoSplt" encoded as the stable STACKMAP identifier for the function-level
+// entry argument pointer map. This record is metadata-only: it is present for
+// both split and nosplit functions and never denotes a callsite.
+inline constexpr uint64_t EntryArgsStackMapID = 0x476f4e6f53706c74ULL;
+
 // "GoStackG" encoded as the stable statepoint identifier for the pre-frame
-// runtime.morestack slow path.
+// runtime.morestack slow path. Unlike EntryArgsStackMapID, this record denotes
+// a real callsite and is absent from nosplit functions.
 inline constexpr uint64_t StackGrowthStatepointID = 0x476f537461636b47ULL;
 
 // GoALLC encodes pointer maps for fixed allocas as a self-describing suffix of

llvm/include/llvm/CodeGen/GoCallingConv.h

@@ -32,12 +32,18 @@ namespace goabi {
 
 inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple";
 inline constexpr StringLiteral PadTypeName = "go.abi.pad";
-// The Go statepoint pass uses this attribute to request that target frame
-// lowering represent the late morestack call with a root-free STATEPOINT.
-inline constexpr StringLiteral StackGrowthStatepointAttr =
-    "go-stack-growth-statepoint";
+// Target frame lowering must not synthesize a morestack edge for such a
+// function. GoObj Go functions otherwise use the native Go default: emit a
+// stack check and represent its late morestack call as a root-free statepoint.
+inline constexpr StringLiteral NoSplitAttr = "go-nosplit";
+// A //go:systemstack function checks g.stackguard1 and traps through
+// runtime.morestackc if it is entered on an ordinary goroutine stack.
+inline constexpr StringLiteral SystemStackAttr = "go-systemstack";
 inline constexpr uint64_t StackGrowthStatepointID =
     GoObj::StackGrowthStatepointID;
+// Every GoObj Go function carries its entry argument pointer map in a
+// zero-byte STACKMAP. It is function metadata, not a stack-growth callsite.
+inline constexpr uint64_t EntryArgsStackMapID = GoObj::EntryArgsStackMapID;
 
 inline bool isGoABIInternalCallingConv(CallingConv::ID CC) {
   return CC == CallingConv::GoABIInternal;

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -464,6 +464,102 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
   return Alignment;
 }
 
+static uint64_t
+getGoObjStackMapNonnegativeConstant(const StackMaps::Location &Location,
+                                    StringRef Description) {
+  if (Location.Type != StackMaps::Location::Constant || Location.Offset < 0)
+    report_fatal_error("malformed GoObj stackmap " + Description);
+  return static_cast<uint64_t>(Location.Offset);
+}
+
+static MCContext::GoObjStackMapLocation::LocationType
+convertGoObjStackMapLocationType(StackMaps::Location::LocationType Type) {
+  using GoLocation = MCContext::GoObjStackMapLocation;
+  switch (Type) {
+  case StackMaps::Location::Unprocessed:
+    return GoLocation::Unprocessed;
+  case StackMaps::Location::Register:
+    return GoLocation::Register;
+  case StackMaps::Location::Direct:
+    return GoLocation::Direct;
+  case StackMaps::Location::Indirect:
+    return GoLocation::Indirect;
+  case StackMaps::Location::Constant:
+    return GoLocation::Constant;
+  case StackMaps::Location::ConstantIndex:
+    return GoLocation::ConstantIndex;
+  }
+  llvm_unreachable("unknown StackMaps location type");
+}
+
+static void emitGoObjStackMaps(StackMaps &SM, AsmPrinter &AP) {
+  auto &Callsites = SM.getCSInfos();
+  auto Callsite = Callsites.begin();
+  uint32_t PointerSize = AP.getPointerSize();
+  if (!PointerSize)
+    report_fatal_error("GoObj statepoint target has no pointer size");
+
+  for (const auto &[Function, Info] : SM.getFnInfos()) {
+    for (uint64_t I = 0; I != Info.RecordCount; ++I) {
+      if (Callsite == Callsites.end())
+        report_fatal_error(
+            "GoObj stackmap function record count exceeds callsites");
+
+      const StackMaps::CallsiteInfo &CSI = *Callsite++;
+      bool IsEntryArgs = CSI.ID == GoObj::EntryArgsStackMapID;
+      uint64_t NumDeopts = 0;
+      ArrayRef<StackMaps::Location> Locations = CSI.Locations;
+      if (!IsEntryArgs) {
+        // LLVM's statepoint parser prefixes locations with the calling
+        // convention, flags, deopt count, and then the deopt operands. These
+        // entries are not GC roots. EntryArgsStackMapID is a plain STACKMAP
+        // and contains only function-level argument pointer homes.
+        if (Locations.size() < 3)
+          report_fatal_error("malformed GoObj statepoint location list");
+        (void)getGoObjStackMapNonnegativeConstant(Locations[0],
+                                                  "calling convention");
+        (void)getGoObjStackMapNonnegativeConstant(Locations[1], "flags");
+        NumDeopts =
+            getGoObjStackMapNonnegativeConstant(Locations[2], "deopt count");
+        if (NumDeopts > Locations.size() - 3)
+          report_fatal_error("malformed GoObj statepoint deopt operands");
+        if (NumDeopts > std::numeric_limits<uint32_t>::max())
+          report_fatal_error("GoObj statepoint has too many deopt operands");
+        Locations = Locations.drop_front(3);
+      }
+
+      MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr,
+                                          CSI.ID,
+                                          CSI.IsIndirectCall,
+                                          Info.StackSize,
+                                          PointerSize,
+                                          static_cast<uint32_t>(NumDeopts),
+                                          {}};
+      Entry.Locations.reserve(Locations.size());
+      for (const StackMaps::Location &Location : Locations) {
+        auto Type = convertGoObjStackMapLocationType(Location.Type);
+        int64_t Offset = Location.Offset;
+        if (Location.Type == StackMaps::Location::Constant ||
+            Location.Type == StackMaps::Location::ConstantIndex) {
+          std::optional<int64_t> Constant = SM.getConstantValue(Location);
+          if (!Constant)
+            report_fatal_error(
+                "GoObj statepoint contains an invalid constant-pool index");
+          Type = MCContext::GoObjStackMapLocation::Constant;
+          Offset = *Constant;
+        }
+        Entry.Locations.push_back({Type, Location.Size, Location.Reg, Offset});
+      }
+      AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry));
+    }
+  }
+
+  if (Callsite != Callsites.end())
+    report_fatal_error(
+        "GoObj stackmap callsites exceed function record counts");
+  SM.reset();
+}
+
 AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer,
                        char &ID)
     : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()),
@@ -503,6 +599,10 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer,
         MP->finishAssembly(M, *MI, *this);
   };
   EmitStackMaps = [this](Module &M) {
+    if (OutContext.isGoObj()) {
+      emitGoObjStackMaps(SM, *this);
+      return;
+    }
     GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>();
     assert(MI && "AsmPrinter didn't require GCModuleInfo?");
     bool NeedsDefault = false;
@@ -698,7 +798,13 @@ bool AsmPrinter::doInitialization(Module &M) {
     if ((Target.isOSWindows() || (Target.isUEFI() && EmitCodeView)) &&
         M.getNamedMetadata("llvm.dbg.cu"))
       Handlers.push_back(std::make_unique<CodeViewDebug>(this));
-    if (!EmitCodeView || M.getDwarfVersion()) {
+    // GoObj consumes source locations through GoObjDebug to build pcfile,
+    // pcline, and pcinline after final machine layout. Generic monolithic
+    // DWARF sections are not Go linker carriers: their relocations can point
+    // at functions removed by Go dead-code elimination. Keep target-object
+    // DWARF disabled until it is represented by Go's per-function carriers.
+    if (!Target.isOSBinFormatGoObj() &&
+        (!EmitCodeView || M.getDwarfVersion())) {
       if (hasDebugInfo()) {
         DD = createDwarfDebug();
         Handlers.push_back(std::unique_ptr<DwarfDebug>(DD));
@@ -872,6 +978,9 @@ getGoObjSymbolFlags(const GlobalObject *GO) {
         Flag2 |= GoObj::SymFlagItab;
     }
   }
+  if (const auto *F = dyn_cast<Function>(GO);
+      F && F->hasFnAttribute(goabi::NoSplitAttr))
+    Flag |= GoObj::SymFlagNoSplit;
 
   if (const MDNode *MD = GO->getMetadata("goobj.symbol.flags")) {
     if (MD->getNumOperands() != 2)
@@ -1081,12 +1190,12 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
     const MDNode *MD = GO.getMetadata("goobj.symbol.nonpackage");
     if (!MD)
       continue;
-    const auto *Marker = MD->getNumOperands() == 1
-                             ? mdconst::dyn_extract<ConstantInt>(
-                                   MD->getOperand(0))
-                             : nullptr;
-    if (!Marker || !Marker->getType()->isIntegerTy(1) ||
-        !Marker->isOne() || GO.isDeclaration())
+    const auto *Marker =
+        MD->getNumOperands() == 1
+            ? mdconst::dyn_extract<ConstantInt>(MD->getOperand(0))
+            : nullptr;
+    if (!Marker || !Marker->getType()->isIntegerTy(1) || !Marker->isOne() ||
+        GO.isDeclaration())
       report_fatal_error("invalid !goobj.symbol.nonpackage attachment");
     AP.OutContext.setGoObjSymbolNonPackage(AP.getSymbol(&GO));
   }
@@ -1165,20 +1274,17 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
     }
   }
 
-  if (const NamedMDNode *Markers =
-          M.getNamedMetadata("goobj.marker_relocs")) {
+  if (const NamedMDNode *Markers = M.getNamedMetadata("goobj.marker_relocs")) {
     DenseMap<const GlobalValue *, std::vector<MCContext::GoObjMarkerReloc>>
         Relocs;
     for (const MDNode *Entry : Markers->operands()) {
       if (Entry->getNumOperands() != 4)
         report_fatal_error(
             "expected !goobj.marker_relocs entries to have four operands");
       const GlobalValue *Source =
-          getGoObjMetadataGlobal(Entry->getOperand(0),
-                                 "goobj.marker_relocs");
+          getGoObjMetadataGlobal(Entry->getOperand(0), "goobj.marker_relocs");
       const GlobalValue *Target =
-          getGoObjMetadataGlobal(Entry->getOperand(1),
-                                 "goobj.marker_relocs");
+          getGoObjMetadataGlobal(Entry->getOperand(1), "goobj.marker_relocs");
       const auto *Type =
           mdconst::dyn_extract<ConstantInt>(Entry->getOperand(2));
       const auto *Addend =
@@ -1195,9 +1301,9 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
       default:
         report_fatal_error("unsupported !goobj.marker_relocs type");
       }
-      Relocs[Source].push_back(
-          {AP.getSymbol(Target), static_cast<uint16_t>(Type->getZExtValue()),
-           Addend->getSExtValue()});
+      Relocs[Source].push_back({AP.getSymbol(Target),
+                                static_cast<uint16_t>(Type->getZExtValue()),
+                                Addend->getSExtValue()});
     }
     for (auto &[Source, SourceRelocs] : Relocs)
       AP.OutContext.setGoObjMarkerRelocs(AP.getSymbol(Source),
@@ -3746,8 +3852,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   if (TM.getTargetTriple().isOSBinFormatGoObj()) {
     if (std::optional<std::pair<uint8_t, uint8_t>> Flags =
             getGoObjSymbolFlags(&F))
-      OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first,
-                                     Flags->second);
+      OutContext.setGoObjSymbolFlags(CurrentFnSym, Flags->first, Flags->second);
     if (std::optional<std::pair<uint8_t, uint8_t>> Info =
             getGoObjFunctionInfo(F))
       OutContext.setGoObjFunctionInfo(CurrentFnSym, Info->first, Info->second);

llvm/lib/MC/GoObjObjectWriter.cpp

@@ -691,17 +691,18 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
   SmallVector<FunctionAllocaRecord, 4> FunctionAllocaRecords;
   uint64_t OrdinaryEntryCount =
       llvm::count_if(ResolvedEntries, [](const ResolvedEntry &Resolved) {
-        return Resolved.Entry->ID != GoObj::StackGrowthStatepointID;
+        return Resolved.Entry->ID != GoObj::EntryArgsStackMapID &&
+               Resolved.Entry->ID != GoObj::StackGrowthStatepointID;
       });
   std::optional<GoObjOpenDeferRecord> FunctionOpenDefer;
   uint64_t OpenDeferEntryCount = 0;
   for (const ResolvedEntry &Resolved : ResolvedEntries) {
     std::optional<GoObjOpenDeferRecord> Record =
         parseOpenDeferRecord(*Resolved.Entry);
-    if (Resolved.Entry->ID == GoObj::StackGrowthStatepointID) {
+    if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID ||
+        Resolved.Entry->ID == GoObj::StackGrowthStatepointID) {
       if (Record)
-        report_fatal_error(
-            "GoObj stack-growth statepoint contains open-defer state");
+        report_fatal_error("GoObj entry metadata contains open-defer state");
       continue;
     }
     if (!Record)
@@ -757,6 +758,7 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
   }
 
   auto BuildPair = [&](const MCContext::GoObjStackMapEntry &Entry) {
+    bool IsEntryArgs = Entry.ID == GoObj::EntryArgsStackMapID;
     bool IsStackGrowth = Entry.ID == GoObj::StackGrowthStatepointID;
     GoObjStackMapPair Pair{SmallVector<uint8_t, 8>(ArgsBytesPerBitmap, 0),
                            SmallVector<uint8_t, 8>(LocalsBytesPerBitmap, 0)};
@@ -769,9 +771,8 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
         ArrayRef(Entry.Locations).drop_front(Entry.NumDeoptLocations);
     for (const GoObjAllocaPtrMapRecord &Record :
          parseAllocaPtrMapRecords(Entry)) {
-      if (IsStackGrowth)
-        report_fatal_error(
-            "GoObj stack-growth statepoint contains an alloca ptrmap");
+      if (IsEntryArgs || IsStackGrowth)
+        report_fatal_error("GoObj entry metadata contains an alloca ptrmap");
       MCContext::GoObjStackMapLocation RecordBase =
           NormalizeFrameLocation(Record.Base);
       if (RecordBase.Size != PointerSize ||
@@ -914,16 +915,18 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
             ", offset=" + Twine(Loc.Offset));
 
       for (int64_t WordOffset : *PointerWordOffsets) {
-        if (IsStackGrowth) {
+        if (IsEntryArgs) {
           std::optional<uint32_t> Bit = goobj::classifyStackGrowthStackMapSlot(
               WordOffset, PointerSize, FrameLayout.EntryArgsStart, ArgSize);
           if (Loc.Type != MCContext::GoObjStackMapLocation::Indirect || !Bit)
             report_fatal_error(
-                "GoObj stack-growth statepoint contains an invalid argument "
+                "GoObj entry argument stack map contains an invalid argument "
                 "pointer slot");
           Pair.Args[*Bit / 8] |= uint8_t(1u << (*Bit % 8));
           continue;
         }
+        if (IsStackGrowth)
+          report_fatal_error("GoObj stack-growth statepoint is not root-free");
 
         goobj::StackMapSlot Slot = goobj::classifyOrdinaryStackMapSlot(
             WordOffset, Loc.Type == MCContext::GoObjStackMapLocation::Indirect,
@@ -958,24 +961,45 @@ GoObjStatepointStackMaps makeStatepointStackMaps(
     return Pair;
   };
 
+  const ResolvedEntry *EntryArgsEntry = nullptr;
   const ResolvedEntry *StackGrowthEntry = nullptr;
   for (const ResolvedEntry &Resolved : ResolvedEntries) {
-    if (Resolved.Entry->ID != GoObj::StackGrowthStatepointID)
+    if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID) {
+      if (EntryArgsEntry)
+        report_fatal_error(
+            "GoObj function contains multiple entry argument stack maps");
+      EntryArgsEntry = &Resolved;
       continue;
-    if (StackGrowthEntry)
-      report_fatal_error(
-          "GoObj function contains multiple stack-growth statepoints");
-    StackGrowthEntry = &Resolved;
+    }
+    if (Resolved.Entry->ID == GoObj::StackGrowthStatepointID) {
+      if (StackGrowthEntry)
+        report_fatal_error(
+            "GoObj function contains multiple stack-growth statepoints");
+      StackGrowthEntry = &Resolved;
+    }
   }
-  if (!StackGrowthEntry)
-    report_fatal_error("GoObj function has no stack-growth statepoint");
+  if (!EntryArgsEntry)
+    report_fatal_error("GoObj function has no entry argument stack map");
+  if (EntryArgsEntry->Entry->IsIndirectCall)
+    report_fatal_error("GoObj entry argument stack map is a callsite");
+
+  bool IsNoSplit = (Function.Flag & GoObj::SymFlagNoSplit) != 0;
+  if (IsNoSplit && StackGrowthEntry)
+    report_fatal_error("GoObj nosplit function has a stack-growth statepoint");
+  if (!IsNoSplit && !StackGrowthEntry)
+    report_fatal_error("GoObj split function has no stack-growth statepoint");
+  if (StackGrowthEntry && (StackGrowthEntry->Entry->NumDeoptLocations != 0 ||
+                           !StackGrowthEntry->Entry->Locations.empty()))
+    report_fatal_error("GoObj stack-growth statepoint is not root-free");
 
   SmallVector<GoObjStackMapPair, 8> Pairs;
-  Pairs.push_back(BuildPair(*StackGrowthEntry->Entry));
+  Pairs.push_back(BuildPair(*EntryArgsEntry->Entry));
   SmallVector<GoObjPCTabEntry, 16> PCDataEntries;
   std::optional<uint64_t> PreviousCallsitePC;
   SmallVector<uint32_t, 4> IndirectCallOffsets;
   for (const ResolvedEntry &Resolved : ResolvedEntries) {
+    if (Resolved.Entry->ID == GoObj::EntryArgsStackMapID)
+      continue;
     if (PreviousCallsitePC && *PreviousCallsitePC == Resolved.CallsitePC)
       report_fatal_error("GoObj statepoint callsites have duplicate PCs");
     PreviousCallsitePC = Resolved.CallsitePC;