[Go] Lower stack ABI inputs with typed byval

.github/workflows/goallc-ci.yml

@@ -100,6 +100,7 @@ jobs:
           build="$RUNNER_TEMP/goallc-llvm-build"
           tests=(
             "$build/test/CodeGen/AArch64/go-callconv.ll"
+            "$build/test/CodeGen/AArch64/go-stack-byval.ll"
             "$build/test/CodeGen/AArch64/go-statepoint-stack-args.ll"
             "$build/test/CodeGen/AArch64/goobj-ir-config.ll"
             "$build/test/CodeGen/AArch64/goobj-register-argument-homes.ll"
@@ -110,6 +111,7 @@ jobs:
             "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll"
             "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll"
             "$build/test/CodeGen/X86/go-callconv.ll"
+            "$build/test/CodeGen/X86/go-stack-byval.ll"
             "$build/test/CodeGen/X86/go-gc-write-barrier.ll"
             "$build/test/CodeGen/X86/go-statepoint-stack-results.ll"
             "$build/test/CodeGen/X86/goobj-filetype.ll"

llvm/include/llvm/CodeGen/GoCallingConv.h

@@ -106,12 +106,16 @@ SmallBitVector getPaddingPieces(Type *Ty);
 void getReturnTypes(Type *ReturnType, bool TupleResults,
                     SmallVectorImpl<Type *> &ResultTys);
 
-CallLayout computeCallLayout(ArrayRef<Type *> ArgTys,
+/// Complete the Go ABI frame layout after the target calling-convention rules
+/// have assigned every input to either registers or a stack offset and
+/// computed the input stack extent. This helper does not classify inputs.
+/// Result classification remains here until stack results have an explicit IR
+/// carrier of their own.
+CallLayout computeCallLayout(ArrayRef<ValueLayout> Args, uint64_t StackArgsSize,
                              ArrayRef<Type *> ResultTys, const DataLayout &DL,
                              const ABIConfig &Config);
 
-EntryArgsInfo computeEntryArgsInfo(ArrayRef<Type *> ArgTys,
-                                   const CallLayout &Layout,
+EntryArgsInfo computeEntryArgsInfo(const CallLayout &Layout,
                                    const DataLayout &DL,
                                    const ABIConfig &Config);
 

llvm/include/llvm/CodeGen/GoISelLowering.h

@@ -0,0 +1,38 @@
+//===- GoISelLowering.h - Go SelectionDAG lowering helpers -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_GOISELLOWERING_H
+#define LLVM_CODEGEN_GOISELLOWERING_H
+
+#include "llvm/CodeGen/GoCallingConv.h"
+#include "llvm/CodeGen/TargetLowering.h"
+
+namespace llvm {
+
+class CCValAssign;
+
+namespace goabi {
+
+/// Derive the logical Go function layout from LLVM formal arguments and the
+/// physical locations assigned by the target calling convention.
+CallLayout computeFormalArgLayout(const Function &F,
+                                  ArrayRef<ISD::InputArg> Ins,
+                                  ArrayRef<CCValAssign> ArgLocs,
+                                  uint64_t StackArgsSize, const DataLayout &DL,
+                                  const ABIConfig &Config);
+
+/// Derive the logical Go call layout from the original LLVM call operands and
+/// the physical locations assigned by the target calling convention.
+CallLayout computeCallLayout(TargetLowering::CallLoweringInfo &CLI,
+                             ArrayRef<CCValAssign> ArgLocs,
+                             uint64_t StackArgsSize, const ABIConfig &Config);
+
+} // namespace goabi
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_GOISELLOWERING_H

llvm/include/llvm/CodeGen/MIRYamlMapping.h

@@ -732,6 +732,8 @@ struct MachineFrameInfo {
   StringValue StackProtector;
   StringValue FunctionContext;
   unsigned MaxCallFrameSize = ~0u; ///< ~0u means: not computed yet.
+  uint64_t GoABIStackArgsSize = ~UINT64_C(0);
+  uint64_t GoABIArgSize = ~UINT64_C(0);
   unsigned CVBytesOfCalleeSavedRegisters = 0;
   bool HasOpaqueSPAdjustment = false;
   bool HasVAStart = false;
@@ -755,6 +757,8 @@ struct MachineFrameInfo {
            StackProtector == Other.StackProtector &&
            FunctionContext == Other.FunctionContext &&
            MaxCallFrameSize == Other.MaxCallFrameSize &&
+           GoABIStackArgsSize == Other.GoABIStackArgsSize &&
+           GoABIArgSize == Other.GoABIArgSize &&
            CVBytesOfCalleeSavedRegisters ==
                Other.CVBytesOfCalleeSavedRegisters &&
            HasOpaqueSPAdjustment == Other.HasOpaqueSPAdjustment &&
@@ -785,6 +789,9 @@ template <> struct MappingTraits<MachineFrameInfo> {
     YamlIO.mapOptional("functionContext", MFI.FunctionContext,
                        StringValue()); // Don't print it out when it's empty.
     YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize, (unsigned)~0);
+    YamlIO.mapOptional("goABIStackArgsSize", MFI.GoABIStackArgsSize,
+                       ~UINT64_C(0));
+    YamlIO.mapOptional("goABIArgSize", MFI.GoABIArgSize, ~UINT64_C(0));
     YamlIO.mapOptional("cvBytesOfCalleeSavedRegisters",
                        MFI.CVBytesOfCalleeSavedRegisters, 0U);
     YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment,

llvm/include/llvm/CodeGen/MachineFrameInfo.h

@@ -294,6 +294,13 @@ class MachineFrameInfo {
   /// It is only valid during and after prolog/epilog code insertion.
   uint64_t MaxCallFrameSize = ~UINT64_C(0);
 
+  /// Logical Go ABI input stack extent and complete argument/home area size.
+  /// Target formal-argument lowering records the values after CCState has
+  /// assigned every input. Late GoObj emission and return lowering consume the
+  /// cached result instead of reconstructing the calling convention from IR.
+  uint64_t GoABIStackArgsSize = ~UINT64_C(0);
+  uint64_t GoABIArgSize = ~UINT64_C(0);
+
   /// The number of bytes of callee saved registers that the target wants to
   /// report for the current function in the CodeView S_FRAMEPROC record.
   unsigned CVBytesOfCalleeSavedRegisters = 0;
@@ -703,6 +710,22 @@ class MachineFrameInfo {
   }
   void setMaxCallFrameSize(uint64_t S) { MaxCallFrameSize = S; }
 
+  bool hasGoABIArgSizes() const {
+    return GoABIStackArgsSize != ~UINT64_C(0) && GoABIArgSize != ~UINT64_C(0);
+  }
+  uint64_t getGoABIStackArgsSize() const {
+    assert(hasGoABIArgSizes() && "Go ABI argument sizes are not initialized");
+    return GoABIStackArgsSize;
+  }
+  uint64_t getGoABIArgSize() const {
+    assert(hasGoABIArgSizes() && "Go ABI argument sizes are not initialized");
+    return GoABIArgSize;
+  }
+  void setGoABIArgSizes(uint64_t StackArgsSize, uint64_t ArgSize) {
+    GoABIStackArgsSize = StackArgsSize;
+    GoABIArgSize = ArgSize;
+  }
+
   /// Returns how many bytes of callee-saved registers the target pushed in the
   /// prologue. Only used for debug info.
   unsigned getCVBytesOfCalleeSavedRegisters() const {

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -226,54 +226,14 @@ class AddrLabelMapCallbackPtr final : CallbackVH {
   void allUsesReplacedWith(Value *V2) override;
 };
 
-static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL,
-                                const Triple &TT) {
+static uint32_t getGoObjArgSize(const MachineFunction &MF) {
+  const Function &F = MF.getFunction();
   if (!goabi::isGoCallingConv(F.getCallingConv()))
     return 0;
-
-  goabi::ABIConfig Config;
-  if (TT.getArch() == Triple::x86_64) {
-    static constexpr unsigned X86GoIntRegs[] = {0, 1, 2, 3, 4, 5, 6, 7, 8};
-    static constexpr unsigned X86GoFPRegs[] = {0, 1, 2,  3,  4,  5,  6, 7,
-                                               8, 9, 10, 11, 12, 13, 14};
-    if (goabi::isGoABI0CallingConv(F.getCallingConv()))
-      Config = {ArrayRef<unsigned>(),
-                ArrayRef<unsigned>(),
-                8,
-                Align(8),
-                Align(8),
-                false};
-    else
-      Config = {X86GoIntRegs, X86GoFPRegs, 8, Align(8), Align(8), false};
-  } else if (TT.getArch() == Triple::aarch64) {
-    static constexpr unsigned AArch64GoIntRegs[] = {
-        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
-    static constexpr unsigned AArch64GoFPRegs[] = {
-        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
-    if (goabi::isGoABI0CallingConv(F.getCallingConv()))
-      Config = {ArrayRef<unsigned>(),
-                ArrayRef<unsigned>(),
-                8,
-                Align(8),
-                Align(16),
-                false};
-    else
-      Config = {AArch64GoIntRegs, AArch64GoFPRegs, 8,
-                Align(8),         Align(16),       false};
-  } else {
-    return 0;
-  }
-
-  SmallVector<Type *, 8> ArgTys;
-  for (const Argument &Arg : F.args())
-    if (!Arg.hasNestAttr())
-      ArgTys.push_back(Arg.getType());
-
-  SmallVector<Type *, 8> ResultTys;
-  goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F),
-                        ResultTys);
-  uint64_t Size =
-      goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize;
+  const MachineFrameInfo &MFI = MF.getFrameInfo();
+  if (!MFI.hasGoABIArgSizes())
+    report_fatal_error("GoObj function is missing lowered Go ABI frame info");
+  uint64_t Size = MFI.getGoABIArgSize();
   if (Size > std::numeric_limits<uint32_t>::max())
     report_fatal_error("GoObj function argument size exceeds uint32 limit");
   return static_cast<uint32_t>(Size);
@@ -1076,8 +1036,7 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) {
       continue;
     bool HasABI0Suffix = F.getName().ends_with(GoObj::ABI0SymbolSuffix);
     bool IsABI0 = goabi::isGoABI0CallingConv(F.getCallingConv());
-    bool IsABIInternal =
-        goabi::isGoABIInternalCallingConv(F.getCallingConv());
+    bool IsABIInternal = goabi::isGoABIInternalCallingConv(F.getCallingConv());
     if (HasABI0Suffix != IsABI0)
       report_fatal_error(
           "Go ABI0 calling convention and <ABI0> symbol suffix disagree");
@@ -3766,9 +3725,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
       report_fatal_error("GoObj function stack size exceeds uint32 limit");
     OutContext.setGoObjSymbolStackSize(CurrentFnSym,
                                        static_cast<uint32_t>(StackSize));
-    OutContext.setGoObjSymbolArgSize(
-        CurrentFnSym,
-        getGoObjArgSize(F, MF.getDataLayout(), TM.getTargetTriple()));
+    OutContext.setGoObjSymbolArgSize(CurrentFnSym, getGoObjArgSize(MF));
     OutContext.setGoObjSymbolHasFramePointer(
         CurrentFnSym, MF.getSubtarget().getFrameLowering()->hasFP(MF));
     OutContext.setGoObjSymbolAsyncUnsafe(CurrentFnSym,

llvm/lib/CodeGen/GoCallingConv.cpp

@@ -245,28 +245,44 @@ void getReturnTypes(Type *ReturnType, bool TupleResults,
   ResultTys.push_back(ReturnType);
 }
 
-CallLayout computeCallLayout(ArrayRef<Type *> ArgTys,
+static uint64_t getDirectValueSize(Type *Ty, const DataLayout &DL) {
+  SmallBitVector PaddingPieces = getPaddingPieces(Ty);
+  if (PaddingPieces.any() && PaddingPieces.count() == PaddingPieces.size())
+    return 0;
+  return DL.getTypeAllocSize(Ty);
+}
+
+CallLayout computeCallLayout(ArrayRef<ValueLayout> Args, uint64_t StackArgsSize,
                              ArrayRef<Type *> ResultTys, const DataLayout &DL,
                              const ABIConfig &Config) {
   CallLayout Layout;
-  Layout.Args.reserve(ArgTys.size());
+  Layout.Args.append(Args.begin(), Args.end());
   Layout.Results.reserve(ResultTys.size());
+  Layout.StackArgsSize = StackArgsSize;
+
+  for (ValueLayout &Arg : Layout.Args) {
+    if (!Arg.Ty)
+      report_fatal_error("Go ABI argument layout has no logical type");
+    uint64_t ExpectedSize = Arg.InRegs ? getDirectValueSize(Arg.Ty, DL)
+                                       : DL.getTypeAllocSize(Arg.Ty);
+    Arg.Size = ExpectedSize;
+    Align ABIAlignment = DL.getABITypeAlign(Arg.Ty);
+    if (Arg.InRegs) {
+      Arg.Alignment = ABIAlignment;
+      continue;
+    }
+    if (Arg.Alignment < ABIAlignment)
+      report_fatal_error("invalid preassigned Go ABI argument layout");
+    if (Arg.StackOffset % Arg.Alignment.value() != 0 ||
+        Arg.StackOffset > StackArgsSize ||
+        Arg.Size > StackArgsSize - Arg.StackOffset)
+      report_fatal_error(
+          "Go ABI stack argument is outside its assigned input area");
+  }
 
   unsigned NextInt = 0;
   unsigned NextFP = 0;
-  uint64_t StackArgsEnd = 0;
-  for (Type *ArgTy : ArgTys) {
-    ValueLayout ArgLayout =
-        computeValueLayout(ArgTy, DL, Config, NextInt, NextFP);
-    if (!ArgLayout.InRegs)
-      StackArgsEnd = layoutStackValue(StackArgsEnd, ArgLayout);
-    Layout.Args.push_back(ArgLayout);
-  }
-  Layout.StackArgsSize = StackArgsEnd;
-
-  NextInt = 0;
-  NextFP = 0;
-  uint64_t StackResultsEnd = alignToValue(StackArgsEnd, Config.PtrAlign);
+  uint64_t StackResultsEnd = alignToValue(StackArgsSize, Config.PtrAlign);
   uint64_t StackResultsStart = StackResultsEnd;
   for (Type *ResultTy : ResultTys) {
     ValueLayout ResultLayout =
@@ -316,22 +332,18 @@ static void collectPointerOffsets(Type *Ty, uint64_t BaseOffset,
     report_fatal_error("Go entry argument maps do not support pointer vectors");
 }
 
-EntryArgsInfo computeEntryArgsInfo(ArrayRef<Type *> ArgTys,
-                                   const CallLayout &Layout,
+EntryArgsInfo computeEntryArgsInfo(const CallLayout &Layout,
                                    const DataLayout &DL,
                                    const ABIConfig &Config) {
   if (!Config.PtrSize || Layout.ArgSize % Config.PtrSize != 0 ||
       Layout.ArgSize / Config.PtrSize > std::numeric_limits<uint32_t>::max())
     report_fatal_error("invalid Go entry argument map dimensions");
-  if (ArgTys.size() != Layout.Args.size())
-    report_fatal_error("Go entry argument types do not match ABI layout");
-
   EntryArgsInfo Info;
   Info.PointerSize = Config.PtrSize;
   Info.ArgSize = Layout.ArgSize;
   Info.NumBits = static_cast<uint32_t>(Layout.ArgSize / Config.PtrSize);
 
-  SmallVector<uint64_t, 8> HomeOffsets(ArgTys.size());
+  SmallVector<uint64_t, 8> HomeOffsets(Layout.Args.size());
   uint64_t SpillOffset = Layout.SpillAreaOffset;
   for (auto [Index, ArgLayout] : llvm::enumerate(Layout.Args)) {
     if (ArgLayout.InRegs) {
@@ -346,8 +358,8 @@ EntryArgsInfo computeEntryArgsInfo(ArrayRef<Type *> ArgTys,
     report_fatal_error("Go entry argument homes do not match spill area");
 
   SmallVector<uint64_t, 16> PointerOffsets;
-  for (auto [Index, ArgTy] : llvm::enumerate(ArgTys))
-    collectPointerOffsets(ArgTy, HomeOffsets[Index], DL, PointerOffsets);
+  for (auto [Index, ArgLayout] : llvm::enumerate(Layout.Args))
+    collectPointerOffsets(ArgLayout.Ty, HomeOffsets[Index], DL, PointerOffsets);
   llvm::sort(PointerOffsets);
 
   for (uint64_t Offset : PointerOffsets) {

llvm/lib/CodeGen/MIRParser/MIRParser.cpp

@@ -944,6 +944,14 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
     MFI.setFramePointerPolicy(YamlMFI.FramePointerPolicy);
   if (YamlMFI.MaxCallFrameSize != ~0u)
     MFI.setMaxCallFrameSize(YamlMFI.MaxCallFrameSize);
+  if (YamlMFI.GoABIStackArgsSize != ~UINT64_C(0) ||
+      YamlMFI.GoABIArgSize != ~UINT64_C(0)) {
+    if (YamlMFI.GoABIStackArgsSize == ~UINT64_C(0) ||
+        YamlMFI.GoABIArgSize == ~UINT64_C(0))
+      return error(Twine("Go ABI frame info requires both input and total "
+                         "argument sizes"));
+    MFI.setGoABIArgSizes(YamlMFI.GoABIStackArgsSize, YamlMFI.GoABIArgSize);
+  }
   MFI.setCVBytesOfCalleeSavedRegisters(YamlMFI.CVBytesOfCalleeSavedRegisters);
   MFI.setHasOpaqueSPAdjustment(YamlMFI.HasOpaqueSPAdjustment);
   MFI.setHasVAStart(YamlMFI.HasVAStart);

llvm/lib/CodeGen/MIRPrinter.cpp

@@ -374,6 +374,10 @@ static void convertMFI(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
   YamlMFI.FramePointerPolicy = MFI.getFramePointerPolicy();
   YamlMFI.MaxCallFrameSize = MFI.isMaxCallFrameSizeComputed()
     ? MFI.getMaxCallFrameSize() : ~0u;
+  if (MFI.hasGoABIArgSizes()) {
+    YamlMFI.GoABIStackArgsSize = MFI.getGoABIStackArgsSize();
+    YamlMFI.GoABIArgSize = MFI.getGoABIArgSize();
+  }
   YamlMFI.CVBytesOfCalleeSavedRegisters =
       MFI.getCVBytesOfCalleeSavedRegisters();
   YamlMFI.HasOpaqueSPAdjustment = MFI.hasOpaqueSPAdjustment();

llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt

@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMSelectionDAG
   DAGCombiner.cpp
   FastISel.cpp
   FunctionLoweringInfo.cpp
+  GoISelLowering.cpp
   InstrEmitter.cpp
   LegalizeDAG.cpp
   LegalizeFloatTypes.cpp