[Go] Model stack ABI values with preallocated and goret

.github/workflows/goallc-ci.yml

@@ -99,6 +99,7 @@ jobs:
           set -euo pipefail
           build="$RUNNER_TEMP/goallc-llvm-build"
           tests=(
+            "$build/test/CodeGen/AArch64/go-argument-homes.ll"
             "$build/test/CodeGen/AArch64/go-callconv.ll"
             "$build/test/CodeGen/AArch64/go-statepoint-stack-args.ll"
             "$build/test/CodeGen/AArch64/goobj-ir-config.ll"
@@ -109,6 +110,7 @@ jobs:
             "$build/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll"
             "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll"
             "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll"
+            "$build/test/CodeGen/X86/go-argument-homes.ll"
             "$build/test/CodeGen/X86/go-callconv.ll"
             "$build/test/CodeGen/X86/go-gc-write-barrier.ll"
             "$build/test/CodeGen/X86/go-statepoint-stack-results.ll"

llvm/docs/LangRef.md

@@ -1408,6 +1408,29 @@ Currently, only the following parameter attributes are defined:
     This is intended for representing ABI constraints, and is not
     intended to be inferred for optimization use.
 
+(attr_goret)=
+
+`goret(<ty>)` and `"goretindex"="<n>"`
+:   The `goret` argument attribute marks a pointer parameter as the logical
+    destination of a Go ABI result whose in-memory type is `<ty>`. The paired
+    `"goretindex"="<n>"` string attribute gives that result's zero-based index
+    in the complete Go result sequence. Both attributes are required on the
+    same parameter.
+
+    These attributes are valid only with the `goabiinternal` and `goabi0`
+    calling conventions. The `goret` parameter is an IR-level carrier and is
+    not passed as a machine pointer argument. For a callee, the target binds it
+    to the result's Go ABI stack home. For a call, the target copies the value
+    from that physical result home to the pointer operand after the call.
+    Consequently, the pointer is dereferenceable for the storage size of
+    `<ty>` and the callee may write that storage as its result.
+
+    `goretindex` values must be unique and increasing in parameter order, and
+    each value must be smaller than the number of direct LLVM results plus the
+    number of `goret` results. The type and index are ABI-affecting and must be
+    present consistently on the function and its call sites. Neither attribute
+    is valid for return values.
+
 (attr_preallocated)=
 
 `preallocated(<ty>)`

llvm/include/llvm/Bitcode/LLVMBitCodes.h

@@ -826,6 +826,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NOOUTLINE = 107,
   ATTR_KIND_FLATTEN = 108,
   ATTR_KIND_NOIPA = 109,
+  ATTR_KIND_GORET = 110,
 };
 
 enum ComdatSelectionKindCodes {
@@ -844,7 +845,7 @@ enum SymtabCodes {
   SYMTAB_BLOB = 1,
 };
 
-} // End bitc namespace
-} // End llvm namespace
+} // namespace bitc
+} // namespace llvm
 
 #endif

llvm/include/llvm/CodeGen/GoCallingConv.h

@@ -32,6 +32,7 @@ class Type;
 namespace goabi {
 
 inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple";
+inline constexpr StringLiteral GoRetIndexAttr = "goretindex";
 inline constexpr StringLiteral PadTypeName = "go.abi.pad";
 // Target frame lowering must not synthesize a morestack edge for such a
 // function. GoObj Go functions otherwise use the native Go default: emit a
@@ -90,6 +91,11 @@ struct CallLayout {
   uint64_t TotalStackSize = 0;
 };
 
+struct MemoryResult {
+  unsigned Index = 0;
+  Type *Ty = nullptr;
+};
+
 struct EntryArgsInfo {
   uint32_t PointerSize = 0;
   uint64_t ArgSize = 0;
@@ -100,15 +106,32 @@ struct EntryArgsInfo {
 bool hasTupleResultsAttr(const AttributeList &Attrs);
 bool hasTupleResultsAttr(const Function &F);
 bool hasTupleResultsAttr(const CallBase &CB);
+unsigned getGoRetIndex(const Argument &Arg);
+unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo);
+/// Return the logical Go ABI type of an IR parameter. Non-empty values that
+/// the Go frontend assigned wholly to memory use a typed preallocated pointer
+/// as the LLVM carrier; direct parameters retain their logical type.
+Type *getParameterType(const Argument &Arg);
+/// Return one bit per logical Go input parameter. A set bit means that the Go
+/// frontend assigned the complete value to memory and represented it with a
+/// typed preallocated carrier. Nest and goret carrier parameters are omitted.
+SmallBitVector getMemoryArgMask(const Function &F);
 // Mirrors ComputeValueTypes and marks leaves originating in %go.abi.pad.
 SmallBitVector getPaddingPieces(Type *Ty);
 
 void getReturnTypes(Type *ReturnType, bool TupleResults,
                     SmallVectorImpl<Type *> &ResultTys);
+void getReturnTypes(Type *ReturnType, bool TupleResults,
+                    ArrayRef<MemoryResult> MemoryResults,
+                    SmallVectorImpl<Type *> &ResultTys);
+SmallBitVector getMemoryResultMask(unsigned NumResults,
+                                   ArrayRef<MemoryResult> MemoryResults);
 
 CallLayout computeCallLayout(ArrayRef<Type *> ArgTys,
-                             ArrayRef<Type *> ResultTys, const DataLayout &DL,
-                             const ABIConfig &Config);
+                             ArrayRef<Type *> ResultTys,
+                             const SmallBitVector &MemoryArgs,
+                             const SmallBitVector &MemoryResults,
+                             const DataLayout &DL, const ABIConfig &Config);
 
 EntryArgsInfo computeEntryArgsInfo(ArrayRef<Type *> ArgTys,
                                    const CallLayout &Layout,

llvm/include/llvm/CodeGen/TargetCallingConv.h

@@ -33,6 +33,7 @@ namespace ISD {
     unsigned IsSRet : 1;     ///< Hidden struct-ret ptr
     unsigned IsByVal : 1;    ///< Struct passed by value
     unsigned IsByRef : 1;    ///< Passed in memory
+    unsigned IsGoRet : 1;    ///< Go result written to its ABI result home
     unsigned IsNest : 1;     ///< Nested fn static chain
     unsigned IsReturned : 1; ///< Always returned
     unsigned IsSplit : 1;
@@ -57,17 +58,17 @@ namespace ISD {
     /// Whether this is part of a variable argument list (non-fixed).
     unsigned IsVarArg : 1;
 
-    unsigned ByValOrByRefSize = 0; ///< Byval or byref struct size
+    unsigned ByValOrIndirectSize = 0; ///< Byval, byref, or goret size
 
     unsigned PointerAddrSpace = 0; ///< Address space of pointer argument
 
   public:
     ArgFlagsTy()
         : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsSRet(0), IsByVal(0),
-          IsByRef(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0),
-          IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0),
-          IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
-          IsSecArgPass(0), MemAlign(0), OrigAlign(0),
+          IsByRef(0), IsGoRet(0), IsNest(0), IsReturned(0), IsSplit(0),
+          IsInAlloca(0), IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0),
+          IsSwiftAsync(0), IsSwiftError(0), IsCFGuardTarget(0), IsHva(0),
+          IsHvaStart(0), IsSecArgPass(0), MemAlign(0), OrigAlign(0),
           IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
           IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) {
       static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big");
@@ -94,6 +95,9 @@ namespace ISD {
     bool isByRef() const { return IsByRef; }
     void setByRef() { IsByRef = 1; }
 
+    bool isGoRet() const { return IsGoRet; }
+    void setGoRet() { IsGoRet = 1; }
+
     bool isInAlloca() const { return IsInAlloca; }
     void setInAlloca() { IsInAlloca = 1; }
 
@@ -176,21 +180,30 @@ namespace ISD {
     }
 
     unsigned getByValSize() const {
-      assert(isByVal() && !isByRef());
-      return ByValOrByRefSize;
+      assert(isByVal() && !isByRef() && !isGoRet());
+      return ByValOrIndirectSize;
     }
     void setByValSize(unsigned S) {
-      assert(isByVal() && !isByRef());
-      ByValOrByRefSize = S;
+      assert(isByVal() && !isByRef() && !isGoRet());
+      ByValOrIndirectSize = S;
     }
 
     unsigned getByRefSize() const {
       assert(!isByVal() && isByRef());
-      return ByValOrByRefSize;
+      return ByValOrIndirectSize;
     }
     void setByRefSize(unsigned S) {
       assert(!isByVal() && isByRef());
-      ByValOrByRefSize = S;
+      ByValOrIndirectSize = S;
+    }
+
+    unsigned getGoRetSize() const {
+      assert(!isByVal() && !isByRef() && isGoRet());
+      return ByValOrIndirectSize;
+    }
+    void setGoRetSize(unsigned S) {
+      assert(!isByVal() && !isByRef() && isGoRet());
+      ByValOrIndirectSize = S;
     }
 
     unsigned getPointerAddrSpace() const { return PointerAddrSpace; }

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -332,6 +332,7 @@ class LLVM_ABI TargetLoweringBase {
     bool IsNest : 1;
     bool IsByVal : 1;
     bool IsByRef : 1;
+    bool IsGoRet : 1;
     bool IsInAlloca : 1;
     bool IsPreallocated : 1;
     bool IsReturned : 1;
@@ -341,13 +342,15 @@ class LLVM_ABI TargetLoweringBase {
     bool IsCFGuardTarget : 1;
     MaybeAlign Alignment = std::nullopt;
     Type *IndirectType = nullptr;
+    unsigned GoRetIndex = 0;
 
     ArgListEntry(Value *Val, SDValue Node, Type *Ty)
         : Val(Val), Node(Node), OrigTy(Ty), Ty(Ty), IsSExt(false),
           IsZExt(false), IsNoExt(false), IsInReg(false), IsSRet(false),
-          IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
-          IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
-          IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {}
+          IsNest(false), IsByVal(false), IsByRef(false), IsGoRet(false),
+          IsInAlloca(false), IsPreallocated(false), IsReturned(false),
+          IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false),
+          IsCFGuardTarget(false) {}
 
     explicit ArgListEntry(Value *Val, SDValue Node = SDValue())
         : ArgListEntry(Val, Node, Val->getType()) {}
@@ -1689,8 +1692,8 @@ class LLVM_ABI TargetLoweringBase {
   /// extending
   virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const { return false; }
 
-  // Returns true if Extend can be folded into the index of a masked gathers/scatters
-  // on this target.
+  // Returns true if Extend can be folded into the index of a masked
+  // gathers/scatters on this target.
   virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const {
     return false;
   }

llvm/include/llvm/IR/Argument.h

@@ -84,6 +84,9 @@ class Argument final : public Value {
   /// Return true if this argument has the byref attribute.
   LLVM_ABI bool hasByRefAttr() const;
 
+  /// Return true if this argument has the goret attribute.
+  LLVM_ABI bool hasGoRetAttr() const;
+
   /// Return true if this argument has the swiftself attribute.
   LLVM_ABI bool hasSwiftSelfAttr() const;
 
@@ -100,8 +103,8 @@ class Argument final : public Value {
   LLVM_ABI uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const;
 
   /// Return true if this argument has the byval, sret, inalloca, preallocated,
-  /// or byref attribute. These attributes represent arguments being passed by
-  /// value (which may or may not involve a stack copy)
+  /// byref, or goret attribute. These attributes represent values with an
+  /// associated in-memory ABI type.
   LLVM_ABI bool hasPointeeInMemoryValueAttr() const;
 
   /// If hasPointeeInMemoryValueAttr returns true, the in-memory ABI type is
@@ -122,6 +125,9 @@ class Argument final : public Value {
   /// If this is a byref argument, return its type.
   LLVM_ABI Type *getParamByRefType() const;
 
+  /// If this is a goret argument, return its logical result type.
+  LLVM_ABI Type *getParamGoRetType() const;
+
   /// If this is an inalloca argument, return its type.
   LLVM_ABI Type *getParamInAllocaType() const;
 
@@ -188,6 +194,6 @@ class Argument final : public Value {
   }
 };
 
-} // End llvm namespace
+} // namespace llvm
 
 #endif

llvm/include/llvm/IR/Attributes.h

@@ -205,6 +205,7 @@ class Attribute {
   LLVM_ABI static Attribute getWithStructRetType(LLVMContext &Context,
                                                  Type *Ty);
   LLVM_ABI static Attribute getWithByRefType(LLVMContext &Context, Type *Ty);
+  LLVM_ABI static Attribute getWithGoRetType(LLVMContext &Context, Type *Ty);
   LLVM_ABI static Attribute getWithPreallocatedType(LLVMContext &Context,
                                                     Type *Ty);
   LLVM_ABI static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty);
@@ -497,6 +498,7 @@ class AttributeSet {
   LLVM_ABI Type *getByValType() const;
   LLVM_ABI Type *getStructRetType() const;
   LLVM_ABI Type *getByRefType() const;
+  LLVM_ABI Type *getGoRetType() const;
   LLVM_ABI Type *getPreallocatedType() const;
   LLVM_ABI Type *getInAllocaType() const;
   LLVM_ABI Type *getElementType() const;
@@ -975,6 +977,9 @@ class AttributeList {
   /// Return the byref type for the specified function parameter.
   LLVM_ABI Type *getParamByRefType(unsigned ArgNo) const;
 
+  /// Return the goret type for the specified function parameter.
+  LLVM_ABI Type *getParamGoRetType(unsigned ArgNo) const;
+
   /// Return the preallocated type for the specified function parameter.
   LLVM_ABI Type *getParamPreallocatedType(unsigned ArgNo) const;
 
@@ -1225,6 +1230,9 @@ class AttrBuilder {
   /// Retrieve the byref type.
   Type *getByRefType() const { return getTypeAttr(Attribute::ByRef); }
 
+  /// Retrieve the goret type.
+  Type *getGoRetType() const { return getTypeAttr(Attribute::GoRet); }
+
   /// Retrieve the preallocated type.
   Type *getPreallocatedType() const {
     return getTypeAttr(Attribute::Preallocated);

llvm/include/llvm/IR/Attributes.td

@@ -108,6 +108,11 @@ def ByVal : TypeAttr<"byval", IntersectPreserve, [ParamAttr]>;
 /// Mark in-memory ABI type.
 def ByRef : TypeAttr<"byref", IntersectPreserve, [ParamAttr]>;
 
+/// Go ABI result written through the caller's logical result home. The
+/// companion "goretindex" string attribute identifies this result in the
+/// complete Go result sequence.
+def GoRet : TypeAttr<"goret", IntersectPreserve, [ParamAttr]>;
+
 /// Parameter or return value may not contain uninitialized or poison bits.
 def NoUndef : EnumAttr<"noundef", IntersectAnd, [ParamAttr, RetAttr]>;
 

llvm/include/llvm/IR/Function.h

@@ -488,6 +488,11 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
     return AttributeSets.getParamByRefType(ArgNo);
   }
 
+  /// Extract the goret type for a parameter.
+  Type *getParamGoRetType(unsigned ArgNo) const {
+    return AttributeSets.getParamGoRetType(ArgNo);
+  }
+
   /// Extract the preallocated type for a parameter.
   Type *getParamPreallocatedType(unsigned ArgNo) const {
     return AttributeSets.getParamPreallocatedType(ArgNo);

llvm/include/llvm/IR/InstrTypes.h

@@ -1861,6 +1861,15 @@ class CallBase : public Instruction {
     return nullptr;
   }
 
+  /// Extract the goret type for a call or parameter.
+  Type *getParamGoRetType(unsigned ArgNo) const {
+    if (auto *Ty = Attrs.getParamGoRetType(ArgNo))
+      return Ty;
+    if (const Function *F = getCalledFunction())
+      return F->getAttributes().getParamGoRetType(ArgNo);
+    return nullptr;
+  }
+
   /// Extract the byval type for a call or parameter.
   Type *getParamByValType(unsigned ArgNo) const {
     if (auto *Ty = Attrs.getParamByValType(ArgNo))

llvm/include/llvm/IR/Intrinsics.td

@@ -1118,7 +1118,7 @@ def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[],
 
 def int_call_preallocated_setup
     : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty],
-                            [ImmArg<ArgIndex<0>>]>;
+                            [ImmArg<ArgIndex<0>>, IntrNoMerge]>;
 def int_call_preallocated_arg
     : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty],
                             [ImmArg<ArgIndex<1>>]>;