diff --git a/.github/workflows/goallc-ci.yml b/.github/workflows/goallc-ci.yml index ce649abdd462f..506fbdb8022e6 100644 --- a/.github/workflows/goallc-ci.yml +++ b/.github/workflows/goallc-ci.yml @@ -99,6 +99,7 @@ jobs: set -euo pipefail build="$RUNNER_TEMP/goallc-llvm-build" tests=( + "$build/test/CodeGen/AArch64/go-argument-homes.ll" "$build/test/CodeGen/AArch64/go-callconv.ll" "$build/test/CodeGen/AArch64/go-statepoint-stack-args.ll" "$build/test/CodeGen/AArch64/goobj-ir-config.ll" @@ -109,6 +110,7 @@ jobs: "$build/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll" "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll" "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll" + "$build/test/CodeGen/X86/go-argument-homes.ll" "$build/test/CodeGen/X86/go-callconv.ll" "$build/test/CodeGen/X86/go-gc-write-barrier.ll" "$build/test/CodeGen/X86/go-statepoint-stack-results.ll" diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md index f555ca543ffdc..f91ebc74c0d9d 100644 --- a/llvm/docs/LangRef.md +++ b/llvm/docs/LangRef.md @@ -1408,6 +1408,29 @@ Currently, only the following parameter attributes are defined: This is intended for representing ABI constraints, and is not intended to be inferred for optimization use. +(attr_goret)= + +`goret()` and `"goretindex"=""` +: The `goret` argument attribute marks a pointer parameter as the logical + destination of a Go ABI result whose in-memory type is ``. The paired + `"goretindex"=""` string attribute gives that result's zero-based index + in the complete Go result sequence. Both attributes are required on the + same parameter. + + These attributes are valid only with the `goabiinternal` and `goabi0` + calling conventions. The `goret` parameter is an IR-level carrier and is + not passed as a machine pointer argument. For a callee, the target binds it + to the result's Go ABI stack home. For a call, the target copies the value + from that physical result home to the pointer operand after the call. + Consequently, the pointer is dereferenceable for the storage size of + `` and the callee may write that storage as its result. + + `goretindex` values must be unique and increasing in parameter order, and + each value must be smaller than the number of direct LLVM results plus the + number of `goret` results. The type and index are ABI-affecting and must be + present consistently on the function and its call sites. Neither attribute + is valid for return values. + (attr_preallocated)= `preallocated()` diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 358f9a65a80af..e1cbbf80c923b 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -826,6 +826,7 @@ enum AttributeKindCodes { ATTR_KIND_NOOUTLINE = 107, ATTR_KIND_FLATTEN = 108, ATTR_KIND_NOIPA = 109, + ATTR_KIND_GORET = 110, }; enum ComdatSelectionKindCodes { @@ -844,7 +845,7 @@ enum SymtabCodes { SYMTAB_BLOB = 1, }; -} // End bitc namespace -} // End llvm namespace +} // namespace bitc +} // namespace llvm #endif diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index a94c652859816..16dce37182465 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -32,6 +32,7 @@ class Type; namespace goabi { inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple"; +inline constexpr StringLiteral GoRetIndexAttr = "goretindex"; inline constexpr StringLiteral PadTypeName = "go.abi.pad"; // Target frame lowering must not synthesize a morestack edge for such a // function. GoObj Go functions otherwise use the native Go default: emit a @@ -90,6 +91,11 @@ struct CallLayout { uint64_t TotalStackSize = 0; }; +struct MemoryResult { + unsigned Index = 0; + Type *Ty = nullptr; +}; + struct EntryArgsInfo { uint32_t PointerSize = 0; uint64_t ArgSize = 0; @@ -100,15 +106,32 @@ struct EntryArgsInfo { bool hasTupleResultsAttr(const AttributeList &Attrs); bool hasTupleResultsAttr(const Function &F); bool hasTupleResultsAttr(const CallBase &CB); +unsigned getGoRetIndex(const Argument &Arg); +unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo); +/// Return the logical Go ABI type of an IR parameter. Non-empty values that +/// the Go frontend assigned wholly to memory use a typed preallocated pointer +/// as the LLVM carrier; direct parameters retain their logical type. +Type *getParameterType(const Argument &Arg); +/// Return one bit per logical Go input parameter. A set bit means that the Go +/// frontend assigned the complete value to memory and represented it with a +/// typed preallocated carrier. Nest and goret carrier parameters are omitted. +SmallBitVector getMemoryArgMask(const Function &F); // Mirrors ComputeValueTypes and marks leaves originating in %go.abi.pad. SmallBitVector getPaddingPieces(Type *Ty); void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys); +void getReturnTypes(Type *ReturnType, bool TupleResults, + ArrayRef MemoryResults, + SmallVectorImpl &ResultTys); +SmallBitVector getMemoryResultMask(unsigned NumResults, + ArrayRef MemoryResults); CallLayout computeCallLayout(ArrayRef ArgTys, - ArrayRef ResultTys, const DataLayout &DL, - const ABIConfig &Config); + ArrayRef ResultTys, + const SmallBitVector &MemoryArgs, + const SmallBitVector &MemoryResults, + const DataLayout &DL, const ABIConfig &Config); EntryArgsInfo computeEntryArgsInfo(ArrayRef ArgTys, const CallLayout &Layout, diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h index f197c7f1645ec..f968033b8a472 100644 --- a/llvm/include/llvm/CodeGen/TargetCallingConv.h +++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -33,6 +33,7 @@ namespace ISD { unsigned IsSRet : 1; ///< Hidden struct-ret ptr unsigned IsByVal : 1; ///< Struct passed by value unsigned IsByRef : 1; ///< Passed in memory + unsigned IsGoRet : 1; ///< Go result written to its ABI result home unsigned IsNest : 1; ///< Nested fn static chain unsigned IsReturned : 1; ///< Always returned unsigned IsSplit : 1; @@ -57,17 +58,17 @@ namespace ISD { /// Whether this is part of a variable argument list (non-fixed). unsigned IsVarArg : 1; - unsigned ByValOrByRefSize = 0; ///< Byval or byref struct size + unsigned ByValOrIndirectSize = 0; ///< Byval, byref, or goret size unsigned PointerAddrSpace = 0; ///< Address space of pointer argument public: ArgFlagsTy() : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsSRet(0), IsByVal(0), - IsByRef(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), - IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0), - IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0), - IsSecArgPass(0), MemAlign(0), OrigAlign(0), + IsByRef(0), IsGoRet(0), IsNest(0), IsReturned(0), IsSplit(0), + IsInAlloca(0), IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), + IsSwiftAsync(0), IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), + IsHvaStart(0), IsSecArgPass(0), MemAlign(0), OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0), IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) { static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big"); @@ -94,6 +95,9 @@ namespace ISD { bool isByRef() const { return IsByRef; } void setByRef() { IsByRef = 1; } + bool isGoRet() const { return IsGoRet; } + void setGoRet() { IsGoRet = 1; } + bool isInAlloca() const { return IsInAlloca; } void setInAlloca() { IsInAlloca = 1; } @@ -176,21 +180,30 @@ namespace ISD { } unsigned getByValSize() const { - assert(isByVal() && !isByRef()); - return ByValOrByRefSize; + assert(isByVal() && !isByRef() && !isGoRet()); + return ByValOrIndirectSize; } void setByValSize(unsigned S) { - assert(isByVal() && !isByRef()); - ByValOrByRefSize = S; + assert(isByVal() && !isByRef() && !isGoRet()); + ByValOrIndirectSize = S; } unsigned getByRefSize() const { assert(!isByVal() && isByRef()); - return ByValOrByRefSize; + return ByValOrIndirectSize; } void setByRefSize(unsigned S) { assert(!isByVal() && isByRef()); - ByValOrByRefSize = S; + ByValOrIndirectSize = S; + } + + unsigned getGoRetSize() const { + assert(!isByVal() && !isByRef() && isGoRet()); + return ByValOrIndirectSize; + } + void setGoRetSize(unsigned S) { + assert(!isByVal() && !isByRef() && isGoRet()); + ByValOrIndirectSize = S; } unsigned getPointerAddrSpace() const { return PointerAddrSpace; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 125b73e59b218..ee15d0904d28d 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -332,6 +332,7 @@ class LLVM_ABI TargetLoweringBase { bool IsNest : 1; bool IsByVal : 1; bool IsByRef : 1; + bool IsGoRet : 1; bool IsInAlloca : 1; bool IsPreallocated : 1; bool IsReturned : 1; @@ -341,13 +342,15 @@ class LLVM_ABI TargetLoweringBase { bool IsCFGuardTarget : 1; MaybeAlign Alignment = std::nullopt; Type *IndirectType = nullptr; + unsigned GoRetIndex = 0; ArgListEntry(Value *Val, SDValue Node, Type *Ty) : Val(Val), Node(Node), OrigTy(Ty), Ty(Ty), IsSExt(false), IsZExt(false), IsNoExt(false), IsInReg(false), IsSRet(false), - IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false), - IsPreallocated(false), IsReturned(false), IsSwiftSelf(false), - IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {} + IsNest(false), IsByVal(false), IsByRef(false), IsGoRet(false), + IsInAlloca(false), IsPreallocated(false), IsReturned(false), + IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false), + IsCFGuardTarget(false) {} explicit ArgListEntry(Value *Val, SDValue Node = SDValue()) : ArgListEntry(Val, Node, Val->getType()) {} @@ -1689,8 +1692,8 @@ class LLVM_ABI TargetLoweringBase { /// extending virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const { return false; } - // Returns true if Extend can be folded into the index of a masked gathers/scatters - // on this target. + // Returns true if Extend can be folded into the index of a masked + // gathers/scatters on this target. virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const { return false; } diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h index 05fd7ff4df326..be7b0c34294ed 100644 --- a/llvm/include/llvm/IR/Argument.h +++ b/llvm/include/llvm/IR/Argument.h @@ -84,6 +84,9 @@ class Argument final : public Value { /// Return true if this argument has the byref attribute. LLVM_ABI bool hasByRefAttr() const; + /// Return true if this argument has the goret attribute. + LLVM_ABI bool hasGoRetAttr() const; + /// Return true if this argument has the swiftself attribute. LLVM_ABI bool hasSwiftSelfAttr() const; @@ -100,8 +103,8 @@ class Argument final : public Value { LLVM_ABI uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const; /// Return true if this argument has the byval, sret, inalloca, preallocated, - /// or byref attribute. These attributes represent arguments being passed by - /// value (which may or may not involve a stack copy) + /// byref, or goret attribute. These attributes represent values with an + /// associated in-memory ABI type. LLVM_ABI bool hasPointeeInMemoryValueAttr() const; /// If hasPointeeInMemoryValueAttr returns true, the in-memory ABI type is @@ -122,6 +125,9 @@ class Argument final : public Value { /// If this is a byref argument, return its type. LLVM_ABI Type *getParamByRefType() const; + /// If this is a goret argument, return its logical result type. + LLVM_ABI Type *getParamGoRetType() const; + /// If this is an inalloca argument, return its type. LLVM_ABI Type *getParamInAllocaType() const; @@ -188,6 +194,6 @@ class Argument final : public Value { } }; -} // End llvm namespace +} // namespace llvm #endif diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h index 049f699b2980b..fb7c25e0e19f1 100644 --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -205,6 +205,7 @@ class Attribute { LLVM_ABI static Attribute getWithStructRetType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithByRefType(LLVMContext &Context, Type *Ty); + LLVM_ABI static Attribute getWithGoRetType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty); @@ -497,6 +498,7 @@ class AttributeSet { LLVM_ABI Type *getByValType() const; LLVM_ABI Type *getStructRetType() const; LLVM_ABI Type *getByRefType() const; + LLVM_ABI Type *getGoRetType() const; LLVM_ABI Type *getPreallocatedType() const; LLVM_ABI Type *getInAllocaType() const; LLVM_ABI Type *getElementType() const; @@ -975,6 +977,9 @@ class AttributeList { /// Return the byref type for the specified function parameter. LLVM_ABI Type *getParamByRefType(unsigned ArgNo) const; + /// Return the goret type for the specified function parameter. + LLVM_ABI Type *getParamGoRetType(unsigned ArgNo) const; + /// Return the preallocated type for the specified function parameter. LLVM_ABI Type *getParamPreallocatedType(unsigned ArgNo) const; @@ -1225,6 +1230,9 @@ class AttrBuilder { /// Retrieve the byref type. Type *getByRefType() const { return getTypeAttr(Attribute::ByRef); } + /// Retrieve the goret type. + Type *getGoRetType() const { return getTypeAttr(Attribute::GoRet); } + /// Retrieve the preallocated type. Type *getPreallocatedType() const { return getTypeAttr(Attribute::Preallocated); diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 4e45100b54d38..9575e79a80e9c 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -108,6 +108,11 @@ def ByVal : TypeAttr<"byval", IntersectPreserve, [ParamAttr]>; /// Mark in-memory ABI type. def ByRef : TypeAttr<"byref", IntersectPreserve, [ParamAttr]>; +/// Go ABI result written through the caller's logical result home. The +/// companion "goretindex" string attribute identifies this result in the +/// complete Go result sequence. +def GoRet : TypeAttr<"goret", IntersectPreserve, [ParamAttr]>; + /// Parameter or return value may not contain uninitialized or poison bits. def NoUndef : EnumAttr<"noundef", IntersectAnd, [ParamAttr, RetAttr]>; diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index bd28f0d9902da..97a2d55263f35 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -488,6 +488,11 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node { return AttributeSets.getParamByRefType(ArgNo); } + /// Extract the goret type for a parameter. + Type *getParamGoRetType(unsigned ArgNo) const { + return AttributeSets.getParamGoRetType(ArgNo); + } + /// Extract the preallocated type for a parameter. Type *getParamPreallocatedType(unsigned ArgNo) const { return AttributeSets.getParamPreallocatedType(ArgNo); diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 5f7df6a4eb6f8..5b9ed55426b01 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1861,6 +1861,15 @@ class CallBase : public Instruction { return nullptr; } + /// Extract the goret type for a call or parameter. + Type *getParamGoRetType(unsigned ArgNo) const { + if (auto *Ty = Attrs.getParamGoRetType(ArgNo)) + return Ty; + if (const Function *F = getCalledFunction()) + return F->getAttributes().getParamGoRetType(ArgNo); + return nullptr; + } + /// Extract the byval type for a call or parameter. Type *getParamByValType(unsigned ArgNo) const { if (auto *Ty = Attrs.getParamByValType(ArgNo)) diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 900dc31967845..e9dad4488fee3 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1118,7 +1118,7 @@ def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty], - [ImmArg>]>; + [ImmArg>, IntrNoMerge]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty], [ImmArg>]>; diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index c57323ed41525..771bd24eb0f3c 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2302,6 +2302,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoUndef; case bitc::ATTR_KIND_BYREF: return Attribute::ByRef; + case bitc::ATTR_KIND_GORET: + return Attribute::GoRet; case bitc::ATTR_KIND_MUSTPROGRESS: return Attribute::MustProgress; case bitc::ATTR_KIND_HOT: @@ -3658,8 +3660,8 @@ Error BitcodeReader::parseConstants() { {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2]}); break; } - case bitc::CST_CODE_CE_EXTRACTELT - : { // CE_EXTRACTELT: [opty, opval, opty, opval] + case bitc::CST_CODE_CE_EXTRACTELT: { // CE_EXTRACTELT: [opty, opval, opty, + // opval] if (Record.size() < 3) return error("Invalid extractelement constexpr record"); unsigned OpTyID = Record[0]; @@ -3682,8 +3684,8 @@ Error BitcodeReader::parseConstants() { {(unsigned)Record[1], IdxRecord}); break; } - case bitc::CST_CODE_CE_INSERTELT - : { // CE_INSERTELT: [opval, opval, opty, opval] + case bitc::CST_CODE_CE_INSERTELT: { // CE_INSERTELT: [opval, opval, opty, + // opval] VectorType *OpTy = dyn_cast(CurTy); if (Record.size() < 3 || !OpTy) return error("Invalid insertelement constexpr record"); @@ -5566,7 +5568,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_VSELECT: {// VSELECT: [ty,opval,opval,predty,pred] + case bitc::FUNC_CODE_INST_VSELECT: { // VSELECT: + // [ty,opval,opval,predty,pred] // new form of select // handles select i1 or select [N x i1] unsigned OpNum = 0; @@ -5636,7 +5639,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_SHUFFLEVEC: {// SHUFFLEVEC: [opval,ty,opval,opval] + case bitc::FUNC_CODE_INST_SHUFFLEVEC: { // SHUFFLEVEC: + // [opval,ty,opval,opval] unsigned OpNum = 0; Value *Vec1, *Vec2, *Mask; unsigned Vec1TypeID; @@ -5659,10 +5663,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_CMP: // CMP: [opty, opval, opval, pred] - // Old form of ICmp/FCmp returning bool - // Existed to differentiate between icmp/fcmp and vicmp/vfcmp which were - // both legal on vectors but had different behaviour. + case bitc::FUNC_CODE_INST_CMP: // CMP: [opty, opval, opval, pred] + // Old form of ICmp/FCmp returning bool + // Existed to differentiate between icmp/fcmp + // and vicmp/vfcmp which were both legal on + // vectors but had different behaviour. case bitc::FUNC_CODE_INST_CMP2: { // CMP2: [opty, opval, opval, pred] // FCmp/ICmp returning bool or vector of bool @@ -6499,7 +6504,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_STORE: - case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, vol] + case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, + // vol] unsigned OpNum = 0; Value *Val, *Ptr; unsigned PtrTypeID, ValTypeID; diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 492240515b489..f4cab1bd67c7c 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -984,6 +984,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NOUNDEF; case Attribute::ByRef: return bitc::ATTR_KIND_BYREF; + case Attribute::GoRet: + return bitc::ATTR_KIND_GORET; case Attribute::MustProgress: return bitc::ATTR_KIND_MUSTPROGRESS; case Attribute::PresplitCoroutine: diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index 2a1ad62c2804c..1e2f7a3e2d299 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -1084,6 +1084,8 @@ void ValueEnumerator::incorporateFunction(const Function &F) { EnumerateType(I.getParamStructRetType()); else if (I.hasAttribute(Attribute::ByRef)) EnumerateType(I.getParamByRefType()); + else if (I.hasAttribute(Attribute::GoRet)) + EnumerateType(I.getAttribute(Attribute::GoRet).getValueAsType()); } FirstFuncConstantID = Values.size(); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 88435712831e8..1d60e032cc2f9 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -265,15 +265,23 @@ static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL, } SmallVector ArgTys; + SmallVector MemoryResults; for (const Argument &Arg : F.args()) - if (!Arg.hasNestAttr()) - ArgTys.push_back(Arg.getType()); + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + else if (!Arg.hasNestAttr()) + ArgTys.push_back(goabi::getParameterType(Arg)); SmallVector ResultTys; goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), - ResultTys); + MemoryResults, ResultTys); uint64_t Size = - goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize; + goabi::computeCallLayout( + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), DL, + Config) + .ArgSize; if (Size > std::numeric_limits::max()) report_fatal_error("GoObj function argument size exceeds uint32 limit"); return static_cast(Size); diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index a580b78462453..3bfc0a95bc936 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -51,6 +51,8 @@ static void addFlagsFromAttrSet(ISD::ArgFlagsTy &Flags, AttributeSet Attrs) { Flags.setByVal(); if (Attrs.hasAttribute(Attribute::ByRef)) Flags.setByRef(); + if (Attrs.hasAttribute(Attribute::GoRet)) + Flags.setGoRet(); if (Attrs.hasAttribute(Attribute::InAlloca)) { Flags.setInAlloca(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -250,23 +252,28 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, Align MemAlign = DL.getABITypeAlign(Arg.Ty); if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || - Flags.isByRef()) { + Flags.isByRef() || Flags.isGoRet()) { assert(OpIdx >= AttributeList::FirstArgIndex); unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamByRefType(ParamIdx); + if (!ElementTy) + ElementTy = FuncInfo.getParamGoRetType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); - assert(ElementTy && "Must have byval, inalloca or preallocated type"); + assert(ElementTy && + "Must have byval, byref, goret, inalloca or preallocated type"); uint64_t MemSize = DL.getTypeAllocSize(ElementTy); if (Flags.isByRef()) Flags.setByRefSize(MemSize); + else if (Flags.isGoRet()) + Flags.setGoRetSize(MemSize); else Flags.setByValSize(MemSize); diff --git a/llvm/lib/CodeGen/GoCallingConv.cpp b/llvm/lib/CodeGen/GoCallingConv.cpp index 73491bbe2fa58..10b97e2368967 100644 --- a/llvm/lib/CodeGen/GoCallingConv.cpp +++ b/llvm/lib/CodeGen/GoCallingConv.cpp @@ -180,7 +180,8 @@ static uint64_t alignToValue(uint64_t Value, Align Alignment) { } static ValueLayout computeValueLayout(Type *Ty, const DataLayout &DL, - const ABIConfig &Config, unsigned &IntReg, + const ABIConfig &Config, + bool MustUseMemory, unsigned &IntReg, unsigned &FPReg) { ValueLayout Layout; Layout.Ty = Ty; @@ -197,7 +198,7 @@ static ValueLayout computeValueLayout(Type *Ty, const DataLayout &DL, unsigned IntAfter = IntReg; unsigned FPAfter = FPReg; - if (classifyType(Ty, DL, Config, IntAfter, FPAfter)) { + if (!MustUseMemory && classifyType(Ty, DL, Config, IntAfter, FPAfter)) { Layout.InRegs = true; Layout.IntRegCount = IntAfter - IntReg; Layout.FPRegCount = FPAfter - FPReg; @@ -222,6 +223,22 @@ bool hasTupleResultsAttr(const Function &F) { return F.hasFnAttribute(TupleResultsAttr); } +static unsigned getGoRetIndex(Attribute Attr) { + unsigned Index; + if (!Attr.isStringAttribute() || + Attr.getValueAsString().getAsInteger(10, Index)) + report_fatal_error("invalid or missing goretindex parameter attribute"); + return Index; +} + +unsigned getGoRetIndex(const Argument &Arg) { + return getGoRetIndex(Arg.getAttributes().getAttribute(GoRetIndexAttr)); +} + +unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo) { + return getGoRetIndex(CB.getParamAttr(ArgNo, GoRetIndexAttr)); +} + bool hasTupleResultsAttr(const CallBase &CB) { if (CB.hasFnAttr(TupleResultsAttr)) return true; @@ -230,6 +247,22 @@ bool hasTupleResultsAttr(const CallBase &CB) { return false; } +Type *getParameterType(const Argument &Arg) { + if (Arg.hasPreallocatedAttr()) + return Arg.getAttributes().getPreallocatedType(); + return Arg.getType(); +} + +SmallBitVector getMemoryArgMask(const Function &F) { + SmallBitVector MemoryArgs; + for (const Argument &Arg : F.args()) { + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) + continue; + MemoryArgs.push_back(Arg.hasPreallocatedAttr()); + } + return MemoryArgs; +} + void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys) { if (ReturnType->isVoidTy()) @@ -245,9 +278,57 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, ResultTys.push_back(ReturnType); } +void getReturnTypes(Type *ReturnType, bool TupleResults, + ArrayRef MemoryResults, + SmallVectorImpl &ResultTys) { + SmallVector DirectResultTys; + getReturnTypes(ReturnType, TupleResults, DirectResultTys); + if (MemoryResults.empty()) { + ResultTys.append(DirectResultTys.begin(), DirectResultTys.end()); + return; + } + + unsigned TotalResultCount = DirectResultTys.size() + MemoryResults.size(); + ResultTys.assign(TotalResultCount, nullptr); + std::optional Previous; + for (const MemoryResult &Result : MemoryResults) { + if (!Result.Ty || Result.Index >= TotalResultCount || + ResultTys[Result.Index] || (Previous && *Previous >= Result.Index)) + report_fatal_error("invalid Go memory result carrier"); + ResultTys[Result.Index] = Result.Ty; + Previous = Result.Index; + } + auto Direct = DirectResultTys.begin(); + for (Type *&Ty : ResultTys) + if (!Ty) + Ty = *Direct++; + if (Direct != DirectResultTys.end()) + report_fatal_error("Go direct result index count mismatch"); +} + +SmallBitVector getMemoryResultMask(unsigned NumResults, + ArrayRef MemoryResults) { + SmallBitVector Mask(NumResults); + std::optional Previous; + for (const MemoryResult &Result : MemoryResults) { + if (Result.Index >= NumResults || Mask.test(Result.Index) || + (Previous && *Previous >= Result.Index)) + report_fatal_error("invalid Go memory result carrier"); + Mask.set(Result.Index); + Previous = Result.Index; + } + return Mask; +} + CallLayout computeCallLayout(ArrayRef ArgTys, - ArrayRef ResultTys, const DataLayout &DL, - const ABIConfig &Config) { + ArrayRef ResultTys, + const SmallBitVector &MemoryArgs, + const SmallBitVector &MemoryResults, + const DataLayout &DL, const ABIConfig &Config) { + if (MemoryArgs.size() != ArgTys.size() || + MemoryResults.size() != ResultTys.size()) + report_fatal_error("Go ABI memory constraints do not match value counts"); + CallLayout Layout; Layout.Args.reserve(ArgTys.size()); Layout.Results.reserve(ResultTys.size()); @@ -255,9 +336,9 @@ CallLayout computeCallLayout(ArrayRef ArgTys, unsigned NextInt = 0; unsigned NextFP = 0; uint64_t StackArgsEnd = 0; - for (Type *ArgTy : ArgTys) { - ValueLayout ArgLayout = - computeValueLayout(ArgTy, DL, Config, NextInt, NextFP); + for (auto [Index, ArgTy] : llvm::enumerate(ArgTys)) { + ValueLayout ArgLayout = computeValueLayout( + ArgTy, DL, Config, MemoryArgs.test(Index), NextInt, NextFP); if (!ArgLayout.InRegs) StackArgsEnd = layoutStackValue(StackArgsEnd, ArgLayout); Layout.Args.push_back(ArgLayout); @@ -268,9 +349,9 @@ CallLayout computeCallLayout(ArrayRef ArgTys, NextFP = 0; uint64_t StackResultsEnd = alignToValue(StackArgsEnd, Config.PtrAlign); uint64_t StackResultsStart = StackResultsEnd; - for (Type *ResultTy : ResultTys) { - ValueLayout ResultLayout = - computeValueLayout(ResultTy, DL, Config, NextInt, NextFP); + for (auto [Index, ResultTy] : llvm::enumerate(ResultTys)) { + ValueLayout ResultLayout = computeValueLayout( + ResultTy, DL, Config, MemoryResults.test(Index), NextInt, NextFP); if (!ResultLayout.InRegs) StackResultsEnd = layoutStackValue(StackResultsEnd, ResultLayout); Layout.Results.push_back(ResultLayout); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 449bfa27dbda4..6e8f07428a873 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -11742,6 +11742,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { Flags.setByVal(); if (Args[i].IsByRef) Flags.setByRef(); + if (Args[i].IsGoRet) + Flags.setGoRet(); if (Args[i].IsPreallocated) { Flags.setPreallocated(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -11770,6 +11772,20 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { MemAlign = *MA; else MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); + } else if (Args[i].IsByRef) { + unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); + Flags.setByRefSize(FrameSize); + if (auto MA = Args[i].Alignment) + MemAlign = *MA; + else + MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); + } else if (Args[i].IsGoRet) { + unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); + Flags.setGoRetSize(FrameSize); + if (auto MA = Args[i].Alignment) + MemAlign = *MA; + else + MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); } else if (auto MA = Args[i].Alignment) { MemAlign = *MA; } else { @@ -11792,8 +11808,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { else if (Args[i].IsZExt) ExtendKind = ISD::ZERO_EXTEND; - // Conservatively only handle 'returned' on non-vectors that can be lowered, - // for now. + // Conservatively only handle 'returned' on non-vectors that can be + // lowered, for now. if (Args[i].IsReturned && !Op.getValueType().isVector() && CanLowerReturn) { assert((CLI.RetTy == Args[i].Ty || @@ -12306,6 +12322,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { Flags.setByVal(); if (Arg.hasAttribute(Attribute::ByRef)) Flags.setByRef(); + if (Arg.hasAttribute(Attribute::GoRet)) + Flags.setGoRet(); if (Arg.hasAttribute(Attribute::InAlloca)) { Flags.setInAlloca(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -12335,7 +12353,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) { Align MemAlign; Type *ArgMemTy = nullptr; if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || - Flags.isByRef()) { + Flags.isByRef() || Flags.isGoRet()) { if (!ArgMemTy) ArgMemTy = Arg.getPointeeInMemoryValueType(); @@ -12352,6 +12370,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL); if (Flags.isByRef()) Flags.setByRefSize(MemSize); + else if (Flags.isGoRet()) + Flags.setGoRetSize(MemSize); else Flags.setByValSize(MemSize); } else if (auto ParamAlign = Arg.getParamStackAlign()) { @@ -12477,8 +12497,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { ArrayRef(&InVals[i], NumParts), *TLI, ArgHasUses); } - // If this argument is unused then remember its value. It is used to generate - // debugging information. + // If this argument is unused then remember its value. It is used to + // generate debugging information. bool isSwiftErrorArg = TLI->supportSwiftError() && Arg.hasAttribute(Attribute::SwiftError); @@ -12886,7 +12906,8 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } else { Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); CurMF->insert(BBI, Fallthrough); - // Put Cond in a virtual register to make it available from the new blocks. + // Put Cond in a virtual register to make it available from the new + // blocks. ExportFromCurrentBlock(Cond); } UnhandledProbs -= I->Prob; diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 259fa480ec1e5..fbd3dbcda1b6f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallBitVector.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -78,7 +79,45 @@ static cl::opt MaxRegistersForGCPointers( typedef FunctionLoweringInfo::StatepointRelocationRecord RecordType; -static std::optional> +struct ArgumentValueOffset { + const Argument *Arg; + uint64_t Offset; +}; + +/// Return true when the pointer carrier for an incoming memory argument is +/// only used to read that argument. In that case the loaded SSA values remain +/// available in their fixed incoming homes across a Go stack-growing call. +/// +/// Preallocated itself does not imply immutability. Reject stores, captures, +/// and all pointer transformations other than constant-addressing operations; +/// values from a mutable or escaped home must use an ordinary relocation slot. +static bool isReadOnlyArgumentHome(const Argument *Arg) { + SmallVector Worklist(1, Arg); + SmallPtrSet Seen; + while (!Worklist.empty()) { + const Value *Pointer = Worklist.pop_back_val(); + if (!Seen.insert(Pointer).second) + continue; + + for (const User *U : Pointer->users()) { + if (const auto *Load = dyn_cast(U)) { + if (Load->getPointerOperand() != Pointer || Load->isVolatile() || + Load->isAtomic()) + return false; + continue; + } + if (isa(U) || isa(U) || + isa(U)) { + Worklist.push_back(cast(U)); + continue; + } + return false; + } + } + return true; +} + +static std::optional getArgumentValueOffset(const Value *V, const DataLayout &DL) { uint64_t Offset = 0; while (const auto *Extract = dyn_cast(V)) { @@ -99,10 +138,30 @@ getArgumentValueOffset(const Value *V, const DataLayout &DL) { V = Extract->getAggregateOperand(); } + // A first-class parameter exposes its pointer pieces directly. A typed + // preallocated parameter instead exposes the same logical value through a + // load from its incoming home. Recover that carrier and any constant byte + // offset so both IR forms can use the same fixed-home optimization. + if (const auto *Load = dyn_cast(V)) { + if (Load->isVolatile() || Load->isAtomic()) + return std::nullopt; + const Value *Pointer = Load->getPointerOperand(); + APInt PointerOffset(DL.getIndexTypeSizeInBits(Pointer->getType()), 0); + V = Pointer->stripAndAccumulateConstantOffsets(DL, PointerOffset, + /*AllowNonInbounds=*/true); + if (PointerOffset.isNegative() || PointerOffset.getActiveBits() > 64) + return std::nullopt; + auto NewOffset = checkedAddUnsigned(Offset, PointerOffset.getZExtValue()); + if (!NewOffset) + return std::nullopt; + Offset = *NewOffset; + } + const auto *Arg = dyn_cast(V); - if (!Arg) + if (!Arg || ((Arg->hasPreallocatedAttr() || Arg->hasGoRetAttr()) && + !isReadOnlyArgumentHome(Arg))) return std::nullopt; - return std::pair(Arg, Offset); + return ArgumentValueOffset{Arg, Offset}; } static void pushStackMapConstant(SmallVectorImpl& Ops, @@ -358,9 +417,53 @@ static void reservePreviousStackSlotForValue(const Value *IncomingValue, /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result static SDNode *peelCallResultChain(SDNode *Node) { - while (Node->getOpcode() == ISD::LOAD || - Node->getOpcode() == ISD::CopyFromReg) - Node = Node->getOperand(0).getNode(); + while (true) { + if (Node->getOpcode() == ISD::CALLSEQ_START || + Node->getOpcode() == ISD::CALLSEQ_END) + break; + // Target call nodes carry the preserved-register mask. They may also have + // explicit memory operands and no glue result, so recognize this semantic + // boundary before considering target memory-copy nodes below. + if (llvm::any_of(Node->ops(), [](SDValue Operand) { + return isa(Operand.getNode()); + })) + break; + if (Node->getOpcode() == ISD::CopyFromReg) { + Node = Node->getOperand(0).getNode(); + continue; + } + if (auto *Mem = dyn_cast(Node)) { + Node = Mem->getChain().getNode(); + continue; + } + + // A target may represent a large memory-result copy with a custom node + // rather than a MemSDNode. Peel a single chain input/result after the + // register-mask check above has established that this is not the call. + unsigned ChainResults = 0; + for (unsigned I = 0; I != Node->getNumValues(); ++I) { + ChainResults += Node->getValueType(I) == MVT::Other; + } + if (Node->getOpcode() != ISD::TokenFactor) { + SDNode *ChainInput = nullptr; + unsigned ChainInputs = 0; + for (SDValue Operand : Node->ops()) { + if (Operand.getValueType() != MVT::Other) + continue; + // Target nodes may use an MVT::Other VTSDNode as an immediate type + // descriptor (for example X86ISD::REP_MOVS). It is not a chain edge. + if (isa(Operand.getNode())) + continue; + ChainInput = Operand.getNode(); + ++ChainInputs; + } + if (ChainInputs == 1 && ChainResults == 1) { + Node = ChainInput; + continue; + } + } + break; + } if (Node->getOpcode() != ISD::TokenFactor) return Node; @@ -368,8 +471,7 @@ static SDNode *peelCallResultChain(SDNode *Node) { SDNode *CommonCallEnd = nullptr; for (SDValue Operand : Node->ops()) { SDNode *CallEnd = peelCallResultChain(Operand.getNode()); - if (CallEnd->getOpcode() != ISD::CALLSEQ_END || - (CommonCallEnd && CommonCallEnd != CallEnd)) + if (CommonCallEnd && CommonCallEnd != CallEnd) return Node; CommonCallEnd = CallEnd; } @@ -646,8 +748,31 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, if (auto ArgValue = getArgumentValueOffset(V, Builder.DAG.getDataLayout())) { uint64_t Size = PtrSD.getValueType().getStoreSize().getKnownMinValue(); - int FI = Builder.FuncInfo.getArgumentValueHome(ArgValue->first, - ArgValue->second, Size); + int FI = Builder.FuncInfo.getArgumentValueHome(ArgValue->Arg, + ArgValue->Offset, Size); + if (FI == INT_MAX && ArgValue->Arg->hasPreallocatedAttr()) { + int HomeFI = Builder.FuncInfo.getArgumentFrameIndex(ArgValue->Arg); + MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo(); + if (HomeFI != INT_MAX && MFI.isFixedObjectIndex(HomeFI) && + ArgValue->Offset <= uint64_t(MFI.getObjectSize(HomeFI)) && + Size <= uint64_t(MFI.getObjectSize(HomeFI)) - ArgValue->Offset) { + FI = HomeFI; + if (ArgValue->Offset != 0 || + MFI.getObjectSize(HomeFI) != int64_t(Size)) { + // Stack maps name one exact frame object. Make a narrow aliasing + // view for this pointer field while retaining the complete typed + // preallocated object for ordinary parameter accesses. + int64_t ViewOffset = MFI.getObjectOffset(HomeFI) + ArgValue->Offset; + MFI.setIsImmutableObjectIndex(HomeFI, false); + MFI.setIsAliasedObjectIndex(HomeFI, true); + FI = MFI.CreateFixedObject(Size, ViewOffset, + /*IsImmutable=*/false, + /*IsAliased=*/true); + } + Builder.FuncInfo.addArgumentValueHome(ArgValue->Arg, ArgValue->Offset, + Size, FI); + } + } if (FI != INT_MAX) { MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo(); MFI.setIsImmutableObjectIndex(FI, false); @@ -838,6 +963,17 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Glue is always last operand Glue = CallNode->getOperand(CallNode->getNumOperands() - 1); } + unsigned FixedCallOperands = CallHasIncomingGlue ? 4 : 3; + if (CallNode->getNumOperands() < FixedCallOperands) + report_fatal_error( + Twine("statepoint call chain did not resolve to a call in ") + + DAG.getMachineFunction().getName()); + unsigned RegMaskOperand = + CallNode->getNumOperands() - (CallHasIncomingGlue ? 2 : 1); + if (!isa(CallNode->getOperand(RegMaskOperand).getNode())) + report_fatal_error( + Twine("statepoint call chain did not resolve to a call in ") + + DAG.getMachineFunction().getName()); // Build the GC_TRANSITION_START node if necessary. // @@ -888,7 +1024,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Calculate and push starting position of vmstate arguments // Get number of arguments incoming directly into call node unsigned NumCallRegArgs = - CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3); + CallNode->getNumOperands() - FixedCallOperands; Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32)); // Add call target @@ -1008,7 +1144,13 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( } } else if (goabi::isGoCallingConv( DAG.getMachineFunction().getFunction().getCallingConv()) && - isa(V) && isa(SDV)) { + isa(SDV) && + (isa(V) || + (isa(V) && + (cast(V)->hasPreallocatedAttr() || + cast(V)->hasGoRetAttr()) && + FuncInfo.getArgumentFrameIndex(cast(V)) == + cast(SDV)->getIndex()))) { Record.type = RecordType::FrameIndexRemat; Record.payload.FI = cast(SDV)->getIndex(); } else if (Loc.getNode()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 5d93472a9f410..01fe54938a9a4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -17,6 +17,7 @@ #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/CodeGenCommonISel.h" +#include "llvm/CodeGen/GoCallingConv.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" @@ -126,6 +127,8 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet); IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest); IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal); + IsByRef = Call->paramHasAttr(ArgIdx, Attribute::ByRef); + IsGoRet = Call->paramHasAttr(ArgIdx, Attribute::GoRet); IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated); IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); @@ -134,13 +137,25 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); Alignment = Call->getParamStackAlign(ArgIdx); IndirectType = nullptr; - assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 && + assert(IsByVal + IsByRef + IsGoRet + IsPreallocated + IsInAlloca + IsSRet <= + 1 && "multiple ABI attributes?"); if (IsByVal) { IndirectType = Call->getParamByValType(ArgIdx); if (!Alignment) Alignment = Call->getParamAlign(ArgIdx); } + if (IsByRef) { + IndirectType = Call->getParamByRefType(ArgIdx); + if (!Alignment) + Alignment = Call->getParamAlign(ArgIdx); + } + if (IsGoRet) { + IndirectType = Call->getParamGoRetType(ArgIdx); + GoRetIndex = goabi::getGoRetIndex(*Call, ArgIdx); + if (!Alignment) + Alignment = Call->getParamAlign(ArgIdx); + } if (IsPreallocated) IndirectType = Call->getParamPreallocatedType(ArgIdx); if (IsInAlloca) @@ -1061,11 +1076,12 @@ static SDValue combineShiftToAVG(SDValue Op, } return false; }; - bool IsCeil = - (ExtOpA.getOpcode() == ISD::ADD && - MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) || - (ExtOpB.getOpcode() == ISD::ADD && - MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB)); + bool IsCeil = (ExtOpA.getOpcode() == ISD::ADD && + MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), + ExtOpB, ExtOpA)) || + (ExtOpB.getOpcode() == ISD::ADD && + MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), + ExtOpA, ExtOpB)); // If the shift is signed (sra): // - Needs >= 2 sign bit for both operands. @@ -2339,12 +2355,14 @@ bool TargetLowering::SimplifyDemandedBits( // See if we don't demand either half of the rotated bits. if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) { - Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType()); + Op1 = + TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType()); return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); } if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) { - Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType()); + Op1 = + TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType()); return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); } } @@ -2458,8 +2476,8 @@ bool TargetLowering::SimplifyDemandedBits( // op legalization. // FIXME: Limit to scalars for now. if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector()) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, - Op.getOperand(0))); + return TLO.CombineTo( + Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, Op.getOperand(0))); Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); break; @@ -2612,7 +2630,8 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); unsigned InBits = SrcVT.getScalarSizeInBits(); - unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; + unsigned InElts = + SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; APInt InDemandedElts = DemandedElts.zext(InElts); @@ -2675,7 +2694,8 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); unsigned InBits = SrcVT.getScalarSizeInBits(); - unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; + unsigned InElts = + SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; // If we only need the bottom element then we can just bitcast. @@ -4104,10 +4124,10 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( return SDValue(); } -SDValue -TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, - SDValue N1, MutableArrayRef Mask, - SelectionDAG &DAG) const { +SDValue TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, + SDValue N0, SDValue N1, + MutableArrayRef Mask, + SelectionDAG &DAG) const { bool LegalMask = isShuffleMaskLegal(Mask, VT); if (!LegalMask) { std::swap(N0, N1); @@ -4615,7 +4635,8 @@ static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, // FIXME: Add vector support? Need to be careful with setcc result type below. SDValue CTPOP = N0; if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && - N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits())) + N0.getScalarValueSizeInBits() > + Log2_32(N0.getOperand(0).getScalarValueSizeInBits())) CTPOP = N0.getOperand(0); if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse()) @@ -4911,8 +4932,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // (zext x) == C --> x == (trunc C) // (sext x) == C --> x == (trunc C) - if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && - DCI.isBeforeLegalize() && N0->hasOneUse()) { + if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && DCI.isBeforeLegalize() && + N0->hasOneUse()) { unsigned MinBits = N0.getValueSizeInBits(); SDValue PreExt; bool Signed = false; @@ -4948,9 +4969,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits(); // Make sure we're not losing bits from the constant. - if (MinBits > 0 && - MinBits < C1.getBitWidth() && - MinBits >= ReqdBits) { + if (MinBits > 0 && MinBits < C1.getBitWidth() && MinBits >= ReqdBits) { EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits); if (isTypeDesirableForOp(ISD::SETCC, MinVT)) { // Will get folded away. @@ -4990,8 +5009,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, cast(TopSetCC.getOperand(2))->get(), TopSetCC.getOperand(0).getValueType()); return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), - TopSetCC.getOperand(1), - InvCond); + TopSetCC.getOperand(1), InvCond); } } } @@ -4999,10 +5017,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // If the LHS is '(and load, const)', the RHS is 0, the test is for // equality or unsigned, and all 1 bits of the const are in the same // partial word, see if we can shorten the load. - if (DCI.isBeforeLegalize() && - !ISD::isSignedIntSetCC(Cond) && - N0.getOpcode() == ISD::AND && C1 == 0 && - N0.getNode()->hasOneUse() && + if (DCI.isBeforeLegalize() && !ISD::isSignedIntSetCC(Cond) && + N0.getOpcode() == ISD::AND && C1 == 0 && N0.getNode()->hasOneUse() && isa(N0.getOperand(0)) && N0.getOperand(0).getNode()->hasOneUse() && isa(N0.getOperand(1))) { @@ -5116,8 +5132,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT); SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT); - SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0), - NewConst, Cond); + SDValue NewSetCC = + DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0), NewConst, Cond); return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); } break; @@ -5180,9 +5196,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We // can only do this if the top bits are known zero. unsigned BitWidth = N0.getValueSizeInBits(); - if (DAG.MaskedValueIsZero(N0, - APInt::getHighBitsSet(BitWidth, - BitWidth-1))) { + if (DAG.MaskedValueIsZero( + N0, APInt::getHighBitsSet(BitWidth, BitWidth - 1))) { // Okay, get the un-inverted input value. SDValue Val; if (N0.getOpcode() == ISD::XOR) { @@ -5192,8 +5207,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, N0.getOperand(0).getOpcode() == ISD::XOR); // ((X^1)&1)^1 -> X & 1 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), - N0.getOperand(0).getOperand(0), - N0.getOperand(1)); + N0.getOperand(0).getOperand(0), N0.getOperand(1)); } return DAG.getSetCC(dl, VT, Val, N1, @@ -5223,13 +5237,15 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) { // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0. if (Op0.getValueType().bitsGT(VT)) - Op0 = DAG.getNode(ISD::AND, dl, VT, - DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), - DAG.getConstant(1, dl, VT)); + Op0 = DAG.getNode( + ISD::AND, dl, VT, + DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), + DAG.getConstant(1, dl, VT)); else if (Op0.getValueType().bitsLT(VT)) - Op0 = DAG.getNode(ISD::AND, dl, VT, - DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), - DAG.getConstant(1, dl, VT)); + Op0 = DAG.getNode( + ISD::AND, dl, VT, + DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), + DAG.getConstant(1, dl, VT)); return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, dl, Op0.getValueType()), @@ -5316,8 +5332,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, (!N1C->isOpaque() || (C.getBitWidth() <= 64 && isLegalICmpImmediate(C.getSExtValue())))) { return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(C, dl, N1.getValueType()), - NewCC); + DAG.getConstant(C, dl, N1.getValueType()), NewCC); } } } @@ -5336,8 +5351,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, (!N1C->isOpaque() || (C.getBitWidth() <= 64 && isLegalICmpImmediate(C.getSExtValue())))) { return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(C, dl, N1.getValueType()), - NewCC); + DAG.getConstant(C, dl, N1.getValueType()), NewCC); } } } @@ -5455,9 +5469,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // SETUGE X, SINTMIN -> SETLT X, 0 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) || (Cond == ISD::SETUGE && C1.isMinSignedValue())) - return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(0, dl, N1.getValueType()), - ISD::SETLT); + return DAG.getSetCC( + dl, VT, N0, DAG.getConstant(0, dl, N1.getValueType()), ISD::SETLT); // SETULT X, SINTMIN -> SETGT X, -1 // SETULE X, SINTMAX -> SETGT X, -1 @@ -5619,11 +5632,20 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, bool IsNegInf = CFP->getValueAPF().isNegative(); ISD::CondCode NewCond = ISD::SETCC_INVALID; switch (Cond) { - case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break; - case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break; - case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break; - case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break; - default: break; + case ISD::SETOEQ: + NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; + break; + case ISD::SETUEQ: + NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; + break; + case ISD::SETUNE: + NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; + break; + case ISD::SETONE: + NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; + break; + default: + break; } if (NewCond != ISD::SETCC_INVALID && isCondCodeLegal(NewCond, N0.getSimpleValueType())) @@ -5647,8 +5669,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO // if it is not already. ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; - if (NewCond != Cond && - (DCI.isBeforeLegalizeOps() || + if (NewCond != Cond && (DCI.isBeforeLegalizeOps() || isCondCodeLegal(NewCond, N0.getSimpleValueType()))) return DAG.getSetCC(dl, VT, N0, N1, NewCond); } @@ -5770,7 +5791,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) { SDValue Temp; switch (Cond) { - default: llvm_unreachable("Unknown integer setcc!"); + default: + llvm_unreachable("Unknown integer setcc!"); case ISD::SETEQ: // X == Y -> ~(X^Y) Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); N0 = DAG.getNOT(dl, Temp, OpVT); @@ -5887,7 +5909,8 @@ TargetLowering::getConstraintType(StringRef Constraint) const { if (S == 1) { switch (Constraint[0]) { - default: break; + default: + break; case 'r': return C_RegisterClass; case 'm': // memory @@ -5954,7 +5977,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, char ConstraintLetter = Constraint[0]; switch (ConstraintLetter) { - default: break; + default: + break; case 'X': // Allows any operand case 'i': // Simple Integer or Relocatable Constant case 'n': // Simple Integer @@ -6164,7 +6188,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, if (!OpTy->isSingleValueType() && OpTy->isSized()) { unsigned BitSize = DL.getTypeSizeInBits(OpTy); switch (BitSize) { - default: break; + default: + break; case 1: case 8: case 16: @@ -6302,8 +6327,8 @@ static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) { /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight - TargetLowering::getMultipleConstraintMatchWeight( - AsmOperandInfo &info, int maIndex) const { +TargetLowering::getMultipleConstraintMatchWeight(AsmOperandInfo &info, + int maIndex) const { InlineAsm::ConstraintCodeVector *rCodes; if (maIndex >= (int)info.multipleAlternatives.size()) rCodes = &info.Codes; @@ -6326,8 +6351,8 @@ TargetLowering::ConstraintWeight /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight - TargetLowering::getSingleConstraintMatchWeight( - AsmOperandInfo &info, const char *constraint) const { +TargetLowering::getSingleConstraintMatchWeight(AsmOperandInfo &info, + const char *constraint) const { ConstraintWeight weight = CW_Invalid; Value *CallOperandVal = info.CallOperandVal; // If we don't have a value, we can't do a match, @@ -6444,8 +6469,7 @@ static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, /// Determines the constraint code and constraint type to use for the specific /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType. -void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, - SDValue Op, +void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG) const { assert(!OpInfo.Codes.empty() && "Must have at least one constraint"); @@ -6623,7 +6647,8 @@ static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N, return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); } -SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, +SDValue +TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl &Created) const { AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes(); @@ -9944,8 +9969,7 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, } bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, - SDValue &Chain, - SelectionDAG &DAG) const { + SDValue &Chain, SelectionDAG &DAG) const { SDLoc dl(SDValue(Node, 0)); unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; SDValue Src = Node->getOperand(OpNo); @@ -9958,8 +9982,8 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT); // Only expand vector types if we have the appropriate vector bit operations. - unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : - ISD::FP_TO_SINT; + unsigned SIntOpcode = + Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) || !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT))) return false; @@ -9990,8 +10014,8 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, SDValue Sel; if (Node->isStrictFPOpcode()) { - Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, - Node->getOperand(0), /*IsSignaling*/ true); + Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, Node->getOperand(0), + /*IsSignaling*/ true); Chain = Sel.getValue(1); } else { Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); @@ -10009,12 +10033,12 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, // Result = fp_to_sint(Src - FltOfs) ^ IntOfs // TODO: Should any fast-math-flags be set for the FSUB? - SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel, - DAG.getConstantFP(0.0, dl, SrcVT), Cst); + SDValue FltOfs = + DAG.getSelect(dl, SrcVT, Sel, DAG.getConstantFP(0.0, dl, SrcVT), Cst); Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); - SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel, - DAG.getConstant(0, dl, DstVT), - DAG.getConstant(SignMask, dl, DstVT)); + SDValue IntOfs = + DAG.getSelect(dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT), + DAG.getConstant(SignMask, dl, DstVT)); SDValue SInt; if (Node->isStrictFPOpcode()) { SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, @@ -10150,12 +10174,12 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node, // Insert canonicalizes if it's possible we need to quiet to get correct // sNaN behavior. if (!DAG.isKnownNeverSNaN(Quiet0)) { - Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, - Node->getFlags()); + Quiet0 = + DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, Node->getFlags()); } if (!DAG.isKnownNeverSNaN(Quiet1)) { - Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, - Node->getFlags()); + Quiet1 = + DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, Node->getFlags()); } } @@ -11013,12 +11037,11 @@ SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG, if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON) return ExtLoad; - EVT SetCCVT = - getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); + EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); SDValue Zero = DAG.getConstant(0, DL, VT); SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ); - return DAG.getSelect(DL, VT, SrcIsZero, - DAG.getConstant(BitWidth, DL, VT), ExtLoad); + return DAG.getSelect(DL, VT, SrcIsZero, DAG.getConstant(BitWidth, DL, VT), + ExtLoad); } SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const { @@ -11542,8 +11565,7 @@ SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { return DAG.getNode(ISD::XOR, dl, VT, Tmp, Rotr8); } Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); - Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, - DAG.getConstant(0xFF00, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT)); Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); @@ -11553,8 +11575,8 @@ SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); case MVT::i64: Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); - Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, - DAG.getConstant(255ULL<<8, dl, VT)); + Tmp7 = + DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(255ULL << 8, dl, VT)); Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(255ULL<<16, dl, VT)); @@ -11782,8 +11804,7 @@ SDValue TargetLowering::expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const { } std::pair -TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, - SelectionDAG &DAG) const { +TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const { SDLoc SL(LD); SDValue Chain = LD->getChain(); SDValue BasePTR = LD->getBasePtr(); @@ -11959,20 +11980,19 @@ TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const { if (VT.isFloatingPoint() || VT.isVector()) { EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) { - if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && - LoadedVT.isVector()) { + if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && LoadedVT.isVector()) { // Scalarize the load and let the individual components be handled. return scalarizeVectorLoad(LD, DAG); } // Expand to a (misaligned) integer load of the same size, // then bitconvert to floating point or vector. - SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, - LD->getMemOperand()); + SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getMemOperand()); SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); if (LoadedVT != VT) - Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : - ISD::ANY_EXTEND, dl, VT, Result); + Result = + DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND, + dl, VT, Result); return std::make_pair(Result, newLoad.getValue(1)); } @@ -12225,11 +12245,10 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, return Result; } -SDValue -TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, - const SDLoc &DL, EVT DataVT, - SelectionDAG &DAG, - bool IsCompressedMemory) const { +SDValue TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, + const SDLoc &DL, EVT DataVT, + SelectionDAG &DAG, + bool IsCompressedMemory) const { SDValue Increment; EVT AddrVT = Addr.getValueType(); EVT MaskVT = Mask.getValueType(); @@ -12253,8 +12272,8 @@ TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg); } // Scale is an element size in bytes. - SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL, - AddrVT); + SDValue Scale = + DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL, AddrVT); Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT); Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale); } else @@ -12551,7 +12570,8 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { unsigned BitWidth = LHS.getScalarValueSizeInBits(); EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); - SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); + SDValue Result = + DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); SDValue SumDiff = Result.getValue(0); SDValue Overflow = Result.getValue(1); SDValue Zero = DAG.getConstant(0, dl, VT); @@ -12948,8 +12968,8 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { // Saturated to SatMin if wide product is negative, and SatMax if wide // product is positive ... SDValue Zero = DAG.getConstant(0, dl, VT); - SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax, - ISD::SETLT); + SDValue ResultIfOverflow = + DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax, ISD::SETLT); // ... but only if we overflowed. return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); } @@ -12958,22 +12978,21 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { // Saturate to max if ((Hi >> (Scale - 1)) > 0), // which is the same as if (Hi > (1 << (Scale - 1)) - 1) - SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), - dl, VT); + SDValue LowMask = + DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), dl, VT); Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); // Saturate to min if (Hi >> (Scale - 1)) < -1), // which is the same as if (HI < (-1 << (Scale - 1)) - SDValue HighMask = - DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), - dl, VT); + SDValue HighMask = DAG.getConstant( + APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), dl, VT); Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); return Result; } -SDValue -TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, - SDValue LHS, SDValue RHS, - unsigned Scale, SelectionDAG &DAG) const { +SDValue TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, + SDValue LHS, SDValue RHS, + unsigned Scale, + SelectionDAG &DAG) const { assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && "Expected a fixed point division opcode"); @@ -13211,8 +13230,8 @@ bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result, SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); } else { - Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, - DAG.getConstant(0, dl, VT), ISD::SETNE); + Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, DAG.getConstant(0, dl, VT), + ISD::SETNE); } // Truncate the result if SetCC returns a larger type than needed. @@ -13307,7 +13326,8 @@ SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const { return Res; } -SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const { +SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, + SelectionDAG &DAG) const { SDLoc dl(Node); SDValue AccOp = Node->getOperand(0); SDValue VecOp = Node->getOperand(1); diff --git a/llvm/lib/FuzzMutate/IRMutator.cpp b/llvm/lib/FuzzMutate/IRMutator.cpp index b56f3704d4305..5dd216dc5c845 100644 --- a/llvm/lib/FuzzMutate/IRMutator.cpp +++ b/llvm/lib/FuzzMutate/IRMutator.cpp @@ -415,13 +415,19 @@ static bool isUnsupportedFunction(Function *F) { // behavior may be undefined. // We don't call those functions for now to prevent UB from happening. auto IsABIAttribute = [](AttributeSet A) { - static const Attribute::AttrKind ABIAttrs[] = { - Attribute::StructRet, Attribute::ByVal, - Attribute::InAlloca, Attribute::InReg, - Attribute::StackAlignment, Attribute::SwiftSelf, - Attribute::SwiftAsync, Attribute::SwiftError, - Attribute::Preallocated, Attribute::ByRef, - Attribute::ZExt, Attribute::SExt}; + static const Attribute::AttrKind ABIAttrs[] = {Attribute::StructRet, + Attribute::ByVal, + Attribute::InAlloca, + Attribute::InReg, + Attribute::StackAlignment, + Attribute::SwiftSelf, + Attribute::SwiftAsync, + Attribute::SwiftError, + Attribute::Preallocated, + Attribute::ByRef, + Attribute::GoRet, + Attribute::ZExt, + Attribute::SExt}; return llvm::any_of(ABIAttrs, [&](Attribute::AttrKind kind) { return A.hasAttribute(kind); diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 4087b25951a1c..2347125a62576 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -265,6 +265,10 @@ Attribute Attribute::getWithByRefType(LLVMContext &Context, Type *Ty) { return get(Context, ByRef, Ty); } +Attribute Attribute::getWithGoRetType(LLVMContext &Context, Type *Ty) { + return get(Context, GoRet, Ty); +} + Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) { return get(Context, Preallocated, Ty); } @@ -1216,6 +1220,10 @@ Type *AttributeSet::getByRefType() const { return SetNode ? SetNode->getAttributeType(Attribute::ByRef) : nullptr; } +Type *AttributeSet::getGoRetType() const { + return SetNode ? SetNode->getAttributeType(Attribute::GoRet) : nullptr; +} + Type *AttributeSet::getByValType() const { return SetNode ? SetNode->getAttributeType(Attribute::ByVal) : nullptr; } @@ -1489,9 +1497,7 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const { /// Map from AttributeList index to the internal array index. Adding one happens /// to work, because -1 wraps around to 0. -static unsigned attrIdxToArrayIdx(unsigned Index) { - return Index + 1; -} +static unsigned attrIdxToArrayIdx(unsigned Index) { return Index + 1; } AttributeListImpl::AttributeListImpl(ArrayRef Sets) : NumAttrSets(Sets.size()) { @@ -1539,7 +1545,6 @@ bool AttributeListImpl::hasAttrSomewhere(Attribute::AttrKind Kind, return true; } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void AttributeListImpl::dump() const { AttributeList(const_cast(this)).dump(); @@ -1596,7 +1601,8 @@ AttributeList::get(LLVMContext &C, // list. SmallVector, 8> AttrPairVec; for (ArrayRef>::iterator I = Attrs.begin(), - E = Attrs.end(); I != E; ) { + E = Attrs.end(); + I != E;) { unsigned Index = I->first; SmallVector AttrVec; while (I != E && I->first == Index) { @@ -2003,6 +2009,10 @@ Type *AttributeList::getParamByRefType(unsigned Index) const { return getAttributes(Index + FirstArgIndex).getByRefType(); } +Type *AttributeList::getParamGoRetType(unsigned Index) const { + return getAttributes(Index + FirstArgIndex).getGoRetType(); +} + Type *AttributeList::getParamPreallocatedType(unsigned Index) const { return getAttributes(Index + FirstArgIndex).getPreallocatedType(); } @@ -2256,7 +2266,8 @@ AttrBuilder &AttrBuilder::addStackAlignmentAttr(MaybeAlign Align) { } AttrBuilder &AttrBuilder::addDereferenceableAttr(uint64_t Bytes) { - if (Bytes == 0) return *this; + if (Bytes == 0) + return *this; return addRawIntAttr(Attribute::Dereferenceable, Bytes); } @@ -2680,8 +2691,8 @@ static void adjustCallerStackProbes(Function &Caller, const Function &Callee) { /// If the inlined function defines the size of guard region /// on the stack, then ensure that the calling function defines a guard region /// that is no larger. -static void -adjustCallerStackProbeSize(Function &Caller, const Function &Callee) { +static void adjustCallerStackProbeSize(Function &Caller, + const Function &Callee) { Attribute CalleeAttr = Callee.getFnAttribute("stack-probe-size"); if (CalleeAttr.isValid()) { Attribute CallerAttr = Caller.getFnAttribute("stack-probe-size"); @@ -2708,8 +2719,8 @@ adjustCallerStackProbeSize(Function &Caller, const Function &Callee) { /// to merge the attribute this way. Heuristics that would use /// min-legal-vector-width to determine inline compatibility would need to be /// handled as part of inline cost analysis. -static void -adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) { +static void adjustMinLegalVectorWidth(Function &Caller, + const Function &Callee) { Attribute CallerAttr = Caller.getFnAttribute("min-legal-vector-width"); if (CallerAttr.isValid()) { Attribute CalleeAttr = Callee.getFnAttribute("min-legal-vector-width"); @@ -2729,21 +2740,19 @@ adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) { /// If the inlined function has null_pointer_is_valid attribute, /// set this attribute in the caller post inlining. -static void -adjustNullPointerValidAttr(Function &Caller, const Function &Callee) { +static void adjustNullPointerValidAttr(Function &Caller, + const Function &Callee) { if (Callee.nullPointerIsDefined() && !Caller.nullPointerIsDefined()) { Caller.addFnAttr(Attribute::NullPointerIsValid); } } struct EnumAttr { - static bool isSet(const Function &Fn, - Attribute::AttrKind Kind) { + static bool isSet(const Function &Fn, Attribute::AttrKind Kind) { return Fn.hasFnAttribute(Kind); } - static void set(Function &Fn, - Attribute::AttrKind Kind, bool Val) { + static void set(Function &Fn, Attribute::AttrKind Kind, bool Val) { if (Val) Fn.addFnAttr(Kind); else @@ -2752,14 +2761,12 @@ struct EnumAttr { }; struct StrBoolAttr { - static bool isSet(const Function &Fn, - StringRef Kind) { + static bool isSet(const Function &Fn, StringRef Kind) { auto A = Fn.getFnAttribute(Kind); return A.getValueAsString() == "true"; } - static void set(Function &Fn, - StringRef Kind, bool Val) { + static void set(Function &Fn, StringRef Kind, bool Val) { Fn.addFnAttr(Kind, Val ? "true" : "false"); } }; diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 567949d194726..7eefb7d08f725 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -140,6 +140,12 @@ bool Argument::hasByRefAttr() const { return hasAttribute(Attribute::ByRef); } +bool Argument::hasGoRetAttr() const { + if (!getType()->isPointerTy()) + return false; + return hasAttribute(Attribute::GoRet); +} + bool Argument::hasSwiftSelfAttr() const { return getParent()->hasParamAttribute(getArgNo(), Attribute::SwiftSelf); } @@ -175,11 +181,11 @@ bool Argument::hasPointeeInMemoryValueAttr() const { Attrs.hasAttribute(Attribute::StructRet) || Attrs.hasAttribute(Attribute::InAlloca) || Attrs.hasAttribute(Attribute::Preallocated) || - Attrs.hasAttribute(Attribute::ByRef); + Attrs.hasAttribute(Attribute::ByRef) || + Attrs.hasAttribute(Attribute::GoRet); } -/// For a byval, sret, inalloca, or preallocated parameter, get the in-memory -/// parameter type. +/// For a parameter with an in-memory ABI type, get that type. static Type *getMemoryParamAllocType(AttributeSet ParamAttrs) { // FIXME: All the type carrying attributes are mutually exclusive, so there // should be a single query to get the stored type that handles any of them. @@ -187,6 +193,8 @@ static Type *getMemoryParamAllocType(AttributeSet ParamAttrs) { return ByValTy; if (Type *ByRefTy = ParamAttrs.getByRefType()) return ByRefTy; + if (Type *GoRetTy = ParamAttrs.getGoRetType()) + return GoRetTy; if (Type *PreAllocTy = ParamAttrs.getPreallocatedType()) return PreAllocTy; if (Type *InAllocaTy = ParamAttrs.getInAllocaType()) @@ -231,6 +239,11 @@ Type *Argument::getParamByRefType() const { return getParent()->getParamByRefType(getArgNo()); } +Type *Argument::getParamGoRetType() const { + assert(getType()->isPointerTy() && "Only pointers have goret types"); + return getParent()->getParamGoRetType(getArgNo()); +} + Type *Argument::getParamInAllocaType() const { assert(getType()->isPointerTy() && "Only pointers have inalloca types"); return getParent()->getParamInAllocaType(getArgNo()); diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index 6d0170dd23e75..a13e923bd4c11 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -843,7 +843,7 @@ bool Value::canBeFreed() const { if (isa(this)) return false; - // Handle byval/byref/sret/inalloca/preallocated arguments. The storage + // Handle byval/byref/goret/sret/inalloca/preallocated arguments. The storage // lifetime is guaranteed to be longer than the callee's lifetime. if (auto *A = dyn_cast(this)) { if (A->hasPointeeInMemoryValueAttr()) @@ -917,7 +917,7 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, if (const Argument *A = dyn_cast(this)) { DerefBytes = A->getDereferenceableBytes(); if (DerefBytes == 0) { - // Handle byval/byref/inalloca/preallocated arguments + // Handle byval/byref/goret/inalloca/preallocated arguments. if (Type *ArgMemTy = A->getPointeeInMemoryValueType()) { if (ArgMemTy->isSized()) { // FIXME: Why isn't this the type alloc size? diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index e35f0d5dcca4a..bf6d9aa565bfd 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -55,6 +55,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -459,6 +460,8 @@ class Verifier : public InstVisitor, VerifierSupport { bool verifyAttributeCount(AttributeList Attrs, unsigned Params); void verifyAttributeTypes(AttributeSet Attrs, const Value *V); void verifyParameterAttrs(AttributeSet Attrs, Type *Ty, const Value *V); + void verifyGoRetAttrs(FunctionType *FT, AttributeList Attrs, + CallingConv::ID CC, const Value *V); void checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr, const Value *V); void verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, @@ -548,7 +551,8 @@ void Verifier::visit(Instruction &I) { InstVisitor::visit(I); } -// Helper to iterate over indirect users. By returning false, the callback can ask to stop traversing further. +// Helper to iterate over indirect users. By returning false, the callback can +// ask to stop traversing further. static void forEachUser(const Value *User, SmallPtrSet &Visited, llvm::function_ref Callback) { @@ -764,8 +768,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { } } - if (GV.hasName() && (GV.getName() == "llvm.used" || - GV.getName() == "llvm.compiler.used")) { + if (GV.hasName() && + (GV.getName() == "llvm.used" || GV.getName() == "llvm.compiler.used")) { Check(!GV.hasInitializer() || GV.hasAppendingLinkage(), "invalid linkage for intrinsic global variable", &GV); Check(GV.materialized_use_empty(), @@ -2098,6 +2102,16 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, "'byref', and 'sret' are incompatible!", V); + if (Attrs.hasAttribute(Attribute::GoRet)) + Check(AttrCount == 0, + "Attribute 'goret' is incompatible with other ABI parameter " + "attributes!", + V); + + Check(Attrs.hasAttribute(Attribute::GoRet) == + Attrs.hasAttribute("goretindex"), + "Attributes 'goret' and 'goretindex' must be used together!", V); + Check(!(Attrs.hasAttribute(Attribute::InAlloca) && Attrs.hasAttribute(Attribute::ReadOnly)), "Attributes " @@ -2184,6 +2198,16 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, (1ULL << 32), "huge 'byref' arguments are unsupported", V); } + if (Attrs.hasAttribute(Attribute::GoRet)) { + Type *GoRetTy = Attrs.getAttribute(Attribute::GoRet).getValueAsType(); + SmallPtrSet Visited; + Check(GoRetTy->isSized(&Visited), + "Attribute 'goret' does not support unsized types!", V); + Check(!GoRetTy->containsNonLocalTargetExtType(), + "'goret' argument has illegal target extension type", V); + Check(DL.getTypeAllocSize(GoRetTy).getKnownMinValue() < (1ULL << 32), + "huge 'goret' arguments are unsupported", V); + } if (Attrs.hasAttribute(Attribute::InAlloca)) { SmallPtrSet Visited; Check(Attrs.getInAllocaType()->isSized(&Visited), @@ -2226,6 +2250,55 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, } } +void Verifier::verifyGoRetAttrs(FunctionType *FT, AttributeList Attrs, + CallingConv::ID CC, const Value *V) { + unsigned MemoryResultCount = 0; + for (unsigned I = 0; I != FT->getNumParams(); ++I) + MemoryResultCount += Attrs.hasParamAttr(I, Attribute::GoRet); + if (MemoryResultCount == 0) + return; + + Check(CC == CallingConv::GoABIInternal || CC == CallingConv::GoABI0, + "'goret' is only valid with a Go calling convention", V); + + unsigned DirectResultCount = 0; + if (!FT->getReturnType()->isVoidTy()) { + if (Attrs.hasFnAttr("go_results_tuple")) { + auto *ST = dyn_cast(FT->getReturnType()); + Check(ST, "'go_results_tuple' requires a struct return type", V); + if (!ST) + return; + DirectResultCount = ST->getNumElements(); + } else { + DirectResultCount = 1; + } + } + + unsigned TotalResultCount = DirectResultCount + MemoryResultCount; + SmallBitVector Seen(TotalResultCount); + std::optional Previous; + for (unsigned I = 0; I != FT->getNumParams(); ++I) { + AttributeSet ParamAttrs = Attrs.getParamAttrs(I); + if (!ParamAttrs.hasAttribute(Attribute::GoRet)) + continue; + StringRef IndexValue = + ParamAttrs.getAttribute("goretindex").getValueAsString(); + uint64_t Index; + Check(!IndexValue.getAsInteger(10, Index), + "'goretindex' must be an unsigned decimal integer", V); + if (IndexValue.getAsInteger(10, Index)) + continue; + Check(Index < TotalResultCount, "'goretindex' is out of range", V); + if (Index >= TotalResultCount) + continue; + Check(!Seen.test(Index), "duplicate 'goretindex'", V); + Check(!Previous || *Previous < Index, + "'goretindex' values must be in increasing parameter order", V); + Seen.set(Index); + Previous = Index; + } +} + void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr, const Value *V) { if (Attrs.hasFnAttr(Attr)) { @@ -3056,6 +3129,7 @@ void Verifier::visitFunction(const Function &F) { // Check function attributes. verifyFunctionAttrs(FT, Attrs, &F, IsIntrinsic, /* IsInlineAsm */ false); + verifyGoRetAttrs(FT, Attrs, F.getCallingConv(), &F); // On function declarations/definitions, we do not support the builtin // attribute. We do not check this in VerifyFunctionAttrs since that is @@ -3413,8 +3487,7 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { } // Check that all instructions have their parent pointers set up correctly. - for (auto &I : BB) - { + for (auto &I : BB) { Check(I.getParent() == &BB, "Instruction has bogus parent pointer!"); } @@ -3889,6 +3962,7 @@ void Verifier::visitCallBase(CallBase &Call) { // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic, Call.isInlineAsm()); + verifyGoRetAttrs(FTy, Attrs, Call.getCallingConv(), &Call); // Conservatively check the inalloca argument. // We have a bug if we can find that there is an underlying alloca without @@ -3906,7 +3980,8 @@ void Verifier::visitCallBase(CallBase &Call) { for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) { if (Call.paramHasAttr(i, Attribute::SwiftError)) { Value *SwiftErrorArg = Call.getArgOperand(i); - if (auto AI = dyn_cast(SwiftErrorArg->stripInBoundsOffsets())) { + if (auto AI = + dyn_cast(SwiftErrorArg->stripInBoundsOffsets())) { Check(AI->isSwiftError(), "swifterror argument for call has mismatched alloca", AI, Call); continue; @@ -4137,25 +4212,34 @@ void Verifier::verifyTailCCMustTailAttrs(const AttrBuilder &Attrs, Twine("preallocated attribute not allowed in ") + Context); Check(!Attrs.contains(Attribute::ByRef), Twine("byref attribute not allowed in ") + Context); + Check(!Attrs.contains(Attribute::GoRet), + Twine("goret attribute not allowed in ") + Context); + Check(!Attrs.contains("goretindex"), + Twine("goretindex attribute not allowed in ") + Context); } -static AttrBuilder getParameterABIAttributes(LLVMContext& C, unsigned I, AttributeList Attrs) { +static AttrBuilder getParameterABIAttributes(LLVMContext &C, unsigned I, + AttributeList Attrs) { static const Attribute::AttrKind ABIAttrs[] = { Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, Attribute::InReg, Attribute::StackAlignment, Attribute::SwiftSelf, Attribute::SwiftAsync, Attribute::SwiftError, Attribute::Preallocated, - Attribute::ByRef}; + Attribute::ByRef, Attribute::GoRet}; AttrBuilder Copy(C); for (auto AK : ABIAttrs) { Attribute Attr = Attrs.getParamAttrs(I).getAttribute(AK); if (Attr.isValid()) Copy.addAttribute(Attr); } + Attribute GoRetIndex = Attrs.getParamAttrs(I).getAttribute("goretindex"); + if (GoRetIndex.isValid()) + Copy.addAttribute(GoRetIndex); - // `align` is ABI-affecting only in combination with `byval` or `byref`. + // `align` is ABI-affecting only in combination with an indirect ABI value. if (Attrs.hasParamAttr(I, Attribute::Alignment) && (Attrs.hasParamAttr(I, Attribute::ByVal) || - Attrs.hasParamAttr(I, Attribute::ByRef))) + Attrs.hasParamAttr(I, Attribute::ByRef) || + Attrs.hasParamAttr(I, Attribute::GoRet))) Copy.addAlignmentAttr(Attrs.getParamAlignment(I)); return Copy; } @@ -4196,12 +4280,14 @@ void Verifier::verifyMustTailCall(CallInst &CI) { // - Only sret, byval, swiftself, and swiftasync ABI-impacting attributes // are allowed in swifttailcc call for (unsigned I = 0, E = CallerTy->getNumParams(); I != E; ++I) { - AttrBuilder ABIAttrs = getParameterABIAttributes(F->getContext(), I, CallerAttrs); + AttrBuilder ABIAttrs = + getParameterABIAttributes(F->getContext(), I, CallerAttrs); SmallString<32> Context{CCName, StringRef(" musttail caller")}; verifyTailCCMustTailAttrs(ABIAttrs, Context); } for (unsigned I = 0, E = CalleeTy->getNumParams(); I != E; ++I) { - AttrBuilder ABIAttrs = getParameterABIAttributes(F->getContext(), I, CalleeAttrs); + AttrBuilder ABIAttrs = + getParameterABIAttributes(F->getContext(), I, CalleeAttrs); SmallString<32> Context{CCName, StringRef(" musttail callee")}; verifyTailCCMustTailAttrs(ABIAttrs, Context); } @@ -4225,8 +4311,10 @@ void Verifier::verifyMustTailCall(CallInst &CI) { // - All ABI-impacting function attributes, such as sret, byval, inreg, // returned, preallocated, and inalloca, must match. for (unsigned I = 0, E = CallerTy->getNumParams(); I != E; ++I) { - AttrBuilder CallerABIAttrs = getParameterABIAttributes(F->getContext(), I, CallerAttrs); - AttrBuilder CalleeABIAttrs = getParameterABIAttributes(F->getContext(), I, CalleeAttrs); + AttrBuilder CallerABIAttrs = + getParameterABIAttributes(F->getContext(), I, CallerAttrs); + AttrBuilder CalleeABIAttrs = + getParameterABIAttributes(F->getContext(), I, CalleeAttrs); Check(CallerABIAttrs == CalleeABIAttrs, "cannot guarantee tail call due to mismatched ABI impacting " "function attributes", diff --git a/llvm/lib/MC/GoObjObjectWriter.cpp b/llvm/lib/MC/GoObjObjectWriter.cpp index 3878c954139a5..de33312ca07df 100644 --- a/llvm/lib/MC/GoObjObjectWriter.cpp +++ b/llvm/lib/MC/GoObjObjectWriter.cpp @@ -37,9 +37,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -58,6 +60,25 @@ struct GoObjSymRef { uint32_t SymIdx = 0; }; +std::string goObjPathToPrefix(StringRef Path) { + size_t Slash = Path.rfind('/'); + size_t LastSegment = Slash == StringRef::npos ? 0 : Slash + 1; + SmallString<128> Prefix; + constexpr char Hex[] = "0123456789abcdef"; + for (size_t I = 0; I != Path.size(); ++I) { + uint8_t C = static_cast(Path[I]); + if (C <= ' ' || (C == '.' && I >= LastSegment) || C == '%' || C == '"' || + C >= 0x7f) { + Prefix.push_back('%'); + Prefix.push_back(Hex[C >> 4]); + Prefix.push_back(Hex[C & 0xf]); + } else { + Prefix.push_back(static_cast(C)); + } + } + return Prefix.str().str(); +} + struct GoObjSymbol { struct Relocation { uint32_t Offset = 0; @@ -1771,7 +1792,8 @@ uint64_t GoObjObjectWriter::writeObject() { GoObj::SRODATA, GoObj::SymFlagDupok | GoObj::SymFlagLocal, 0, GoObj::SymABI0, *ExactSize, Align, Data); - Symbols.back().ContentHash = makeGoObjContentHash(0, Data); + // The native Go object identity includes relocations as well as data. + // Compute it after all MC fixups have been assigned to their carriers. } } @@ -2338,14 +2360,15 @@ uint64_t GoObjObjectWriter::writeObject() { IndexedRefs.push_back({Ref, Name.str(), Flags2}); }; - auto GetTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, - int64_t &Addend) { + auto ResolveTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, + int64_t &Addend) { if (!Target) report_fatal_error("GoObj relocation without a target symbol"); if (auto It = DefinedSymbolIndexes.find(Target); It != DefinedSymbolIndexes.end()) - return DefinedSymRefs[It->second]; + return std::make_pair(DefinedSymRefs[It->second], + std::optional(It->second)); if (Target->isInSection()) { uint64_t TargetOffset = Asm->getSymbolOffset(*Target); @@ -2353,7 +2376,7 @@ uint64_t GoObjObjectWriter::writeObject() { FindContainingSymbol(&Target->getSection(), TargetOffset)) { Addend += static_cast(TargetOffset - Symbols[*SymIdx].SectionBegin); - return DefinedSymRefs[*SymIdx]; + return std::make_pair(DefinedSymRefs[*SymIdx], SymIdx); } } @@ -2366,7 +2389,7 @@ uint64_t GoObjObjectWriter::writeObject() { GoObjSymRef Ref{GetPackageIndex(Metadata->PackagePrefix), Metadata->SymIdx}; RecordIndexedRef(Ref, TrimInlineHash(Identity.Name), Metadata->Flags2); - return Ref; + return std::make_pair(Ref, std::optional()); } bool IsFunction; switch (RelocType) { @@ -2384,11 +2407,15 @@ uint64_t GoObjObjectWriter::writeObject() { if (Identity.BuiltinIndex && !Identity.IsLinknameRef) { if (std::optional Ref = FindDefinedSymRef( Identity.Name, GetSymbolABI(Target, IsFunction))) - return *Ref; - return GoObjSymRef{GoObj::PkgIdxBuiltin, *Identity.BuiltinIndex}; + return std::make_pair(*Ref, std::optional()); + return std::make_pair( + GoObjSymRef{GoObj::PkgIdxBuiltin, *Identity.BuiltinIndex}, + std::optional()); } - return GoObjSymRef{GoObj::PkgIdxNone, - GetNonPkgRefSymIdx(Target, IsFunction)}; + return std::make_pair( + GoObjSymRef{GoObj::PkgIdxNone, + GetNonPkgRefSymIdx(Target, IsFunction)}, + std::optional()); } report_fatal_error( @@ -2400,6 +2427,10 @@ uint64_t GoObjObjectWriter::writeObject() { " in-section=" + Twine(static_cast(Target->isInSection())) + " undefined=" + Twine(static_cast(Target->isUndefined()))); }; + auto GetTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, + int64_t &Addend) { + return ResolveTargetSymRef(Target, RelocType, Addend).first; + }; SmallVector MergedRelocations; for (const GoObjRelocationEntry &Reloc : Relocations) { @@ -2431,11 +2462,6 @@ uint64_t GoObjObjectWriter::writeObject() { if (LocalOffset > static_cast(std::numeric_limits::max())) report_fatal_error("GoObj relocation offset exceeds int32 range"); - if (Source.DefinedBlock == GoObj::DefinedSymbolBlock::Hasheddef && - Source.Symbol && Source.Symbol->isTemporary()) - report_fatal_error( - "GoObj private constants with relocations are not supported"); - int64_t Addend = getGoObjRelocAddend(Reloc); uint16_t RelocType = checkedUint16(Reloc.Type, "relocation type"); if (Source.Symbol) { @@ -2465,13 +2491,15 @@ uint64_t GoObjObjectWriter::writeObject() { const Triple::ArchType Arch = Asm->getContext().getTargetTriple().getArch(); const bool IsX86TLSLE = (Arch == Triple::x86 || Arch == Triple::x86_64) && (RelocType & ~GoObj::R_WEAK) == GoObj::R_TLS_LE; - GoObjSymRef TargetSymRef = - IsX86TLSLE ? GoObjSymRef{} - : GetTargetSymRef(Reloc.Symbol, Reloc.Type, Addend); + GoObjSymRef TargetSymRef; + std::optional TargetSymbolIndex; + if (!IsX86TLSLE) + std::tie(TargetSymRef, TargetSymbolIndex) = + ResolveTargetSymRef(Reloc.Symbol, Reloc.Type, Addend); Source.Relocations.push_back( {static_cast(LocalOffset), Reloc.Size, RelocType, Addend, - TargetSymRef.PkgIdx, TargetSymRef.SymIdx, std::nullopt}); + TargetSymRef.PkgIdx, TargetSymRef.SymIdx, TargetSymbolIndex}); } // R_KEEP has no bytes or MC fixup. It is a Go linker reachability edge @@ -2560,6 +2588,133 @@ uint64_t GoObjObjectWriter::writeObject() { }); } + // LLVM optimizations can synthesize private pointer lookup tables. Match + // cmd/internal/obj's content identity: hash the symbol bytes and each + // relocation's shape and globally stable target identity. Hashing only the + // zero relocation placeholders would incorrectly merge different tables. + DenseSet ContentHashInProgress; + std::function ComputeContentHash = [&](uint32_t SymbolIndex) { + if (SymbolIndex >= Symbols.size()) + report_fatal_error("GoObj hashed definition index is invalid"); + GoObjSymbol &Symbol = Symbols[SymbolIndex]; + if (Symbol.ContentHash) + return; + if (Symbol.DefinedBlock != GoObj::DefinedSymbolBlock::Hasheddef) + report_fatal_error( + "GoObj content hash requested for a non-hashed symbol"); + if (!Symbol.Symbol || !Symbol.Symbol->isTemporary() || + Symbol.Type != GoObj::SRODATA) + report_fatal_error( + "GoObj frontend hashed definition has no content hash"); + if (!ContentHashInProgress.insert(SymbolIndex).second) + report_fatal_error("circular GoObj content-addressable relocation"); + + SHA256 Hasher; + const char Version = 1; + Hasher.update(StringRef(&Version, 1)); + SmallString<16> Encoded; + raw_svector_ostream EncodedOS(Encoded); + support::endian::Writer EncodedWriter(EncodedOS, llvm::endianness::little); + EncodedWriter.write(Symbol.Size); + EncodedWriter.write(0); // Default read-only data section. + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + + ArrayRef Data(Symbol.Data); + while (!Data.empty() && Data.back() == 0) + Data = Data.drop_back(); + if (!Data.empty()) + Hasher.update(StringRef(Data.data(), Data.size())); + + for (const GoObjSymbol::Relocation &Reloc : Symbol.Relocations) { + Encoded.clear(); + EncodedWriter.write(Reloc.Offset); + EncodedWriter.write(Reloc.Size); + if (Reloc.Type > std::numeric_limits::max()) + report_fatal_error("GoObj content hash relocation type is too large"); + EncodedWriter.write(static_cast(Reloc.Type)); + EncodedWriter.write(static_cast(Reloc.Addend)); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + + if (Reloc.TargetSymbolIndex && *Reloc.TargetSymbolIndex == SymbolIndex) { + Hasher.update("self symbol"); + continue; + } + if (Reloc.PkgIdx == GoObj::PkgIdxInvalid) { + Hasher.update("nil symbol"); + continue; + } + + switch (Reloc.PkgIdx) { + case GoObj::PkgIdxHashed64: { + if (Reloc.SymIdx >= Hashed64defSymbols.size()) + report_fatal_error("invalid GoObj short-hashed relocation target"); + const GoObjSymbol &Target = Symbols[Hashed64defSymbols[Reloc.SymIdx]]; + if (Target.Data.size() != GoObj::Hash64Size) + report_fatal_error("GoObj short-hashed target has invalid size"); + const char Kind = 0; + Hasher.update(StringRef(&Kind, 1)); + Hasher.update(StringRef(Target.Data.data(), Target.Data.size())); + break; + } + case GoObj::PkgIdxHashed: { + if (Reloc.SymIdx >= HasheddefSymbols.size()) + report_fatal_error("invalid GoObj hashed relocation target"); + uint32_t TargetIndex = HasheddefSymbols[Reloc.SymIdx]; + ComputeContentHash(TargetIndex); + const char Kind = 1; + Hasher.update(StringRef(&Kind, 1)); + Hasher.update(ArrayRef(*Symbols[TargetIndex].ContentHash)); + break; + } + case GoObj::PkgIdxNone: { + const char Kind = 2; + Hasher.update(StringRef(&Kind, 1)); + if (Reloc.SymIdx < NonpkgdefSymbols.size()) { + Hasher.update(Symbols[NonpkgdefSymbols[Reloc.SymIdx]].Name); + } else { + uint32_t RefIndex = Reloc.SymIdx - NonpkgdefSymbols.size(); + if (RefIndex >= NonPkgRefs.size()) + report_fatal_error("invalid GoObj non-package relocation target"); + Hasher.update(NonPkgRefs[RefIndex].Name); + } + break; + } + case GoObj::PkgIdxBuiltin: { + const char Kind = 3; + Hasher.update(StringRef(&Kind, 1)); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + case GoObj::PkgIdxSelf: { + Hasher.update(goObjPathToPrefix(Config.PackagePath)); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + default: + if (Reloc.PkgIdx >= PackagePrefixes.size()) + report_fatal_error("invalid GoObj imported relocation package"); + Hasher.update(PackagePrefixes[Reloc.PkgIdx]); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + } + + std::array FullHash = Hasher.final(); + std::array Hash; + std::copy_n(FullHash.begin(), Hash.size(), Hash.begin()); + Symbol.ContentHash = Hash; + ContentHashInProgress.erase(SymbolIndex); + }; + for (uint32_t SymbolIndex : HasheddefSymbols) + if (!Symbols[SymbolIndex].ContentHash) + ComputeContentHash(SymbolIndex); + SmallString<0> Body; raw_svector_ostream BodyOS(Body); support::endian::Writer W(BodyOS, llvm::endianness::little); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index e18f0450ac094..a0cd4fc629b17 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -145,6 +145,8 @@ class AArch64AsmPrinter : public AsmPrinter { void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI); + void LowerGoMemoryCopy(MCStreamer &OutStreamer, const MachineInstr &MI); + void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI); void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, @@ -1762,6 +1764,148 @@ void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer, } } +void AArch64AsmPrinter::LowerGoMemoryCopy(MCStreamer &OutStreamer, + const MachineInstr &MI) { + assert(MI.getOpcode() == AArch64::GoMemoryCopyPseudo); + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + Register Size = MI.getOperand(2).getReg(); + + assert(Dst == MI.getOperand(3).getReg() && Src == MI.getOperand(4).getReg() && + Size == MI.getOperand(5).getReg() && + "Go memory-copy operands must be tied"); + + MCSymbol *Loop = OutContext.createTempSymbol(); + MCSymbol *Tail8 = OutContext.createTempSymbol(); + MCSymbol *Tail4 = OutContext.createTempSymbol(); + MCSymbol *Tail2 = OutContext.createTempSymbol(); + MCSymbol *Done = OutContext.createTempSymbol(); + auto Ref = [&](MCSymbol *Sym) { + return MCSymbolRefExpr::create(Sym, OutContext); + }; + + // Copy full 16-byte units. + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::SUBSXri) + .addReg(AArch64::XZR) + .addReg(Size) + .addImm(16) + .addImm(0)); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::Bcc).addImm(AArch64CC::LO).addExpr(Ref(Tail8))); + OutStreamer.emitLabel(Loop); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRQui) + .addReg(AArch64::Q31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRQui) + .addReg(AArch64::Q31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(16) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(16) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::SUBSXri) + .addReg(Size) + .addReg(Size) + .addImm(16) + .addImm(0)); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::Bcc).addImm(AArch64CC::HS).addExpr(Ref(Loop))); + + // Copy the remaining 8/4/2/1-byte units. Advancing both pointers after + // each unit keeps every access at offset zero and valid at any alignment. + OutStreamer.emitLabel(Tail8); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(3).addExpr(Ref(Tail4))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRDui) + .addReg(AArch64::D31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRDui) + .addReg(AArch64::D31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(8) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(8) + .addImm(0)); + + OutStreamer.emitLabel(Tail4); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(2).addExpr(Ref(Tail2))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRSui) + .addReg(AArch64::S31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRSui) + .addReg(AArch64::S31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(4) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(4) + .addImm(0)); + + OutStreamer.emitLabel(Tail2); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(1).addExpr(Ref(Done))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRHHui) + .addReg(AArch64::H31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRHHui) + .addReg(AArch64::H31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(2) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(2) + .addImm(0)); + + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(0).addExpr(Ref(Done))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRBBui) + .addReg(AArch64::B31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRBBui) + .addReg(AArch64::B31) + .addReg(Dst) + .addImm(0)); + OutStreamer.emitLabel(Done); +} + void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI) { unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); @@ -3703,6 +3847,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { LowerMOPS(*OutStreamer, *MI); return; + case AArch64::GoMemoryCopyPseudo: + LowerGoMemoryCopy(*OutStreamer, *MI); + return; + case TargetOpcode::STACKMAP: return LowerSTACKMAP(*OutStreamer, SM, *MI); diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 3f23e34d54c55..293200e6784e0 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -5108,6 +5108,39 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { default: break; + case ISD::PREALLOCATED_SETUP: { + auto *MFI = CurDAG->getMachineFunction().getInfo(); + auto CallId = MFI->getPreallocatedIdForCallSite( + cast(Node->getOperand(1))->getValue()); + SDValue Chain = Node->getOperand(0); + SDValue CallIdValue = + CurDAG->getTargetConstant(CallId, SDLoc(Node), MVT::i32); + MachineSDNode *New = + CurDAG->getMachineNode(TargetOpcode::PREALLOCATED_SETUP, SDLoc(Node), + MVT::Other, CallIdValue, Chain); + ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); + CurDAG->RemoveDeadNode(Node); + return; + } + + case ISD::PREALLOCATED_ARG: { + auto *MFI = CurDAG->getMachineFunction().getInfo(); + auto CallId = MFI->getPreallocatedIdForCallSite( + cast(Node->getOperand(1))->getValue()); + SDValue Ops[] = {CurDAG->getTargetConstant(CallId, SDLoc(Node), MVT::i32), + Node->getOperand(2), Node->getOperand(0)}; + MachineSDNode *New = CurDAG->getMachineNode( + TargetOpcode::PREALLOCATED_ARG, SDLoc(Node), + CurDAG->getVTList(CurDAG->getTargetLoweringInfo().getPointerTy( + CurDAG->getDataLayout()), + MVT::Other), + Ops); + ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); + ReplaceUses(SDValue(Node, 1), SDValue(New, 1)); + CurDAG->RemoveDeadNode(Node); + return; + } + case ISD::ATOMIC_CMP_SWAP: if (SelectCMP_SWAP(Node)) return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index b3a69f3134f28..564bdeba73d3b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -13,6 +13,7 @@ #include "AArch64ISelLowering.h" #include "AArch64CallingConvention.h" #include "AArch64ExpandImm.h" +#include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" #include "AArch64PerfectShuffle.h" #include "AArch64RegisterInfo.h" @@ -3604,6 +3605,30 @@ MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter( MI.dump(); #endif llvm_unreachable("Unexpected instruction for custom inserter!"); + case TargetOpcode::PREALLOCATED_SETUP: { + auto *FuncInfo = MI.getMF()->getInfo(); + int64_t PreallocatedId = MI.getOperand(0).getImm(); + if (!FuncInfo->preallocatedUsesReservedCallFrame(PreallocatedId)) + report_fatal_error( + "AArch64 preallocated call requires a reserved Go call frame"); + // Go reserves its maximum outgoing argument area in the function frame. + // The preallocated setup token therefore owns that existing area and must + // not adjust SP a second time. + MI.eraseFromParent(); + return BB; + } + case TargetOpcode::PREALLOCATED_ARG: { + int64_t PreallocatedId = MI.getOperand(1).getImm(); + int64_t ArgIndex = MI.getOperand(2).getImm(); + auto *FuncInfo = MI.getMF()->getInfo(); + size_t ArgOffset = + FuncInfo->getPreallocatedArgOffsets(PreallocatedId)[ArgIndex]; + emitFrameOffset(*BB, MI, MI.getDebugLoc(), MI.getOperand(0).getReg(), + AArch64::SP, StackOffset::getFixed(ArgOffset), + Subtarget->getInstrInfo()); + MI.eraseFromParent(); + return BB; + } case AArch64::EntryPStateSM: return EmitEntryPStateSM(MI, BB); case AArch64::F128CSEL: @@ -9058,10 +9083,10 @@ getAArch64GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) continue; LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); + ArgTys.push_back(goabi::getParameterType(Arg)); } return ArgTys; } @@ -9072,21 +9097,81 @@ getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) + if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); + ArgTys.push_back(Args[I].IsPreallocated ? Args[I].IndirectType + : Args[I].OrigTy); } return ArgTys; } +static SmallBitVector +getAArch64GoCallMemoryArgMask(const TargetLowering::ArgListTy &Args) { + SmallBitVector MemoryArgs; + for (const TargetLowering::ArgListEntry &Arg : Args) { + if (Arg.IsNest || Arg.IsGoRet) + continue; + MemoryArgs.push_back(Arg.IsPreallocated); + } + return MemoryArgs; +} + +static SmallVector getAArch64GoReturnTypes(const Function &F) { + SmallVector MemoryResults; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + SmallVector ResultTys; + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + MemoryResults, ResultTys); + return ResultTys; +} + static SmallVector -getAArch64GoReturnTypes(Type *RetTy, const AttributeList &Attrs) { +getAArch64GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, + const TargetLowering::ArgListTy &Args) { + SmallVector MemoryResults; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + MemoryResults.push_back({Arg.GoRetIndex, Arg.IndirectType}); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), ResultTys); + goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), MemoryResults, + ResultTys); return ResultTys; } +static SmallVector +getAArch64GoDirectResultIndices(ArrayRef MemoryResults, + unsigned DirectResultCount) { + SmallVector DirectIndices; + for (unsigned I = 0, E = DirectResultCount + MemoryResults.size(); I != E; + ++I) + if (!llvm::any_of(MemoryResults, + [I](const auto &Result) { return Result.Index == I; })) + DirectIndices.push_back(I); + return DirectIndices; +} + +static SmallVector +getAArch64GoMemoryResults(const Function &F) { + SmallVector Results; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + Results.push_back({goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + return Results; +} + +static SmallVector +getAArch64GoMemoryResults(const TargetLowering::ArgListTy &Args) { + SmallVector Results; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + Results.push_back({Arg.GoRetIndex, Arg.IndirectType}); + return Results; +} + static SDValue lowerAArch64GoFormalArguments( const AArch64TargetLowering &TLI, SDValue Chain, MachineFunction &MF, const SmallVectorImpl &Ins, const SDLoc &DL, @@ -9099,10 +9184,14 @@ static SDValue lowerAArch64GoFormalArguments( SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getAArch64GoReturnTypes(F); + SmallVector MemoryResults = + getAArch64GoMemoryResults(F); goabi::ABIConfig ABIConfig = getAArch64GoABIConfig(TLI, Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( @@ -9125,6 +9214,14 @@ static SDValue lowerAArch64GoFormalArguments( FuncInfo->clearGoArgPointerSlots(); unsigned StackBias = getAArch64GoStackBias(F.getCallingConv()); + SmallVector DirectResultTys; + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + DirectResultTys); + SmallVector MemoryResultMap(F.arg_size(), -1); + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + MemoryResultMap[Arg.getArgNo()] = goabi::getGoRetIndex(Arg); + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { uint64_t PointerSize = EntryArgs.PointerSize; @@ -9153,6 +9250,22 @@ static SDValue lowerAArch64GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); + if (Arg->hasGoRetAttr()) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Ins[Group.Start].Flags.isGoRet()) + report_fatal_error("invalid AArch64 Go memory result carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Ins[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) + report_fatal_error( + "AArch64 Go memory result carrier disagrees with Go ABI layout"); + int FI = MFI.CreateFixedObject(ResultLayout.Size, + StackBias + ResultLayout.StackOffset, + /*IsImmutable=*/false, /*isAliased=*/true); + InVals.push_back(DAG.getFrameIndex(FI, PtrVT)); + continue; + } if (Arg->hasNestAttr()) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); MVT CopyVT = getAArch64GoCopyVT(Ins[Group.Start].VT); @@ -9166,15 +9279,26 @@ static SDValue lowerAArch64GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; + bool IsPreallocated = Ins[Group.Start].Flags.isPreallocated(); + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasPreallocatedAttr() != IsPreallocated || + IsPreallocated != MustBePreallocated) + report_fatal_error( + "AArch64 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; int64_t FixedHomeOffset = StackBias + LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false); + else if (IsPreallocated) + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/true); AArch64FunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); // LLVM may replace an unused incoming pointer with poison at every call @@ -9186,6 +9310,14 @@ static SDValue lowerAArch64GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); + if (IsPreallocated) { + if (Group.End != Group.Start + 1 || + Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid AArch64 Go preallocated formal argument"); + InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9243,17 +9375,26 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(MF.getFunction(), LayoutMap); - SmallVector ResultTys = getAArch64GoReturnTypes( - MF.getFunction().getReturnType(), MF.getFunction().getAttributes()); + SmallVector ResultTys = getAArch64GoReturnTypes(MF.getFunction()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(MF.getFunction()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), + ArgTys, ResultTys, goabi::getMemoryArgMask(MF.getFunction()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, MF.getFunction().getCallingConv())); unsigned StackBias = getAArch64GoStackBias(MF.getFunction().getCallingConv()); + SmallVector DirectResultIndices = + getAArch64GoDirectResultIndices(MemoryResults, Layout.Results.size() - + MemoryResults.size()); SmallVector MemOps; SmallVector, 8> RetRegs; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid AArch64 Go direct result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9328,14 +9469,25 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoCallArgTypes(CLI.getArgs(), LayoutMap); - SmallVector ResultTys; - goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); + AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); + SmallVector ResultTys = + getAArch64GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(CLI.getArgs()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); + ArgTys, ResultTys, getAArch64GoCallMemoryArgMask(CLI.getArgs()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); + SmallVector DirectResultTys; + goabi::getReturnTypes(CLI.RetTy, + CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), + DirectResultTys); + SmallVector MemoryResultMap(CLI.getArgs().size(), -1); + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) + if (CLI.getArgs()[I].IsGoRet) + MemoryResultMap[I] = CLI.getArgs()[I].GoRetIndex; + SmallVector DirectResultIndices = + getAArch64GoDirectResultIndices(MemoryResults, DirectResultTys.size()); unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); // Layout.TotalStackSize rounds the logical Go argument area to the target @@ -9345,6 +9497,26 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, // The caller frame itself remains stack-aligned; reserve only the bytes // through the last physical argument home here. unsigned NumBytes = Layout.ArgSize + StackBias; + SmallVector PreallocatedOffsets; + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) { + if (!CLI.getArgs()[I].IsPreallocated) + continue; + int LayoutIndex = LayoutMap[I]; + if (LayoutIndex < 0 || Layout.Args[LayoutIndex].InRegs) + report_fatal_error( + "AArch64 Go preallocated call argument has no stack home"); + PreallocatedOffsets.push_back(StackBias + + Layout.Args[LayoutIndex].StackOffset); + } + if (CLI.IsPreallocated != !PreallocatedOffsets.empty()) + report_fatal_error( + "AArch64 Go preallocated call bundle disagrees with its arguments"); + if (CLI.IsPreallocated) { + auto *FuncInfo = MF.getInfo(); + size_t PreallocatedId = FuncInfo->getPreallocatedIdForCallSite(CLI.CB); + FuncInfo->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets); + FuncInfo->setPreallocatedUsesReservedCallFrame(PreallocatedId); + } Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; @@ -9353,12 +9525,28 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector MemOpChains; SmallVector, 8> RegsToPass; + SmallVector MemoryResultDests(Layout.Results.size()); MachineFunction::CallSiteInfo CSInfo; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { if (Group.Index >= CLI.getArgs().size()) continue; + if (CLI.getArgs()[Group.Index].IsGoRet) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Outs[Group.Start].Flags.isGoRet()) + report_fatal_error("invalid AArch64 Go memory result call carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Outs[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) + report_fatal_error( + "AArch64 Go memory result call carrier disagrees with Go ABI " + "layout"); + MemoryResultDests[ResultIndex] = OutVals[Group.Start]; + continue; + } + if (CLI.getArgs()[Group.Index].IsNest) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); RegsToPass.emplace_back(AArch64::X26, OutVals[Group.Start]); @@ -9366,6 +9554,21 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, } const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutMap[Group.Index]]; + bool IsPreallocated = CLI.getArgs()[Group.Index].IsPreallocated; + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isPreallocated() != IsPreallocated || + IsPreallocated != MustBePreallocated) + report_fatal_error( + "AArch64 Go call argument carrier disagrees with Go ABI layout"); + if (IsPreallocated) { + if (Group.End != Group.Start + 1 || + Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid AArch64 Go preallocated call argument"); + // llvm.call.preallocated.arg exposes the final outgoing home. The + // frontend has already initialized it before this call. + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9454,7 +9657,10 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector ResultVals(Ins.size()); for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid AArch64 Go direct call result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (!ResultLayout.InRegs) continue; unsigned IntPiece = 0; @@ -9488,10 +9694,8 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, // Keep the call sequence active while reading stack results. The load chain // is after the call, so the physical SP is the current post-growth Go stack; // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + bool HasStackResults = llvm::any_of( + Layout.Results, [](const auto &Result) { return !Result.InRegs; }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, PtrVT, InGlue); @@ -9499,7 +9703,8 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, InGlue = ResultStackPtr.getValue(2); } for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (ResultLayout.InRegs) continue; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9517,6 +9722,25 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, } } + SmallVector MemoryResultCopies; + for (unsigned I = 0; I != MemoryResultDests.size(); ++I) { + if (!MemoryResultDests[I].getNode()) + continue; + const goabi::ValueLayout &ResultLayout = Layout.Results[I]; + SDValue Src = DAG.getNode( + ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(StackBias + ResultLayout.StackOffset, DL)); + MemoryResultCopies.push_back(DAG.getMemcpy( + Chain, DL, MemoryResultDests[I], Src, + DAG.getConstant(ResultLayout.Size, DL, PtrVT), ResultLayout.Alignment, + ResultLayout.Alignment, /*isVol=*/false, /*AlwaysInline=*/false, + /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), + MachinePointerInfo::getStack(MF, + StackBias + ResultLayout.StackOffset))); + } + if (!MemoryResultCopies.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemoryResultCopies); + Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); @@ -9531,13 +9755,17 @@ AArch64TargetLowering::getArgumentCopyElisionFrameInfo( const Argument &Arg, MachineFunction &MF) const { if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) return std::nullopt; + // Memory carriers already denote their canonical fixed ABI homes. This + // hook only redirects the frontend spill of a direct register parameter. + if (Arg.hasPreallocatedAttr() || Arg.hasGoRetAttr()) + return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const AArch64FunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) - if (Home.ArgNo == Arg.getArgNo() && + if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) return ArgumentCopyElisionFrameInfo{Home.FrameIndex, - Home.valueAlreadyInFrame()}; + /*ValueAlreadyInFrame=*/false}; return std::nullopt; } @@ -9552,8 +9780,12 @@ int AArch64TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); const AArch64Subtarget &Subtarget = MF.getSubtarget(); + SmallVector ResultTys = getAArch64GoReturnTypes(F); + SmallVector MemoryResults = + getAArch64GoMemoryResults(F); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), MF.getDataLayout(), getAArch64GoABIConfig(*this, Subtarget, F.getCallingConv())); if (Layout.ArgSize == 0) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 0532218514007..5dc1bbeb88c7c 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -11498,6 +11498,21 @@ let Predicates = [HasMOPS], Defs = [NZCV], Size = 12, mayStore = 1 in { [], "$Rd = $Rd_wb,$Rn = $Rn_wb,@earlyclobber $Rn_wb">, Sched<[]>; } } + +// Populate a Go call's outgoing stack-argument area without introducing a +// libcall. The AsmPrinter expands this to a baseline-AArch64 vector loop with +// scalar tails. All three GPR operands are tied because the expansion advances +// them; Q31 and NZCV are explicit clobbers so register allocation preserves any +// live values. +let Defs = [Q31, NZCV], Size = 104, mayLoad = 1, mayStore = 1 in { + def GoMemoryCopyPseudo + : Pseudo<(outs GPR64common:$Rd_wb, GPR64common:$Rs_wb, + GPR64:$Rn_wb), + (ins GPR64common:$Rd, GPR64common:$Rs, GPR64:$Rn), [], + "$Rd = $Rd_wb,$Rs = $Rs_wb,$Rn = $Rn_wb">, + Sched<[]>; +} + let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, mayStore = 1 in { def MOPSMemorySetTaggingPseudo : Pseudo<(outs GPR64common:$Rd_wb, GPR64:$Rn_wb), (ins GPR64common:$Rd, GPR64:$Rn, GPR64:$Rm), diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h index c7d02a93e090f..2e64791d852b6 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -16,6 +16,7 @@ #include "AArch64SMEAttributes.h" #include "AArch64Subtarget.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" @@ -84,6 +85,12 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { int GoABI0FrameIndex = 0; bool HasGoABI0FrameIndex = false; + /// Preallocated call-frame layouts, keyed by the consuming IR call. These + /// fields are used only during instruction selection. + DenseMap PreallocatedIds; + SmallVector, 0> PreallocatedArgOffsets; + SmallVector PreallocatedUsesReservedCallFrame; + /// Number of bytes of arguments this function has on the stack. If the callee /// is expected to restore the argument stack this should be a multiple of 16, /// all usable during a tail call. @@ -329,6 +336,30 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { HasGoABI0FrameIndex = true; } + size_t getPreallocatedIdForCallSite(const Value *Call) { + auto Insert = PreallocatedIds.insert({Call, PreallocatedIds.size()}); + if (Insert.second) { + PreallocatedArgOffsets.emplace_back(); + PreallocatedUsesReservedCallFrame.push_back(false); + } + return Insert.first->second; + } + + void setPreallocatedArgOffsets(size_t Id, ArrayRef Offsets) { + PreallocatedArgOffsets[Id].assign(Offsets.begin(), Offsets.end()); + } + ArrayRef getPreallocatedArgOffsets(size_t Id) const { + assert(!PreallocatedArgOffsets[Id].empty() && "arg offsets not set"); + return PreallocatedArgOffsets[Id]; + } + + void setPreallocatedUsesReservedCallFrame(size_t Id) { + PreallocatedUsesReservedCallFrame[Id] = true; + } + bool preallocatedUsesReservedCallFrame(size_t Id) const { + return PreallocatedUsesReservedCallFrame[Id]; + } + void clearGoArgPointerSlots() { GoArgPointerSlots.clear(); } void addGoArgPointerSlot(int FrameIndex, uint32_t OffsetWithinObject, uint32_t ArgWord) { diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp index 286e5638fc088..56e4946eebd90 100644 --- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp @@ -12,6 +12,7 @@ #include "AArch64SelectionDAGInfo.h" #include "AArch64MachineFunctionInfo.h" +#include "llvm/CodeGen/GoCallingConv.h" #define GET_SDNODE_DESC #include "AArch64GenSDNodeInfo.inc" @@ -234,6 +235,16 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( Size, DstAlign, SrcAlign, isVolatile, DstPtrInfo, SrcPtrInfo); + // A Go call cannot use a C memcpy libcall to populate its outgoing argument + // area: the copy is part of the call sequence and must not introduce another + // call or safepoint. Small constant copies have already been expanded by + // SelectionDAG. Keep large copies compact by expanding this pseudo to a + // baseline-AArch64 loop in the AsmPrinter. + if (goabi::isGoCallingConv( + DAG.getMachineFunction().getFunction().getCallingConv())) + return EmitMOPS(AArch64::GoMemoryCopyPseudo, DAG, DL, Chain, Dst, Src, Size, + DstAlign, SrcAlign, isVolatile, DstPtrInfo, SrcPtrInfo); + auto *AFI = DAG.getMachineFunction().getInfo(); SMEAttrs Attrs = AFI->getSMEFnAttrs(); if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index d3fff2d4b54e2..9c63ae2fa4445 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -38838,22 +38838,28 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } case TargetOpcode::PREALLOCATED_SETUP: { - assert(Subtarget.is32Bit() && "preallocated only used in 32-bit"); auto *MFI = MF->getInfo(); - MFI->setHasPreallocatedCall(true); int64_t PreallocatedId = MI.getOperand(0).getImm(); + if (MFI->preallocatedUsesReservedCallFrame(PreallocatedId)) { + // Go's outgoing argument area is already part of the fixed function + // frame. The setup token owns that area without moving SP again. + MI.eraseFromParent(); + return BB; + } + MFI->setHasPreallocatedCall(true); size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId); assert(StackAdjustment != 0 && "0 stack adjustment"); LLVM_DEBUG(dbgs() << "PREALLOCATED_SETUP stack adjustment " << StackAdjustment << "\n"); - BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP) - .addReg(X86::ESP) + Register StackPtr = Subtarget.is64Bit() ? X86::RSP : X86::ESP; + unsigned SubOpcode = Subtarget.is64Bit() ? X86::SUB64ri32 : X86::SUB32ri; + BuildMI(*BB, MI, MIMD, TII->get(SubOpcode), StackPtr) + .addReg(StackPtr) .addImm(StackAdjustment); MI.eraseFromParent(); return BB; } case TargetOpcode::PREALLOCATED_ARG: { - assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit"); int64_t PreallocatedId = MI.getOperand(1).getImm(); int64_t ArgIdx = MI.getOperand(2).getImm(); auto *MFI = MF->getInfo(); @@ -38861,9 +38867,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx << ", arg offset " << ArgOffset << "\n"); // stack pointer + offset - addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r), - MI.getOperand(0).getReg()), - X86::ESP, false, ArgOffset); + Register StackPtr = Subtarget.is64Bit() ? X86::RSP : X86::ESP; + unsigned LeaOpcode = Subtarget.is64Bit() ? X86::LEA64r : X86::LEA32r; + addRegOffset( + BuildMI(*BB, MI, MIMD, TII->get(LeaOpcode), MI.getOperand(0).getReg()), + StackPtr, false, ArgOffset); MI.eraseFromParent(); return BB; } diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index f48a98b2437e2..dc1eca74f611b 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -153,10 +153,10 @@ getX86GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) continue; LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); + ArgTys.push_back(goabi::getParameterType(Arg)); } return ArgTys; } @@ -167,21 +167,81 @@ getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) + if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); + ArgTys.push_back(Args[I].IsPreallocated ? Args[I].IndirectType + : Args[I].OrigTy); } return ArgTys; } -static SmallVector getX86GoReturnTypes(Type *RetTy, - const AttributeList &Attrs) { +static SmallBitVector +getX86GoCallMemoryArgMask(const TargetLowering::ArgListTy &Args) { + SmallBitVector MemoryArgs; + for (const TargetLowering::ArgListEntry &Arg : Args) { + if (Arg.IsNest || Arg.IsGoRet) + continue; + MemoryArgs.push_back(Arg.IsPreallocated); + } + return MemoryArgs; +} + +static SmallVector getX86GoReturnTypes(const Function &F) { + SmallVector MemoryResults; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), ResultTys); + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + MemoryResults, ResultTys); return ResultTys; } +static SmallVector +getX86GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, + const TargetLowering::ArgListTy &Args) { + SmallVector MemoryResults; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + MemoryResults.push_back({Arg.GoRetIndex, Arg.IndirectType}); + SmallVector ResultTys; + goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), MemoryResults, + ResultTys); + return ResultTys; +} + +static SmallVector +getX86GoDirectResultIndices(ArrayRef MemoryResults, + unsigned DirectResultCount) { + SmallVector DirectIndices; + for (unsigned I = 0, E = DirectResultCount + MemoryResults.size(); I != E; + ++I) + if (!llvm::any_of(MemoryResults, + [I](const auto &Result) { return Result.Index == I; })) + DirectIndices.push_back(I); + return DirectIndices; +} + +static SmallVector +getX86GoMemoryResults(const Function &F) { + SmallVector Results; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + Results.push_back({goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + return Results; +} + +static SmallVector +getX86GoMemoryResults(const TargetLowering::ArgListTy &Args) { + SmallVector Results; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + Results.push_back({Arg.GoRetIndex, Arg.IndirectType}); + return Results; +} + static SDValue lowerX86GoFormalArguments( const X86TargetLowering &TLI, SDValue Chain, MachineFunction &MF, const SmallVectorImpl &Ins, const SDLoc &DL, @@ -199,10 +259,13 @@ static SDValue lowerX86GoFormalArguments( SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getX86GoReturnTypes(F); + SmallVector MemoryResults = getX86GoMemoryResults(F); goabi::ABIConfig ABIConfig = getX86GoABIConfig(Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( @@ -226,6 +289,14 @@ static SDValue lowerX86GoFormalArguments( FuncInfo->clearGoArgHomes(); FuncInfo->clearGoArgPointerSlots(); + SmallVector DirectResultTys; + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + DirectResultTys); + SmallVector MemoryResultMap(F.arg_size(), -1); + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + MemoryResultMap[Arg.getArgNo()] = goabi::getGoRetIndex(Arg); + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { uint64_t PointerSize = EntryArgs.PointerSize; @@ -264,6 +335,23 @@ static SDValue lowerX86GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); + if (Arg->hasGoRetAttr()) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Ins[Group.Start].Flags.isGoRet()) + report_fatal_error("invalid X86 Go memory result carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Ins[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) + report_fatal_error( + "X86 Go memory result carrier disagrees with Go ABI layout"); + int FI = + MFI.CreateFixedObject(ResultLayout.Size, ResultLayout.StackOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + InVals.push_back(DAG.getFrameIndex(FI, PtrVT)); + continue; + } if (Arg->hasNestAttr()) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); Register VReg = @@ -279,15 +367,26 @@ static SDValue lowerX86GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; + bool IsPreallocated = Ins[Group.Start].Flags.isPreallocated(); + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasPreallocatedAttr() != IsPreallocated || + IsPreallocated != MustBePreallocated) + report_fatal_error( + "X86 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; int64_t FixedHomeOffset = LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false); + else if (IsPreallocated) + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/true); X86MachineFunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); // LLVM may replace an unused incoming pointer with poison at every call @@ -299,6 +398,14 @@ static SDValue lowerX86GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); + if (IsPreallocated) { + if (Group.End != Group.Start + 1 || + Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid X86 Go preallocated formal argument"); + InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -355,17 +462,24 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, MachineFrameInfo &MFI = MF.getFrameInfo(); SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(MF.getFunction(), LayoutMap); - SmallVector ResultTys = - getX86GoReturnTypes(MF.getFunction().getReturnType(), - MF.getFunction().getAttributes()); + SmallVector ResultTys = getX86GoReturnTypes(MF.getFunction()); + SmallVector MemoryResults = + getX86GoMemoryResults(MF.getFunction()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), + ArgTys, ResultTys, goabi::getMemoryArgMask(MF.getFunction()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); + SmallVector DirectResultIndices = getX86GoDirectResultIndices( + MemoryResults, Layout.Results.size() - MemoryResults.size()); SmallVector MemOps; SmallVector, 8> RetRegs; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid X86 Go direct result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -375,9 +489,9 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, MVT CopyVT = getX86GoCopyVT(Val.getSimpleValueType()); if (Val.getSimpleValueType() != CopyVT) Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); + MCPhysReg PReg = getX86GoPhysReg(Out.VT, Out.OrigTy, + ResultLayout.IntRegStart + IntPiece, + ResultLayout.FPRegStart + FPPiece); if (isX86GoFloatPiece(Out.OrigTy)) ++FPPiece; else @@ -393,10 +507,11 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, /*IsImmutable=*/false); if (MF.getInfo()->hasGoABI0FrameIndex()) MFI.setIsAliasedObjectIndex(FI, true); - SDValue Addr = DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); - MemOps.push_back(storeX86GoStackPiece( - DAG, Chain, DL, Val, Out.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, FI))); + SDValue Addr = + DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); + MemOps.push_back( + storeX86GoStackPiece(DAG, Chain, DL, Val, Out.ArgVT, Addr, + MachinePointerInfo::getFixedStack(MF, FI))); } } @@ -440,38 +555,101 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, report_fatal_error("Go calling convention does not support varargs"); SmallVector ArgLayoutMap; - SmallVector ArgTys = getX86GoCallArgTypes(CLI.getArgs(), ArgLayoutMap); - SmallVector ResultTys; - goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); + SmallVector ArgTys = + getX86GoCallArgTypes(CLI.getArgs(), ArgLayoutMap); + AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); + SmallVector ResultTys = + getX86GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); + SmallVector MemoryResults = + getX86GoMemoryResults(CLI.getArgs()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getX86GoABIConfig(Subtarget, CLI.CallConv)); + ArgTys, ResultTys, getX86GoCallMemoryArgMask(CLI.getArgs()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getX86GoABIConfig(Subtarget, CLI.CallConv)); + SmallVector DirectResultTys; + goabi::getReturnTypes(CLI.RetTy, + CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), + DirectResultTys); + SmallVector MemoryResultMap(CLI.getArgs().size(), -1); + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) + if (CLI.getArgs()[I].IsGoRet) + MemoryResultMap[I] = CLI.getArgs()[I].GoRetIndex; + SmallVector DirectResultIndices = + getX86GoDirectResultIndices(MemoryResults, DirectResultTys.size()); unsigned NumBytes = Layout.TotalStackSize; + SmallVector PreallocatedOffsets; + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) { + if (!CLI.getArgs()[I].IsPreallocated) + continue; + int LayoutIndex = ArgLayoutMap[I]; + if (LayoutIndex < 0 || Layout.Args[LayoutIndex].InRegs) + report_fatal_error("X86 Go preallocated call argument has no stack home"); + PreallocatedOffsets.push_back(Layout.Args[LayoutIndex].StackOffset); + } + if (CLI.IsPreallocated != !PreallocatedOffsets.empty()) + report_fatal_error( + "X86 Go preallocated call bundle disagrees with its arguments"); + if (CLI.IsPreallocated) { + auto *FuncInfo = MF.getInfo(); + size_t PreallocatedId = FuncInfo->getPreallocatedIdForCallSite(CLI.CB); + FuncInfo->setPreallocatedStackSize(PreallocatedId, NumBytes); + FuncInfo->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets); + FuncInfo->setPreallocatedUsesReservedCallFrame(PreallocatedId); + } Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; if (NumBytes != 0) - StackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), PtrVT); + StackPtr = + DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), PtrVT); SmallVector MemOpChains; SmallVector, 8> RegsToPass; + SmallVector MemoryResultDests(Layout.Results.size()); MachineFunction::CallSiteInfo CSInfo; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { if (Group.Index >= CLI.getArgs().size()) continue; + if (CLI.getArgs()[Group.Index].IsGoRet) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Outs[Group.Start].Flags.isGoRet()) + report_fatal_error("invalid X86 Go memory result call carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Outs[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) + report_fatal_error( + "X86 Go memory result call carrier disagrees with Go ABI layout"); + MemoryResultDests[ResultIndex] = OutVals[Group.Start]; + continue; + } + if (CLI.getArgs()[Group.Index].IsNest) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); RegsToPass.emplace_back(X86::RDX, OutVals[Group.Start]); continue; } - const goabi::ValueLayout &ArgLayout = Layout.Args[ArgLayoutMap[Group.Index]]; + const goabi::ValueLayout &ArgLayout = + Layout.Args[ArgLayoutMap[Group.Index]]; + bool IsPreallocated = CLI.getArgs()[Group.Index].IsPreallocated; + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isPreallocated() != IsPreallocated || + IsPreallocated != MustBePreallocated) + report_fatal_error( + "X86 Go call argument carrier disagrees with Go ABI layout"); + if (IsPreallocated) { + if (Group.End != Group.Start + 1 || + Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid X86 Go preallocated call argument"); + // llvm.call.preallocated.arg exposes the final outgoing home. The + // frontend has already initialized it before this call. + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -481,9 +659,9 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, MVT CopyVT = getX86GoCopyVT(Arg.getSimpleValueType()); if (Arg.getSimpleValueType() != CopyVT) Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Arg); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); + MCPhysReg PReg = getX86GoPhysReg(Out.VT, Out.OrigTy, + ArgLayout.IntRegStart + IntPiece, + ArgLayout.FPRegStart + FPPiece); if (isX86GoFloatPiece(Out.OrigTy)) ++FPPiece; else @@ -495,14 +673,13 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, } assert(StackPtr && "missing Go call stack pointer"); - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant(ArgLayout.StackOffset + Out.PartOffset, - DL)); + SDValue Addr = DAG.getNode( + ISD::ADD, DL, PtrVT, StackPtr, + DAG.getIntPtrConstant(ArgLayout.StackOffset + Out.PartOffset, DL)); MemOpChains.push_back(storeX86GoStackPiece( DAG, Chain, DL, Arg, Out.ArgVT, Addr, - MachinePointerInfo::getStack(MF, - ArgLayout.StackOffset + Out.PartOffset))); + MachinePointerInfo::getStack(MF, ArgLayout.StackOffset + + Out.PartOffset))); } } @@ -529,7 +706,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, Ops.push_back(Callee); for (const auto &[Reg, Val] : RegsToPass) Ops.push_back(DAG.getRegister(Reg, Val.getValueType())); - Ops.push_back(DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); + Ops.push_back( + DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); if (InGlue.getNode()) Ops.push_back(InGlue); @@ -541,7 +719,10 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, SmallVector ResultVals(Ins.size()); for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid X86 Go direct call result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (!ResultLayout.InRegs) continue; unsigned IntPiece = 0; @@ -549,9 +730,9 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, for (unsigned I = Group.Start; I != Group.End; ++I) { const ISD::InputArg &In = Ins[I]; MVT CopyVT = getX86GoCopyVT(In.VT); - MCPhysReg PReg = getX86GoPhysReg( - In.VT, In.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); + MCPhysReg PReg = + getX86GoPhysReg(In.VT, In.OrigTy, ResultLayout.IntRegStart + IntPiece, + ResultLayout.FPRegStart + FPPiece); SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); Chain = Val.getValue(1); InGlue = Val.getValue(2); @@ -575,10 +756,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, // Keep the call sequence active while reading stack results. The load chain // is after the call, so the physical SP is the current post-growth Go stack; // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + bool HasStackResults = llvm::any_of( + Layout.Results, [](const auto &Result) { return !Result.InRegs; }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), @@ -587,24 +766,42 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, InGlue = ResultStackPtr.getValue(2); } for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (ResultLayout.InRegs) continue; for (unsigned I = Group.Start; I != Group.End; ++I) { const ISD::InputArg &In = Ins[I]; - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, - DAG.getIntPtrConstant(ResultLayout.StackOffset + In.PartOffset, - DL)); + SDValue Addr = DAG.getNode( + ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(ResultLayout.StackOffset + In.PartOffset, DL)); SDValue Load = loadX86GoStackPiece( DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getStack( - MF, ResultLayout.StackOffset + In.PartOffset)); + MachinePointerInfo::getStack(MF, ResultLayout.StackOffset + + In.PartOffset)); Chain = Load.getValue(1); ResultVals[I] = Load; } } + SmallVector MemoryResultCopies; + for (unsigned I = 0; I != MemoryResultDests.size(); ++I) { + if (!MemoryResultDests[I].getNode()) + continue; + const goabi::ValueLayout &ResultLayout = Layout.Results[I]; + SDValue Src = + DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(ResultLayout.StackOffset, DL)); + MemoryResultCopies.push_back(DAG.getMemcpy( + Chain, DL, MemoryResultDests[I], Src, + DAG.getConstant(ResultLayout.Size, DL, PtrVT), ResultLayout.Alignment, + ResultLayout.Alignment, /*isVol=*/false, /*AlwaysInline=*/false, + /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), + MachinePointerInfo::getStack(MF, ResultLayout.StackOffset))); + } + if (!MemoryResultCopies.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemoryResultCopies); + Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); @@ -619,14 +816,18 @@ X86TargetLowering::getArgumentCopyElisionFrameInfo(const Argument &Arg, MachineFunction &MF) const { if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) return std::nullopt; + // Memory carriers already denote their canonical fixed ABI homes. This + // hook only redirects the frontend spill of a direct register parameter. + if (Arg.hasPreallocatedAttr() || Arg.hasGoRetAttr()) + return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const X86MachineFunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) - if (Home.ArgNo == Arg.getArgNo() && + if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) return ArgumentCopyElisionFrameInfo{Home.FrameIndex, - Home.valueAlreadyInFrame()}; + /*ValueAlreadyInFrame=*/false}; return std::nullopt; } @@ -640,8 +841,11 @@ int X86TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getX86GoReturnTypes(F); + SmallVector MemoryResults = getX86GoMemoryResults(F); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), MF.getDataLayout(), getX86GoABIConfig(MF.getSubtarget(), F.getCallingConv())); if (Layout.ArgSize == 0) @@ -823,13 +1027,12 @@ unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv( if (VT.isVectorOf(MVT::bf16) && isTypeLegal(MVT::f16)) VT = VT.changeVectorElementType(Context, MVT::f16); - return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT, - NumIntermediates, RegisterVT); + return TargetLowering::getVectorTypeBreakdownForCallingConv( + Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); } EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL, - LLVMContext& Context, - EVT VT) const { + LLVMContext &Context, EVT VT) const { if (!VT.isVector()) return MVT::i8; @@ -1121,9 +1324,8 @@ SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, /// This returns the relocation base for the given PIC jumptable, /// the same as getPICJumpTableRelocBase, but as an MCExpr. -const MCExpr *X86TargetLowering:: -getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, - MCContext &Ctx) const { +const MCExpr *X86TargetLowering::getPICJumpTableRelocBaseExpr( + const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { // X86-64 uses RIP relative addressing based on the jump table label. if (Subtarget.isPICStyleRIPRel() || (Subtarget.is64Bit() && @@ -1142,19 +1344,35 @@ X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, switch (VT.SimpleTy) { default: return TargetLowering::findRepresentativeClass(TRI, VT); - case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: + case MVT::i8: + case MVT::i16: + case MVT::i32: + case MVT::i64: RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass; break; case MVT::x86mmx: RRC = &X86::VR64RegClass; break; - case MVT::f32: case MVT::f64: - case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: - case MVT::v4f32: case MVT::v2f64: - case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64: - case MVT::v8f32: case MVT::v4f64: - case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64: - case MVT::v16f32: case MVT::v8f64: + case MVT::f32: + case MVT::f64: + case MVT::v16i8: + case MVT::v8i16: + case MVT::v4i32: + case MVT::v2i64: + case MVT::v4f32: + case MVT::v2f64: + case MVT::v32i8: + case MVT::v16i16: + case MVT::v8i32: + case MVT::v4i64: + case MVT::v8f32: + case MVT::v4f64: + case MVT::v64i8: + case MVT::v32i16: + case MVT::v16i32: + case MVT::v8i64: + case MVT::v16f32: + case MVT::v8f64: RRC = &X86::VR128XRegClass; break; } @@ -1173,8 +1391,8 @@ static bool hasStackGuardSlotTLS(const Triple &TargetTriple) { TargetTriple.isOSFuchsia() || TargetTriple.isAndroid(); } -static Constant* SegmentOffset(IRBuilderBase &IRB, - int Offset, unsigned AddressSpace) { +static Constant *SegmentOffset(IRBuilderBase &IRB, int Offset, + unsigned AddressSpace) { return ConstantExpr::getIntToPtr( ConstantInt::getSigned(Type::getInt32Ty(IRB.getContext()), Offset), IRB.getPtrTy(AddressSpace)); @@ -1494,8 +1712,8 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, if (ValVT == MVT::x86mmx) { if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy); - ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, - ValToCopy); + ValToCopy = + DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, ValToCopy); // If we don't have SSE2 available, convert to v4f32 so the generated // register is legal. if (!Subtarget.hasSSE2()) @@ -1574,9 +1792,9 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg, getPointerTy(MF.getDataLayout())); - Register RetValReg - = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? - X86::RAX : X86::EAX; + Register RetValReg = + (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? X86::RAX + : X86::EAX; Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Glue); Glue = Chain.getValue(1); @@ -1626,7 +1844,8 @@ bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { if (Copy->getOpcode() == ISD::CopyToReg) { // If the copy has a glue operand, we conservatively assume it isn't safe to // perform a tail call. - if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) + if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == + MVT::Glue) return false; TCChain = Copy->getOperand(0); } else if (Copy->getOpcode() != ISD::FP_EXTEND) @@ -1992,7 +2211,8 @@ X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv, // could be overwritten by lowering of arguments in case of a tail call. if (Flags.isByVal()) { unsigned Bytes = Flags.getByValSize(); - if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects. + if (Bytes == 0) + Bytes = 1; // Don't create zero-sized stack objects. // FIXME: For now, all byval parameter objects are marked as aliasing. This // can be improved with deeper analysis. @@ -2084,15 +2304,13 @@ static ArrayRef get64BitArgumentGPRs(CallingConv::ID CallConv, assert(Subtarget.is64Bit()); if (Subtarget.isCallingConvWin64(CallConv)) { - static const MCPhysReg GPR64ArgRegsWin64[] = { - X86::RCX, X86::RDX, X86::R8, X86::R9 - }; + static const MCPhysReg GPR64ArgRegsWin64[] = {X86::RCX, X86::RDX, X86::R8, + X86::R9}; return GPR64ArgRegsWin64; } - static const MCPhysReg GPR64ArgRegs64Bit[] = { - X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 - }; + static const MCPhysReg GPR64ArgRegs64Bit[] = {X86::RDI, X86::RSI, X86::RDX, + X86::RCX, X86::R8, X86::R9}; return GPR64ArgRegs64Bit; } @@ -2115,10 +2333,9 @@ static ArrayRef get64BitArgumentXMMs(MachineFunction &MF, // registers. return {}; - static const MCPhysReg XMMArgRegs64Bit[] = { - X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, - X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 - }; + static const MCPhysReg XMMArgRegs64Bit[] = {X86::XMM0, X86::XMM1, X86::XMM2, + X86::XMM3, X86::XMM4, X86::XMM5, + X86::XMM6, X86::XMM7}; return XMMArgRegs64Bit; } @@ -2343,8 +2560,8 @@ SDValue X86TargetLowering::LowerFormalArguments( const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { if (goabi::isGoCallingConv(CallConv)) - return lowerX86GoFormalArguments(*this, Chain, DAG.getMachineFunction(), Ins, - dl, DAG, InVals); + return lowerX86GoFormalArguments(*this, Chain, DAG.getMachineFunction(), + Ins, dl, DAG, InVals); MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo(); @@ -2651,11 +2868,11 @@ static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl) { // Store the return address to the appropriate stack slot. - if (!FPDiff) return Chain; + if (!FPDiff) + return Chain; // Calculate the new stack slot for the return address. - int NewReturnAddrFI = - MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize, - false); + int NewReturnAddrFI = MF.getFrameInfo().CreateFixedObject( + SlotSize, (int64_t)FPDiff - SlotSize, false); SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT); Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, MachinePointerInfo::getFixedStack( @@ -2718,9 +2935,8 @@ X86TargetLowering::ByValCopyKind X86TargetLowering::ByValNeedsCopyForTailCall( return CopyViaTemp; } -SDValue -X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl &InVals) const { +SDValue X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, + SmallVectorImpl &InVals) const { if (goabi::isGoCallingConv(CLI.CallConv)) return lowerX86GoCall(*this, CLI, InVals); SelectionDAG &DAG = CLI.DAG; @@ -2741,8 +2957,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool ShouldGuaranteeTCO = shouldGuaranteeTCO( CallConv, MF.getTarget().Options.GuaranteedTailCallOpt); X86MachineFunctionInfo *X86Info = MF.getInfo(); - bool HasNCSR = (CB && isa(CB) && - CB->hasFnAttr("no_caller_saved_registers")); + bool HasNCSR = + (CB && isa(CB) && CB->hasFnAttr("no_caller_saved_registers")); bool IsIndirectCall = (CB && isa(CB) && CB->isIndirectCall()); bool IsCFICall = IsIndirectCall && CLI.CFIType; const Module *M = MF.getFunction().getParent(); @@ -2976,8 +3192,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // Promote the value if needed. switch (VA.getLocInfo()) { - default: llvm_unreachable("Unknown loc info!"); - case CCValAssign::Full: break; + default: + llvm_unreachable("Unknown loc info!"); + case CCValAssign::Full: + break; case CCValAssign::SExt: Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); break; @@ -3042,10 +3260,18 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // shadow reg if callee is a varargs function. Register ShadowReg; switch (VA.getLocReg()) { - case X86::XMM0: ShadowReg = X86::RCX; break; - case X86::XMM1: ShadowReg = X86::RDX; break; - case X86::XMM2: ShadowReg = X86::R8; break; - case X86::XMM3: ShadowReg = X86::R9; break; + case X86::XMM0: + ShadowReg = X86::RCX; + break; + case X86::XMM1: + ShadowReg = X86::RDX; + break; + case X86::XMM2: + ShadowReg = X86::R8; + break; + case X86::XMM3: + ShadowReg = X86::R9; + break; } if (ShadowReg) RegsToPass.push_back(std::make_pair(ShadowReg, Arg)); @@ -3055,8 +3281,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (!StackPtr.getNode()) StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(), getPointerTy(DAG.getDataLayout())); - MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, - dl, DAG, VA, Flags, isByVal)); + MemOpChains.push_back( + LowerMemOpCallTo(Chain, StackPtr, Arg, dl, DAG, VA, Flags, isByVal)); } } @@ -3072,9 +3298,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // go through the PLT and do not need EBX to hold the GOT base. if ((Callee->getOpcode() == ISD::GlobalAddress || Callee->getOpcode() == ISD::ExternalSymbol)) - RegsToPass.push_back(std::make_pair( - Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), - getPointerTy(DAG.getDataLayout())))); + RegsToPass.push_back( + std::make_pair(Register(X86::EBX), + DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), + getPointerTy(DAG.getDataLayout())))); } else { // If we are tail calling and generating PIC/GOT style code load the // address of the callee into ECX. The value in ecx is used as target of @@ -3105,16 +3332,14 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // registers used and is in the range 0 - 8 inclusive. // Count the number of XMM registers allocated. - static const MCPhysReg XMMArgRegs[] = { - X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, - X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 - }; + static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, + X86::XMM3, X86::XMM4, X86::XMM5, + X86::XMM6, X86::XMM7}; unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs); - assert((Subtarget.hasSSE1() || !NumXMMRegs) - && "SSE registers cannot be used when SSE is disabled"); - RegsToPass.push_back(std::make_pair(Register(X86::AL), - DAG.getConstant(NumXMMRegs, dl, - MVT::i8))); + assert((Subtarget.hasSSE1() || !NumXMMRegs) && + "SSE registers cannot be used when SSE is disabled"); + RegsToPass.push_back(std::make_pair( + Register(X86::AL), DAG.getConstant(NumXMMRegs, dl, MVT::i8))); } if (isVarArg && IsMustTail) { @@ -3784,8 +4009,11 @@ bool X86TargetLowering::isEligibleForSiblingCallOpt( continue; Register Reg = VA.getLocReg(); switch (Reg) { - default: break; - case X86::EAX: case X86::EDX: case X86::ECX: + default: + break; + case X86::EAX: + case X86::EDX: + case X86::ECX: if (++NumInRegs == MaxInRegs) return false; break; diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h index a5d182aba7f8b..380f363410696 100644 --- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h +++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h @@ -203,6 +203,7 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { DenseMap PreallocatedIds; SmallVector PreallocatedStackSizes; SmallVector, 0> PreallocatedArgOffsets; + SmallVector PreallocatedUsesReservedCallFrame; // True if a function clobbers FP/BP according to its calling convention. bool FPClobberedByCall = false; @@ -366,6 +367,7 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { if (Insert.second) { PreallocatedStackSizes.push_back(0); PreallocatedArgOffsets.emplace_back(); + PreallocatedUsesReservedCallFrame.push_back(false); } return Insert.first->second; } @@ -388,6 +390,13 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { return PreallocatedArgOffsets[Id]; } + void setPreallocatedUsesReservedCallFrame(size_t Id) { + PreallocatedUsesReservedCallFrame[Id] = true; + } + bool preallocatedUsesReservedCallFrame(size_t Id) const { + return PreallocatedUsesReservedCallFrame[Id]; + } + bool getFPClobberedByCall() const { return FPClobberedByCall; } void setFPClobberedByCall(bool C) { FPClobberedByCall = C; } diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp index 4736dca6ba57d..cc59c317d5c54 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp @@ -400,6 +400,8 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy( MachinePointerInfo SrcPtrInfo) const { const X86Subtarget &Subtarget = DAG.getMachineFunction().getSubtarget(); + const bool IsGo = goabi::isGoCallingConv( + DAG.getMachineFunction().getFunction().getCallingConv()); // If to a segment-relative address space, use the default lowering. if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256) @@ -427,7 +429,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy( return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Size.getValueType(), Alignment, isVolatile, - AlwaysInline, DstPtrInfo, SrcPtrInfo); + AlwaysInline || IsGo, DstPtrInfo, SrcPtrInfo); } return SDValue(); diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 4c33848dacf51..c4d35831ccaef 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -285,7 +285,8 @@ static bool definedInRegion(const SetVector &Blocks, Value *V) { /// function being code extracted, but not in the region being extracted. /// These values must be passed in as live-ins to the function. static bool definedInCaller(const SetVector &Blocks, Value *V) { - if (isa(V)) return true; + if (isa(V)) + return true; if (Instruction *I = dyn_cast(V)) if (!Blocks.count(I->getParent())) return true; @@ -725,7 +726,8 @@ void CodeExtractor::severSplitPHINodesOfEntry(BasicBlock *&Header) { if (Header != &Header->getParent()->getEntryBlock()) { PHINode *PN = dyn_cast(Header->begin()); - if (!PN) return; // No PHI nodes. + if (!PN) + return; // No PHI nodes. // If the header node contains any PHI nodes, check to see if there is more // than one entry from outside the region. If so, we need to sever the @@ -766,8 +768,8 @@ void CodeExtractor::severSplitPHINodesOfEntry(BasicBlock *&Header) { TI->replaceUsesOfWith(OldPred, NewBB); } - // Okay, everything within the region is now branching to the right block, we - // just have to update the PHI nodes now, inserting PHI nodes into NewBB. + // Okay, everything within the region is now branching to the right block, + // we just have to update the PHI nodes now, inserting PHI nodes into NewBB. BasicBlock::iterator AfterPHIs; for (AfterPHIs = OldPred->begin(); isa(AfterPHIs); ++AfterPHIs) { PHINode *PN = cast(AfterPHIs); @@ -1046,6 +1048,7 @@ Function *CodeExtractor::constructFunctionDeclaration( case Attribute::ZExt: case Attribute::ImmArg: case Attribute::ByRef: + case Attribute::GoRet: case Attribute::WriteOnly: case Attribute::Writable: case Attribute::DeadOnUnwind: diff --git a/llvm/test/Assembler/goret-attribute.ll b/llvm/test/Assembler/goret-attribute.ll new file mode 100644 index 0000000000000..94fb1e18f48a6 --- /dev/null +++ b/llvm/test/Assembler/goret-attribute.ll @@ -0,0 +1,27 @@ +; RUN: llvm-as < %s | llvm-dis | FileCheck %s + +%stack.result = type { ptr, i64 } + +; CHECK: define goabiinternal { i64, i64 } @roundtrip( +; CHECK-SAME: ptr goret(%stack.result) align 8 "goretindex"="1" %result1, +; CHECK-SAME: ptr goret(i32) align 4 "goretindex"="3" %result3) +define goabiinternal { i64, i64 } @roundtrip( + ptr goret(%stack.result) "goretindex"="1" align 8 %result1, + ptr goret(i32) "goretindex"="3" align 4 %result3) #0 { + ret { i64, i64 } zeroinitializer +} + +; CHECK: call goabiinternal { i64, i64 } @roundtrip( +; CHECK-SAME: ptr goret(%stack.result) align 8 "goretindex"="1" %result1, +; CHECK-SAME: ptr goret(i32) align 4 "goretindex"="3" %result3) +define goabiinternal void @roundtrip_callsite() { +entry: + %result1 = alloca %stack.result, align 8 + %result3 = alloca i32, align 4 + %direct = call goabiinternal { i64, i64 } @roundtrip( + ptr goret(%stack.result) "goretindex"="1" align 8 %result1, + ptr goret(i32) "goretindex"="3" align 4 %result3) #0 + ret void +} + +attributes #0 = { "go_results_tuple" } diff --git a/llvm/test/CodeGen/AArch64/go-argument-homes.ll b/llvm/test/CodeGen/AArch64/go-argument-homes.ll new file mode 100644 index 0000000000000..797be356f46fc --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-argument-homes.ll @@ -0,0 +1,27 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 < %s | \ +; RUN: FileCheck %s --check-prefix=ASM + +%pair = type { i64, i64 } + +; Exhaust AArch64's integer register budget so the pair arrives in one typed +; stack home at the target's Go stack bias. The carrier is bound directly to +; that home and needs no frontend alloca or copy. +define goabiinternal ptr @aarch64_stack_pair_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, + i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, + ptr preallocated(%pair) align 8 %value.home) { +entry: + ret ptr %value.home +} + +; MIR-LABEL: name: aarch64_stack_pair_home +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 8, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +; MIR-NOT: STRXui {{.*}}%fixed-stack.0 +; MIR: ADDXri %fixed-stack.0 +; ASM-LABEL: aarch64_stack_pair_home: +; ASM: add x0, sp, #8 diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index d41426c27aa01..1fa6bd7a6b1fd 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -1,6 +1,9 @@ ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 < %s | FileCheck %s --check-prefix=A64 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 < %s | FileCheck %s --check-prefix=A64-O2 +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) + define goabiinternal i64 @second_int(i64 %a, i64 %b) { ; A64-LABEL: second_int: ; A64: mov x0, x1 @@ -53,28 +56,42 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @abi0_second_int(i64 %a, i64 %b) { +define goabi0 i64 @abi0_second_int( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) { ; A64-LABEL: abi0_second_int: ; Go's arm64 stack ABI reserves 0(SP) for the return PC. ; A64: ldr x[[REG:[0-9]+]], [sp, #16] ; A64: str x[[REG]], [sp, #24] ; A64: ret entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } define goabi0 i64 @abi0_call_second_int() { ; A64-LABEL: abi0_call_second_int: -; A64: mov x[[BASE:[0-9]+]], sp -; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #8] -; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #16] +; A64-DAG: add x[[AHOME:[0-9]+]], sp, #8 +; A64-DAG: add x[[BHOME:[0-9]+]], sp, #16 +; A64-DAG: str x{{[0-9]+}}, [x[[AHOME]]] +; A64-DAG: str x{{[0-9]+}}, [x[[BHOME]]] ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] ; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. ; A64: str x[[RET]], [sp, #56] entry: - %ret = call goabi0 i64 @abi0_second_int(i64 11, i64 22) + %setup = call token @llvm.call.preallocated.setup(i32 2) + %a.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(i64) + %b.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated(i64) + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @abi0_second_int( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) + ["preallocated"(token %setup)] ret i64 %ret } @@ -111,30 +128,38 @@ define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { + ptr preallocated(%pair) align 8 %value.home) { ; A64-LABEL: stack_pair: ; A64: ldr x0, [sp, #16] ; A64: ret entry: + %value = load %pair, ptr %value.home, align 8 %right = extractvalue %pair %value, 1 ret i64 %right } define goabiinternal i64 @call_stack_pair() { ; A64-LABEL: call_stack_pair: -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #16] +; A64: add x[[BASE:[0-9]+]], sp, #8 +; A64-DAG: str x{{[0-9]+}}, [x[[BASE]]] +; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #8] ; A64: bl stack_pair entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%pair) + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, - %pair { i64 13, i64 17 }) + ptr preallocated(%pair) align 8 %value.home) + ["preallocated"(token %setup)] ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr preallocated([8 x i8]) align 1 %value.home) { ; A64-LABEL: stack_bytes: ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #8] ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #15] @@ -142,13 +167,15 @@ define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { ; A64-DAG: strb w{{[0-9]+}}, [sp, #23] ; A64: ret entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; A64-LABEL: call_stack_bytes: -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #15] +; A64: add x[[BASE:[0-9]+]], sp, #8 +; A64-DAG: strb w{{[0-9]+}}, [x[[BASE]]] +; A64-DAG: strb w{{[0-9]+}}, [x[[BASE]], #7] ; A64: bl stack_bytes ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #16] ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #23] @@ -158,8 +185,14 @@ define goabiinternal i16 @call_stack_bytes() { ; A64-O2: ldrb w{{[0-9]+}}, [sp, #16] ; A64-O2: add sp, sp, #{{[0-9]+}} entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([8 x i8]) + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr preallocated([8 x i8]) align 1 %value.home) + ["preallocated"(token %setup)] %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll b/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll new file mode 100644 index 0000000000000..2a166dee09efb --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll @@ -0,0 +1,100 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] +%go.abi.pad = type { i8 } +%go_memory = type { i64, [0 x i64], [2 x [0 x i64]], %go.abi.pad } + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1 immarg) +declare goabiinternal void @consume(ptr preallocated(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +; CHECK: bl consume +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%large) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 320000, i1 false) + call goabiinternal void @consume(ptr preallocated(%large) align 8 %home) + ["preallocated"(token %setup)] + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr preallocated(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: ldr x0, [sp, #8] +; CHECK-NEXT: ret +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +; The Go ABI rejects arrays with length greater than one from register +; decomposition even when their element type is zero-sized. The preallocated +; carrier is authoritative; reconstructing the ABI from the remaining LLVM +; leaves alone would incorrectly assign this value to x1. +define goabiinternal i64 @read_go_assigned_memory_argument( + i8 %head, ptr preallocated(%go_memory) align 8 %value, i64 %tail) { +; CHECK-LABEL: read_go_assigned_memory_argument: +; CHECK: ldr x8, [sp, #8] +; CHECK-NEXT: add x0, x8, x1 +; CHECK-NEXT: ret +entry: + %word = load i64, ptr %value, align 8 + %result = add i64 %word, %tail + ret i64 %result +} + +declare goabiinternal i64 @consume_go_assigned_memory_argument( + i8, ptr preallocated(%go_memory) align 8, i64) + +define goabiinternal i64 @pass_go_assigned_memory_argument(ptr %source) { +; CHECK-LABEL: pass_go_assigned_memory_argument: +; CHECK: add x8, sp, #8 +; CHECK: mov w0, #7 +; CHECK: mov w1, #11 +; CHECK: str q0, [x8] +; CHECK: bl consume_go_assigned_memory_argument +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%go_memory) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 16, i1 false) + %result = call goabiinternal i64 @consume_go_assigned_memory_argument( + i8 7, ptr preallocated(%go_memory) align 8 %home, i64 11) + ["preallocated"(token %setup)] + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) { +; CHECK-LABEL: write_memory_result: +; CHECK: str x{{[0-9]+}}, [sp, #8] +; CHECK-NEXT: ret +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: bl write_memory_result +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) + %value = load i64, ptr %result, align 8 + ret i64 %value +} diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll index 00961b87513ae..83c5813205152 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll @@ -11,8 +11,9 @@ define goabiinternal ptr addrspace(1) @scalar_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -27,8 +28,9 @@ define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + ptr preallocated(%aggregate) align 8 %value.home) gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -45,13 +47,38 @@ entry: ret ptr addrspace(1) %result } +; A preallocated carrier is writable. Once the home is modified, the earlier +; loaded SSA value must use a distinct relocation slot. +define goabiinternal ptr addrspace(1) @mutable_stack_arg( + ptr addrspace(1) %replacement, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, + i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, + i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %value.home) + gc "statepoint-example" { +entry: + %original = load ptr addrspace(1), ptr %value.home, align 8 + store ptr addrspace(1) %replacement, ptr %value.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %original) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + define goabiinternal ptr addrspace(1) @merged_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -69,8 +96,11 @@ define goabiinternal ptr addrspace(1) @relocated_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -115,6 +145,7 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK: fixedStack: ; CHECK: - { id: 0, type: default, offset: 8, size: 8, ; CHECK: isImmutable: false +; A read-only scalar remains in its exact typed incoming home. ; CHECK: stack: [] ; CHECK: STATEPOINT 1, ; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.0, 0, @@ -123,13 +154,26 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 ; CHECK-LABEL: name: aggregate_stack_arg +; The aggregate remains one typed fixed incoming object. Its two read-only +; pointer fields use narrow fixed views of that object. +; CHECK: fixedStack: +; CHECK: - { id: 2, type: default, offset: 8, size: 24, ; CHECK: stack: [] ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1) +; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.1, 0, 1, 8, %fixed-stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1), +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 + +; CHECK-LABEL: name: mutable_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: STRXui {{.*}}, %fixed-stack.0, 0 +; CHECK: STRXui {{.*}}, %stack.0, 0 +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll index 0c68fc30db9ea..4ad54d0586df5 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll @@ -6,8 +6,12 @@ i64, i64, i64, i64, i64, i64, i64, ptr, ptr, ptr } +%memory_result = type [722 x i64] declare goabiinternal %results @overflow_results() #0 +declare goabiinternal void @two_memory_results( + ptr goret(%memory_result) "goretindex"="0" align 8, + ptr goret(%memory_result) "goretindex"="1" align 8) define void @statepoint_with_register_and_stack_results() gc "statepoint-example" { @@ -41,6 +45,33 @@ entry: ; CHECK-NEXT: bl _use declare void @use(ptr) +declare void @use_i64(i64) + +define goabiinternal void @statepoint_with_two_memory_results() + gc "statepoint-example" { +entry: + %first = alloca %memory_result, align 8 + %second = alloca %memory_result, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, + ptr elementtype(void (ptr, ptr)) @two_memory_results, + i32 2, i32 0, + ptr goret(%memory_result) "goretindex"="0" align 8 %first, + ptr goret(%memory_result) "goretindex"="1" align 8 %second, + i32 0, i32 0) + %first_value = load i64, ptr %first, align 8 + %second_value = load i64, ptr %second, align 8 + call void @use_i64(i64 %first_value) + call void @use_i64(i64 %second_value) + ret void +} + +; CHECK-LABEL: _statepoint_with_two_memory_results: +; CHECK: bl _two_memory_results +; CHECK: bl _use_i64 +; CHECK: bl _use_i64 + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare %results @llvm.experimental.gc.result.results(token) diff --git a/llvm/test/CodeGen/AArch64/goobj-abi.ll b/llvm/test/CodeGen/AArch64/goobj-abi.ll index 12803449afa58..6cc665717b7ec 100644 --- a/llvm/test/CodeGen/AArch64/goobj-abi.ll +++ b/llvm/test/CodeGen/AArch64/goobj-abi.ll @@ -11,8 +11,12 @@ entry: ret i64 %sum } -define goabi0 i64 @"stackadd"(i64 %a, i64 %b) #0 { +define goabi0 i64 @"stackadd"( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) #0 { entry: + %a = load i64, ptr %a.home, align 8 + %b = load i64, ptr %b.home, align 8 %sum = add i64 %a, %b ret i64 %sum } diff --git a/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll new file mode 100644 index 0000000000000..2f3f24ed45683 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll @@ -0,0 +1,37 @@ +; REQUIRES: aarch64-registered-target +; RUN: llc -filetype=obj < %s -o %t.o +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.o | FileCheck %s + +; Optimized Go IR can contain private lookup tables whose bytes are identical +; before fixups but whose relocations name different symbols. Their Go content +; hashes must include the relocation target identity, matching cmd/internal/obj, +; so the linker cannot fold the tables together. + +target triple = "aarch64-apple-darwin-goobj" + +@runtime.targetA = global i8 1, section ".rodata", !goobj.symbol.nonpackage !2 +@runtime.targetB = global i8 2, section ".rodata", !goobj.symbol.nonpackage !2 +@table.a = private constant [1 x ptr] [ptr @runtime.targetA], align 8 +@table.b = private constant [1 x ptr] [ptr @runtime.targetB], align 8 +@llvm.compiler.used = appending global [2 x ptr] [ptr @table.a, ptr @table.b], section "llvm.metadata" + +define ptr @p.lookup(i1 %which) !goobj.symbol.index !3 { +entry: + %table = select i1 %which, ptr @table.a, ptr @table.b + ret ptr %table +} + +!goobj.config = !{!0} +!0 = !{!"goallc.goobj", !"darwin", !"arm64", !"go1.27", !"GOARM64", !"v8.0", !"", !"p", !"0", !"0", !"0", !1} +!1 = !{!"regabiwrappers", !"regabiargs"} +!2 = !{i1 true} +!3 = !{i32 0} + +; CHECK: hasheddef 0: .Ltable.b abi=0 type=3 size=8 align=8 +; CHECK-NEXT: hasheddef 1: .Ltable.a abi=0 type=3 size=8 align=8 +; These are cmd/internal/obj's native SHA-256-based hashes for size 8, +; default rodata, one R_ADDR relocation, and each non-package target name. +; CHECK: hash 0: 0f368520f5468968253a6a8c2a0e2f05 +; CHECK-NEXT: hash 1: 05c6989a6d8dc052c2e96991d2f58be6 +; CHECK: reloc 2.{{[0-9]+}}: off=0 size=8 type=1 add=0 target=runtime.targetB +; CHECK: reloc 3.{{[0-9]+}}: off=0 size=8 type=1 add=0 target=runtime.targetA diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index 50ee4a27d6226..3a735546dba92 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -11,16 +11,16 @@ entry: ret void } -define goabiinternal i64 @large_home_offset([4096 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_offset( + ptr preallocated([4096 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg } -define goabiinternal i64 @large_home_boundary([4094 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_boundary( + ptr preallocated([4094 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll index 1bec5369e70a4..1a230bb5faa18 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll @@ -32,8 +32,9 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %p16) "frame-pointer"="non-leaf" { + ptr preallocated(ptr) align 8 %p16.home) "frame-pointer"="non-leaf" { entry: + %p16 = load ptr, ptr %p16.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 diff --git a/llvm/test/CodeGen/Generic/go-abi0-frame.ll b/llvm/test/CodeGen/Generic/go-abi0-frame.ll index fd7d23c8cbe9d..777556f787fc9 100644 --- a/llvm/test/CodeGen/Generic/go-abi0-frame.ll +++ b/llvm/test/CodeGen/Generic/go-abi0-frame.ll @@ -12,7 +12,9 @@ declare ptr @llvm.go.abi0.frame() ; The intrinsic denotes one mutable, aliased object spanning the two arguments ; and one result slot. It deliberately overlaps the per-value fixed objects. -define goabi0 ptr @abi0_frame(i64 %a, i64 %b) noinline { +define goabi0 ptr @abi0_frame( + ptr preallocated(i64) align 8 %a, + ptr preallocated(i64) align 8 %b) noinline { entry: %frame = call ptr @llvm.go.abi0.frame() ret ptr %frame diff --git a/llvm/test/CodeGen/Generic/go-argument-homes.ll b/llvm/test/CodeGen/Generic/go-argument-homes.ll index 67e7696baafdd..b66cfa5c6831d 100644 --- a/llvm/test/CodeGen/Generic/go-argument-homes.ll +++ b/llvm/test/CodeGen/Generic/go-argument-homes.ll @@ -9,7 +9,6 @@ ; RUN: FileCheck %s --check-prefix=A64-ASM %padded = type { i8, i64 } -%pair = type { i64, i64 } ; A register argument's canonical alloca is remapped to its ABI spill home and ; the IR store initializes that home. @@ -66,48 +65,3 @@ entry: ; A64-ASM-DAG: strb w0, [sp, #8] ; A64-ASM-DAG: str x1, [sp, #16] ; A64-ASM: add x0, sp, #8 - -; Exhaust X86's integer register budget so the pair arrives in one stack home. -define goabiinternal ptr @x86_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; X86-LABEL: name: x86_stack_pair_home -; X86: fixedStack: -; X86-NEXT: - { id: 0, type: default, offset: 0, size: 16 -; X86-NEXT: isImmutable: false, isAliased: true -; X86: MOV64rm %fixed-stack.0 -; X86: MOV64rm %fixed-stack.0{{.*}}8 -; X86-NOT: MOV64mr %fixed-stack.0 -; X86: LEA64r %fixed-stack.0 -; X86-ASM-LABEL: x86_stack_pair_home: -; X86-ASM-DAG: movq 8(%rsp), %rax -; X86-ASM-DAG: movq 16(%rsp), %rax -; X86-ASM: leaq 8(%rsp), %rax - -; Exhaust AArch64's integer register budget so the pair arrives in one stack -; home at the target's Go stack bias. -define goabiinternal ptr @aarch64_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; A64-LABEL: name: aarch64_stack_pair_home -; A64: fixedStack: -; A64-NEXT: - { id: 0, type: default, offset: 8, size: 16 -; A64-NEXT: isImmutable: false, isAliased: true -; A64-NOT: STRXui {{.*}}%fixed-stack.0 -; A64: ADDXri %fixed-stack.0 -; A64-ASM-LABEL: aarch64_stack_pair_home: -; A64-ASM: add x0, sp, #8 diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll index 428cdd6055b65..010502bdce050 100644 --- a/llvm/test/CodeGen/Generic/goobj-nosplit.ll +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -7,7 +7,10 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s declare goabiinternal void @callee(ptr) -declare goabi0 void @"callee.abi0"(ptr) +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare goabi0 void @"callee.abi0"( + ptr preallocated(ptr) align 8) define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { entry: @@ -32,7 +35,13 @@ entry: %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 store volatile ptr @callee, ptr %code, align 8 store volatile ptr %pointer, ptr %context, align 8 - call goabi0 void @"callee.abi0"(ptr %closure) + %setup = call token @llvm.call.preallocated.setup(i32 1) + %argument = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(ptr) + store ptr %closure, ptr %argument, align 8 + call goabi0 void @"callee.abi0"( + ptr preallocated(ptr) align 8 %argument) + ["preallocated"(token %setup)] ret void } diff --git a/llvm/test/CodeGen/X86/go-argument-homes.ll b/llvm/test/CodeGen/X86/go-argument-homes.ll new file mode 100644 index 0000000000000..d7abbc70b8951 --- /dev/null +++ b/llvm/test/CodeGen/X86/go-argument-homes.ll @@ -0,0 +1,26 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 < %s | \ +; RUN: FileCheck %s --check-prefix=ASM + +%pair = type { i64, i64 } + +; Exhaust X86's integer register budget so the pair arrives in one typed stack +; home. The preallocated carrier is the home itself; no frontend alloca or copy +; exists. +define goabiinternal ptr @x86_stack_pair_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(%pair) align 8 %value.home) { +entry: + ret ptr %value.home +} + +; MIR-LABEL: name: x86_stack_pair_home +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 0, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +; MIR-NOT: MOV64mr %fixed-stack.0 +; MIR: LEA64r %fixed-stack.0 +; ASM-LABEL: x86_stack_pair_home: +; ASM: leaq 8(%rsp), %rax diff --git a/llvm/test/CodeGen/X86/go-callconv.ll b/llvm/test/CodeGen/X86/go-callconv.ll index 226f7007968ad..237e112017122 100644 --- a/llvm/test/CodeGen/X86/go-callconv.ll +++ b/llvm/test/CodeGen/X86/go-callconv.ll @@ -2,6 +2,9 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 < %s | FileCheck %s --check-prefix=X86-O2 ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -O0 -filetype=null < %s +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) + define goabiinternal i64 @second_int(i64 %a, i64 %b) { ; X86-LABEL: second_int: ; X86: movq %rbx, %rax @@ -61,25 +64,40 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @"abi0_second_int"(i64 %a, i64 %b) { +define goabi0 i64 @"abi0_second_int"( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) { ; X86-LABEL: "abi0_second_int": -; X86: movq 16(%rsp), %rax -; X86: movq %rax, 24(%rsp) +; X86: leaq 16(%rsp), %[[BHOME:r[a-z0-9]+]] +; X86: movq (%[[BHOME]]), %[[B:r[a-z0-9]+]] +; X86: movq %[[B]], 24(%rsp) ; X86: retq entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } define goabi0 i64 @"abi0_call_second_int"() { ; X86-LABEL: "abi0_call_second_int": -; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86: movq $22, 8(%[[BASE]]) -; X86: movq $11, (%[[BASE]]) +; X86-DAG: leaq (%rsp), %[[AHOME:r[a-z0-9]+]] +; X86-DAG: leaq 8(%rsp), %[[BHOME:r[a-z0-9]+]] +; X86-DAG: movq $11, (%[[AHOME]]) +; X86-DAG: movq $22, (%[[BHOME]]) ; X86: callq "abi0_second_int" ; X86: movq %rsp, %[[RELOAD:r[a-z0-9]+]] ; X86: movq 16(%[[RELOAD]]), %rax entry: - %ret = call goabi0 i64 @"abi0_second_int"(i64 11, i64 22) + %setup = call token @llvm.call.preallocated.setup(i32 2) + %a.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(i64) + %b.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated(i64) + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @"abi0_second_int"( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) + ["preallocated"(token %setup)] ret i64 %ret } @@ -125,9 +143,19 @@ define goabiinternal i64 @call_method_results(ptr %callee, ptr %recv, ; X86-O2: addq 56(%rsp), %rax ; X86-O2: addq ${{[0-9]+}}, %rsp entry: + %setup = call token @llvm.call.preallocated.setup(i32 2) + %integers.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([2 x i64]) + %floats.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated([2 x double]) + store [2 x i64] [i64 456, i64 789], ptr %integers.home, align 8 + store [2 x double] [double 3.4, double 5.6], ptr %floats.home, align 8 %result = call goabiinternal %method.results %callee( - ptr %recv, i64 123, [2 x i64] [i64 456, i64 789], - double 1.2, [2 x double] [double 3.4, double 5.6], ptr nest %ctxt) #0 + ptr %recv, i64 123, + ptr preallocated([2 x i64]) align 8 %integers.home, + double 1.2, + ptr preallocated([2 x double]) align 8 %floats.home, + ptr nest %ctxt) #0 ["preallocated"(token %setup)] %s = extractvalue %method.results %result, 0 %a = extractvalue %method.results %result, 1 %x = extractvalue %method.results %result, 2 @@ -148,13 +176,16 @@ entry: define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(%pair) align 8 %value.home) { ; X86-LABEL: stack_pair: -; X86-DAG: movq 8(%rsp), %[[LEFT:r[a-z0-9]+]] -; X86-DAG: movq 16(%rsp), %[[RIGHT:r[a-z0-9]+]] +; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] +; X86-DAG: movq (%[[HOME]]), %[[LEFT:r[a-z0-9]+]] +; X86-DAG: movq 8(%[[HOME]]), %[[RIGHT:r[a-z0-9]+]] ; X86: addq %[[RIGHT]], %[[LEFT]] ; X86: retq entry: + %value = load %pair, ptr %value.home, align 8 %left = extractvalue %pair %value, 0 %right = extractvalue %pair %value, 1 %sum = add i64 %left, %right @@ -163,39 +194,55 @@ entry: define goabiinternal i64 @call_stack_pair() { ; X86-LABEL: call_stack_pair: -; X86-DAG: movq $13, (%[[BASE:r[a-z0-9]+]]) +; X86: leaq (%rsp), %[[BASE:r[a-z0-9]+]] +; X86-DAG: movq $13, (%[[BASE]]) ; X86-DAG: movq $17, 8(%[[BASE]]) ; X86: callq stack_pair entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%pair) + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, - %pair { i64 13, i64 17 }) + ptr preallocated(%pair) align 8 %value.home) + ["preallocated"(token %setup)] ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr preallocated([8 x i8]) align 1 %value.home) { ; X86-LABEL: stack_bytes: -; X86-DAG: movb 8(%rsp), %{{[a-z0-9]+}} -; X86-DAG: movb 15(%rsp), %{{[a-z0-9]+}} +; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] +; X86-DAG: movb (%[[HOME]]), %{{[a-z0-9]+}} +; X86-DAG: movb 7(%[[HOME]]), %{{[a-z0-9]+}} ; X86-DAG: movb %{{[a-z0-9]+}}, 16(%rsp) ; X86-DAG: movb %{{[a-z0-9]+}}, 23(%rsp) ; X86: retq entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; X86-LABEL: call_stack_bytes: -; X86-DAG: movb $1, (%[[BASE:r[a-z0-9]+]]) +; X86: leaq (%rsp), %[[BASE:r[a-z0-9]+]] +; X86-DAG: movb $1, (%[[BASE]]) ; X86-DAG: movb $8, 7(%[[BASE]]) ; X86: callq stack_bytes ; X86: movq %rsp, %[[RESULT_BASE:r[a-z0-9]+]] ; X86-DAG: movzbl 8(%[[RESULT_BASE]]), %{{[a-z0-9]+}} ; X86-DAG: movzbl 15(%[[RESULT_BASE]]), %{{[a-z0-9]+}} entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([8 x i8]) + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr preallocated([8 x i8]) align 1 %value.home) + ["preallocated"(token %setup)] %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/X86/go-memset-inline.ll b/llvm/test/CodeGen/X86/go-memset-inline.ll index 378aec5f18ece..821e6c9cd6d64 100644 --- a/llvm/test/CodeGen/X86/go-memset-inline.ll +++ b/llvm/test/CodeGen/X86/go-memset-inline.ll @@ -4,20 +4,26 @@ declare void @llvm.memset.p0.i64(ptr writeonly, i8, i64, i1 immarg) define goabiinternal void @constant_memset(ptr %dst) { ; CHECK-LABEL: constant_memset: -; CHECK: rep -; CHECK-SAME: stos ; CHECK-NOT: callq memset +; CHECK: movb $0, +; CHECK: cmpq $800, +; CHECK: jb ; CHECK: retq call void @llvm.memset.p0.i64(ptr align 8 %dst, i8 0, i64 800, i1 false) ret void } -define goabi0 void @"dynamic_memset"(ptr %dst, i64 %size) { +define goabi0 void @"dynamic_memset"( + ptr preallocated(ptr) align 8 %dst.home, + ptr preallocated(i64) align 8 %size.home) { ; CHECK-LABEL: "dynamic_memset": -; CHECK: rep -; CHECK-SAME: stosb ; CHECK-NOT: callq memset +; CHECK: movb $0, +; CHECK: cmpq +; CHECK: jb ; CHECK: retq + %dst = load ptr, ptr %dst.home, align 8 + %size = load i64, ptr %size.home, align 8 call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 %size, i1 false) ret void } diff --git a/llvm/test/CodeGen/X86/go-stack-preallocated.ll b/llvm/test/CodeGen/X86/go-stack-preallocated.ll new file mode 100644 index 0000000000000..c204f28ca96fc --- /dev/null +++ b/llvm/test/CodeGen/X86/go-stack-preallocated.ll @@ -0,0 +1,96 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] +%go.abi.pad = type { i8 } +%go_memory = type { i64, [0 x i64], [2 x [0 x i64]], %go.abi.pad } + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1 immarg) +declare goabiinternal void @consume(ptr preallocated(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: rep;movsq +; CHECK: callq consume +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%large) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 320000, i1 false) + call goabiinternal void @consume(ptr preallocated(%large) align 8 %home) + ["preallocated"(token %setup)] + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr preallocated(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: movq 8(%rsp), %rax +; CHECK-NEXT: retq +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +; The Go ABI rejects arrays with length greater than one from register +; decomposition even when their element type is zero-sized. The preallocated +; carrier is authoritative; reconstructing the ABI from the remaining LLVM +; leaves alone would incorrectly assign this value to RBX. +define goabiinternal i64 @read_go_assigned_memory_argument( + i8 %head, ptr preallocated(%go_memory) align 8 %value, i64 %tail) { +; CHECK-LABEL: read_go_assigned_memory_argument: +; CHECK: movq %rbx, %rax +; CHECK-NEXT: addq 8(%rsp), %rax +; CHECK-NEXT: retq +entry: + %word = load i64, ptr %value, align 8 + %result = add i64 %word, %tail + ret i64 %result +} + +declare goabiinternal i64 @consume_go_assigned_memory_argument( + i8, ptr preallocated(%go_memory) align 8, i64) + +define goabiinternal i64 @pass_go_assigned_memory_argument(ptr %source) { +; CHECK-LABEL: pass_go_assigned_memory_argument: +; CHECK: leaq (%rsp), %rcx +; CHECK: movq {{.*}}, (%rcx) +; CHECK: movl $11, %ebx +; CHECK: movb $7, %al +; CHECK: callq consume_go_assigned_memory_argument +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%go_memory) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 16, i1 false) + %result = call goabiinternal i64 @consume_go_assigned_memory_argument( + i8 7, ptr preallocated(%go_memory) align 8 %home, i64 11) + ["preallocated"(token %setup)] + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) { +; CHECK-LABEL: write_memory_result: +; CHECK: movq $42, 8(%rsp) +; CHECK-NEXT: retq +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: callq write_memory_result +; CHECK: rep;movsq +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) + %value = load i64, ptr %result, align 8 + ret i64 %value +} diff --git a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll index 17a4bcf33b224..4fa58509c74b3 100644 --- a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll @@ -9,10 +9,18 @@ declare goabiinternal void @safepoint() define goabiinternal ptr addrspace(1) @scalar_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home) + gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -25,10 +33,17 @@ entry: define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(%aggregate) align 8 %value.home) + gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -45,13 +60,43 @@ entry: ret ptr addrspace(1) %result } +; A preallocated carrier is writable. Once the home is modified, the earlier +; loaded SSA value must use a distinct relocation slot. +define goabiinternal ptr addrspace(1) @mutable_stack_arg( + ptr addrspace(1) %replacement, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(ptr addrspace(1)) align 8 %value.home) + gc "statepoint-example" { +entry: + %original = load ptr addrspace(1), ptr %value.home, align 8 + store ptr addrspace(1) %replacement, ptr %value.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %original) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + define goabiinternal ptr addrspace(1) @merged_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -67,10 +112,19 @@ entry: define goabiinternal ptr addrspace(1) @relocated_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -125,11 +179,20 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-LABEL: name: aggregate_stack_arg ; CHECK: stack: [] ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0{{[^)]*}}), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1{{[^)]*}}) +; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.1, 0, 1, 8, %fixed-stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1{{[^)]*}}), +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0{{[^)]*}}) ; CHECK-NEXT: ADJCALLSTACKUP64 -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0 + +; CHECK-LABEL: name: mutable_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: MOV64mr %fixed-stack.0, +; CHECK: MOV64mr %stack.0, +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: diff --git a/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll b/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll index 0b383ba9a4858..793e45632bbb9 100644 --- a/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll +++ b/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll @@ -5,8 +5,12 @@ ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr } +%memory_result = type [722 x i64] declare goabiinternal %results @overflow_results() #0 +declare goabiinternal void @two_memory_results( + ptr goret(%memory_result) "goretindex"="0" align 8, + ptr goret(%memory_result) "goretindex"="1" align 8) define void @statepoint_with_register_and_stack_results() gc "statepoint-example" { @@ -33,6 +37,34 @@ entry: ; CHECK-DAG: movq 8(%rsp), [[STACK1:%r[a-z0-9]+]] declare void @use(ptr) +declare void @use_i64(i64) + +define goabiinternal void @statepoint_with_two_memory_results() + gc "statepoint-example" { +entry: + %first = alloca %memory_result, align 8 + %second = alloca %memory_result, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, + ptr elementtype(void (ptr, ptr)) @two_memory_results, + i32 2, i32 0, + ptr goret(%memory_result) "goretindex"="0" align 8 %first, + ptr goret(%memory_result) "goretindex"="1" align 8 %second, + i32 0, i32 0) + %first_value = load i64, ptr %first, align 8 + %second_value = load i64, ptr %second, align 8 + call void @use_i64(i64 %first_value) + call void @use_i64(i64 %second_value) + ret void +} + +; CHECK-LABEL: statepoint_with_two_memory_results: +; CHECK: callq two_memory_results +; CHECK-COUNT-2: rep;movsq +; CHECK: callq use_i64 +; CHECK: callq use_i64 + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare %results @llvm.experimental.gc.result.results(token) diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index 0547d1a782189..d40120808f559 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -73,18 +73,16 @@ entry: ; An aggregate assigned wholly to the stack reuses its caller-populated slot; ; the lifetime marker must not make SelectionDAG allocate and copy a local. -define goabiinternal void @inactive_stack_aggregate(%stack_aggregate %value) +define goabiinternal void @inactive_stack_aggregate( + ptr preallocated(%stack_aggregate) align 8 %value.home) gc "statepoint-example" { entry: - %home = alloca %stack_aggregate, align 8 - call void @llvm.lifetime.start.p0(i64 16, ptr %home) - store %stack_aggregate %value, ptr %home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 3, i32 0, ptr elementtype(void ()) @safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, - i64 1095520067, i64 11, ptr %home, i64 0, i64 16, + i64 1095520067, i64 11, ptr %value.home, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 15) ] ret void diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index e5f7a24173352..1b912f4fefdd0 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -34,11 +34,13 @@ entry: ret i64 %sum } +%stack.args = type [23 x i64] + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) declare goabiinternal void @many_stack_args( - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64) + i64, i64, i64, i64, i64, i64, i64, i64, i64, + ptr preallocated(%stack.args) align 8) @condition = external global i1 ; X86 lowers the stack arguments below to push sequences instead of reserving @@ -50,11 +52,18 @@ entry: br i1 %cond, label %call, label %join call: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%stack.args) + store %stack.args [ + i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, + i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, + i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, + i64 27, i64 28, i64 29, i64 30, i64 31], ptr %home, align 8 call goabiinternal void @many_stack_args( - i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, - i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, - i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, - i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31) + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, + ptr preallocated(%stack.args) align 8 %home) + ["preallocated"(token %setup)] br label %join join: @@ -72,10 +81,10 @@ join: ; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1] ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack_noctxt ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack -; The outgoing frame is 72 bytes of alignment/register-argument space plus 23 -; 8-byte PUSHes. PCSP must reach 256 bytes at the call, return to zero before -; the CFG join, and thereby let funcMaxSPDelta size a grown Go stack correctly. -; CHECK: aux 2.{{[0-9]+}}: type=pcsp target= pc=[{{.*}}:256,{{[0-9]+}}-{{[0-9]+}}:0] +; The outgoing frame is 72 bytes of register-argument homes plus one 184-byte +; preallocated value. Go reserves it in the containing frame, so PCSP is stable +; across the call while funcMaxSPDelta still sees the full 256-byte frame. +; CHECK: aux 2.{{[0-9]+}}: type=pcsp target= pc=[0-[[PROLOG:[0-9]+]]:0,[[PROLOG]]-[[EPILOG:[0-9]+]]:256,[[EPILOG]]-{{[0-9]+}}:0] ; ASM-LABEL: big_frame: ; Match Go's single stack-check loop: the hot entry executes the check @@ -122,7 +131,8 @@ join: ; PEI-SAME: implicit $rsp, implicit $ssp, implicit $rdx ; PEI-LABEL: name: large_outgoing_frame -; PEI: stackSize: 0 +; PEI: stackSize: 256 +; PEI-NEXT: offsetAdjustment: -256 ; PEI: maxCallFrameSize: 256 ; PEI: $r12 = LEA64r $rsp, 1, $noreg, -128, $noreg ; PEI: CMP64rm $r12, $r14, 1, $noreg, 16, $noreg diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth.ll b/llvm/test/CodeGen/X86/goobj-stack-growth.ll index 691d27e10d43e..8b4b7e9e4ff52 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth.ll @@ -23,9 +23,15 @@ entry: ret i64 %value } -define goabi0 void @"abi0_pointer_arguments"(ptr %first, ptr %second, ptr %third) +define goabi0 void @"abi0_pointer_arguments"( + ptr preallocated(ptr) align 8 %first.home, + ptr preallocated(ptr) align 8 %second.home, + ptr preallocated(ptr) align 8 %third.home) "frame-pointer"="non-leaf" { entry: + %first = load ptr, ptr %first.home, align 8 + %second = load ptr, ptr %second.home, align 8 + %third = load ptr, ptr %third.home, align 8 call goabiinternal void @use_three_pointers( ptr %first, ptr %second, ptr %third) ret void @@ -52,10 +58,17 @@ entry: define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr) align 8 %pointer.home) { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1 @@ -64,10 +77,16 @@ entry: define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, %pointer.aggregate %value) + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(%pointer.aggregate) align 8 %value.home) "go_results_tuple" { entry: + %value = load %pointer.aggregate, ptr %value.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1 diff --git a/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll b/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll new file mode 100644 index 0000000000000..3be5f938c7b75 --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll @@ -0,0 +1,38 @@ +; RUN: opt -passes='default' -S %s | FileCheck %s + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @left_callee(ptr preallocated(i32)) +declare void @right_callee(ptr preallocated(i32)) + +; Each setup owns one physical outgoing call frame. SimplifyCFG must not merge +; identical setup/arg pairs from mutually exclusive branches and then let two +; calls consume the same token. +define void @branch_local_call_frames(i1 %condition) { +; CHECK-LABEL: define void @branch_local_call_frames( +; CHECK: %left.setup = {{(tail )?}}call token @llvm.call.preallocated.setup(i32 1) +; CHECK: call void @left_callee +; CHECK: %right.setup = {{(tail )?}}call token @llvm.call.preallocated.setup(i32 1) +; CHECK: call void @right_callee +entry: + br i1 %condition, label %left, label %right + +left: + %left.setup = call token @llvm.call.preallocated.setup(i32 1) + %left.home = call ptr @llvm.call.preallocated.arg(token %left.setup, i32 0) preallocated(i32) + store i32 1, ptr %left.home, align 4 + call void @left_callee(ptr preallocated(i32) %left.home) + ["preallocated"(token %left.setup)] + br label %exit + +right: + %right.setup = call token @llvm.call.preallocated.setup(i32 1) + %right.home = call ptr @llvm.call.preallocated.arg(token %right.setup, i32 0) preallocated(i32) + store i32 2, ptr %right.home, align 4 + call void @right_callee(ptr preallocated(i32) %right.home) + ["preallocated"(token %right.setup)] + br label %exit + +exit: + ret void +} diff --git a/llvm/test/Verifier/goret.ll b/llvm/test/Verifier/goret.ll new file mode 100644 index 0000000000000..ac2482002b045 --- /dev/null +++ b/llvm/test/Verifier/goret.ll @@ -0,0 +1,54 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s + +%opaque = type opaque + +; CHECK: Attributes 'goret' and 'goretindex' must be used together! +define goabiinternal void @missing_index(ptr goret(i64) %result) { + ret void +} + +; CHECK: Attributes 'goret' and 'goretindex' must be used together! +define goabiinternal void @missing_type(ptr "goretindex"="0" %result) { + ret void +} + +; CHECK: 'goret' is only valid with a Go calling convention +define void @wrong_cc(ptr goret(i64) "goretindex"="0" %result) { + ret void +} + +; CHECK: duplicate 'goretindex' +define goabiinternal void @duplicate( + ptr goret(i64) "goretindex"="0" %result0, + ptr goret(i64) "goretindex"="0" %result1) { + ret void +} + +; CHECK: 'goretindex' values must be in increasing parameter order +define goabiinternal void @unordered( + ptr goret(i64) "goretindex"="1" %result1, + ptr goret(i64) "goretindex"="0" %result0) { + ret void +} + +; CHECK: 'goretindex' is out of range +define goabiinternal void @out_of_range(ptr goret(i64) "goretindex"="1" %result) { + ret void +} + +; CHECK: Attribute 'goret' does not support unsized types! +define goabiinternal void @unsized(ptr goret(%opaque) "goretindex"="0" %result) { + ret void +} + +; CHECK: Attribute 'goret' is incompatible with other ABI parameter attributes! +define goabiinternal void @incompatible( + ptr byval(i64) goret(i64) "goretindex"="0" %result) { + ret void +} + +; CHECK: 'goretindex' must be an unsigned decimal integer +define goabiinternal void @invalid_index( + ptr goret(i64) "goretindex"="not-a-number" %result) { + ret void +}