diff --git a/.github/workflows/goallc-ci.yml b/.github/workflows/goallc-ci.yml index ce649abdd462f..eb3d2d0a0f33b 100644 --- a/.github/workflows/goallc-ci.yml +++ b/.github/workflows/goallc-ci.yml @@ -100,6 +100,7 @@ jobs: build="$RUNNER_TEMP/goallc-llvm-build" tests=( "$build/test/CodeGen/AArch64/go-callconv.ll" + "$build/test/CodeGen/AArch64/go-stack-byval.ll" "$build/test/CodeGen/AArch64/go-statepoint-stack-args.ll" "$build/test/CodeGen/AArch64/goobj-ir-config.ll" "$build/test/CodeGen/AArch64/goobj-register-argument-homes.ll" @@ -110,6 +111,7 @@ jobs: "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll" "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll" "$build/test/CodeGen/X86/go-callconv.ll" + "$build/test/CodeGen/X86/go-stack-byval.ll" "$build/test/CodeGen/X86/go-gc-write-barrier.ll" "$build/test/CodeGen/X86/go-statepoint-stack-results.ll" "$build/test/CodeGen/X86/goobj-filetype.ll" diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index a94c652859816..03f0639d5c046 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -106,12 +106,16 @@ SmallBitVector getPaddingPieces(Type *Ty); void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys); -CallLayout computeCallLayout(ArrayRef ArgTys, +/// Complete the Go ABI frame layout after the target calling-convention rules +/// have assigned every input to either registers or a stack offset and +/// computed the input stack extent. This helper does not classify inputs. +/// Result classification remains here until stack results have an explicit IR +/// carrier of their own. +CallLayout computeCallLayout(ArrayRef Args, uint64_t StackArgsSize, ArrayRef ResultTys, const DataLayout &DL, const ABIConfig &Config); -EntryArgsInfo computeEntryArgsInfo(ArrayRef ArgTys, - const CallLayout &Layout, +EntryArgsInfo computeEntryArgsInfo(const CallLayout &Layout, const DataLayout &DL, const ABIConfig &Config); diff --git a/llvm/include/llvm/CodeGen/GoISelLowering.h b/llvm/include/llvm/CodeGen/GoISelLowering.h new file mode 100644 index 0000000000000..0d3c96d224538 --- /dev/null +++ b/llvm/include/llvm/CodeGen/GoISelLowering.h @@ -0,0 +1,38 @@ +//===- GoISelLowering.h - Go SelectionDAG lowering helpers -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GOISELLOWERING_H +#define LLVM_CODEGEN_GOISELLOWERING_H + +#include "llvm/CodeGen/GoCallingConv.h" +#include "llvm/CodeGen/TargetLowering.h" + +namespace llvm { + +class CCValAssign; + +namespace goabi { + +/// Derive the logical Go function layout from LLVM formal arguments and the +/// physical locations assigned by the target calling convention. +CallLayout computeFormalArgLayout(const Function &F, + ArrayRef Ins, + ArrayRef ArgLocs, + uint64_t StackArgsSize, const DataLayout &DL, + const ABIConfig &Config); + +/// Derive the logical Go call layout from the original LLVM call operands and +/// the physical locations assigned by the target calling convention. +CallLayout computeCallLayout(TargetLowering::CallLoweringInfo &CLI, + ArrayRef ArgLocs, + uint64_t StackArgsSize, const ABIConfig &Config); + +} // namespace goabi +} // namespace llvm + +#endif // LLVM_CODEGEN_GOISELLOWERING_H diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h index 76f5e38a045ff..ea37112d267bd 100644 --- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -732,6 +732,8 @@ struct MachineFrameInfo { StringValue StackProtector; StringValue FunctionContext; unsigned MaxCallFrameSize = ~0u; ///< ~0u means: not computed yet. + uint64_t GoABIStackArgsSize = ~UINT64_C(0); + uint64_t GoABIArgSize = ~UINT64_C(0); unsigned CVBytesOfCalleeSavedRegisters = 0; bool HasOpaqueSPAdjustment = false; bool HasVAStart = false; @@ -755,6 +757,8 @@ struct MachineFrameInfo { StackProtector == Other.StackProtector && FunctionContext == Other.FunctionContext && MaxCallFrameSize == Other.MaxCallFrameSize && + GoABIStackArgsSize == Other.GoABIStackArgsSize && + GoABIArgSize == Other.GoABIArgSize && CVBytesOfCalleeSavedRegisters == Other.CVBytesOfCalleeSavedRegisters && HasOpaqueSPAdjustment == Other.HasOpaqueSPAdjustment && @@ -785,6 +789,9 @@ template <> struct MappingTraits { YamlIO.mapOptional("functionContext", MFI.FunctionContext, StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize, (unsigned)~0); + YamlIO.mapOptional("goABIStackArgsSize", MFI.GoABIStackArgsSize, + ~UINT64_C(0)); + YamlIO.mapOptional("goABIArgSize", MFI.GoABIArgSize, ~UINT64_C(0)); YamlIO.mapOptional("cvBytesOfCalleeSavedRegisters", MFI.CVBytesOfCalleeSavedRegisters, 0U); YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment, diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h index c11e6ed102806..c6c10a1cb39a2 100644 --- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -294,6 +294,13 @@ class MachineFrameInfo { /// It is only valid during and after prolog/epilog code insertion. uint64_t MaxCallFrameSize = ~UINT64_C(0); + /// Logical Go ABI input stack extent and complete argument/home area size. + /// Target formal-argument lowering records the values after CCState has + /// assigned every input. Late GoObj emission and return lowering consume the + /// cached result instead of reconstructing the calling convention from IR. + uint64_t GoABIStackArgsSize = ~UINT64_C(0); + uint64_t GoABIArgSize = ~UINT64_C(0); + /// The number of bytes of callee saved registers that the target wants to /// report for the current function in the CodeView S_FRAMEPROC record. unsigned CVBytesOfCalleeSavedRegisters = 0; @@ -703,6 +710,22 @@ class MachineFrameInfo { } void setMaxCallFrameSize(uint64_t S) { MaxCallFrameSize = S; } + bool hasGoABIArgSizes() const { + return GoABIStackArgsSize != ~UINT64_C(0) && GoABIArgSize != ~UINT64_C(0); + } + uint64_t getGoABIStackArgsSize() const { + assert(hasGoABIArgSizes() && "Go ABI argument sizes are not initialized"); + return GoABIStackArgsSize; + } + uint64_t getGoABIArgSize() const { + assert(hasGoABIArgSizes() && "Go ABI argument sizes are not initialized"); + return GoABIArgSize; + } + void setGoABIArgSizes(uint64_t StackArgsSize, uint64_t ArgSize) { + GoABIStackArgsSize = StackArgsSize; + GoABIArgSize = ArgSize; + } + /// Returns how many bytes of callee-saved registers the target pushed in the /// prologue. Only used for debug info. unsigned getCVBytesOfCalleeSavedRegisters() const { diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 88435712831e8..afe248a169f10 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -226,54 +226,14 @@ class AddrLabelMapCallbackPtr final : CallbackVH { void allUsesReplacedWith(Value *V2) override; }; -static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL, - const Triple &TT) { +static uint32_t getGoObjArgSize(const MachineFunction &MF) { + const Function &F = MF.getFunction(); if (!goabi::isGoCallingConv(F.getCallingConv())) return 0; - - goabi::ABIConfig Config; - if (TT.getArch() == Triple::x86_64) { - static constexpr unsigned X86GoIntRegs[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; - static constexpr unsigned X86GoFPRegs[] = {0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14}; - if (goabi::isGoABI0CallingConv(F.getCallingConv())) - Config = {ArrayRef(), - ArrayRef(), - 8, - Align(8), - Align(8), - false}; - else - Config = {X86GoIntRegs, X86GoFPRegs, 8, Align(8), Align(8), false}; - } else if (TT.getArch() == Triple::aarch64) { - static constexpr unsigned AArch64GoIntRegs[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - static constexpr unsigned AArch64GoFPRegs[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - if (goabi::isGoABI0CallingConv(F.getCallingConv())) - Config = {ArrayRef(), - ArrayRef(), - 8, - Align(8), - Align(16), - false}; - else - Config = {AArch64GoIntRegs, AArch64GoFPRegs, 8, - Align(8), Align(16), false}; - } else { - return 0; - } - - SmallVector ArgTys; - for (const Argument &Arg : F.args()) - if (!Arg.hasNestAttr()) - ArgTys.push_back(Arg.getType()); - - SmallVector ResultTys; - goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), - ResultTys); - uint64_t Size = - goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize; + const MachineFrameInfo &MFI = MF.getFrameInfo(); + if (!MFI.hasGoABIArgSizes()) + report_fatal_error("GoObj function is missing lowered Go ABI frame info"); + uint64_t Size = MFI.getGoABIArgSize(); if (Size > std::numeric_limits::max()) report_fatal_error("GoObj function argument size exceeds uint32 limit"); return static_cast(Size); @@ -1076,8 +1036,7 @@ static void collectGoObjModuleMetadata(AsmPrinter &AP, const Module &M) { continue; bool HasABI0Suffix = F.getName().ends_with(GoObj::ABI0SymbolSuffix); bool IsABI0 = goabi::isGoABI0CallingConv(F.getCallingConv()); - bool IsABIInternal = - goabi::isGoABIInternalCallingConv(F.getCallingConv()); + bool IsABIInternal = goabi::isGoABIInternalCallingConv(F.getCallingConv()); if (HasABI0Suffix != IsABI0) report_fatal_error( "Go ABI0 calling convention and symbol suffix disagree"); @@ -3766,9 +3725,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { report_fatal_error("GoObj function stack size exceeds uint32 limit"); OutContext.setGoObjSymbolStackSize(CurrentFnSym, static_cast(StackSize)); - OutContext.setGoObjSymbolArgSize( - CurrentFnSym, - getGoObjArgSize(F, MF.getDataLayout(), TM.getTargetTriple())); + OutContext.setGoObjSymbolArgSize(CurrentFnSym, getGoObjArgSize(MF)); OutContext.setGoObjSymbolHasFramePointer( CurrentFnSym, MF.getSubtarget().getFrameLowering()->hasFP(MF)); OutContext.setGoObjSymbolAsyncUnsafe(CurrentFnSym, diff --git a/llvm/lib/CodeGen/GoCallingConv.cpp b/llvm/lib/CodeGen/GoCallingConv.cpp index 73491bbe2fa58..44df16bf87d9b 100644 --- a/llvm/lib/CodeGen/GoCallingConv.cpp +++ b/llvm/lib/CodeGen/GoCallingConv.cpp @@ -245,28 +245,44 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, ResultTys.push_back(ReturnType); } -CallLayout computeCallLayout(ArrayRef ArgTys, +static uint64_t getDirectValueSize(Type *Ty, const DataLayout &DL) { + SmallBitVector PaddingPieces = getPaddingPieces(Ty); + if (PaddingPieces.any() && PaddingPieces.count() == PaddingPieces.size()) + return 0; + return DL.getTypeAllocSize(Ty); +} + +CallLayout computeCallLayout(ArrayRef Args, uint64_t StackArgsSize, ArrayRef ResultTys, const DataLayout &DL, const ABIConfig &Config) { CallLayout Layout; - Layout.Args.reserve(ArgTys.size()); + Layout.Args.append(Args.begin(), Args.end()); Layout.Results.reserve(ResultTys.size()); + Layout.StackArgsSize = StackArgsSize; + + for (ValueLayout &Arg : Layout.Args) { + if (!Arg.Ty) + report_fatal_error("Go ABI argument layout has no logical type"); + uint64_t ExpectedSize = Arg.InRegs ? getDirectValueSize(Arg.Ty, DL) + : DL.getTypeAllocSize(Arg.Ty); + Arg.Size = ExpectedSize; + Align ABIAlignment = DL.getABITypeAlign(Arg.Ty); + if (Arg.InRegs) { + Arg.Alignment = ABIAlignment; + continue; + } + if (Arg.Alignment < ABIAlignment) + report_fatal_error("invalid preassigned Go ABI argument layout"); + if (Arg.StackOffset % Arg.Alignment.value() != 0 || + Arg.StackOffset > StackArgsSize || + Arg.Size > StackArgsSize - Arg.StackOffset) + report_fatal_error( + "Go ABI stack argument is outside its assigned input area"); + } unsigned NextInt = 0; unsigned NextFP = 0; - uint64_t StackArgsEnd = 0; - for (Type *ArgTy : ArgTys) { - ValueLayout ArgLayout = - computeValueLayout(ArgTy, DL, Config, NextInt, NextFP); - if (!ArgLayout.InRegs) - StackArgsEnd = layoutStackValue(StackArgsEnd, ArgLayout); - Layout.Args.push_back(ArgLayout); - } - Layout.StackArgsSize = StackArgsEnd; - - NextInt = 0; - NextFP = 0; - uint64_t StackResultsEnd = alignToValue(StackArgsEnd, Config.PtrAlign); + uint64_t StackResultsEnd = alignToValue(StackArgsSize, Config.PtrAlign); uint64_t StackResultsStart = StackResultsEnd; for (Type *ResultTy : ResultTys) { ValueLayout ResultLayout = @@ -316,22 +332,18 @@ static void collectPointerOffsets(Type *Ty, uint64_t BaseOffset, report_fatal_error("Go entry argument maps do not support pointer vectors"); } -EntryArgsInfo computeEntryArgsInfo(ArrayRef ArgTys, - const CallLayout &Layout, +EntryArgsInfo computeEntryArgsInfo(const CallLayout &Layout, const DataLayout &DL, const ABIConfig &Config) { if (!Config.PtrSize || Layout.ArgSize % Config.PtrSize != 0 || Layout.ArgSize / Config.PtrSize > std::numeric_limits::max()) report_fatal_error("invalid Go entry argument map dimensions"); - if (ArgTys.size() != Layout.Args.size()) - report_fatal_error("Go entry argument types do not match ABI layout"); - EntryArgsInfo Info; Info.PointerSize = Config.PtrSize; Info.ArgSize = Layout.ArgSize; Info.NumBits = static_cast(Layout.ArgSize / Config.PtrSize); - SmallVector HomeOffsets(ArgTys.size()); + SmallVector HomeOffsets(Layout.Args.size()); uint64_t SpillOffset = Layout.SpillAreaOffset; for (auto [Index, ArgLayout] : llvm::enumerate(Layout.Args)) { if (ArgLayout.InRegs) { @@ -346,8 +358,8 @@ EntryArgsInfo computeEntryArgsInfo(ArrayRef ArgTys, report_fatal_error("Go entry argument homes do not match spill area"); SmallVector PointerOffsets; - for (auto [Index, ArgTy] : llvm::enumerate(ArgTys)) - collectPointerOffsets(ArgTy, HomeOffsets[Index], DL, PointerOffsets); + for (auto [Index, ArgLayout] : llvm::enumerate(Layout.Args)) + collectPointerOffsets(ArgLayout.Ty, HomeOffsets[Index], DL, PointerOffsets); llvm::sort(PointerOffsets); for (uint64_t Offset : PointerOffsets) { diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp index 6f1e7594f34da..dd5789b0865e2 100644 --- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -944,6 +944,14 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS, MFI.setFramePointerPolicy(YamlMFI.FramePointerPolicy); if (YamlMFI.MaxCallFrameSize != ~0u) MFI.setMaxCallFrameSize(YamlMFI.MaxCallFrameSize); + if (YamlMFI.GoABIStackArgsSize != ~UINT64_C(0) || + YamlMFI.GoABIArgSize != ~UINT64_C(0)) { + if (YamlMFI.GoABIStackArgsSize == ~UINT64_C(0) || + YamlMFI.GoABIArgSize == ~UINT64_C(0)) + return error(Twine("Go ABI frame info requires both input and total " + "argument sizes")); + MFI.setGoABIArgSizes(YamlMFI.GoABIStackArgsSize, YamlMFI.GoABIArgSize); + } MFI.setCVBytesOfCalleeSavedRegisters(YamlMFI.CVBytesOfCalleeSavedRegisters); MFI.setHasOpaqueSPAdjustment(YamlMFI.HasOpaqueSPAdjustment); MFI.setHasVAStart(YamlMFI.HasVAStart); diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 8acd6f14ebc2e..5ccfd07153bfe 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -374,6 +374,10 @@ static void convertMFI(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI, YamlMFI.FramePointerPolicy = MFI.getFramePointerPolicy(); YamlMFI.MaxCallFrameSize = MFI.isMaxCallFrameSizeComputed() ? MFI.getMaxCallFrameSize() : ~0u; + if (MFI.hasGoABIArgSizes()) { + YamlMFI.GoABIStackArgsSize = MFI.getGoABIStackArgsSize(); + YamlMFI.GoABIArgSize = MFI.getGoABIArgSize(); + } YamlMFI.CVBytesOfCalleeSavedRegisters = MFI.getCVBytesOfCalleeSavedRegisters(); YamlMFI.HasOpaqueSPAdjustment = MFI.hasOpaqueSPAdjustment(); diff --git a/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt b/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt index 93a742a19aa79..1a208ed465214 100644 --- a/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt +++ b/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt @@ -2,6 +2,7 @@ add_llvm_component_library(LLVMSelectionDAG DAGCombiner.cpp FastISel.cpp FunctionLoweringInfo.cpp + GoISelLowering.cpp InstrEmitter.cpp LegalizeDAG.cpp LegalizeFloatTypes.cpp diff --git a/llvm/lib/CodeGen/SelectionDAG/GoISelLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/GoISelLowering.cpp new file mode 100644 index 0000000000000..d97865e09893f --- /dev/null +++ b/llvm/lib/CodeGen/SelectionDAG/GoISelLowering.cpp @@ -0,0 +1,97 @@ +//===- GoISelLowering.cpp - Go SelectionDAG lowering helpers -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/GoISelLowering.h" +#include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/TargetCallingConv.h" +#include "llvm/IR/Argument.h" +#include "llvm/Support/ErrorHandling.h" + +using namespace llvm; + +namespace { + +template +static void setByValLocation(unsigned ArgIndex, ArrayRef Pieces, + ArrayRef ArgLocs, + goabi::ValueLayout &Layout, const DataLayout &DL) { + assert(ArgLocs.size() == Pieces.size() && + "argument assignments must match argument pieces"); + + const ArgT *ByValPiece = nullptr; + const CCValAssign *ByValLoc = nullptr; + for (auto [PieceIndex, Piece] : llvm::enumerate(Pieces)) { + if (Piece.OrigArgIndex != ArgIndex) + continue; + if (ByValPiece) + report_fatal_error("invalid Go byval argument piece count"); + ByValPiece = &Piece; + ByValLoc = &ArgLocs[PieceIndex]; + } + + if (!ByValPiece || !ByValPiece->Flags.isByVal() || + ByValPiece->PartOffset != 0 || !ByValLoc->isMemLoc() || + ByValPiece->Flags.getByValSize() != DL.getTypeAllocSize(Layout.Ty)) + report_fatal_error("invalid Go byval argument"); + + Layout.StackOffset = ByValLoc->getLocMemOffset(); + Layout.Alignment = ByValPiece->Flags.getNonZeroByValAlign(); +} + +} // namespace + +goabi::CallLayout goabi::computeFormalArgLayout(const Function &F, + ArrayRef Ins, + ArrayRef ArgLocs, + uint64_t StackArgsSize, + const DataLayout &DL, + const ABIConfig &Config) { + SmallVector ArgLayouts; + for (const Argument &Arg : F.args()) { + if (Arg.hasNestAttr()) + continue; + ValueLayout &Layout = ArgLayouts.emplace_back(); + Layout.Ty = + Arg.hasByValAttr() ? Arg.getPointeeInMemoryValueType() : Arg.getType(); + Layout.InRegs = !Arg.hasByValAttr(); + if (Arg.hasByValAttr()) + setByValLocation(Arg.getArgNo(), Ins, ArgLocs, Layout, DL); + } + + SmallVector ResultTys; + getReturnTypes(F.getReturnType(), hasTupleResultsAttr(F), ResultTys); + return goabi::computeCallLayout(ArgLayouts, StackArgsSize, ResultTys, DL, + Config); +} + +goabi::CallLayout +goabi::computeCallLayout(TargetLowering::CallLoweringInfo &CLI, + ArrayRef ArgLocs, uint64_t StackArgsSize, + const ABIConfig &Config) { + const TargetLowering::ArgListTy &Args = CLI.getArgs(); + SmallVector ArgLayouts; + for (auto [I, Arg] : llvm::enumerate(Args)) { + if (Arg.IsNest) + continue; + Type *Ty = Arg.IsByVal ? Arg.IndirectType : Arg.OrigTy; + if (!Ty) + report_fatal_error("Go call argument has no logical type"); + ValueLayout &Layout = ArgLayouts.emplace_back(); + Layout.Ty = Ty; + Layout.InRegs = !Arg.IsByVal; + if (Arg.IsByVal) + setByValLocation(I, ArrayRef(CLI.Outs), ArgLocs, Layout, + CLI.DAG.getDataLayout()); + } + + SmallVector ResultTys; + getReturnTypes(CLI.RetTy, CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), + ResultTys); + return goabi::computeCallLayout(ArgLayouts, StackArgsSize, ResultTys, + CLI.DAG.getDataLayout(), Config); +} diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 259fa480ec1e5..8807bd7547e72 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -105,6 +105,25 @@ getArgumentValueOffset(const Value *V, const DataLayout &DL) { return std::pair(Arg, Offset); } +static SDValue getStatepointGCValue(const Value *V, + SelectionDAGBuilder &Builder) { + // A typed byval argument denotes its incoming Go stack home, not a heap + // pointer stored in that home. Always use the canonical fixed frame index + // for this address. In particular, do not let an earlier gc.relocate or a + // larger live set turn the address into an ordinary pointer spill: stack + // growth rematerializes frame-index addresses, while the separate object + // layout describes which words in the home are GC roots. + if (const auto *Arg = dyn_cast(V); + Arg && Arg->hasByValAttr() && + goabi::isGoCallingConv( + Builder.DAG.getMachineFunction().getFunction().getCallingConv())) { + int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg); + if (FI != INT_MAX) + return Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy()); + } + return Builder.getValue(V); +} + static void pushStackMapConstant(SmallVectorImpl& Ops, SelectionDAGBuilder &Builder, uint64_t Value) { SDLoc L = Builder.getCurSDLoc(); @@ -489,7 +508,7 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl &Ops, SmallVectorImpl &MemRefs, SelectionDAGBuilder &Builder) { - + if (willLowerDirectly(Incoming)) { if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { // This handles allocas as arguments to the statepoint (this is only @@ -507,7 +526,7 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, } assert(Incoming.getValueType().getSizeInBits() <= 64); - + if (Incoming.isUndef()) { // Put an easily recognized constant that's unlikely to be a valid // value so that uses of undef by the consumer of the stackmap is @@ -533,8 +552,6 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, llvm_unreachable("unhandled direct lowering case"); } - - if (!RequireSpillSlot) { // If this value is live in (not live-on-return, or live-through), we can // treat it the same way patchpoint treats it's "live in" values. We'll @@ -549,7 +566,7 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, // found by the runtime later. Note: We know all of these spills are // independent, but don't bother to exploit that chain wise. DAGCombine // will happily do so as needed, so doing it here would be a small compile - // time win at most. + // time win at most. SDValue Chain = Builder.getRoot(); auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder); Ops.push_back(std::get<0>(Res)); @@ -558,7 +575,6 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, Chain = std::get<1>(Res); Builder.DAG.setRoot(Chain); } - } /// Return true if value V represents the GC value. The behavior is conservative @@ -615,8 +631,10 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, LandingPadInst *LPI = StInvoke->getLandingPadInst(); for (const auto *Relocate : SI.GCRelocates) if (Relocate->getOperand(0) == LPI) { - LPadPointers.insert(Builder.getValue(Relocate->getBasePtr())); - LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr())); + LPadPointers.insert( + getStatepointGCValue(Relocate->getBasePtr(), Builder)); + LPadPointers.insert( + getStatepointGCValue(Relocate->getDerivedPtr(), Builder)); } } @@ -638,13 +656,14 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, }; auto processGCPtr = [&](const Value *V) { - SDValue PtrSD = Builder.getValue(V); + SDValue PtrSD = getStatepointGCValue(V, Builder); if (!LoweredGCPtrs.insert(PtrSD)) return; // skip duplicates GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1; - if (auto ArgValue = - getArgumentValueOffset(V, Builder.DAG.getDataLayout())) { + if (auto ArgValue = getArgumentValueOffset(V, Builder.DAG.getDataLayout()); + ArgValue && + !(V == ArgValue->first && ArgValue->first->hasByValAttr())) { uint64_t Size = PtrSD.getValueType().getStoreSize().getKnownMinValue(); int FI = Builder.FuncInfo.getArgumentValueHome(ArgValue->first, ArgValue->second, Size); @@ -700,13 +719,13 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, } for (const Value *V : SI.Ptrs) { - SDValue SDV = Builder.getValue(V); + SDValue SDV = getStatepointGCValue(V, Builder); if (!LowerAsVReg.count(SDV)) reservePreviousStackSlotForValue(V, Builder); } for (const Value *V : SI.Bases) { - SDValue SDV = Builder.getValue(V); + SDValue SDV = getStatepointGCValue(V, Builder); if (!LowerAsVReg.count(SDV)) reservePreviousStackSlotForValue(V, Builder); } @@ -753,7 +772,7 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, // the alloca SmallVector Allocas; for (Value *V : SI.GCLives) { - SDValue Incoming = Builder.getValue(V); + SDValue Incoming = getStatepointGCValue(V, Builder); if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { // This handles allocas as arguments to the statepoint assert(Incoming.getValueType() == Builder.getFrameIndexTy() && @@ -773,11 +792,11 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, pushStackMapConstant(Ops, Builder, SI.Ptrs.size()); SDLoc L = Builder.getCurSDLoc(); for (unsigned i = 0; i < SI.Ptrs.size(); ++i) { - SDValue Base = Builder.getValue(SI.Bases[i]); + SDValue Base = getStatepointGCValue(SI.Bases[i], Builder); assert(GCPtrIndexMap.count(Base) && "base not found in index map"); Ops.push_back( Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64)); - SDValue Derived = Builder.getValue(SI.Ptrs[i]); + SDValue Derived = getStatepointGCValue(SI.Ptrs[i], Builder); assert(GCPtrIndexMap.count(Derived) && "derived not found in index map"); Ops.push_back( Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64)); @@ -950,7 +969,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( DenseMap VirtRegs; for (const auto *Relocate : SI.GCRelocates) { Value *Derived = Relocate->getDerivedPtr(); - SDValue SD = getValue(Derived); + SDValue SD = getStatepointGCValue(Derived, *this); auto It = LowerAsVReg.find(SD); if (It == LowerAsVReg.end()) continue; @@ -990,7 +1009,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr]; for (const GCRelocateInst *Relocate : SI.GCRelocates) { const Value *V = Relocate->getDerivedPtr(); - SDValue SDV = getValue(V); + SDValue SDV = getStatepointGCValue(V, *this); SDValue Loc = StatepointLowering.getLocation(SDV); bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent()); @@ -1008,7 +1027,11 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( } } else if (goabi::isGoCallingConv( DAG.getMachineFunction().getFunction().getCallingConv()) && - isa(V) && isa(SDV)) { + isa(SDV) && + (isa(V) || + (isa(V) && cast(V)->hasByValAttr() && + FuncInfo.getArgumentFrameIndex(cast(V)) == + cast(SDV)->getIndex()))) { Record.type = RecordType::FrameIndexRemat; Record.payload.FI = cast(SDV)->getIndex(); } else if (Loc.getNode()) { @@ -1026,8 +1049,6 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( RelocationMap[Relocate] = Record; } - - SDNode *SinkNode = StatepointMCNode; // Build the GC_TRANSITION_END node if necessary. @@ -1168,11 +1189,14 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, // pointers passed to deopt are base pointers; relaxing that assumption // would require relatively large changes to how we represent relocations. for (Value *V : I.deopt_operands()) { - // GoALLC uses direct static alloca deopt operands as frame-layout carriers - // for its per-alloca pointer maps. The alloca is a GC root only when it is - // also present in the explicit gc-live bundle; treating the deopt carrier - // as a root would make an inactive lifetime scan uninitialized storage. - if (GFI->getStrategy().getName() == "goallc" && isa(V)) + // GoALLC uses direct static allocas and typed byval parameters as + // frame-layout carriers for its per-object pointer maps. The object is a + // GC root only when its base is also present in the explicit gc-live + // bundle; treating the deopt carrier as a root would make an inactive + // lifetime scan uninitialized or dead storage. + const auto *Arg = dyn_cast(V); + if (GFI->getStrategy().getName() == "goallc" && + (isa(V) || (Arg && Arg->hasByValAttr()))) continue; if (!isGCValue(V, *this)) continue; @@ -1207,7 +1231,7 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, if (GCResultLocality.first) { // Result value will be used in a same basic block. Don't export it or // perform any explicit register copies. The gc_result will simply grab - // this value. + // this value. setValue(&I, ReturnValue); } @@ -1227,7 +1251,7 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, DAG.getDataLayout(), Reg, RetTy, I.getCallingConv()); SDValue Chain = DAG.getEntryNode(); - + RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr); PendingExports.push_back(Chain); FuncInfo.ValueMap[&I] = Reg; @@ -1297,7 +1321,7 @@ void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) { // which is always i32 in our case. Type *RetTy = CI.getType(); SDValue CopyFromReg = getCopyFromRegs(SI, RetTy); - + assert(CopyFromReg.getNode()); setValue(&CI, CopyFromReg); } diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.h b/llvm/lib/Target/AArch64/AArch64CallingConvention.h index 7105fa695334b..2bfa27938b54f 100644 --- a/llvm/lib/Target/AArch64/AArch64CallingConvention.h +++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.h @@ -61,6 +61,9 @@ bool CC_AArch64_Arm64EC_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State); +bool CC_AArch64_Go(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + Type *OrigTy, CCState &State); bool CC_AArch64_Preserve_None(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td index 28ada11cb2aaa..775b260288d96 100644 --- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td +++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td @@ -577,6 +577,43 @@ def CC_AArch64_Preserve_None : CallingConv<[ CCDelegateTo ]>; +// Go's frontend has already made the whole-value register-or-memory decision. +// Direct parameters are ABIInternal register values; typed byval parameters +// are the only ordinary stack-input carrier. Leaving direct values unmatched +// when the register list is exhausted keeps frontend/backend ABI drift +// fail-closed instead of silently splitting one Go value across locations. +let Entry = 1 in +def CC_AArch64_Go : CallingConv<[ + // The closure context is not part of the ordinary Go argument layout. + CCIfNest>>, + + // Preserve the exact size and alignment carried by byval(T). + CCIfByVal>, + + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i1, i8, i16], CCPromoteToType>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i32], + CCAssignToReg<[W0, W1, W2, W3, W4, W5, W6, W7, + W8, W9, W10, W11, W12, W13, W14, W15]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i64], + CCAssignToReg<[X0, X1, X2, X3, X4, X5, X6, X7, + X8, X9, X10, X11, X12, X13, X14, X15]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[f16, bf16], + CCAssignToReg<[H0, H1, H2, H3, H4, H5, H6, H7, + H8, H9, H10, H11, H12, H13, H14, H15]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[f32], + CCAssignToReg<[S0, S1, S2, S3, S4, S5, S6, S7, + S8, S9, S10, S11, S12, S13, S14, S15]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[f64], + CCAssignToReg<[D0, D1, D2, D3, D4, D5, D6, D7, + D8, D9, D10, D11, D12, D13, D14, D15]>>> +]>; + // The order of the callee-saves in this file is important, because the // FrameLowering code will use this order to determine the layout the // callee-save area in the stack frame. As can be observed below, Darwin diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index b3a69f3134f28..3147a9a0b9091 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -45,6 +45,7 @@ #include "llvm/CodeGen/ComplexDeinterleavingPass.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/GoCallingConv.h" +#include "llvm/CodeGen/GoISelLowering.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -8903,31 +8904,6 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectorVT( namespace { -template struct GoArgGroup { - unsigned Start = 0; - unsigned End = 0; - unsigned Index = 0; -}; - -template -static SmallVector, 8> groupGoArgs(ArrayRef Args) { - SmallVector, 8> Groups; - if (Args.empty()) - return Groups; - - unsigned Start = 0; - unsigned Index = Args.front().OrigArgIndex; - for (unsigned I = 1; I != Args.size(); ++I) { - if (Args[I].OrigArgIndex == Index) - continue; - Groups.push_back({Start, I, Index}); - Start = I; - Index = Args[I].OrigArgIndex; - } - Groups.push_back({Start, static_cast(Args.size()), Index}); - return Groups; -} - static constexpr unsigned AArch64GoXRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4, AArch64::X5, AArch64::X6, AArch64::X7, @@ -9034,52 +9010,6 @@ static unsigned getAArch64GoPhysReg(MVT VT, Type *OrigTy, unsigned IntIndex, : AArch64GoWRegs[IntIndex]; } -static const TargetRegisterClass *getAArch64GoRegClass(MVT VT) { - switch (VT.SimpleTy) { - case MVT::i32: - return &AArch64::GPR32RegClass; - case MVT::i64: - return &AArch64::GPR64RegClass; - case MVT::f16: - case MVT::bf16: - return &AArch64::FPR16RegClass; - case MVT::f32: - return &AArch64::FPR32RegClass; - case MVT::f64: - return &AArch64::FPR64RegClass; - default: - llvm_unreachable("unsupported Go ABI register class"); - } -} - -static SmallVector -getAArch64GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { - SmallVector ArgTys; - LayoutMap.assign(F.arg_size(), -1); - for (const Argument &Arg : F.args()) { - unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) - continue; - LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); - } - return ArgTys; -} - -static SmallVector -getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, - SmallVectorImpl &LayoutMap) { - SmallVector ArgTys; - LayoutMap.assign(Args.size(), -1); - for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) - continue; - LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); - } - return ArgTys; -} - static SmallVector getAArch64GoReturnTypes(Type *RetTy, const AttributeList &Attrs) { SmallVector ResultTys; @@ -9087,31 +9017,34 @@ getAArch64GoReturnTypes(Type *RetTy, const AttributeList &Attrs) { return ResultTys; } -static SDValue lowerAArch64GoFormalArguments( - const AArch64TargetLowering &TLI, SDValue Chain, MachineFunction &MF, - const SmallVectorImpl &Ins, const SDLoc &DL, - SelectionDAG &DAG, SmallVectorImpl &InVals) { +struct AArch64GoFormalArgInfo { + goabi::CallLayout Layout; + SmallVector HomeFIs; +}; + +static AArch64GoFormalArgInfo prepareAArch64GoFormalArguments( + const AArch64TargetLowering &TLI, MachineFunction &MF, + ArrayRef Ins, ArrayRef ArgLocs, + uint64_t StackArgsSize, SelectionDAG &DAG) { const Function &F = MF.getFunction(); const AArch64Subtarget &Subtarget = MF.getSubtarget(); auto *FuncInfo = MF.getInfo(); MachineFrameInfo &MFI = MF.getFrameInfo(); - MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); - - SmallVector LayoutMap; - SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); goabi::ABIConfig ABIConfig = getAArch64GoABIConfig(TLI, Subtarget, F.getCallingConv()); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), - DAG.getDataLayout(), ABIConfig); - - goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( - ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + AArch64GoFormalArgInfo Info; + Info.Layout = goabi::computeFormalArgLayout(F, Ins, ArgLocs, StackArgsSize, + DAG.getDataLayout(), ABIConfig); + const goabi::CallLayout &Layout = Info.Layout; + MFI.setGoABIArgSizes(Layout.StackArgsSize, Layout.ArgSize); + + goabi::EntryArgsInfo EntryArgs = + goabi::computeEntryArgsInfo(Layout, DAG.getDataLayout(), ABIConfig); SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); - SmallVector ArgSpillOffsets(ArgTys.size(), 0); + SmallVector ArgSpillOffsets(Layout.Args.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; - for (unsigned I = 0, E = ArgTys.size(); I != E; ++I) { + for (unsigned I = 0, E = Layout.Args.size(); I != E; ++I) { const goabi::ValueLayout &ArgLayout = Layout.Args[I]; if (!ArgLayout.InRegs) continue; @@ -9124,6 +9057,13 @@ static SDValue lowerAArch64GoFormalArguments( FuncInfo->clearGoArgHomes(); FuncInfo->clearGoArgPointerSlots(); unsigned StackBias = getAArch64GoStackBias(F.getCallingConv()); + Info.HomeFIs.assign(F.arg_size(), INT_MAX); + + SmallVector IsLiveAtEntry(F.arg_size(), false); + for (const ISD::InputArg &In : Ins) + if (In.OrigArgIndex != ISD::InputArg::NoArgIndex && + In.OrigArgIndex < F.arg_size()) + IsLiveAtEntry[In.OrigArgIndex] |= In.Used; auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { @@ -9148,89 +9088,62 @@ static SDValue lowerAArch64GoFormalArguments( } }; - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - if (Group.Index == ISD::InputArg::NoArgIndex) - continue; - - const Argument *Arg = F.getArg(Group.Index); - if (Arg->hasNestAttr()) { - assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); - MVT CopyVT = getAArch64GoCopyVT(Ins[Group.Start].VT); - Register VReg = MF.addLiveIn(AArch64::X26, getAArch64GoRegClass(CopyVT)); - SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, CopyVT); - if (Ins[Group.Start].VT != CopyVT) - Val = DAG.getNode(ISD::TRUNCATE, DL, Ins[Group.Start].VT, Val); - InVals.push_back(Val); + unsigned NextLayoutIndex = 0; + for (const Argument &Arg : F.args()) { + if (Arg.hasNestAttr()) continue; - } - unsigned LayoutIndex = LayoutMap[Group.Index]; + if (NextLayoutIndex >= Layout.Args.size()) + report_fatal_error("AArch64 Go argument has no logical layout"); + unsigned LayoutIndex = NextLayoutIndex++; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; int64_t FixedHomeOffset = StackBias + LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false, + /*IsAliased=*/true); + Info.HomeFIs[Arg.getArgNo()] = HomeFI; AArch64FunctionInfo::GoArgHome &Home = - FuncInfo->addGoArgHome(Group.Index, HomeFI); + FuncInfo->addGoArgHome(Arg.getArgNo(), HomeFI); // LLVM may replace an unused incoming pointer with poison at every call // edge. Keep its ABI home so morestack can preserve the complete register // assignment, but do not expose that uninitialized word as a GC root. - bool IsLiveAtEntry = llvm::any_of( - ArrayRef(Ins).slice(Group.Start, Group.End - Group.Start), - [](const ISD::InputArg &In) { return In.Used; }); RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, - IsLiveAtEntry); - - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - if (ArgLayout.InRegs) { - MVT CopyVT = getAArch64GoCopyVT(In.VT); - unsigned PReg = getAArch64GoPhysReg(In.VT, In.OrigTy, - ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); - Register VReg = MF.addLiveIn(PReg, getAArch64GoRegClass(CopyVT)); - SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, CopyVT); - if (isAArch64GoFloatPiece(In.OrigTy)) - ++FPPiece; - else - ++IntPiece; - - // The physical register copy may widen a sub-word integer to i32, but - // its Go ABI home retains the original piece's size and offset. - unsigned Size = static_cast( - std::max(1, In.ArgVT.getStoreSize().getKnownMinValue())); - Home.addRegisterPiece(PReg, In.PartOffset, Size, - isAArch64GoFloatPiece(In.OrigTy)); - - if (In.VT != CopyVT) - Val = DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); - InVals.push_back(Val); + IsLiveAtEntry[Arg.getArgNo()]); + + if (!ArgLayout.InRegs) + continue; + for (auto [I, In] : llvm::enumerate(Ins)) { + if (In.OrigArgIndex != Arg.getArgNo()) continue; - } + const CCValAssign &VA = ArgLocs[I]; + if (!VA.isRegLoc()) + report_fatal_error("invalid AArch64 Go register argument location"); + unsigned PReg = VA.getLocReg(); - SDValue Addr = DAG.getFrameIndex(HomeFI, PtrVT); - if (In.PartOffset != 0) - Addr = - DAG.getObjectPtrOffset(DL, Addr, TypeSize::getFixed(In.PartOffset)); - InVals.push_back(loadAArch64GoStackPiece( - DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, HomeFI, In.PartOffset))); + // The physical register copy may widen a sub-word integer to i32, but + // its Go ABI home retains the original piece's size and offset. + unsigned Size = static_cast( + std::max(1, In.ArgVT.getStoreSize().getKnownMinValue())); + Home.addRegisterPiece(PReg, In.PartOffset, Size, + isAArch64GoFloatPiece(In.OrigTy)); } } + if (NextLayoutIndex != Layout.Args.size()) + report_fatal_error("AArch64 Go ABI layout has unmatched arguments"); for (uint32_t Word : EntryArgs.PointerWords) if (!MatchedEntryArgWords.test(Word)) report_fatal_error( "Go entry argument pointer word has no AArch64 fixed object"); - return Chain; + return Info; } static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, @@ -9240,53 +9153,50 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, const SDLoc &DL, SelectionDAG &DAG) { const AArch64Subtarget &Subtarget = MF.getSubtarget(); MachineFrameInfo &MFI = MF.getFrameInfo(); - SmallVector LayoutMap; - SmallVector ArgTys = - getAArch64GoArgTypes(MF.getFunction(), LayoutMap); + if (!MFI.hasGoABIArgSizes()) + report_fatal_error("missing AArch64 Go ABI argument layout"); SmallVector ResultTys = getAArch64GoReturnTypes( MF.getFunction().getReturnType(), MF.getFunction().getAttributes()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), + {}, MFI.getGoABIStackArgsSize(), ResultTys, DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, MF.getFunction().getCallingConv())); unsigned StackBias = getAArch64GoStackBias(MF.getFunction().getCallingConv()); SmallVector MemOps; SmallVector, 8> RetRegs; - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::OutputArg &Out = Outs[I]; - SDValue Val = OutVals[I]; - if (ResultLayout.InRegs) { - MVT CopyVT = getAArch64GoCopyVT(Val.getSimpleValueType()); - if (Val.getSimpleValueType() != CopyVT) - Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); - unsigned PReg = getAArch64GoPhysReg(Out.VT, Out.OrigTy, - ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); - if (isAArch64GoFloatPiece(Out.OrigTy)) - ++FPPiece; - else - ++IntPiece; - RetRegs.emplace_back(PReg, Val); - continue; - } - - unsigned Size = static_cast( - std::max(1, Out.ArgVT.getStoreSize().getKnownMinValue())); - int FI = MFI.CreateFixedObject( - Size, StackBias + ResultLayout.StackOffset + Out.PartOffset, - /*IsImmutable=*/false); - if (MF.getInfo()->hasGoABI0FrameIndex()) - MFI.setIsAliasedObjectIndex(FI, true); - SDValue Addr = - DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); - MemOps.push_back( - storeAArch64GoStackPiece(DAG, Chain, DL, Val, Out.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, FI))); + SmallVector IntPieces(Layout.Results.size(), 0); + SmallVector FPPieces(Layout.Results.size(), 0); + for (auto [I, Out] : llvm::enumerate(Outs)) { + unsigned ResultIndex = Out.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("AArch64 Go return piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + SDValue Val = OutVals[I]; + if (ResultLayout.InRegs) { + MVT CopyVT = getAArch64GoCopyVT(Val.getSimpleValueType()); + if (Val.getSimpleValueType() != CopyVT) + Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); + bool IsFP = isAArch64GoFloatPiece(Out.OrigTy); + unsigned Piece = + IsFP ? FPPieces[ResultIndex]++ : IntPieces[ResultIndex]++; + unsigned PReg = getAArch64GoPhysReg( + Out.VT, Out.OrigTy, ResultLayout.IntRegStart + (IsFP ? 0 : Piece), + ResultLayout.FPRegStart + (IsFP ? Piece : 0)); + RetRegs.emplace_back(PReg, Val); + continue; } + + unsigned Size = static_cast( + std::max(1, Out.ArgVT.getStoreSize().getKnownMinValue())); + int FI = MFI.CreateFixedObject( + Size, StackBias + ResultLayout.StackOffset + Out.PartOffset, + /*IsImmutable=*/false); + if (MF.getInfo()->hasGoABI0FrameIndex()) + MFI.setIsAliasedObjectIndex(FI, true); + SDValue Addr = DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); + MemOps.push_back( + storeAArch64GoStackPiece(DAG, Chain, DL, Val, Out.ArgVT, Addr, + MachinePointerInfo::getFixedStack(MF, FI))); } if (!MemOps.empty()) @@ -9306,220 +9216,71 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, return DAG.getNode(AArch64ISD::RET_GLUE, DL, MVT::Other, RetOps); } -static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, - TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl &InVals) { - SelectionDAG &DAG = CLI.DAG; - SDLoc &DL = CLI.DL; - auto &Outs = CLI.Outs; - auto &OutVals = CLI.OutVals; - auto &Ins = CLI.Ins; - SDValue Chain = CLI.Chain; - SDValue Callee = CLI.Callee; +static SDValue lowerAArch64GoCallResults(const AArch64TargetLowering &TLI, + SDValue Chain, SDValue InGlue, + ArrayRef Ins, + const goabi::CallLayout &Layout, + unsigned StackBias, unsigned NumBytes, + const SDLoc &DL, SelectionDAG &DAG, + SmallVectorImpl &InVals) { MachineFunction &MF = DAG.getMachineFunction(); - const AArch64Subtarget &Subtarget = MF.getSubtarget(); - const AArch64RegisterInfo *TRI = Subtarget.getRegisterInfo(); MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); - - CLI.IsTailCall = false; - if (CLI.IsVarArg) - report_fatal_error("Go calling convention does not support varargs"); - - SmallVector LayoutMap; - SmallVector ArgTys = - getAArch64GoCallArgTypes(CLI.getArgs(), LayoutMap); - SmallVector ResultTys; - goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); - - unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); - // Layout.TotalStackSize rounds the logical Go argument area to the target - // stack alignment before the physical entry-SP bias is applied. Adding the - // bias after that rounding reserves an extra word for one-word calls (the - // common ABI0 funcval case) and inflates every containing nosplit frame. - // The caller frame itself remains stack-aligned; reserve only the bytes - // through the last physical argument home here. - unsigned NumBytes = Layout.ArgSize + StackBias; - Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); - - SDValue StackPtr; - if (NumBytes != 0) - StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, PtrVT); - - SmallVector MemOpChains; - SmallVector, 8> RegsToPass; - MachineFunction::CallSiteInfo CSInfo; - - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - if (Group.Index >= CLI.getArgs().size()) - continue; - - if (CLI.getArgs()[Group.Index].IsNest) { - assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); - RegsToPass.emplace_back(AArch64::X26, OutVals[Group.Start]); - continue; - } - - const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutMap[Group.Index]]; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - SDValue Arg = OutVals[I]; - const ISD::OutputArg &Out = Outs[I]; - if (ArgLayout.InRegs) { - MVT CopyVT = getAArch64GoCopyVT(Arg.getSimpleValueType()); - if (Arg.getSimpleValueType() != CopyVT) - Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Arg); - unsigned PReg = getAArch64GoPhysReg(Out.VT, Out.OrigTy, - ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); - if (isAArch64GoFloatPiece(Out.OrigTy)) - ++FPPiece; - else - ++IntPiece; - RegsToPass.emplace_back(PReg, Arg); - if (DAG.getTarget().Options.EmitCallSiteInfo) - CSInfo.ArgRegPairs.emplace_back(PReg, I); - continue; - } - - assert(StackPtr && "missing Go call stack pointer"); - SDValue Addr = DAG.getNode( - ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant( - StackBias + ArgLayout.StackOffset + Out.PartOffset, DL)); - MemOpChains.push_back(storeAArch64GoStackPiece( - DAG, Chain, DL, Arg, Out.ArgVT, Addr, - MachinePointerInfo::getStack(MF, StackBias + ArgLayout.StackOffset + - Out.PartOffset))); - } - } - - if (!MemOpChains.empty()) - Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); - - SDValue InGlue; - for (const auto &[Reg, Val] : RegsToPass) { - Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue); - InGlue = Chain.getValue(1); - } - - const GlobalValue *CalledGlobal = nullptr; - unsigned OpFlags = 0; - if (auto *G = dyn_cast(Callee)) { - CalledGlobal = G->getGlobal(); - OpFlags = Subtarget.classifyGlobalFunctionReference(CalledGlobal, - TLI.getTargetMachine()); - if (OpFlags & AArch64II::MO_GOT) { - Callee = DAG.getTargetGlobalAddress(CalledGlobal, DL, PtrVT, 0, OpFlags); - Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); - } else { - Callee = DAG.getTargetGlobalAddress(CalledGlobal, DL, PtrVT, 0, OpFlags); - } - } else if (auto *S = dyn_cast(Callee)) { - bool UseGot = (TLI.getTargetMachine().getCodeModel() == CodeModel::Large && - Subtarget.isTargetMachO()) || - MF.getFunction().getParent()->getRtLibUseGOT(); - if (UseGot) { - Callee = - DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, AArch64II::MO_GOT); - Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); - } else { - Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, 0); - } - } - - SmallVector Ops; - Ops.push_back(Chain); - Ops.push_back(Callee); - for (const auto &[Reg, Val] : RegsToPass) - Ops.push_back(DAG.getRegister(Reg, Val.getValueType())); - Ops.push_back( - DAG.getRegisterMask(TRI->getCallPreservedMask(MF, CLI.CallConv))); - if (InGlue.getNode()) - Ops.push_back(InGlue); - - Chain = DAG.getNode(AArch64ISD::CALL, DL, - DAG.getVTList(MVT::Other, MVT::Glue), Ops); - DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); - if (CalledGlobal && - MF.getFunction().getParent()->getModuleFlag("import-call-optimization")) - DAG.addCalledGlobal(Chain.getNode(), CalledGlobal, OpFlags); - InGlue = Chain.getValue(1); - SmallVector ResultVals(Ins.size()); - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + + SmallVector IntPieces(Layout.Results.size(), 0); + SmallVector FPPieces(Layout.Results.size(), 0); + for (auto [I, In] : llvm::enumerate(Ins)) { + unsigned ResultIndex = In.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("AArch64 Go call result piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (!ResultLayout.InRegs) continue; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - MVT CopyVT = getAArch64GoCopyVT(In.VT); - unsigned PReg = getAArch64GoPhysReg(In.VT, In.OrigTy, - ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); - SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); - Chain = Val.getValue(1); - InGlue = Val.getValue(2); - if (isAArch64GoFloatPiece(In.OrigTy)) - ++FPPiece; - else - ++IntPiece; - if (In.VT != CopyVT) - ResultVals[I] = DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); - else - ResultVals[I] = Val; - } - } - - // Read every register result before introducing stack loads into the chain. - // A Go result that does not fit in the remaining register budget is placed - // on the stack without consuming that budget, so a later result can still - // be register-assigned. Interleaving those stack loads with glued physical - // register copies can create a cyclic scheduler dependency at statepoints. - // - // Keep the call sequence active while reading stack results. The load chain - // is after the call, so the physical SP is the current post-growth Go stack; - // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + MVT CopyVT = getAArch64GoCopyVT(In.VT); + bool IsFP = isAArch64GoFloatPiece(In.OrigTy); + unsigned Piece = IsFP ? FPPieces[ResultIndex]++ : IntPieces[ResultIndex]++; + unsigned PReg = getAArch64GoPhysReg( + In.VT, In.OrigTy, ResultLayout.IntRegStart + (IsFP ? 0 : Piece), + ResultLayout.FPRegStart + (IsFP ? Piece : 0)); + SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); + Chain = Val.getValue(1); + InGlue = Val.getValue(2); + ResultVals[I] = + In.VT == CopyVT ? Val : DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); + } + + // Register results must be read before stack result loads. Keeping the call + // sequence active also keeps the outgoing Go frame, and therefore the + // post-growth result addresses, valid until every stack result is consumed. + bool HasStackResults = llvm::any_of(Ins, [&](const ISD::InputArg &In) { + return In.OrigArgIndex >= Layout.Results.size() || + !Layout.Results[In.OrigArgIndex].InRegs; + }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, PtrVT, InGlue); Chain = ResultStackPtr.getValue(1); InGlue = ResultStackPtr.getValue(2); } - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + for (auto [I, In] : llvm::enumerate(Ins)) { + unsigned ResultIndex = In.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("AArch64 Go call result piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs) continue; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - SDValue Addr = DAG.getNode( - ISD::ADD, DL, PtrVT, ResultStackPtr, - DAG.getIntPtrConstant( - StackBias + ResultLayout.StackOffset + In.PartOffset, DL)); - SDValue Load = loadAArch64GoStackPiece( - DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getStack( - MF, StackBias + ResultLayout.StackOffset + In.PartOffset)); - Chain = Load.getValue(1); - ResultVals[I] = Load; - } + uint64_t Offset = StackBias + ResultLayout.StackOffset + In.PartOffset; + SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(Offset, DL)); + SDValue Load = + loadAArch64GoStackPiece(DAG, Chain, DL, In.VT, In.ArgVT, Addr, + MachinePointerInfo::getStack(MF, Offset)); + Chain = Load.getValue(1); + ResultVals[I] = Load; } Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); - InVals.append(ResultVals.begin(), ResultVals.end()); return Chain; } @@ -9529,7 +9290,8 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, std::optional AArch64TargetLowering::getArgumentCopyElisionFrameInfo( const Argument &Arg, MachineFunction &MF) const { - if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) + if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv()) || + Arg.hasByValAttr()) return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); @@ -9549,19 +9311,15 @@ int AArch64TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { if (FuncInfo->hasGoABI0FrameIndex()) return FuncInfo->getGoABI0FrameIndex(); - SmallVector LayoutMap; - SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); - const AArch64Subtarget &Subtarget = MF.getSubtarget(); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), - MF.getDataLayout(), - getAArch64GoABIConfig(*this, Subtarget, F.getCallingConv())); - if (Layout.ArgSize == 0) + MachineFrameInfo &MFI = MF.getFrameInfo(); + if (!MFI.hasGoABIArgSizes()) + report_fatal_error("missing AArch64 Go ABI argument layout"); + if (MFI.getGoABIArgSize() == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); - int FI = MF.getFrameInfo().CreateFixedObject( - Layout.ArgSize, getAArch64GoStackBias(F.getCallingConv()), - /*IsImmutable=*/false, /*IsAliased=*/true); + int FI = MFI.CreateFixedObject(MFI.getGoABIArgSize(), + getAArch64GoStackBias(F.getCallingConv()), + /*IsImmutable=*/false, /*IsAliased=*/true); for (const AArch64FunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) { MF.getFrameInfo().setIsImmutableObjectIndex(Home.FrameIndex, false); MF.getFrameInfo().setIsAliasedObjectIndex(Home.FrameIndex, true); @@ -9618,8 +9376,6 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, [[fallthrough]]; case CallingConv::C: case CallingConv::Fast: - case CallingConv::GoABIInternal: - case CallingConv::GoABI0: case CallingConv::PreserveMost: case CallingConv::PreserveAll: case CallingConv::CXX_FAST_TLS: @@ -9641,6 +9397,11 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, return CC_AArch64_DarwinPCS; return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg : CC_AArch64_DarwinPCS_VarArg; + case CallingConv::GoABIInternal: + case CallingConv::GoABI0: + if (IsVarArg) + reportFatalUsageError("Go calling convention does not support varargs"); + return CC_AArch64_Go; case CallingConv::Win64: if (IsVarArg) { if (Subtarget->isWindowsArm64EC()) @@ -9724,9 +9485,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl &InVals) const { - if (goabi::isGoCallingConv(CallConv)) - return lowerAArch64GoFormalArguments(*this, Chain, DAG.getMachineFunction(), - Ins, DL, DAG, InVals); + bool IsGo = goabi::isGoCallingConv(CallConv); MachineFunction &MF = DAG.getMachineFunction(); const Function &F = MF.getFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); @@ -9736,54 +9495,66 @@ SDValue AArch64TargetLowering::LowerFormalArguments( (isVarArg && Subtarget->isWindowsArm64EC()); AArch64FunctionInfo *FuncInfo = MF.getInfo(); - SmallVector Outs; - GetReturnInfo(CallConv, F.getReturnType(), F.getAttributes(), Outs, - DAG.getTargetLoweringInfo(), MF.getDataLayout()); - if (any_of(Outs, - [](ISD::OutputArg &Out) { return Out.VT.isScalableVector(); })) - FuncInfo->setIsSVECC(true); + if (IsGo && isVarArg) + report_fatal_error("AArch64 Go calling conventions do not support varargs"); + + if (!IsGo) { + SmallVector Outs; + GetReturnInfo(CallConv, F.getReturnType(), F.getAttributes(), Outs, + DAG.getTargetLoweringInfo(), MF.getDataLayout()); + if (any_of(Outs, + [](ISD::OutputArg &Out) { return Out.VT.isScalableVector(); })) + FuncInfo->setIsSVECC(true); + } // Assign locations to all of the incoming arguments. SmallVector ArgLocs; CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); - // At this point, Ins[].VT may already be promoted to i32. To correctly - // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and - // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT. - // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here - // we use a special version of AnalyzeFormalArguments to pass in ValVT and - // LocVT. - unsigned NumArgs = Ins.size(); - Function::const_arg_iterator CurOrigArg = F.arg_begin(); - unsigned CurArgIdx = 0; - bool UseVarArgCC = false; - if (IsWin64) - UseVarArgCC = isVarArg; - - CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC); - - for (unsigned i = 0; i != NumArgs; ++i) { - MVT ValVT = Ins[i].VT; - if (Ins[i].isOrigArg()) { - std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx); - CurArgIdx = Ins[i].getOrigArgIndex(); + if (IsGo) { + CCInfo.AnalyzeArguments(Ins, CC_AArch64_Go); + } else { + // At this point, Ins[].VT may already be promoted to i32. To correctly + // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and + // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT. + // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, + // here we use a special version of AnalyzeFormalArguments to pass in ValVT + // and LocVT. + unsigned NumArgs = Ins.size(); + Function::const_arg_iterator CurOrigArg = F.arg_begin(); + unsigned CurArgIdx = 0; + bool UseVarArgCC = IsWin64 && isVarArg; + CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC); - // Get type of the original argument. - EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(), - /*AllowUnknown*/ true); - MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other; - // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16. - if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) - ValVT = MVT::i8; - else if (ActualMVT == MVT::i16) - ValVT = MVT::i16; + for (unsigned i = 0; i != NumArgs; ++i) { + MVT ValVT = Ins[i].VT; + if (Ins[i].isOrigArg()) { + std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx); + CurArgIdx = Ins[i].getOrigArgIndex(); + + // Get type of the original argument. + EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(), + /*AllowUnknown*/ true); + MVT ActualMVT = + ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other; + // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16. + if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) + ValVT = MVT::i8; + else if (ActualMVT == MVT::i16) + ValVT = MVT::i16; + } + bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, + Ins[i].OrigTy, CCInfo); + assert(!Res && "Call operand has unhandled type"); + (void)Res; } - bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, - Ins[i].OrigTy, CCInfo); - assert(!Res && "Call operand has unhandled type"); - (void)Res; } + std::optional GoInfo; + if (IsGo) + GoInfo = prepareAArch64GoFormalArguments(*this, MF, Ins, ArgLocs, + CCInfo.getStackSize(), DAG); + SMEAttrs Attrs = FuncInfo->getSMEFnAttrs(); bool IsLocallyStreaming = !Attrs.hasStreamingInterface() && Attrs.hasStreamingBody(); @@ -9795,6 +9566,16 @@ SDValue AArch64TargetLowering::LowerFormalArguments( CCValAssign &VA = ArgLocs[i - ExtraArgLocs]; if (Ins[i].Flags.isByVal()) { + if (IsGo) { + unsigned ArgIndex = Ins[i].OrigArgIndex; + if (ArgIndex == ISD::InputArg::NoArgIndex || + ArgIndex >= GoInfo->HomeFIs.size() || + GoInfo->HomeFIs[ArgIndex] == INT_MAX) + report_fatal_error("AArch64 Go byval argument has no incoming home"); + InVals.push_back(DAG.getFrameIndex(GoInfo->HomeFIs[ArgIndex], + getPointerTy(DAG.getDataLayout()))); + continue; + } // Byval is used for HFAs in the PCS, but the system should work in a // non-compliant manner for larger structs. EVT PtrVT = getPointerTy(DAG.getDataLayout()); @@ -10111,7 +9892,8 @@ SDValue AArch64TargetLowering::LowerFormalArguments( } } - unsigned StackArgSize = CCInfo.getStackSize(); + unsigned StackArgSize = + IsGo ? GoInfo->Layout.TotalStackSize : CCInfo.getStackSize(); bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) { // This is a non-standard ABI so by fiat I say we're allowed to make full @@ -10845,8 +10627,6 @@ static bool shouldLowerTailCallStackArg(const MachineFunction &MF, SDValue AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, SmallVectorImpl &InVals) const { - if (goabi::isGoCallingConv(CLI.CallConv)) - return lowerAArch64GoCall(*this, CLI, InVals); SelectionDAG &DAG = CLI.DAG; SDLoc &DL = CLI.DL; SmallVector &Outs = CLI.Outs; @@ -10858,6 +10638,13 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, CallingConv::ID &CallConv = CLI.CallConv; bool IsVarArg = CLI.IsVarArg; const CallBase *CB = CLI.CB; + bool IsGo = goabi::isGoCallingConv(CallConv); + + if (IsGo) { + IsTailCall = false; + if (IsVarArg) + report_fatal_error("Go calling convention does not support varargs"); + } MachineFunction &MF = DAG.getMachineFunction(); MachineFunction::CallSiteInfo CSInfo; @@ -10888,14 +10675,28 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, } } - analyzeCallOperands(*this, Subtarget, CLI, CCInfo); + if (IsGo) + CCInfo.AnalyzeArguments(Outs, CC_AArch64_Go); + else + analyzeCallOperands(*this, Subtarget, CLI, CCInfo); + + goabi::CallLayout GoLayout; + unsigned GoStackBias = 0; + if (IsGo) { + GoLayout = goabi::computeCallLayout( + CLI, ArgLocs, CCInfo.getStackSize(), + getAArch64GoABIConfig(*this, *Subtarget, CallConv)); + GoStackBias = getAArch64GoStackBias(CallConv); + } - CCAssignFn *RetCC = CCAssignFnForReturn(CallConv); // Assign locations to each value returned by this call. SmallVector RVLocs; - CCState RetCCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, - *DAG.getContext()); - RetCCInfo.AnalyzeCallResult(Ins, RetCC); + if (!IsGo) { + CCAssignFn *RetCC = CCAssignFnForReturn(CallConv); + CCState RetCCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, + *DAG.getContext()); + RetCCInfo.AnalyzeCallResult(Ins, RetCC); + } // Set type id for call site info. setTypeIdForCallsiteInfo(CB, MF, CSInfo); @@ -10944,7 +10745,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, "site marked musttail"); // Get a count of how many bytes are to be pushed on the stack. - unsigned NumBytes = CCInfo.getStackSize(); + unsigned NumBytes = + IsGo ? GoLayout.ArgSize + GoStackBias : CCInfo.getStackSize(); if (IsSibCall) { // Since we're not changing the ABI to make this a tail call, the memory @@ -11259,7 +11061,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, BEAlign = 8 - OpSize; } unsigned LocMemOffset = VA.getLocMemOffset(); - int32_t Offset = LocMemOffset + BEAlign; + int32_t Offset = LocMemOffset + BEAlign + GoStackBias; if (IsTailCall) { // When the frame pointer is perfectly aligned for the tail call and the @@ -11281,7 +11083,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL); DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); - DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset); + DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset + GoStackBias); } if (Outs[i].Flags.isByVal()) { @@ -11291,7 +11093,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getNonZeroByValAlign(), Outs[i].Flags.getNonZeroByValAlign(), - /*isVol = */ false, /*AlwaysInline = */ false, + /*isVol = */ false, /*AlwaysInline = */ IsGo, /*CI=*/nullptr, std::nullopt, DstInfo, MachinePointerInfo()); MemOpChains.push_back(Cpy); @@ -11507,6 +11309,10 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, MF.getFunction().getParent()->getModuleFlag("import-call-optimization")) DAG.addCalledGlobal(Chain.getNode(), CalledGlobal, OpFlags); + if (IsGo) + return lowerAArch64GoCallResults(*this, Chain, InGlue, Ins, GoLayout, + GoStackBias, NumBytes, DL, DAG, InVals); + uint64_t CalleePopBytes = DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0; diff --git a/llvm/lib/Target/X86/X86CallingConv.h b/llvm/lib/Target/X86/X86CallingConv.h index 8e37f345ed9e6..917fd530743df 100644 --- a/llvm/lib/Target/X86/X86CallingConv.h +++ b/llvm/lib/Target/X86/X86CallingConv.h @@ -26,8 +26,10 @@ bool RetCC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, bool CC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State); +bool CC_X86_64_Go(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + Type *OrigTy, CCState &State); } // End llvm namespace #endif - diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index cc73cc14a4abd..33abcfc368a26 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -1057,6 +1057,40 @@ def CC_X86_64_Preserve_None : CallingConv<[ CCDelegateTo ]>; +// Go's frontend is the source of truth for whole-value assignment. A direct +// parameter is guaranteed to fit completely in the ABIInternal register +// budget; a non-empty value assigned wholly to memory is represented by a +// typed byval pointer. Do not add a stack fallback for direct values here: an +// exhausted register list is an ABI carrier mismatch and must fail closed. +let Entry = 1 in +def CC_X86_64_Go : CallingConv<[ + // The closure context is not part of the ordinary Go argument layout. + CCIfNest>>, + + // Preserve the exact size and alignment carried by byval(T). + CCIfByVal>, + + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i1], CCPromoteToType>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i8], CCAssignToReg<[AL, BL, CL, DIL, SIL, + R8B, R9B, R10B, R11B]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i16], CCAssignToReg<[AX, BX, CX, DI, SI, + R8W, R9W, R10W, R11W]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i32], CCAssignToReg<[EAX, EBX, ECX, EDI, ESI, + R8D, R9D, R10D, R11D]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[i64], CCAssignToReg<[RAX, RBX, RCX, RDI, RSI, + R8, R9, R10, R11]>>>, + CCIfCC<"CallingConv::GoABIInternal", + CCIfType<[f32, f64], + CCAssignToReg<[XMM0, XMM1, XMM2, XMM3, XMM4, + XMM5, XMM6, XMM7, XMM8, XMM9, + XMM10, XMM11, XMM12, XMM13, XMM14]>>> +]>; + //===----------------------------------------------------------------------===// // X86 Root Argument Calling Conventions //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index f48a98b2437e2..9c57e8982afa9 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -24,6 +24,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/CodeGen/GoCallingConv.h" +#include "llvm/CodeGen/GoISelLowering.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/WinEHFuncInfo.h" @@ -40,31 +41,6 @@ STATISTIC(NumTailCalls, "Number of tail calls"); namespace { -template struct GoArgGroup { - unsigned Start = 0; - unsigned End = 0; - unsigned Index = 0; -}; - -template -static SmallVector, 8> groupGoArgs(ArrayRef Args) { - SmallVector, 8> Groups; - if (Args.empty()) - return Groups; - - unsigned Start = 0; - unsigned Index = Args.front().OrigArgIndex; - for (unsigned I = 1; I != Args.size(); ++I) { - if (Args[I].OrigArgIndex == Index) - continue; - Groups.push_back({Start, I, Index}); - Start = I; - Index = Args[I].OrigArgIndex; - } - Groups.push_back({Start, static_cast(Args.size()), Index}); - return Groups; -} - static constexpr unsigned X86GoIntRegs[] = {X86::RAX, X86::RBX, X86::RCX, X86::RDI, X86::RSI, X86::R8, X86::R9, X86::R10, X86::R11}; @@ -127,54 +103,6 @@ static MCPhysReg getX86GoPhysReg(MVT VT, Type *OrigTy, unsigned IntIndex, return getX86GoIntPhysReg(getX86GoCopyVT(VT), IntIndex); } -static const TargetRegisterClass *getX86GoRegClass(MVT VT, - const X86Subtarget &Subtarget) { - switch (VT.SimpleTy) { - case MVT::i8: - return &X86::GR8RegClass; - case MVT::i16: - return &X86::GR16RegClass; - case MVT::i32: - return &X86::GR32RegClass; - case MVT::i64: - return &X86::GR64RegClass; - case MVT::f32: - return Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; - case MVT::f64: - return Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; - default: - llvm_unreachable("unsupported Go ABI value type"); - } -} - -static SmallVector -getX86GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { - SmallVector ArgTys; - LayoutMap.assign(F.arg_size(), -1); - for (const Argument &Arg : F.args()) { - unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) - continue; - LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); - } - return ArgTys; -} - -static SmallVector -getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, - SmallVectorImpl &LayoutMap) { - SmallVector ArgTys; - LayoutMap.assign(Args.size(), -1); - for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) - continue; - LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); - } - return ArgTys; -} - static SmallVector getX86GoReturnTypes(Type *RetTy, const AttributeList &Attrs) { SmallVector ResultTys; @@ -182,36 +110,39 @@ static SmallVector getX86GoReturnTypes(Type *RetTy, return ResultTys; } -static SDValue lowerX86GoFormalArguments( - const X86TargetLowering &TLI, SDValue Chain, MachineFunction &MF, - const SmallVectorImpl &Ins, const SDLoc &DL, - SelectionDAG &DAG, SmallVectorImpl &InVals) { +struct X86GoFormalArgInfo { + goabi::CallLayout Layout; + SmallVector HomeFIs; +}; + +static X86GoFormalArgInfo +prepareX86GoFormalArguments(MachineFunction &MF, ArrayRef Ins, + ArrayRef ArgLocs, + uint64_t StackArgsSize, SelectionDAG &DAG) { auto *FuncInfo = MF.getInfo(); MachineFrameInfo &MFI = MF.getFrameInfo(); const Function &F = MF.getFunction(); const X86Subtarget &Subtarget = MF.getSubtarget(); - MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); // Fixed argument homes use offsets in the logical Go argument area. Stack // map locations are instead relative to the physical entry RSP, which // points at the return address for both Go calling conventions. int64_t EntryStackMapBias = - static_cast(PtrVT.getStoreSize()); + static_cast(DAG.getDataLayout().getPointerSize()); - SmallVector LayoutMap; - SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); - goabi::ABIConfig ABIConfig = - getX86GoABIConfig(Subtarget, F.getCallingConv()); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), - DAG.getDataLayout(), ABIConfig); + goabi::ABIConfig ABIConfig = getX86GoABIConfig(Subtarget, F.getCallingConv()); + X86GoFormalArgInfo Info; + Info.Layout = goabi::computeFormalArgLayout( + F, Ins, ArgLocs, StackArgsSize, DAG.getDataLayout(), ABIConfig); + const goabi::CallLayout &Layout = Info.Layout; + MFI.setGoABIArgSizes(Layout.StackArgsSize, Layout.ArgSize); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( - ArgTys, Layout, DAG.getDataLayout(), ABIConfig); + Layout, DAG.getDataLayout(), ABIConfig); SmallBitVector MatchedEntryArgWords(EntryArgs.NumBits); - SmallVector ArgSpillOffsets(ArgTys.size(), 0); + SmallVector ArgSpillOffsets(Layout.Args.size(), 0); uint64_t SpillOffset = Layout.SpillAreaOffset; - for (unsigned I = 0, E = ArgTys.size(); I != E; ++I) { + for (unsigned I = 0, E = Layout.Args.size(); I != E; ++I) { const goabi::ValueLayout &ArgLayout = Layout.Args[I]; if (!ArgLayout.InRegs) continue; @@ -225,6 +156,13 @@ static SDValue lowerX86GoFormalArguments( FuncInfo->setRegSaveFrameIndex(0xAAAAAAA); FuncInfo->clearGoArgHomes(); FuncInfo->clearGoArgPointerSlots(); + Info.HomeFIs.assign(F.arg_size(), INT_MAX); + + SmallVector IsLiveAtEntry(F.arg_size(), false); + for (const ISD::InputArg &In : Ins) + if (In.OrigArgIndex != ISD::InputArg::NoArgIndex && + In.OrigArgIndex < F.arg_size()) + IsLiveAtEntry[In.OrigArgIndex] |= In.Used; auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { @@ -259,91 +197,62 @@ static SDValue lowerX86GoFormalArguments( } }; - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - if (Group.Index == ISD::InputArg::NoArgIndex) - continue; - - const Argument *Arg = F.getArg(Group.Index); - if (Arg->hasNestAttr()) { - assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); - Register VReg = - MF.addLiveIn(X86::RDX, getX86GoRegClass(getX86GoCopyVT(Ins[Group.Start].VT), - Subtarget)); - SDValue Val = - DAG.getCopyFromReg(Chain, DL, VReg, getX86GoCopyVT(Ins[Group.Start].VT)); - if (Ins[Group.Start].VT != getX86GoCopyVT(Ins[Group.Start].VT)) - Val = DAG.getNode(ISD::TRUNCATE, DL, Ins[Group.Start].VT, Val); - InVals.push_back(Val); + unsigned NextLayoutIndex = 0; + for (const Argument &Arg : F.args()) { + if (Arg.hasNestAttr()) continue; - } - unsigned LayoutIndex = LayoutMap[Group.Index]; + if (NextLayoutIndex >= Layout.Args.size()) + report_fatal_error("X86 Go argument has no logical layout"); + unsigned LayoutIndex = NextLayoutIndex++; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; - int64_t FixedHomeOffset = LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = + MFI.CreateFixedSpillStackObject(ArgLayout.Size, LogicalHomeOffset, + /*IsImmutable=*/false); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, LogicalHomeOffset, + /*IsImmutable=*/false, + /*IsAliased=*/true); + Info.HomeFIs[Arg.getArgNo()] = HomeFI; X86MachineFunctionInfo::GoArgHome &Home = - FuncInfo->addGoArgHome(Group.Index, HomeFI); + FuncInfo->addGoArgHome(Arg.getArgNo(), HomeFI); // LLVM may replace an unused incoming pointer with poison at every call // edge. Keep its ABI home so morestack can preserve the complete register // assignment, but do not expose that uninitialized word as a GC root. - bool IsLiveAtEntry = llvm::any_of( - ArrayRef(Ins).slice(Group.Start, Group.End - Group.Start), - [](const ISD::InputArg &In) { return In.Used; }); RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, - IsLiveAtEntry); - - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - if (ArgLayout.InRegs) { - MVT CopyVT = getX86GoCopyVT(In.VT); - MCPhysReg PReg = getX86GoPhysReg( - In.VT, In.OrigTy, ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); - Register VReg = MF.addLiveIn(PReg, getX86GoRegClass(CopyVT, Subtarget)); - SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, CopyVT); - bool IsFP = isX86GoFloatPiece(In.OrigTy); - if (IsFP) - ++FPPiece; - else - ++IntPiece; + IsLiveAtEntry[Arg.getArgNo()]); - // The physical register copy may widen i1 to i8, but its Go ABI home - // retains the original piece's size and offset. - unsigned Size = static_cast( - std::max(1, In.ArgVT.getStoreSize().getKnownMinValue())); - Home.addRegisterPiece(PReg, In.PartOffset, Size, IsFP); - - if (In.VT != CopyVT) - Val = DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); - InVals.push_back(Val); + if (!ArgLayout.InRegs) + continue; + for (auto [I, In] : llvm::enumerate(Ins)) { + if (In.OrigArgIndex != Arg.getArgNo()) continue; - } + const CCValAssign &VA = ArgLocs[I]; + if (!VA.isRegLoc()) + report_fatal_error("invalid X86 Go register argument location"); + MCPhysReg PReg = VA.getLocReg(); + bool IsFP = isX86GoFloatPiece(In.OrigTy); - SDValue Addr = DAG.getFrameIndex(HomeFI, PtrVT); - if (In.PartOffset != 0) - Addr = - DAG.getObjectPtrOffset(DL, Addr, TypeSize::getFixed(In.PartOffset)); - InVals.push_back(loadX86GoStackPiece( - DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, HomeFI, In.PartOffset))); + // The physical register copy may widen i1 to i8, but its Go ABI home + // retains the original piece's size and offset. + unsigned Size = static_cast( + std::max(1, In.ArgVT.getStoreSize().getKnownMinValue())); + Home.addRegisterPiece(PReg, In.PartOffset, Size, IsFP); } } + if (NextLayoutIndex != Layout.Args.size()) + report_fatal_error("X86 Go ABI layout has unmatched arguments"); for (uint32_t Word : EntryArgs.PointerWords) if (!MatchedEntryArgWords.test(Word)) report_fatal_error( "Go entry argument pointer word has no X86 fixed object"); - return Chain; + return Info; } static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, @@ -353,51 +262,50 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) { const X86Subtarget &Subtarget = MF.getSubtarget(); MachineFrameInfo &MFI = MF.getFrameInfo(); - SmallVector LayoutMap; - SmallVector ArgTys = getX86GoArgTypes(MF.getFunction(), LayoutMap); + if (!MFI.hasGoABIArgSizes()) + report_fatal_error("missing X86 Go ABI argument layout"); SmallVector ResultTys = getX86GoReturnTypes(MF.getFunction().getReturnType(), MF.getFunction().getAttributes()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), + {}, MFI.getGoABIStackArgsSize(), ResultTys, DAG.getDataLayout(), getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); SmallVector MemOps; SmallVector, 8> RetRegs; - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::OutputArg &Out = Outs[I]; - SDValue Val = OutVals[I]; - if (ResultLayout.InRegs) { - MVT CopyVT = getX86GoCopyVT(Val.getSimpleValueType()); - if (Val.getSimpleValueType() != CopyVT) - Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); - if (isX86GoFloatPiece(Out.OrigTy)) - ++FPPiece; - else - ++IntPiece; - RetRegs.emplace_back(PReg, Val); - continue; - } - - unsigned Size = static_cast( - std::max(1, Out.ArgVT.getStoreSize().getKnownMinValue())); - int FI = - MFI.CreateFixedObject(Size, ResultLayout.StackOffset + Out.PartOffset, - /*IsImmutable=*/false); - if (MF.getInfo()->hasGoABI0FrameIndex()) - MFI.setIsAliasedObjectIndex(FI, true); - SDValue Addr = DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); - MemOps.push_back(storeX86GoStackPiece( - DAG, Chain, DL, Val, Out.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, FI))); + SmallVector IntPieces(Layout.Results.size(), 0); + SmallVector FPPieces(Layout.Results.size(), 0); + for (auto [I, Out] : llvm::enumerate(Outs)) { + unsigned ResultIndex = Out.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("X86 Go return piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + SDValue Val = OutVals[I]; + if (ResultLayout.InRegs) { + MVT CopyVT = getX86GoCopyVT(Val.getSimpleValueType()); + if (Val.getSimpleValueType() != CopyVT) + Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); + bool IsFP = isX86GoFloatPiece(Out.OrigTy); + unsigned Piece = + IsFP ? FPPieces[ResultIndex]++ : IntPieces[ResultIndex]++; + MCPhysReg PReg = getX86GoPhysReg( + Out.VT, Out.OrigTy, ResultLayout.IntRegStart + (IsFP ? 0 : Piece), + ResultLayout.FPRegStart + (IsFP ? Piece : 0)); + RetRegs.emplace_back(PReg, Val); + continue; } + + unsigned Size = static_cast( + std::max(1, Out.ArgVT.getStoreSize().getKnownMinValue())); + int FI = + MFI.CreateFixedObject(Size, ResultLayout.StackOffset + Out.PartOffset, + /*IsImmutable=*/false); + if (MF.getInfo()->hasGoABI0FrameIndex()) + MFI.setIsAliasedObjectIndex(FI, true); + SDValue Addr = DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); + MemOps.push_back( + storeX86GoStackPiece(DAG, Chain, DL, Val, Out.ArgVT, Addr, + MachinePointerInfo::getFixedStack(MF, FI))); } if (!MemOps.empty()) @@ -420,165 +328,48 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, return DAG.getNode(X86ISD::RET_GLUE, DL, MVT::Other, RetOps); } -static SDValue lowerX86GoCall(const X86TargetLowering &TLI, - TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl &InVals) { - SelectionDAG &DAG = CLI.DAG; - SDLoc &DL = CLI.DL; - auto &Outs = CLI.Outs; - auto &OutVals = CLI.OutVals; - auto &Ins = CLI.Ins; - SDValue Chain = CLI.Chain; - SDValue Callee = CLI.Callee; +static SDValue lowerX86GoCallResults(const X86TargetLowering &TLI, + SDValue Chain, SDValue InGlue, + ArrayRef Ins, + const goabi::CallLayout &Layout, + unsigned NumBytes, const SDLoc &DL, + SelectionDAG &DAG, + SmallVectorImpl &InVals) { MachineFunction &MF = DAG.getMachineFunction(); const X86Subtarget &Subtarget = MF.getSubtarget(); const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); - - CLI.IsTailCall = false; - if (CLI.IsVarArg) - report_fatal_error("Go calling convention does not support varargs"); - - SmallVector ArgLayoutMap; - SmallVector ArgTys = getX86GoCallArgTypes(CLI.getArgs(), ArgLayoutMap); - SmallVector ResultTys; - goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getX86GoABIConfig(Subtarget, CLI.CallConv)); - - unsigned NumBytes = Layout.TotalStackSize; - Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); - - SDValue StackPtr; - if (NumBytes != 0) - StackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), PtrVT); - - SmallVector MemOpChains; - SmallVector, 8> RegsToPass; - MachineFunction::CallSiteInfo CSInfo; - - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - if (Group.Index >= CLI.getArgs().size()) - continue; - - if (CLI.getArgs()[Group.Index].IsNest) { - assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); - RegsToPass.emplace_back(X86::RDX, OutVals[Group.Start]); - continue; - } - - const goabi::ValueLayout &ArgLayout = Layout.Args[ArgLayoutMap[Group.Index]]; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - SDValue Arg = OutVals[I]; - const ISD::OutputArg &Out = Outs[I]; - if (ArgLayout.InRegs) { - MVT CopyVT = getX86GoCopyVT(Arg.getSimpleValueType()); - if (Arg.getSimpleValueType() != CopyVT) - Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Arg); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); - if (isX86GoFloatPiece(Out.OrigTy)) - ++FPPiece; - else - ++IntPiece; - RegsToPass.emplace_back(PReg, Arg); - if (DAG.getTarget().Options.EmitCallSiteInfo) - CSInfo.ArgRegPairs.emplace_back(PReg, I); - continue; - } - - assert(StackPtr && "missing Go call stack pointer"); - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant(ArgLayout.StackOffset + Out.PartOffset, - DL)); - MemOpChains.push_back(storeX86GoStackPiece( - DAG, Chain, DL, Arg, Out.ArgVT, Addr, - MachinePointerInfo::getStack(MF, - ArgLayout.StackOffset + Out.PartOffset))); - } - } - - if (!MemOpChains.empty()) - Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); - - bool IsImpCall = false; - if (DAG.getTarget().getCodeModel() != CodeModel::Large && - (Callee->getOpcode() == ISD::GlobalAddress || - Callee->getOpcode() == ISD::ExternalSymbol)) - Callee = - TLI.lowerGlobalOrExternalForCall(Callee, DAG, /*IsImpCall=*/&IsImpCall); - else if (Subtarget.isTarget64BitILP32() && Callee.getValueType() == MVT::i32) - Callee = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Callee); - - SDValue InGlue; - for (const auto &[Reg, Val] : RegsToPass) { - Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue); - InGlue = Chain.getValue(1); - } - - SmallVector Ops; - Ops.push_back(Chain); - Ops.push_back(Callee); - for (const auto &[Reg, Val] : RegsToPass) - Ops.push_back(DAG.getRegister(Reg, Val.getValueType())); - Ops.push_back(DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); - if (InGlue.getNode()) - Ops.push_back(InGlue); - - SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); - Chain = DAG.getNode(IsImpCall ? X86ISD::IMP_CALL : X86ISD::CALL, DL, NodeTys, - Ops); - DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); - InGlue = Chain.getValue(1); - SmallVector ResultVals(Ins.size()); - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + + SmallVector IntPieces(Layout.Results.size(), 0); + SmallVector FPPieces(Layout.Results.size(), 0); + for (auto [I, In] : llvm::enumerate(Ins)) { + unsigned ResultIndex = In.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("X86 Go call result piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (!ResultLayout.InRegs) continue; - unsigned IntPiece = 0; - unsigned FPPiece = 0; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - MVT CopyVT = getX86GoCopyVT(In.VT); - MCPhysReg PReg = getX86GoPhysReg( - In.VT, In.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); - SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); - Chain = Val.getValue(1); - InGlue = Val.getValue(2); - if (isX86GoFloatPiece(In.OrigTy)) - ++FPPiece; - else - ++IntPiece; - if (In.VT != CopyVT) - ResultVals[I] = DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); - else - ResultVals[I] = Val; - } - } - - // Read every register result before introducing stack loads into the chain. - // A result that does not fit in the remaining register budget is placed on - // the stack without consuming that budget, so a later result can still be - // register-assigned. Interleaving stack loads with glued physical-register - // copies can create a cyclic scheduler dependency at statepoints. - // - // Keep the call sequence active while reading stack results. The load chain - // is after the call, so the physical SP is the current post-growth Go stack; - // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + MVT CopyVT = getX86GoCopyVT(In.VT); + bool IsFP = isX86GoFloatPiece(In.OrigTy); + unsigned Piece = IsFP ? FPPieces[ResultIndex]++ : IntPieces[ResultIndex]++; + MCPhysReg PReg = getX86GoPhysReg( + In.VT, In.OrigTy, ResultLayout.IntRegStart + (IsFP ? 0 : Piece), + ResultLayout.FPRegStart + (IsFP ? Piece : 0)); + SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); + Chain = Val.getValue(1); + InGlue = Val.getValue(2); + ResultVals[I] = + In.VT == CopyVT ? Val : DAG.getNode(ISD::TRUNCATE, DL, In.VT, Val); + } + + // Register results must be read before stack result loads. Keeping the call + // sequence active also keeps the outgoing Go frame, and therefore the + // post-growth result addresses, valid until every stack result is consumed. + bool HasStackResults = llvm::any_of(Ins, [&](const ISD::InputArg &In) { + return In.OrigArgIndex >= Layout.Results.size() || + !Layout.Results[In.OrigArgIndex].InRegs; + }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), @@ -586,28 +377,25 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, Chain = ResultStackPtr.getValue(1); InGlue = ResultStackPtr.getValue(2); } - for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + for (auto [I, In] : llvm::enumerate(Ins)) { + unsigned ResultIndex = In.OrigArgIndex; + if (ResultIndex >= Layout.Results.size()) + report_fatal_error("X86 Go call result piece has no logical result"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs) continue; - for (unsigned I = Group.Start; I != Group.End; ++I) { - const ISD::InputArg &In = Ins[I]; - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, - DAG.getIntPtrConstant(ResultLayout.StackOffset + In.PartOffset, - DL)); - SDValue Load = loadX86GoStackPiece( - DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getStack( - MF, ResultLayout.StackOffset + In.PartOffset)); - Chain = Load.getValue(1); - ResultVals[I] = Load; - } + uint64_t Offset = ResultLayout.StackOffset + In.PartOffset; + SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(Offset, DL)); + SDValue Load = + loadX86GoStackPiece(DAG, Chain, DL, In.VT, In.ArgVT, Addr, + MachinePointerInfo::getStack(MF, Offset)); + Chain = Load.getValue(1); + ResultVals[I] = Load; } Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); - InVals.append(ResultVals.begin(), ResultVals.end()); return Chain; } @@ -617,7 +405,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, std::optional X86TargetLowering::getArgumentCopyElisionFrameInfo(const Argument &Arg, MachineFunction &MF) const { - if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) + if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv()) || + Arg.hasByValAttr()) return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); @@ -638,18 +427,15 @@ int X86TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { if (FuncInfo->hasGoABI0FrameIndex()) return FuncInfo->getGoABI0FrameIndex(); - SmallVector LayoutMap; - SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), - MF.getDataLayout(), - getX86GoABIConfig(MF.getSubtarget(), F.getCallingConv())); - if (Layout.ArgSize == 0) + MachineFrameInfo &MFI = MF.getFrameInfo(); + if (!MFI.hasGoABIArgSizes()) + report_fatal_error("missing X86 Go ABI argument layout"); + if (MFI.getGoABIArgSize() == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); - int FI = MF.getFrameInfo().CreateFixedObject(Layout.ArgSize, /*SPOffset=*/0, - /*IsImmutable=*/false, - /*IsAliased=*/true); + int FI = MFI.CreateFixedObject(MFI.getGoABIArgSize(), /*SPOffset=*/0, + /*IsImmutable=*/false, + /*IsAliased=*/true); for (const X86MachineFunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) { MF.getFrameInfo().setIsImmutableObjectIndex(Home.FrameIndex, false); @@ -1903,14 +1689,16 @@ static bool hasCalleePopSRet(const SmallVectorImpl &Args, /// Make a copy of an aggregate at address specified by "Src" to address /// "Dst" with size and alignment information specified by the specific /// parameter attribute. The copy will be passed as a byval function parameter. -static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, - SDValue Chain, ISD::ArgFlagsTy Flags, - SelectionDAG &DAG, const SDLoc &dl) { +static SDValue +CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, + ISD::ArgFlagsTy Flags, SelectionDAG &DAG, + const SDLoc &dl, + MachinePointerInfo DstInfo = MachinePointerInfo()) { SDValue SizeNode = DAG.getIntPtrConstant(Flags.getByValSize(), dl); Align Alignment = Flags.getNonZeroByValAlign(); return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Alignment, Alignment, /*isVolatile*/ false, /*AlwaysInline=*/true, - /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), + /*CI=*/nullptr, std::nullopt, DstInfo, MachinePointerInfo()); } @@ -2342,9 +2130,7 @@ SDValue X86TargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { - if (goabi::isGoCallingConv(CallConv)) - return lowerX86GoFormalArguments(*this, Chain, DAG.getMachineFunction(), Ins, - dl, DAG, InVals); + bool IsGo = goabi::isGoCallingConv(CallConv); MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo(); @@ -2373,6 +2159,8 @@ SDValue X86TargetLowering::LowerFormalArguments( assert( !(IsVarArg && canGuaranteeTCO(CallConv)) && "Var args not supported with calling conv' regcall, fastcc, ghc or hipe"); + if (IsGo && IsVarArg) + report_fatal_error("X86 Go calling conventions do not support varargs"); // Assign locations to all of the incoming arguments. SmallVector ArgLocs; @@ -2382,7 +2170,7 @@ SDValue X86TargetLowering::LowerFormalArguments( if (IsWin64) CCInfo.AllocateStack(32, Align(8)); - CCInfo.AnalyzeArguments(Ins, CC_X86); + CCInfo.AnalyzeArguments(Ins, IsGo ? CC_X86_64_Go : CC_X86); // In vectorcall calling convention a second pass is required for the HVA // types. @@ -2395,6 +2183,11 @@ SDValue X86TargetLowering::LowerFormalArguments( assert(isSortedByValueNo(ArgLocs) && "Argument Location list must be sorted before lowering"); + std::optional GoInfo; + if (IsGo) + GoInfo = prepareX86GoFormalArguments(MF, Ins, ArgLocs, + CCInfo.getStackSize(), DAG); + SDValue ArgValue; for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E; ++I, ++InsIndex) { @@ -2482,6 +2275,14 @@ SDValue X86TargetLowering::LowerFormalArguments( } else ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); } + } else if (IsGo && Ins[InsIndex].Flags.isByVal()) { + unsigned ArgIndex = Ins[InsIndex].OrigArgIndex; + if (ArgIndex == ISD::InputArg::NoArgIndex || + ArgIndex >= GoInfo->HomeFIs.size() || + GoInfo->HomeFIs[ArgIndex] == INT_MAX) + report_fatal_error("X86 Go byval argument has no incoming home"); + ArgValue = DAG.getFrameIndex(GoInfo->HomeFIs[ArgIndex], + getPointerTy(DAG.getDataLayout())); } else { assert(VA.isMemLoc()); ArgValue = @@ -2538,7 +2339,8 @@ SDValue X86TargetLowering::LowerFormalArguments( } } - unsigned StackSize = CCInfo.getStackSize(); + unsigned StackSize = + IsGo ? GoInfo->Layout.TotalStackSize : CCInfo.getStackSize(); // Align stack specially for tail calls. if (shouldGuaranteeTCO(CallConv, MF.getTarget().Options.GuaranteedTailCallOpt)) @@ -2618,7 +2420,9 @@ SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), StackPtr, PtrOff); if (isByVal) - return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); + return CreateCopyOfByValArgument( + Arg, PtrOff, Chain, Flags, DAG, dl, + MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset)); MaybeAlign Alignment; if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() && @@ -2718,13 +2522,10 @@ X86TargetLowering::ByValCopyKind X86TargetLowering::ByValNeedsCopyForTailCall( return CopyViaTemp; } -SDValue -X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl &InVals) const { - if (goabi::isGoCallingConv(CLI.CallConv)) - return lowerX86GoCall(*this, CLI, InVals); - SelectionDAG &DAG = CLI.DAG; - SDLoc &dl = CLI.DL; +SDValue X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, + SmallVectorImpl &InVals) const { + SelectionDAG &DAG = CLI.DAG; + SDLoc &dl = CLI.DL; SmallVectorImpl &Outs = CLI.Outs; SmallVectorImpl &OutVals = CLI.OutVals; SmallVectorImpl &Ins = CLI.Ins; @@ -2736,7 +2537,15 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, const auto *CB = CLI.CB; MachineFunction &MF = DAG.getMachineFunction(); - bool Is64Bit = Subtarget.is64Bit(); + bool Is64Bit = Subtarget.is64Bit(); + bool IsGo = goabi::isGoCallingConv(CallConv); + if (IsGo) { + isTailCall = false; + if (!Is64Bit) + report_fatal_error("Go calling convention requires x86-64"); + if (isVarArg) + report_fatal_error("Go calling convention does not support varargs"); + } bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); bool ShouldGuaranteeTCO = shouldGuaranteeTCO( CallConv, MF.getTarget().Options.GuaranteedTailCallOpt); @@ -2777,7 +2586,13 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (IsWin64) CCInfo.AllocateStack(32, Align(8)); - CCInfo.AnalyzeArguments(Outs, CC_X86); + CCInfo.AnalyzeArguments(Outs, IsGo ? CC_X86_64_Go : CC_X86); + + goabi::CallLayout GoLayout; + if (IsGo) + GoLayout = + goabi::computeCallLayout(CLI, ArgLocs, CCInfo.getStackSize(), + getX86GoABIConfig(Subtarget, CLI.CallConv)); // In vectorcall calling convention a second pass is required for the HVA // types. @@ -2813,7 +2628,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, "Var args not supported with calling convention fastcc, ghc or hipe"); // Get a count of how many bytes are to be pushed on the stack. - unsigned NumBytes = CCInfo.getAlignedCallFrameSize(); + unsigned NumBytes = + IsGo ? GoLayout.TotalStackSize : CCInfo.getAlignedCallFrameSize(); if (IsSibcall) // This is a sibcall. The memory operands are available in caller's // own caller's stack. @@ -3395,6 +3211,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (MDNode *HeapAlloc = CLI.CB->getMetadata("heapallocsite")) DAG.addHeapAllocSite(Chain.getNode(), HeapAlloc); + if (IsGo) + return lowerX86GoCallResults(*this, Chain, InGlue, Ins, GoLayout, NumBytes, + dl, DAG, InVals); + // Create the CALLSEQ_END node. unsigned NumBytesForCalleeToPop = 0; // Callee pops nothing. if (X86::isCalleePop(CallConv, Is64Bit, isVarArg, diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index d41426c27aa01..740be79718780 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -53,13 +53,15 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @abi0_second_int(i64 %a, i64 %b) { +define goabi0 i64 @abi0_second_int( + ptr byval(i64) align 8 %a.home, ptr byval(i64) align 8 %b.home) { ; A64-LABEL: abi0_second_int: ; Go's arm64 stack ABI reserves 0(SP) for the return PC. ; A64: ldr x[[REG:[0-9]+]], [sp, #16] ; A64: str x[[REG]], [sp, #24] ; A64: ret entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } @@ -71,10 +73,16 @@ define goabi0 i64 @abi0_call_second_int() { ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] -; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. -; A64: str x[[RET]], [sp, #56] +; The ordinary source allocas make this a 64-byte caller frame. This function's +; ABI0 result still resides at entry SP+8. +; A64: str x[[RET]], [sp, #72] entry: - %ret = call goabi0 i64 @abi0_second_int(i64 11, i64 22) + %a.home = alloca i64, align 8 + %b.home = alloca i64, align 8 + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @abi0_second_int( + ptr byval(i64) align 8 %a.home, ptr byval(i64) align 8 %b.home) ret i64 %ret } @@ -111,30 +119,33 @@ define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { + ptr byval(%pair) align 8 %value.home) { ; A64-LABEL: stack_pair: ; A64: ldr x0, [sp, #16] ; A64: ret entry: + %value = load %pair, ptr %value.home, align 8 %right = extractvalue %pair %value, 1 ret i64 %right } define goabiinternal i64 @call_stack_pair() { ; A64-LABEL: call_stack_pair: -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #16] +; A64: stur q{{[0-9]+}}, [x{{[0-9]+}}, #8] ; A64: bl stack_pair entry: + %value.home = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, - %pair { i64 13, i64 17 }) + ptr byval(%pair) align 8 %value.home) ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr byval([8 x i8]) align 1 %value.home) { ; A64-LABEL: stack_bytes: ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #8] ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #15] @@ -142,13 +153,13 @@ define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { ; A64-DAG: strb w{{[0-9]+}}, [sp, #23] ; A64: ret entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; A64-LABEL: call_stack_bytes: -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #15] +; A64: str x{{[0-9]+}}, [x{{[0-9]+}}, #8] ; A64: bl stack_bytes ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #16] ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #23] @@ -158,8 +169,11 @@ define goabiinternal i16 @call_stack_bytes() { ; A64-O2: ldrb w{{[0-9]+}}, [sp, #16] ; A64-O2: add sp, sp, #{{[0-9]+}} entry: + %value.home = alloca [8 x i8], align 1 + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr byval([8 x i8]) align 1 %value.home) %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/AArch64/go-stack-byval.ll b/llvm/test/CodeGen/AArch64/go-stack-byval.ll new file mode 100644 index 0000000000000..c0e34090f3845 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-stack-byval.ll @@ -0,0 +1,159 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR + +%pair = type { i64, i64 } + +declare goabiinternal void @consume(i64, i64, i64, i64, i64, i64, i64, i64, + i64, i64, i64, i64, i64, i64, i64, i64, + ptr byval(i64) align 8) +declare goabiinternal void @consume_pair( + i64, i64, i64, i64, i64, i64, i64, i64, + i64, i64, i64, i64, i64, i64, i64, + ptr byval(%pair) align 8) +declare goabiinternal float @consume_memory_float( + ptr byval(float) align 4, float) +declare void @llvm.lifetime.start.p0(ptr captures(none)) + +define goabiinternal void @ssa_stack_argument() { +; CHECK-LABEL: ssa_stack_argument: +; CHECK-NOT: memcpy +; The frontend alloca remains an ordinary source object. Generic byval +; lowering copies its value into the outgoing Go argument area. +; CHECK: mov w{{[0-9]+}}, #42 +; CHECK: str x{{[0-9]+}}, [sp, #8] +; CHECK: bl consume +entry: + %argument = alloca i64, align 8 + store i64 42, ptr %argument, align 8 + call goabiinternal void @consume( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal void @register_argument_byval_source(i64 %value) { +; CHECK-LABEL: register_argument_byval_source: +; Existing incoming argument-copy elision reuses %value's fixed home for the +; temporary, while ordinary byval lowering still writes the outgoing slot. +; CHECK: bl consume +; MIR-LABEL: name: register_argument_byval_source +; MIR: fixedStack: +; MIR: - { id: 0, type: spill-slot, offset: 8, size: 8 +; MIR: stack: [] +; MIR: %[[VALUE:[0-9]+]]:gpr64 = COPY $x0 +; MIR: STRXui %[[VALUE]], %fixed-stack.0, 0 +; MIR: STRXui %[[VALUE]], %{{[0-9]+}}, 1 +entry: + %argument = alloca i64, align 8 + store i64 %value, ptr %argument, align 8 + call goabiinternal void @consume( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal void @memory_stack_argument(ptr %source) { +; CHECK-LABEL: memory_stack_argument: +; CHECK-NOT: memcpy +; CHECK: ldr x{{[0-9]+}}, [x{{[0-9]+}}] +; CHECK: str x{{[0-9]+}}, [sp, #8] +; CHECK: bl consume +entry: + call goabiinternal void @consume( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, + ptr byval(i64) align 8 %source) + ret void +} + +define goabiinternal void @ssa_aggregate_stack_argument() { +; CHECK-LABEL: ssa_aggregate_stack_argument: +; CHECK: ldr q[[VALUE:[0-9]+]], [sp, #{{[1-9][0-9]*}}] +; CHECK: stur q[[VALUE]], [sp, #8] +; CHECK: bl consume_pair +; MIR-LABEL: name: ssa_aggregate_stack_argument +; MIR: stack: +; MIR: - { id: 0, name: argument, type: default, offset: 0, size: 16 +; MIR: LDRQui %stack.0.argument +; MIR: STURQi +; MIR-SAME: store (s128) into stack + 8 +entry: + %argument = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %argument, align 8 + call goabiinternal void @consume_pair( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, + ptr byval(%pair) align 8 %argument) + ret void +} + +define goabiinternal void @noncanonical_stack_argument() { +; Lifetime markers do not change the ordinary source-object plus byval-copy +; semantics. +; MIR-LABEL: name: noncanonical_stack_argument +; MIR: stack: +; MIR-NEXT: - { id: 0, +entry: + %argument = alloca i64, align 8 + call void @llvm.lifetime.start.p0(ptr %argument) + store i64 42, ptr %argument, align 8 + call goabiinternal void @consume( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal i64 @read_stack_argument( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, + i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, + i64 %a15, ptr byval(i64) align 8 %value) { +; CHECK-LABEL: read_stack_argument: +; CHECK: ldr x0, [sp, #8] +; CHECK-NEXT: ret +; MIR-LABEL: name: read_stack_argument +; MIR: frameInfo: +; MIR: goABIStackArgsSize: 8 +; MIR: goABIArgSize: 136 +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +define goabiinternal ptr @address_stack_pair_argument( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, + i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, + ptr byval(%pair) align 8 %value) { +; CHECK-LABEL: address_stack_pair_argument: +; CHECK: add x0, sp, #8 +; MIR-LABEL: name: address_stack_pair_argument +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 8, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +entry: + ret ptr %value +} + +define goabiinternal float @read_memory_float( + ptr byval(float) align 4 %memory, float %register) { +; CHECK-LABEL: read_memory_float: +; CHECK: ldr [[MEMORY:s[0-9]+]], [sp, #8] +; CHECK: fadd s0, [[MEMORY]], s0 +entry: + %loaded = load float, ptr %memory, align 4 + %sum = fadd float %loaded, %register + ret float %sum +} + +define goabiinternal float @call_memory_float(ptr %source, float %register) { +; CHECK-LABEL: call_memory_float: +; CHECK: str {{w[0-9]+}}, [sp, #8] +; CHECK: bl consume_memory_float +entry: + %result = call goabiinternal float @consume_memory_float( + ptr byval(float) align 4 %source, float %register) + ret float %result +} diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll index 00961b87513ae..fde133b454f83 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll @@ -11,8 +11,9 @@ define goabiinternal ptr addrspace(1) @scalar_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + ptr readonly byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -27,8 +28,9 @@ define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + ptr readonly byval(%aggregate) align 8 %value.home) gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -50,8 +52,10 @@ define goabiinternal ptr addrspace(1) @merged_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = icmp ne ptr addrspace(1) %p0, null %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -69,8 +73,10 @@ define goabiinternal ptr addrspace(1) @relocated_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr readonly byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = icmp ne ptr addrspace(1) %p0, null %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -106,6 +112,42 @@ merge: ret ptr addrspace(1) %relocated3 } +define goabiinternal ptr addrspace(1) @mutated_stack_arg( + ptr addrspace(1) %p0, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, + i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, + i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, + ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { +entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + store ptr addrspace(1) null, ptr %p16.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %p16) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + +define goabiinternal ptr addrspace(1) @byval_home_address( + ptr addrspace(1) %heap, + ptr addrspace(1) byval(%aggregate) align 8 %value.home) gc "statepoint-example" { +entry: + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 8, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %heap, + ptr addrspace(1) %value.home) ] + %heap.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + %home.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 1, i32 1) + ret ptr addrspace(1) %home.relocated +} + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( @@ -113,23 +155,29 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-LABEL: name: scalar_stack_arg ; CHECK: fixedStack: -; CHECK: - { id: 0, type: default, offset: 8, size: 8, +; CHECK: - { id: [[SCALAR_HOME:[0-9]+]], type: default, offset: 8, size: 8, ; CHECK: isImmutable: false -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[SCALAR_SLOT:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: STRXui {{.*}}, %stack.[[SCALAR_SLOT]], 0 ; CHECK: STATEPOINT 1, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.0, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0) +; CHECK-SAME: 2, 1, 1, 8, %stack.[[SCALAR_SLOT]], 0, +; CHECK-SAME: (volatile load store (s64) on %stack.[[SCALAR_SLOT]]) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 +; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.[[SCALAR_SLOT]] ; CHECK-LABEL: name: aggregate_stack_arg -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[AGG_FIRST:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: - { id: [[AGG_SECOND:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: STRXui {{.*}}, %stack.[[AGG_FIRST]], 0 +; CHECK: STRXui {{.*}}, %stack.[[AGG_SECOND]], 0 ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1) +; CHECK-SAME: 2, 2, 1, 8, %stack.[[AGG_FIRST]], 0, 1, 8, %stack.[[AGG_SECOND]], 0, +; CHECK-SAME: (volatile load store (s64) on %stack.[[AGG_FIRST]]), +; CHECK-SAME: (volatile load store (s64) on %stack.[[AGG_SECOND]]) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.{{[0-9]+}} ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: @@ -140,10 +188,28 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: relocated_stack_arg -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[HOME:[0-9]+]], name: '', type: default, offset: 0, size: 8, ; CHECK: STATEPOINT 4, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME:[0-9]+]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, ; CHECK: STATEPOINT 5, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, ; CHECK: STATEPOINT 6, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, + +; CHECK-LABEL: name: mutated_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: STRXui {{.*}}, %stack.0, 0 +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, + +; CHECK-LABEL: name: byval_home_address +; CHECK: fixedStack: +; CHECK: - { id: [[BYVAL_HOME:[0-9]+]], type: default, offset: 8, size: 24, +; CHECK: stack: +; CHECK: - { id: [[HEAP_SLOT:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: STATEPOINT 8, +; CHECK-SAME: 2, 2, 0, %fixed-stack.[[BYVAL_HOME]], 0, 1, 8, %stack.[[HEAP_SLOT]], 0, +; CHECK-NEXT: ADJCALLSTACKUP +; CHECK-NEXT: {{%[0-9]+}}:gpr64sp = ADDXri %fixed-stack.[[BYVAL_HOME]], 0, 0 diff --git a/llvm/test/CodeGen/AArch64/goobj-abi.ll b/llvm/test/CodeGen/AArch64/goobj-abi.ll index 12803449afa58..85d537b23bf6c 100644 --- a/llvm/test/CodeGen/AArch64/goobj-abi.ll +++ b/llvm/test/CodeGen/AArch64/goobj-abi.ll @@ -11,8 +11,11 @@ entry: ret i64 %sum } -define goabi0 i64 @"stackadd"(i64 %a, i64 %b) #0 { +define goabi0 i64 @"stackadd"( + ptr byval(i64) align 8 %a.home, ptr byval(i64) align 8 %b.home) #0 { entry: + %a = load i64, ptr %a.home, align 8 + %b = load i64, ptr %b.home, align 8 %sum = add i64 %a, %b ret i64 %sum } diff --git a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-invalid.mir b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-invalid.mir index 7f35f07bc5a4d..3ae5b97577730 100644 --- a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-invalid.mir +++ b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-invalid.mir @@ -22,6 +22,8 @@ name: inconsistent tracksRegLiveness: true frameInfo: + goABIStackArgsSize: 0 + goABIArgSize: 0 stackSize: 16 adjustsStack: true hasCalls: true @@ -58,6 +60,9 @@ body: | --- name: unreachable tracksRegLiveness: true +frameInfo: + goABIStackArgsSize: 0 + goABIArgSize: 0 body: | bb.0: RET undef $lr diff --git a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-layout.mir b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-layout.mir index e39c5f1195827..c2cdf1e76ae86 100644 --- a/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-layout.mir +++ b/llvm/test/CodeGen/AArch64/goobj-pcsp-cfg-layout.mir @@ -72,6 +72,8 @@ liveins: - { reg: '$x0', virtual-reg: '' } - { reg: '$w1', virtual-reg: '' } frameInfo: + goABIStackArgsSize: 0 + goABIArgSize: 16 isFrameAddressTaken: false isReturnAddressTaken: false hasStackMap: false diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index 50ee4a27d6226..a019d3941da05 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -11,16 +11,16 @@ entry: ret void } -define goabiinternal i64 @large_home_offset([4096 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_offset( + ptr byval([4096 x i64]) align 8 %stackarg.home, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg } -define goabiinternal i64 @large_home_boundary([4094 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_boundary( + ptr byval([4094 x i64]) align 8 %stackarg.home, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll index 1bec5369e70a4..695e2007eaf78 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll @@ -32,8 +32,9 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %p16) "frame-pointer"="non-leaf" { + ptr byval(ptr) align 8 %p16.home) "frame-pointer"="non-leaf" { entry: + %p16 = load ptr, ptr %p16.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 diff --git a/llvm/test/CodeGen/Generic/go-abi0-frame.ll b/llvm/test/CodeGen/Generic/go-abi0-frame.ll index fd7d23c8cbe9d..df50214a101d7 100644 --- a/llvm/test/CodeGen/Generic/go-abi0-frame.ll +++ b/llvm/test/CodeGen/Generic/go-abi0-frame.ll @@ -12,7 +12,8 @@ declare ptr @llvm.go.abi0.frame() ; The intrinsic denotes one mutable, aliased object spanning the two arguments ; and one result slot. It deliberately overlaps the per-value fixed objects. -define goabi0 ptr @abi0_frame(i64 %a, i64 %b) noinline { +define goabi0 ptr @abi0_frame(ptr byval(i64) align 8 %a, + ptr byval(i64) align 8 %b) noinline { entry: %frame = call ptr @llvm.go.abi0.frame() ret ptr %frame diff --git a/llvm/test/CodeGen/Generic/go-argument-homes.ll b/llvm/test/CodeGen/Generic/go-argument-homes.ll index 67e7696baafdd..b66cfa5c6831d 100644 --- a/llvm/test/CodeGen/Generic/go-argument-homes.ll +++ b/llvm/test/CodeGen/Generic/go-argument-homes.ll @@ -9,7 +9,6 @@ ; RUN: FileCheck %s --check-prefix=A64-ASM %padded = type { i8, i64 } -%pair = type { i64, i64 } ; A register argument's canonical alloca is remapped to its ABI spill home and ; the IR store initializes that home. @@ -66,48 +65,3 @@ entry: ; A64-ASM-DAG: strb w0, [sp, #8] ; A64-ASM-DAG: str x1, [sp, #16] ; A64-ASM: add x0, sp, #8 - -; Exhaust X86's integer register budget so the pair arrives in one stack home. -define goabiinternal ptr @x86_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; X86-LABEL: name: x86_stack_pair_home -; X86: fixedStack: -; X86-NEXT: - { id: 0, type: default, offset: 0, size: 16 -; X86-NEXT: isImmutable: false, isAliased: true -; X86: MOV64rm %fixed-stack.0 -; X86: MOV64rm %fixed-stack.0{{.*}}8 -; X86-NOT: MOV64mr %fixed-stack.0 -; X86: LEA64r %fixed-stack.0 -; X86-ASM-LABEL: x86_stack_pair_home: -; X86-ASM-DAG: movq 8(%rsp), %rax -; X86-ASM-DAG: movq 16(%rsp), %rax -; X86-ASM: leaq 8(%rsp), %rax - -; Exhaust AArch64's integer register budget so the pair arrives in one stack -; home at the target's Go stack bias. -define goabiinternal ptr @aarch64_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; A64-LABEL: name: aarch64_stack_pair_home -; A64: fixedStack: -; A64-NEXT: - { id: 0, type: default, offset: 8, size: 16 -; A64-NEXT: isImmutable: false, isAliased: true -; A64-NOT: STRXui {{.*}}%fixed-stack.0 -; A64: ADDXri %fixed-stack.0 -; A64-ASM-LABEL: aarch64_stack_pair_home: -; A64-ASM: add x0, sp, #8 diff --git a/llvm/test/CodeGen/Generic/go-direct-argument-overflow.ll b/llvm/test/CodeGen/Generic/go-direct-argument-overflow.ll new file mode 100644 index 0000000000000..2a27d4dc71402 --- /dev/null +++ b/llvm/test/CodeGen/Generic/go-direct-argument-overflow.ll @@ -0,0 +1,16 @@ +; RUN: not --crash llc -mtriple=x86_64-unknown-linux-gnu -filetype=null < %s 2>&1 \ +; RUN: | FileCheck %s +; RUN: not --crash llc -mtriple=aarch64-unknown-linux-gnu -filetype=null < %s 2>&1 \ +; RUN: | FileCheck %s + +; A direct Go ABI value has no stack fallback. The frontend must use a typed +; byval carrier once a whole logical argument no longer fits in registers. +define goabiinternal void @direct_register_overflow( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, + i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, + i64 %a14, i64 %a15, i64 %a16) { +entry: + ret void +} + +; CHECK: LLVM ERROR: unable to allocate function argument # diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll index 428cdd6055b65..3fc47b1205753 100644 --- a/llvm/test/CodeGen/Generic/goobj-nosplit.ll +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -7,7 +7,7 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s declare goabiinternal void @callee(ptr) -declare goabi0 void @"callee.abi0"(ptr) +declare goabi0 void @"callee.abi0"(ptr byval(ptr) align 8) define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { entry: @@ -28,11 +28,13 @@ entry: define goabiinternal void @nosplit_abi0_call(ptr %pointer) "go-nosplit" { entry: %closure = alloca [3 x ptr], align 8 + %closure.arg = alloca ptr, align 8 %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 store volatile ptr @callee, ptr %code, align 8 store volatile ptr %pointer, ptr %context, align 8 - call goabi0 void @"callee.abi0"(ptr %closure) + store ptr %closure, ptr %closure.arg, align 8 + call goabi0 void @"callee.abi0"(ptr byval(ptr) align 8 %closure.arg) ret void } @@ -45,5 +47,5 @@ entry: ; CHECK-DAG: runtime.morestack_noctxt ; A64-LABEL: name: nosplit_abi0_call -; A64: stackSize: 48 +; A64: stackSize: 64 ; A64: maxCallFrameSize: 16 diff --git a/llvm/test/CodeGen/X86/go-callconv.ll b/llvm/test/CodeGen/X86/go-callconv.ll index 226f7007968ad..bcc15ab52562a 100644 --- a/llvm/test/CodeGen/X86/go-callconv.ll +++ b/llvm/test/CodeGen/X86/go-callconv.ll @@ -61,25 +61,33 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @"abi0_second_int"(i64 %a, i64 %b) { +define goabi0 i64 @"abi0_second_int"( + ptr byval(i64) align 8 %a.home, ptr byval(i64) align 8 %b.home) { ; X86-LABEL: "abi0_second_int": -; X86: movq 16(%rsp), %rax +; X86: leaq 16(%rsp), %[[B_HOME:r[a-z0-9]+]] +; X86-NEXT: movq (%[[B_HOME]]), %rax ; X86: movq %rax, 24(%rsp) ; X86: retq entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } define goabi0 i64 @"abi0_call_second_int"() { ; X86-LABEL: "abi0_call_second_int": ; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86: movq $22, 8(%[[BASE]]) -; X86: movq $11, (%[[BASE]]) +; X86: movq %{{r[a-z0-9]+}}, 8(%[[BASE]]) +; X86: movq %{{r[a-z0-9]+}}, (%[[BASE]]) ; X86: callq "abi0_second_int" ; X86: movq %rsp, %[[RELOAD:r[a-z0-9]+]] ; X86: movq 16(%[[RELOAD]]), %rax entry: - %ret = call goabi0 i64 @"abi0_second_int"(i64 11, i64 22) + %a.home = alloca i64, align 8 + %b.home = alloca i64, align 8 + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @"abi0_second_int"( + ptr byval(i64) align 8 %a.home, ptr byval(i64) align 8 %b.home) ret i64 %ret } @@ -125,9 +133,13 @@ define goabiinternal i64 @call_method_results(ptr %callee, ptr %recv, ; X86-O2: addq 56(%rsp), %rax ; X86-O2: addq ${{[0-9]+}}, %rsp entry: + %x.home = alloca [2 x i64], align 8 + %y.home = alloca [2 x double], align 8 + store [2 x i64] [i64 456, i64 789], ptr %x.home, align 8 + store [2 x double] [double 3.4, double 5.6], ptr %y.home, align 8 %result = call goabiinternal %method.results %callee( - ptr %recv, i64 123, [2 x i64] [i64 456, i64 789], - double 1.2, [2 x double] [double 3.4, double 5.6], ptr nest %ctxt) #0 + ptr %recv, i64 123, ptr byval([2 x i64]) align 8 %x.home, + double 1.2, ptr byval([2 x double]) align 8 %y.home, ptr nest %ctxt) #0 %s = extractvalue %method.results %result, 0 %a = extractvalue %method.results %result, 1 %x = extractvalue %method.results %result, 2 @@ -148,13 +160,16 @@ entry: define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr byval(%pair) align 8 %value.home) { ; X86-LABEL: stack_pair: -; X86-DAG: movq 8(%rsp), %[[LEFT:r[a-z0-9]+]] -; X86-DAG: movq 16(%rsp), %[[RIGHT:r[a-z0-9]+]] +; X86: leaq 8(%rsp), %[[PAIR_HOME:r[a-z0-9]+]] +; X86-DAG: movq (%[[PAIR_HOME]]), %[[LEFT:r[a-z0-9]+]] +; X86-DAG: movq 8(%[[PAIR_HOME]]), %[[RIGHT:r[a-z0-9]+]] ; X86: addq %[[RIGHT]], %[[LEFT]] ; X86: retq entry: + %value = load %pair, ptr %value.home, align 8 %left = extractvalue %pair %value, 0 %right = extractvalue %pair %value, 1 %sum = add i64 %left, %right @@ -163,39 +178,46 @@ entry: define goabiinternal i64 @call_stack_pair() { ; X86-LABEL: call_stack_pair: -; X86-DAG: movq $13, (%[[BASE:r[a-z0-9]+]]) -; X86-DAG: movq $17, 8(%[[BASE]]) +; X86-DAG: movq %{{r[a-z0-9]+}}, (%[[BASE:r[a-z0-9]+]]) +; X86-DAG: movq %{{r[a-z0-9]+}}, 8(%[[BASE]]) ; X86: callq stack_pair entry: + %value.home = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, - %pair { i64 13, i64 17 }) + ptr byval(%pair) align 8 %value.home) ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr byval([8 x i8]) align 1 %value.home) { ; X86-LABEL: stack_bytes: -; X86-DAG: movb 8(%rsp), %{{[a-z0-9]+}} -; X86-DAG: movb 15(%rsp), %{{[a-z0-9]+}} +; X86: leaq 8(%rsp), %[[BYTES_HOME:r[a-z0-9]+]] +; X86-DAG: movb (%[[BYTES_HOME]]), %{{[a-z0-9]+}} +; X86-DAG: movb 7(%[[BYTES_HOME]]), %{{[a-z0-9]+}} ; X86-DAG: movb %{{[a-z0-9]+}}, 16(%rsp) ; X86-DAG: movb %{{[a-z0-9]+}}, 23(%rsp) ; X86: retq entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; X86-LABEL: call_stack_bytes: -; X86-DAG: movb $1, (%[[BASE:r[a-z0-9]+]]) -; X86-DAG: movb $8, 7(%[[BASE]]) +; X86: movq %{{r[a-z0-9]+}}, (%[[BASE:r[a-z0-9]+]]) ; X86: callq stack_bytes ; X86: movq %rsp, %[[RESULT_BASE:r[a-z0-9]+]] ; X86-DAG: movzbl 8(%[[RESULT_BASE]]), %{{[a-z0-9]+}} ; X86-DAG: movzbl 15(%[[RESULT_BASE]]), %{{[a-z0-9]+}} entry: + %value.home = alloca [8 x i8], align 1 + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr byval([8 x i8]) align 1 %value.home) %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/X86/go-memset-inline.ll b/llvm/test/CodeGen/X86/go-memset-inline.ll index 378aec5f18ece..06c0c0e9a6b95 100644 --- a/llvm/test/CodeGen/X86/go-memset-inline.ll +++ b/llvm/test/CodeGen/X86/go-memset-inline.ll @@ -1,4 +1,5 @@ -; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-goobj -verify-machineinstrs \ +; RUN: -mem-intrinsic-expand-size=801 < %s | FileCheck %s declare void @llvm.memset.p0.i64(ptr writeonly, i8, i64, i1 immarg) @@ -12,12 +13,17 @@ define goabiinternal void @constant_memset(ptr %dst) { ret void } -define goabi0 void @"dynamic_memset"(ptr %dst, i64 %size) { +define goabi0 void @"dynamic_memset"( + ptr byval(ptr) align 8 %dst.home, + ptr byval(i64) align 8 %size.home) { ; CHECK-LABEL: "dynamic_memset": -; CHECK: rep -; CHECK-SAME: stosb +; CHECK: movb $0, +; CHECK: incq +; CHECK: cmpq ; CHECK-NOT: callq memset ; CHECK: retq + %dst = load ptr, ptr %dst.home, align 8 + %size = load i64, ptr %size.home, align 8 call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 %size, i1 false) ret void } diff --git a/llvm/test/CodeGen/X86/go-stack-byval.ll b/llvm/test/CodeGen/X86/go-stack-byval.ll new file mode 100644 index 0000000000000..a4d5a9137646f --- /dev/null +++ b/llvm/test/CodeGen/X86/go-stack-byval.ll @@ -0,0 +1,152 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR + +%pair = type { i64, i64 } + +declare goabiinternal void @consume(i64, i64, i64, i64, i64, i64, i64, i64, + i64, ptr byval(i64) align 8) +declare goabiinternal void @consume_pair( + i64, i64, i64, i64, i64, i64, i64, i64, + ptr byval(%pair) align 8) +declare goabiinternal float @consume_memory_float( + ptr byval(float) align 4, float) +declare void @llvm.lifetime.start.p0(ptr captures(none)) + +define goabiinternal void @ssa_stack_argument() { +; CHECK-LABEL: ssa_stack_argument: +; CHECK-NOT: memcpy +; The frontend alloca remains an ordinary source object. Generic byval +; lowering copies its value into the outgoing Go argument area. +; CHECK: movq $42, {{[1-9][0-9]*}}(%rsp) +; CHECK: movq $42, (%rsp) +; CHECK: callq consume +entry: + %argument = alloca i64, align 8 + store i64 42, ptr %argument, align 8 + call goabiinternal void @consume(i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, + i64 6, i64 7, i64 8, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal void @register_argument_byval_source(i64 %value) { +; CHECK-LABEL: register_argument_byval_source: +; Existing incoming argument-copy elision reuses %value's fixed home for the +; temporary, while ordinary byval lowering still writes the outgoing slot. +; CHECK: callq consume +; MIR-LABEL: name: register_argument_byval_source +; MIR: fixedStack: +; MIR: - { id: 0, type: spill-slot, offset: 0, size: 8 +; MIR: stack: [] +; MIR: %[[VALUE:[0-9]+]]:gr64 = COPY $rax +; MIR: MOV64mr %fixed-stack.0{{.*}}%[[VALUE]] +; MIR: MOV64mr %{{[0-9]+}}{{.*}}%[[VALUE]] +entry: + %argument = alloca i64, align 8 + store i64 %value, ptr %argument, align 8 + call goabiinternal void @consume(i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, + i64 6, i64 7, i64 8, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal void @memory_stack_argument(ptr %source) { +; CHECK-LABEL: memory_stack_argument: +; CHECK-NOT: memcpy +; CHECK: movq (%rax), %rax +; CHECK: movq %rax, (%rsp) +; CHECK: callq consume +entry: + call goabiinternal void @consume(i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, + i64 6, i64 7, i64 8, + ptr byval(i64) align 8 %source) + ret void +} + +define goabiinternal void @ssa_aggregate_stack_argument() { +; CHECK-LABEL: ssa_aggregate_stack_argument: +; CHECK-NOT: movdqu +; CHECK-NOT: movups +; CHECK: movq $17, {{[1-9][0-9]*}}(%rsp) +; CHECK: movq {{[1-9][0-9]*}}(%rsp), %[[RIGHT:r[a-z0-9]+]] +; CHECK: pushq %[[RIGHT]] +; CHECK: pushq $13 +; CHECK: callq consume_pair +; MIR-LABEL: name: ssa_aggregate_stack_argument +; MIR: stack: +; MIR: - { id: 0, name: argument, type: default, offset: 0, size: 16 +; MIR: MOV64rm %stack.0.argument +; MIR: store (s64) into stack +entry: + %argument = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %argument, align 8 + call goabiinternal void @consume_pair( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, + ptr byval(%pair) align 8 %argument) + ret void +} + +define goabiinternal void @noncanonical_stack_argument() { +; Lifetime markers do not change the ordinary source-object plus byval-copy +; semantics. +; MIR-LABEL: name: noncanonical_stack_argument +; MIR: stack: +; MIR-NEXT: - { id: 0, +entry: + %argument = alloca i64, align 8 + call void @llvm.lifetime.start.p0(ptr %argument) + store i64 42, ptr %argument, align 8 + call goabiinternal void @consume(i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, + i64 6, i64 7, i64 8, + ptr byval(i64) align 8 %argument) + ret void +} + +define goabiinternal i64 @read_stack_argument( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, + i64 %a8, ptr byval(i64) align 8 %value) { +; CHECK-LABEL: read_stack_argument: +; CHECK: movq 8(%rsp), %rax +; CHECK-NEXT: retq +; MIR-LABEL: name: read_stack_argument +; MIR: frameInfo: +; MIR: goABIStackArgsSize: 8 +; MIR: goABIArgSize: 80 +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +define goabiinternal ptr @address_stack_pair_argument( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, + ptr byval(%pair) align 8 %value) { +; CHECK-LABEL: address_stack_pair_argument: +; CHECK: leaq 8(%rsp), %rax +; MIR-LABEL: name: address_stack_pair_argument +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 0, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +entry: + ret ptr %value +} + +define goabiinternal float @read_memory_float( + ptr byval(float) align 4 %memory, float %register) { +; CHECK-LABEL: read_memory_float: +; CHECK: addss 8(%rsp), %xmm0 +entry: + %loaded = load float, ptr %memory, align 4 + %sum = fadd float %loaded, %register + ret float %sum +} + +define goabiinternal float @call_memory_float(ptr %source, float %register) { +; CHECK-LABEL: call_memory_float: +; CHECK: movl {{.*}}, (%rsp) +; CHECK: callq consume_memory_float +entry: + %result = call goabiinternal float @consume_memory_float( + ptr byval(float) align 4 %source, float %register) + ret float %result +} diff --git a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll index 17a4bcf33b224..e3f5fd24f5737 100644 --- a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll @@ -8,11 +8,11 @@ declare goabiinternal void @safepoint() define goabiinternal ptr addrspace(1) @scalar_stack_arg( ptr addrspace(1) %p0, - i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr readonly byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -24,11 +24,11 @@ entry: } define goabiinternal ptr addrspace(1) @aggregate_stack_arg( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + i64 %a0, i64 %a1, i64 %a2, i64 %a3, + i64 %a4, i64 %a5, i64 %a6, i64 %a7, + ptr readonly byval(%aggregate) align 8 %value.home) gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -47,11 +47,12 @@ entry: define goabiinternal ptr addrspace(1) @merged_stack_arg( ptr addrspace(1) %p0, - i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = icmp ne ptr addrspace(1) %p0, null %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -66,11 +67,12 @@ entry: define goabiinternal ptr addrspace(1) @relocated_stack_arg( ptr addrspace(1) %p0, - i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr readonly byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = icmp ne ptr addrspace(1) %p0, null %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -106,6 +108,41 @@ merge: ret ptr addrspace(1) %relocated3 } +define goabiinternal ptr addrspace(1) @mutated_stack_arg( + ptr addrspace(1) %p0, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { +entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + store ptr addrspace(1) null, ptr %p16.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %p16) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + +define goabiinternal ptr addrspace(1) @byval_home_address( + ptr addrspace(1) %heap, + ptr addrspace(1) byval(%aggregate) align 8 %value.home) gc "statepoint-example" { +entry: + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 8, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %heap, + ptr addrspace(1) %value.home) ] + %heap.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + %home.relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 1, i32 1) + ret ptr addrspace(1) %home.relocated +} + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( @@ -113,23 +150,29 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-LABEL: name: scalar_stack_arg ; CHECK: fixedStack: -; CHECK: - { id: 0, type: default, offset: 56, size: 8, +; CHECK: - { id: [[SCALAR_HOME:[0-9]+]], type: default, offset: 0, size: 8, ; CHECK: isImmutable: false -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[SCALAR_SLOT:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: MOV64mr %stack.[[SCALAR_SLOT]], ; CHECK: STATEPOINT 1, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.0, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0 +; CHECK-SAME: 2, 1, 1, 8, %stack.[[SCALAR_SLOT]], 0, +; CHECK-SAME: (volatile load store (s64) on %stack.[[SCALAR_SLOT]]) ; CHECK-NEXT: ADJCALLSTACKUP64 -; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0 +; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gr64 = MOV64rm %stack.[[SCALAR_SLOT]] ; CHECK-LABEL: name: aggregate_stack_arg -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[AGG_FIRST:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: - { id: [[AGG_SECOND:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: MOV64mr %stack.[[AGG_FIRST]], +; CHECK: MOV64mr %stack.[[AGG_SECOND]], ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0{{[^)]*}}), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1{{[^)]*}}) +; CHECK-SAME: 2, 2, 1, 8, %stack.[[AGG_FIRST]], 0, 1, 8, %stack.[[AGG_SECOND]], 0, +; CHECK-SAME: (volatile load store (s64) on %stack.[[AGG_FIRST]]), +; CHECK-SAME: (volatile load store (s64) on %stack.[[AGG_SECOND]]) ; CHECK-NEXT: ADJCALLSTACKUP64 -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %stack.{{[0-9]+}} ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: @@ -140,10 +183,28 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: relocated_stack_arg -; CHECK: stack: [] +; CHECK: stack: +; CHECK: - { id: [[HOME:[0-9]+]], name: '', type: default, offset: 0, size: 8, ; CHECK: STATEPOINT 4, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME:[0-9]+]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, ; CHECK: STATEPOINT 5, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, ; CHECK: STATEPOINT 6, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, + +; CHECK-LABEL: name: mutated_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: MOV64mr %stack.0, +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, + +; CHECK-LABEL: name: byval_home_address +; CHECK: fixedStack: +; CHECK: - { id: [[BYVAL_HOME:[0-9]+]], type: default, offset: 0, size: 24, +; CHECK: stack: +; CHECK: - { id: [[HEAP_SLOT:[0-9]+]], name: '', type: default, offset: 0, size: 8, +; CHECK: STATEPOINT 8, +; CHECK-SAME: 2, 2, 0, %fixed-stack.[[BYVAL_HOME]], 0, 1, 8, %stack.[[HEAP_SLOT]], 0, +; CHECK-NEXT: ADJCALLSTACKUP64 +; CHECK-NEXT: {{%[0-9]+}}:gr64 = LEA64r %fixed-stack.[[BYVAL_HOME]], diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index 0547d1a782189..ad49bb25da00e 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -71,20 +71,18 @@ entry: ; A64-MIR-NEXT: - { id: 0, type: spill-slot, offset: 8, size: 24 ; A64-MIR: STATEPOINT{{.*}}%fixed-stack.0 -; An aggregate assigned wholly to the stack reuses its caller-populated slot; -; the lifetime marker must not make SelectionDAG allocate and copy a local. -define goabiinternal void @inactive_stack_aggregate(%stack_aggregate %value) +; An aggregate assigned wholly to the stack is represented directly by its +; caller-populated typed byval home; SelectionDAG must not allocate a copy. +define goabiinternal void @inactive_stack_aggregate( + ptr byval(%stack_aggregate) align 8 %value.home) gc "statepoint-example" { entry: - %home = alloca %stack_aggregate, align 8 - call void @llvm.lifetime.start.p0(i64 16, ptr %home) - store %stack_aggregate %value, ptr %home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 3, i32 0, ptr elementtype(void ()) @safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, - i64 1095520067, i64 11, ptr %home, i64 0, i64 16, + i64 1095520067, i64 11, ptr %value.home, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 15) ] ret void diff --git a/llvm/test/CodeGen/X86/goobj-pcsp-cfg-invalid.mir b/llvm/test/CodeGen/X86/goobj-pcsp-cfg-invalid.mir index d0805af4e8d11..fb7530e31bed3 100644 --- a/llvm/test/CodeGen/X86/goobj-pcsp-cfg-invalid.mir +++ b/llvm/test/CodeGen/X86/goobj-pcsp-cfg-invalid.mir @@ -22,6 +22,8 @@ name: inconsistent tracksRegLiveness: true frameInfo: + goABIStackArgsSize: 0 + goABIArgSize: 0 stackSize: 16 adjustsStack: true hasCalls: true @@ -58,6 +60,9 @@ body: | --- name: unreachable tracksRegLiveness: true +frameInfo: + goABIStackArgsSize: 0 + goABIArgSize: 0 body: | bb.0: RET64 diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index e5f7a24173352..5b0cfaa9f53dd 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -35,10 +35,24 @@ entry: } declare goabiinternal void @many_stack_args( - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64) + i64, i64, i64, i64, i64, i64, i64, i64, i64, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8) +@stack.args = private constant [23 x i64] [ + i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, + i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, + i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, + i64 27, i64 28, i64 29, i64 30, i64 31] @condition = external global i1 ; X86 lowers the stack arguments below to push sequences instead of reserving @@ -51,10 +65,30 @@ entry: call: call goabiinternal void @many_stack_args( - i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, - i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, - i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, - i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31) + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 0), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 1), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 2), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 3), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 4), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 5), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 6), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 7), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 8), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 9), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 10), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 11), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 12), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 13), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 14), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 15), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 16), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 17), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 18), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 19), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 20), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 21), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @stack.args, i64 0, i64 22)) br label %join join: diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth.ll b/llvm/test/CodeGen/X86/goobj-stack-growth.ll index 691d27e10d43e..5be2177d902f1 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth.ll @@ -23,9 +23,15 @@ entry: ret i64 %value } -define goabi0 void @"abi0_pointer_arguments"(ptr %first, ptr %second, ptr %third) +define goabi0 void @"abi0_pointer_arguments"( + ptr byval(ptr) align 8 %first.home, + ptr byval(ptr) align 8 %second.home, + ptr byval(ptr) align 8 %third.home) "frame-pointer"="non-leaf" { entry: + %first = load ptr, ptr %first.home, align 8 + %second = load ptr, ptr %second.home, align 8 + %third = load ptr, ptr %third.home, align 8 call goabiinternal void @use_three_pointers( ptr %first, ptr %second, ptr %third) ret void @@ -52,10 +58,17 @@ entry: define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) { + i64 %a6, i64 %a7, i64 %a8, + ptr byval(i64) align 8 %a9.home, + ptr byval(i64) align 8 %a10.home, + ptr byval(i64) align 8 %a11.home, + ptr byval(i64) align 8 %a12.home, + ptr byval(i64) align 8 %a13.home, + ptr byval(i64) align 8 %a14.home, + ptr byval(i64) align 8 %a15.home, + ptr byval(ptr) align 8 %pointer.home) { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1 @@ -64,10 +77,16 @@ entry: define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, %pointer.aggregate %value) + i64 %a6, i64 %a7, i64 %a8, + ptr byval(i64) align 8 %a9.home, + ptr byval(i64) align 8 %a10.home, + ptr byval(i64) align 8 %a11.home, + ptr byval(i64) align 8 %a12.home, + ptr byval(i64) align 8 %a13.home, + ptr byval(%pointer.aggregate) align 8 %value.home) "go_results_tuple" { entry: + %value = load %pointer.aggregate, ptr %value.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1