From f78f04bf64a4ab64fe3533351fb19f700073cb84 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 14:18:23 +0800 Subject: [PATCH 1/6] [Go] Model stack ABI values with typed memory carriers --- .github/workflows/goallc-ci.yml | 2 + llvm/include/llvm/CodeGen/GoCallingConv.h | 11 + llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 9 +- llvm/lib/CodeGen/GoCallingConv.cpp | 58 +++++ .../SelectionDAG/SelectionDAGBuilder.cpp | 7 + .../SelectionDAG/StatepointLowering.cpp | 25 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 8 +- llvm/lib/MC/GoObjObjectWriter.cpp | 193 ++++++++++++-- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 148 +++++++++++ .../Target/AArch64/AArch64ISelLowering.cpp | 236 ++++++++++++++--- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 15 ++ .../AArch64/AArch64SelectionDAGInfo.cpp | 11 + llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 244 +++++++++++++++--- llvm/lib/Target/X86/X86SelectionDAGInfo.cpp | 4 +- .../test/CodeGen/AArch64/go-argument-homes.ll | 27 ++ llvm/test/CodeGen/AArch64/go-callconv.ll | 43 ++- llvm/test/CodeGen/AArch64/go-stack-byval.ll | 54 ++++ .../AArch64/go-statepoint-stack-args.ll | 56 ++-- .../AArch64/goobj-private-reloc-constants.ll | 37 +++ .../AArch64/goobj-register-argument-homes.ll | 8 +- .../CodeGen/AArch64/goobj-stack-growth.ll | 3 +- llvm/test/CodeGen/Generic/go-abi0-frame.ll | 3 +- .../test/CodeGen/Generic/go-argument-homes.ll | 46 ---- llvm/test/CodeGen/Generic/goobj-nosplit.ll | 7 +- llvm/test/CodeGen/X86/go-argument-homes.ll | 25 ++ llvm/test/CodeGen/X86/go-callconv.ll | 80 ++++-- llvm/test/CodeGen/X86/go-stack-byval.ll | 50 ++++ .../X86/goobj-alloca-argument-homes.ll | 8 +- .../X86/goobj-stack-growth-metadata.ll | 50 +++- llvm/test/CodeGen/X86/goobj-stack-growth.ll | 26 +- 30 files changed, 1272 insertions(+), 222 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/go-argument-homes.ll create mode 100644 llvm/test/CodeGen/AArch64/go-stack-byval.ll create mode 100644 llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll create mode 100644 llvm/test/CodeGen/X86/go-argument-homes.ll create mode 100644 llvm/test/CodeGen/X86/go-stack-byval.ll diff --git a/.github/workflows/goallc-ci.yml b/.github/workflows/goallc-ci.yml index ce649abdd462f..506fbdb8022e6 100644 --- a/.github/workflows/goallc-ci.yml +++ b/.github/workflows/goallc-ci.yml @@ -99,6 +99,7 @@ jobs: set -euo pipefail build="$RUNNER_TEMP/goallc-llvm-build" tests=( + "$build/test/CodeGen/AArch64/go-argument-homes.ll" "$build/test/CodeGen/AArch64/go-callconv.ll" "$build/test/CodeGen/AArch64/go-statepoint-stack-args.ll" "$build/test/CodeGen/AArch64/goobj-ir-config.ll" @@ -109,6 +110,7 @@ jobs: "$build/test/CodeGen/Generic/goobj-entry-stackmap-cfg.ll" "$build/test/CodeGen/Generic/goobj-entry-stackmap-sentinel.ll" "$build/test/CodeGen/Generic/goobj-stack-check-policy.ll" + "$build/test/CodeGen/X86/go-argument-homes.ll" "$build/test/CodeGen/X86/go-callconv.ll" "$build/test/CodeGen/X86/go-gc-write-barrier.ll" "$build/test/CodeGen/X86/go-statepoint-stack-results.ll" diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index a94c652859816..e6429d463ee2f 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -32,6 +32,7 @@ class Type; namespace goabi { inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple"; +inline constexpr StringLiteral MemoryResultsAttr = "go_memory_results"; inline constexpr StringLiteral PadTypeName = "go.abi.pad"; // Target frame lowering must not synthesize a morestack edge for such a // function. GoObj Go functions otherwise use the native Go default: emit a @@ -100,11 +101,21 @@ struct EntryArgsInfo { bool hasTupleResultsAttr(const AttributeList &Attrs); bool hasTupleResultsAttr(const Function &F); bool hasTupleResultsAttr(const CallBase &CB); +/// Return the logical Go ABI type of an IR parameter. Non-empty values that +/// the Go frontend assigned wholly to memory use a typed byval pointer as the +/// LLVM carrier; direct parameters retain their logical type. +Type *getParameterType(const Argument &Arg); // Mirrors ComputeValueTypes and marks leaves originating in %go.abi.pad. SmallBitVector getPaddingPieces(Type *Ty); void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys); +SmallVector getMemoryResultIndices(const AttributeList &Attrs, + unsigned DirectResultCount, + unsigned MemoryResultCount); +void getReturnTypes(Type *ReturnType, const AttributeList &Attrs, + ArrayRef MemoryResultTys, + SmallVectorImpl &ResultTys); CallLayout computeCallLayout(ArrayRef ArgTys, ArrayRef ResultTys, const DataLayout &DL, diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 88435712831e8..71d360a21590f 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -265,12 +265,15 @@ static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL, } SmallVector ArgTys; + SmallVector MemoryResultTys; for (const Argument &Arg : F.args()) - if (!Arg.hasNestAttr()) - ArgTys.push_back(Arg.getType()); + if (Arg.hasAttribute(Attribute::ByRef)) + MemoryResultTys.push_back(Arg.getParamByRefType()); + else if (!Arg.hasNestAttr()) + ArgTys.push_back(goabi::getParameterType(Arg)); SmallVector ResultTys; - goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, ResultTys); uint64_t Size = goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize; diff --git a/llvm/lib/CodeGen/GoCallingConv.cpp b/llvm/lib/CodeGen/GoCallingConv.cpp index 73491bbe2fa58..50166dfafd555 100644 --- a/llvm/lib/CodeGen/GoCallingConv.cpp +++ b/llvm/lib/CodeGen/GoCallingConv.cpp @@ -230,6 +230,10 @@ bool hasTupleResultsAttr(const CallBase &CB) { return false; } +Type *getParameterType(const Argument &Arg) { + return Arg.hasByValAttr() ? Arg.getParamByValType() : Arg.getType(); +} + void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys) { if (ReturnType->isVoidTy()) @@ -245,6 +249,60 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, ResultTys.push_back(ReturnType); } +SmallVector getMemoryResultIndices(const AttributeList &Attrs, + unsigned DirectResultCount, + unsigned MemoryResultCount) { + Attribute Attr = Attrs.getFnAttr(MemoryResultsAttr); + if (!Attr.isValid()) { + if (MemoryResultCount != 0) + report_fatal_error("Go memory result carriers have no index attribute"); + return {}; + } + if (!Attr.isStringAttribute() || Attr.getValueAsString().empty()) + report_fatal_error("invalid Go memory result index attribute"); + + unsigned TotalResultCount = DirectResultCount + MemoryResultCount; + SmallVector Indices; + SmallVector Fields; + Attr.getValueAsString().split(Fields, ',', /*MaxSplit=*/-1, + /*KeepEmpty=*/true); + for (StringRef Field : Fields) { + unsigned Index; + if (Field.empty() || Field.getAsInteger(10, Index) || + Index >= TotalResultCount || + (!Indices.empty() && Index <= Indices.back())) + report_fatal_error("invalid Go memory result index attribute"); + Indices.push_back(Index); + } + if (Indices.size() != MemoryResultCount) + report_fatal_error("Go memory result index count mismatch"); + return Indices; +} + +void getReturnTypes(Type *ReturnType, const AttributeList &Attrs, + ArrayRef MemoryResultTys, + SmallVectorImpl &ResultTys) { + SmallVector DirectResultTys; + getReturnTypes(ReturnType, hasTupleResultsAttr(Attrs), DirectResultTys); + SmallVector MemoryIndices = getMemoryResultIndices( + Attrs, DirectResultTys.size(), MemoryResultTys.size()); + if (MemoryIndices.empty()) { + ResultTys.append(DirectResultTys.begin(), DirectResultTys.end()); + return; + } + + unsigned TotalResultCount = DirectResultTys.size() + MemoryResultTys.size(); + ResultTys.assign(TotalResultCount, nullptr); + for (auto [Index, Ty] : llvm::zip_equal(MemoryIndices, MemoryResultTys)) + ResultTys[Index] = Ty; + auto Direct = DirectResultTys.begin(); + for (Type *&Ty : ResultTys) + if (!Ty) + Ty = *Direct++; + if (Direct != DirectResultTys.end()) + report_fatal_error("Go direct result index count mismatch"); +} + CallLayout computeCallLayout(ArrayRef ArgTys, ArrayRef ResultTys, const DataLayout &DL, const ABIConfig &Config) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 449bfa27dbda4..7b325dfa6a6e8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -11770,6 +11770,13 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { MemAlign = *MA; else MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); + } else if (Args[i].IsByRef) { + unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); + Flags.setByRefSize(FrameSize); + if (auto MA = Args[i].Alignment) + MemAlign = *MA; + else + MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); } else if (auto MA = Args[i].Alignment) { MemAlign = *MA; } else { diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 259fa480ec1e5..eda649ae8be0a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -358,9 +358,17 @@ static void reservePreviousStackSlotForValue(const Value *IncomingValue, /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result static SDNode *peelCallResultChain(SDNode *Node) { - while (Node->getOpcode() == ISD::LOAD || - Node->getOpcode() == ISD::CopyFromReg) - Node = Node->getOperand(0).getNode(); + while (true) { + if (Node->getOpcode() == ISD::CopyFromReg) { + Node = Node->getOperand(0).getNode(); + continue; + } + if (auto *Mem = dyn_cast(Node)) { + Node = Mem->getChain().getNode(); + continue; + } + break; + } if (Node->getOpcode() != ISD::TokenFactor) return Node; @@ -368,8 +376,7 @@ static SDNode *peelCallResultChain(SDNode *Node) { SDNode *CommonCallEnd = nullptr; for (SDValue Operand : Node->ops()) { SDNode *CallEnd = peelCallResultChain(Operand.getNode()); - if (CallEnd->getOpcode() != ISD::CALLSEQ_END || - (CommonCallEnd && CommonCallEnd != CallEnd)) + if (CommonCallEnd && CommonCallEnd != CallEnd) return Node; CommonCallEnd = CallEnd; } @@ -1008,7 +1015,13 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( } } else if (goabi::isGoCallingConv( DAG.getMachineFunction().getFunction().getCallingConv()) && - isa(V) && isa(SDV)) { + isa(SDV) && + (isa(V) || + (isa(V) && + (cast(V)->hasByValAttr() || + cast(V)->hasAttribute(Attribute::ByRef)) && + FuncInfo.getArgumentFrameIndex(cast(V)) == + cast(SDV)->getIndex()))) { Record.type = RecordType::FrameIndexRemat; Record.payload.FI = cast(SDV)->getIndex(); } else if (Loc.getNode()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 5d93472a9f410..3a46fbb9d7611 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -126,6 +126,7 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet); IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest); IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal); + IsByRef = Call->paramHasAttr(ArgIdx, Attribute::ByRef); IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated); IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); @@ -134,13 +135,18 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); Alignment = Call->getParamStackAlign(ArgIdx); IndirectType = nullptr; - assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 && + assert(IsByVal + IsByRef + IsPreallocated + IsInAlloca + IsSRet <= 1 && "multiple ABI attributes?"); if (IsByVal) { IndirectType = Call->getParamByValType(ArgIdx); if (!Alignment) Alignment = Call->getParamAlign(ArgIdx); } + if (IsByRef) { + IndirectType = Call->getParamByRefType(ArgIdx); + if (!Alignment) + Alignment = Call->getParamAlign(ArgIdx); + } if (IsPreallocated) IndirectType = Call->getParamPreallocatedType(ArgIdx); if (IsInAlloca) diff --git a/llvm/lib/MC/GoObjObjectWriter.cpp b/llvm/lib/MC/GoObjObjectWriter.cpp index 3878c954139a5..de33312ca07df 100644 --- a/llvm/lib/MC/GoObjObjectWriter.cpp +++ b/llvm/lib/MC/GoObjObjectWriter.cpp @@ -37,9 +37,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -58,6 +60,25 @@ struct GoObjSymRef { uint32_t SymIdx = 0; }; +std::string goObjPathToPrefix(StringRef Path) { + size_t Slash = Path.rfind('/'); + size_t LastSegment = Slash == StringRef::npos ? 0 : Slash + 1; + SmallString<128> Prefix; + constexpr char Hex[] = "0123456789abcdef"; + for (size_t I = 0; I != Path.size(); ++I) { + uint8_t C = static_cast(Path[I]); + if (C <= ' ' || (C == '.' && I >= LastSegment) || C == '%' || C == '"' || + C >= 0x7f) { + Prefix.push_back('%'); + Prefix.push_back(Hex[C >> 4]); + Prefix.push_back(Hex[C & 0xf]); + } else { + Prefix.push_back(static_cast(C)); + } + } + return Prefix.str().str(); +} + struct GoObjSymbol { struct Relocation { uint32_t Offset = 0; @@ -1771,7 +1792,8 @@ uint64_t GoObjObjectWriter::writeObject() { GoObj::SRODATA, GoObj::SymFlagDupok | GoObj::SymFlagLocal, 0, GoObj::SymABI0, *ExactSize, Align, Data); - Symbols.back().ContentHash = makeGoObjContentHash(0, Data); + // The native Go object identity includes relocations as well as data. + // Compute it after all MC fixups have been assigned to their carriers. } } @@ -2338,14 +2360,15 @@ uint64_t GoObjObjectWriter::writeObject() { IndexedRefs.push_back({Ref, Name.str(), Flags2}); }; - auto GetTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, - int64_t &Addend) { + auto ResolveTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, + int64_t &Addend) { if (!Target) report_fatal_error("GoObj relocation without a target symbol"); if (auto It = DefinedSymbolIndexes.find(Target); It != DefinedSymbolIndexes.end()) - return DefinedSymRefs[It->second]; + return std::make_pair(DefinedSymRefs[It->second], + std::optional(It->second)); if (Target->isInSection()) { uint64_t TargetOffset = Asm->getSymbolOffset(*Target); @@ -2353,7 +2376,7 @@ uint64_t GoObjObjectWriter::writeObject() { FindContainingSymbol(&Target->getSection(), TargetOffset)) { Addend += static_cast(TargetOffset - Symbols[*SymIdx].SectionBegin); - return DefinedSymRefs[*SymIdx]; + return std::make_pair(DefinedSymRefs[*SymIdx], SymIdx); } } @@ -2366,7 +2389,7 @@ uint64_t GoObjObjectWriter::writeObject() { GoObjSymRef Ref{GetPackageIndex(Metadata->PackagePrefix), Metadata->SymIdx}; RecordIndexedRef(Ref, TrimInlineHash(Identity.Name), Metadata->Flags2); - return Ref; + return std::make_pair(Ref, std::optional()); } bool IsFunction; switch (RelocType) { @@ -2384,11 +2407,15 @@ uint64_t GoObjObjectWriter::writeObject() { if (Identity.BuiltinIndex && !Identity.IsLinknameRef) { if (std::optional Ref = FindDefinedSymRef( Identity.Name, GetSymbolABI(Target, IsFunction))) - return *Ref; - return GoObjSymRef{GoObj::PkgIdxBuiltin, *Identity.BuiltinIndex}; + return std::make_pair(*Ref, std::optional()); + return std::make_pair( + GoObjSymRef{GoObj::PkgIdxBuiltin, *Identity.BuiltinIndex}, + std::optional()); } - return GoObjSymRef{GoObj::PkgIdxNone, - GetNonPkgRefSymIdx(Target, IsFunction)}; + return std::make_pair( + GoObjSymRef{GoObj::PkgIdxNone, + GetNonPkgRefSymIdx(Target, IsFunction)}, + std::optional()); } report_fatal_error( @@ -2400,6 +2427,10 @@ uint64_t GoObjObjectWriter::writeObject() { " in-section=" + Twine(static_cast(Target->isInSection())) + " undefined=" + Twine(static_cast(Target->isUndefined()))); }; + auto GetTargetSymRef = [&](const MCSymbol *Target, unsigned RelocType, + int64_t &Addend) { + return ResolveTargetSymRef(Target, RelocType, Addend).first; + }; SmallVector MergedRelocations; for (const GoObjRelocationEntry &Reloc : Relocations) { @@ -2431,11 +2462,6 @@ uint64_t GoObjObjectWriter::writeObject() { if (LocalOffset > static_cast(std::numeric_limits::max())) report_fatal_error("GoObj relocation offset exceeds int32 range"); - if (Source.DefinedBlock == GoObj::DefinedSymbolBlock::Hasheddef && - Source.Symbol && Source.Symbol->isTemporary()) - report_fatal_error( - "GoObj private constants with relocations are not supported"); - int64_t Addend = getGoObjRelocAddend(Reloc); uint16_t RelocType = checkedUint16(Reloc.Type, "relocation type"); if (Source.Symbol) { @@ -2465,13 +2491,15 @@ uint64_t GoObjObjectWriter::writeObject() { const Triple::ArchType Arch = Asm->getContext().getTargetTriple().getArch(); const bool IsX86TLSLE = (Arch == Triple::x86 || Arch == Triple::x86_64) && (RelocType & ~GoObj::R_WEAK) == GoObj::R_TLS_LE; - GoObjSymRef TargetSymRef = - IsX86TLSLE ? GoObjSymRef{} - : GetTargetSymRef(Reloc.Symbol, Reloc.Type, Addend); + GoObjSymRef TargetSymRef; + std::optional TargetSymbolIndex; + if (!IsX86TLSLE) + std::tie(TargetSymRef, TargetSymbolIndex) = + ResolveTargetSymRef(Reloc.Symbol, Reloc.Type, Addend); Source.Relocations.push_back( {static_cast(LocalOffset), Reloc.Size, RelocType, Addend, - TargetSymRef.PkgIdx, TargetSymRef.SymIdx, std::nullopt}); + TargetSymRef.PkgIdx, TargetSymRef.SymIdx, TargetSymbolIndex}); } // R_KEEP has no bytes or MC fixup. It is a Go linker reachability edge @@ -2560,6 +2588,133 @@ uint64_t GoObjObjectWriter::writeObject() { }); } + // LLVM optimizations can synthesize private pointer lookup tables. Match + // cmd/internal/obj's content identity: hash the symbol bytes and each + // relocation's shape and globally stable target identity. Hashing only the + // zero relocation placeholders would incorrectly merge different tables. + DenseSet ContentHashInProgress; + std::function ComputeContentHash = [&](uint32_t SymbolIndex) { + if (SymbolIndex >= Symbols.size()) + report_fatal_error("GoObj hashed definition index is invalid"); + GoObjSymbol &Symbol = Symbols[SymbolIndex]; + if (Symbol.ContentHash) + return; + if (Symbol.DefinedBlock != GoObj::DefinedSymbolBlock::Hasheddef) + report_fatal_error( + "GoObj content hash requested for a non-hashed symbol"); + if (!Symbol.Symbol || !Symbol.Symbol->isTemporary() || + Symbol.Type != GoObj::SRODATA) + report_fatal_error( + "GoObj frontend hashed definition has no content hash"); + if (!ContentHashInProgress.insert(SymbolIndex).second) + report_fatal_error("circular GoObj content-addressable relocation"); + + SHA256 Hasher; + const char Version = 1; + Hasher.update(StringRef(&Version, 1)); + SmallString<16> Encoded; + raw_svector_ostream EncodedOS(Encoded); + support::endian::Writer EncodedWriter(EncodedOS, llvm::endianness::little); + EncodedWriter.write(Symbol.Size); + EncodedWriter.write(0); // Default read-only data section. + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + + ArrayRef Data(Symbol.Data); + while (!Data.empty() && Data.back() == 0) + Data = Data.drop_back(); + if (!Data.empty()) + Hasher.update(StringRef(Data.data(), Data.size())); + + for (const GoObjSymbol::Relocation &Reloc : Symbol.Relocations) { + Encoded.clear(); + EncodedWriter.write(Reloc.Offset); + EncodedWriter.write(Reloc.Size); + if (Reloc.Type > std::numeric_limits::max()) + report_fatal_error("GoObj content hash relocation type is too large"); + EncodedWriter.write(static_cast(Reloc.Type)); + EncodedWriter.write(static_cast(Reloc.Addend)); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + + if (Reloc.TargetSymbolIndex && *Reloc.TargetSymbolIndex == SymbolIndex) { + Hasher.update("self symbol"); + continue; + } + if (Reloc.PkgIdx == GoObj::PkgIdxInvalid) { + Hasher.update("nil symbol"); + continue; + } + + switch (Reloc.PkgIdx) { + case GoObj::PkgIdxHashed64: { + if (Reloc.SymIdx >= Hashed64defSymbols.size()) + report_fatal_error("invalid GoObj short-hashed relocation target"); + const GoObjSymbol &Target = Symbols[Hashed64defSymbols[Reloc.SymIdx]]; + if (Target.Data.size() != GoObj::Hash64Size) + report_fatal_error("GoObj short-hashed target has invalid size"); + const char Kind = 0; + Hasher.update(StringRef(&Kind, 1)); + Hasher.update(StringRef(Target.Data.data(), Target.Data.size())); + break; + } + case GoObj::PkgIdxHashed: { + if (Reloc.SymIdx >= HasheddefSymbols.size()) + report_fatal_error("invalid GoObj hashed relocation target"); + uint32_t TargetIndex = HasheddefSymbols[Reloc.SymIdx]; + ComputeContentHash(TargetIndex); + const char Kind = 1; + Hasher.update(StringRef(&Kind, 1)); + Hasher.update(ArrayRef(*Symbols[TargetIndex].ContentHash)); + break; + } + case GoObj::PkgIdxNone: { + const char Kind = 2; + Hasher.update(StringRef(&Kind, 1)); + if (Reloc.SymIdx < NonpkgdefSymbols.size()) { + Hasher.update(Symbols[NonpkgdefSymbols[Reloc.SymIdx]].Name); + } else { + uint32_t RefIndex = Reloc.SymIdx - NonpkgdefSymbols.size(); + if (RefIndex >= NonPkgRefs.size()) + report_fatal_error("invalid GoObj non-package relocation target"); + Hasher.update(NonPkgRefs[RefIndex].Name); + } + break; + } + case GoObj::PkgIdxBuiltin: { + const char Kind = 3; + Hasher.update(StringRef(&Kind, 1)); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + case GoObj::PkgIdxSelf: { + Hasher.update(goObjPathToPrefix(Config.PackagePath)); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + default: + if (Reloc.PkgIdx >= PackagePrefixes.size()) + report_fatal_error("invalid GoObj imported relocation package"); + Hasher.update(PackagePrefixes[Reloc.PkgIdx]); + Encoded.clear(); + EncodedWriter.write(Reloc.SymIdx); + Hasher.update(StringRef(Encoded.data(), Encoded.size())); + break; + } + } + + std::array FullHash = Hasher.final(); + std::array Hash; + std::copy_n(FullHash.begin(), Hash.size(), Hash.begin()); + Symbol.ContentHash = Hash; + ContentHashInProgress.erase(SymbolIndex); + }; + for (uint32_t SymbolIndex : HasheddefSymbols) + if (!Symbols[SymbolIndex].ContentHash) + ComputeContentHash(SymbolIndex); + SmallString<0> Body; raw_svector_ostream BodyOS(Body); support::endian::Writer W(BodyOS, llvm::endianness::little); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index e18f0450ac094..a0cd4fc629b17 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -145,6 +145,8 @@ class AArch64AsmPrinter : public AsmPrinter { void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI); + void LowerGoMemoryCopy(MCStreamer &OutStreamer, const MachineInstr &MI); + void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI); void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, @@ -1762,6 +1764,148 @@ void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer, } } +void AArch64AsmPrinter::LowerGoMemoryCopy(MCStreamer &OutStreamer, + const MachineInstr &MI) { + assert(MI.getOpcode() == AArch64::GoMemoryCopyPseudo); + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + Register Size = MI.getOperand(2).getReg(); + + assert(Dst == MI.getOperand(3).getReg() && Src == MI.getOperand(4).getReg() && + Size == MI.getOperand(5).getReg() && + "Go memory-copy operands must be tied"); + + MCSymbol *Loop = OutContext.createTempSymbol(); + MCSymbol *Tail8 = OutContext.createTempSymbol(); + MCSymbol *Tail4 = OutContext.createTempSymbol(); + MCSymbol *Tail2 = OutContext.createTempSymbol(); + MCSymbol *Done = OutContext.createTempSymbol(); + auto Ref = [&](MCSymbol *Sym) { + return MCSymbolRefExpr::create(Sym, OutContext); + }; + + // Copy full 16-byte units. + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::SUBSXri) + .addReg(AArch64::XZR) + .addReg(Size) + .addImm(16) + .addImm(0)); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::Bcc).addImm(AArch64CC::LO).addExpr(Ref(Tail8))); + OutStreamer.emitLabel(Loop); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRQui) + .addReg(AArch64::Q31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRQui) + .addReg(AArch64::Q31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(16) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(16) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::SUBSXri) + .addReg(Size) + .addReg(Size) + .addImm(16) + .addImm(0)); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::Bcc).addImm(AArch64CC::HS).addExpr(Ref(Loop))); + + // Copy the remaining 8/4/2/1-byte units. Advancing both pointers after + // each unit keeps every access at offset zero and valid at any alignment. + OutStreamer.emitLabel(Tail8); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(3).addExpr(Ref(Tail4))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRDui) + .addReg(AArch64::D31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRDui) + .addReg(AArch64::D31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(8) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(8) + .addImm(0)); + + OutStreamer.emitLabel(Tail4); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(2).addExpr(Ref(Tail2))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRSui) + .addReg(AArch64::S31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRSui) + .addReg(AArch64::S31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(4) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(4) + .addImm(0)); + + OutStreamer.emitLabel(Tail2); + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(1).addExpr(Ref(Done))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRHHui) + .addReg(AArch64::H31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRHHui) + .addReg(AArch64::H31) + .addReg(Dst) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Src) + .addReg(Src) + .addImm(2) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(Dst) + .addReg(Dst) + .addImm(2) + .addImm(0)); + + EmitToStreamer( + OutStreamer, + MCInstBuilder(AArch64::TBZX).addReg(Size).addImm(0).addExpr(Ref(Done))); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::LDRBBui) + .addReg(AArch64::B31) + .addReg(Src) + .addImm(0)); + EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::STRBBui) + .addReg(AArch64::B31) + .addReg(Dst) + .addImm(0)); + OutStreamer.emitLabel(Done); +} + void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI) { unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); @@ -3703,6 +3847,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { LowerMOPS(*OutStreamer, *MI); return; + case AArch64::GoMemoryCopyPseudo: + LowerGoMemoryCopy(*OutStreamer, *MI); + return; + case TargetOpcode::STACKMAP: return LowerSTACKMAP(*OutStreamer, SM, *MI); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index b3a69f3134f28..707976695a018 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9058,10 +9058,10 @@ getAArch64GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) + if (Arg.hasNestAttr() || Arg.hasAttribute(Attribute::ByRef)) continue; LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); + ArgTys.push_back(goabi::getParameterType(Arg)); } return ArgTys; } @@ -9072,21 +9072,65 @@ getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) + if (Args[I].IsNest || Args[I].IsByRef) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); + ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); } return ArgTys; } +static SmallVector getAArch64GoReturnTypes(const Function &F) { + SmallVector MemoryResultTys; + for (const Argument &Arg : F.args()) + if (Arg.hasAttribute(Attribute::ByRef)) + MemoryResultTys.push_back(Arg.getParamByRefType()); + SmallVector ResultTys; + goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, + ResultTys); + return ResultTys; +} + static SmallVector -getAArch64GoReturnTypes(Type *RetTy, const AttributeList &Attrs) { +getAArch64GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, + const TargetLowering::ArgListTy &Args) { + SmallVector MemoryResultTys; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsByRef) + MemoryResultTys.push_back(Arg.IndirectType); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), ResultTys); + goabi::getReturnTypes(RetTy, Attrs, MemoryResultTys, ResultTys); return ResultTys; } +static SmallVector +getAArch64GoDirectResultIndices(const AttributeList &Attrs, + unsigned DirectResultCount, + unsigned MemoryResultCount) { + SmallVector MemoryIndices = goabi::getMemoryResultIndices( + Attrs, DirectResultCount, MemoryResultCount); + SmallVector DirectIndices; + for (unsigned I = 0, E = DirectResultCount + MemoryResultCount; I != E; ++I) + if (!llvm::is_contained(MemoryIndices, I)) + DirectIndices.push_back(I); + return DirectIndices; +} + +static SmallVector +getAArch64GoMemoryResultMap(const AttributeList &Attrs, unsigned ParamCount, + ArrayRef IsByRef, + unsigned DirectResultCount) { + unsigned MemoryResultCount = llvm::count(IsByRef, true); + SmallVector MemoryIndices = goabi::getMemoryResultIndices( + Attrs, DirectResultCount, MemoryResultCount); + SmallVector Map(ParamCount, -1); + unsigned Next = 0; + for (unsigned I = 0; I != ParamCount; ++I) + if (IsByRef[I]) + Map[I] = MemoryIndices[Next++]; + return Map; +} + static SDValue lowerAArch64GoFormalArguments( const AArch64TargetLowering &TLI, SDValue Chain, MachineFunction &MF, const SmallVectorImpl &Ins, const SDLoc &DL, @@ -9102,8 +9146,7 @@ static SDValue lowerAArch64GoFormalArguments( goabi::ABIConfig ABIConfig = getAArch64GoABIConfig(TLI, Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), - DAG.getDataLayout(), ABIConfig); + ArgTys, getAArch64GoReturnTypes(F), DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( ArgTys, Layout, DAG.getDataLayout(), ABIConfig); @@ -9125,6 +9168,15 @@ static SDValue lowerAArch64GoFormalArguments( FuncInfo->clearGoArgPointerSlots(); unsigned StackBias = getAArch64GoStackBias(F.getCallingConv()); + SmallVector DirectResultTys; + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + DirectResultTys); + SmallVector IsByRef; + for (const Argument &Arg : F.args()) + IsByRef.push_back(Arg.hasAttribute(Attribute::ByRef)); + SmallVector MemoryResultMap = getAArch64GoMemoryResultMap( + F.getAttributes(), F.arg_size(), IsByRef, DirectResultTys.size()); + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { uint64_t PointerSize = EntryArgs.PointerSize; @@ -9153,6 +9205,22 @@ static SDValue lowerAArch64GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); + if (Arg->hasAttribute(Attribute::ByRef)) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Ins[Group.Start].Flags.isByRef()) + report_fatal_error("invalid AArch64 Go memory result carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Ins[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + report_fatal_error( + "AArch64 Go memory result carrier disagrees with Go ABI layout"); + int FI = MFI.CreateFixedObject(ResultLayout.Size, + StackBias + ResultLayout.StackOffset, + /*IsImmutable=*/false, /*isAliased=*/true); + InVals.push_back(DAG.getFrameIndex(FI, PtrVT)); + continue; + } if (Arg->hasNestAttr()) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); MVT CopyVT = getAArch64GoCopyVT(Ins[Group.Start].VT); @@ -9166,15 +9234,25 @@ static SDValue lowerAArch64GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; + bool IsByVal = Ins[Group.Start].Flags.isByVal(); + bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasByValAttr() != IsByVal || IsByVal != MustBeByVal) + report_fatal_error( + "AArch64 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; int64_t FixedHomeOffset = StackBias + LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false); + else if (IsByVal) + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/true); AArch64FunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); // LLVM may replace an unused incoming pointer with poison at every call @@ -9186,6 +9264,14 @@ static SDValue lowerAArch64GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); + if (IsByVal) { + if (Group.End != Group.Start + 1 || + Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid AArch64 Go byval formal argument"); + InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9243,17 +9329,30 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(MF.getFunction(), LayoutMap); - SmallVector ResultTys = getAArch64GoReturnTypes( - MF.getFunction().getReturnType(), MF.getFunction().getAttributes()); + SmallVector ResultTys = getAArch64GoReturnTypes(MF.getFunction()); goabi::CallLayout Layout = goabi::computeCallLayout( ArgTys, ResultTys, DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, MF.getFunction().getCallingConv())); unsigned StackBias = getAArch64GoStackBias(MF.getFunction().getCallingConv()); + SmallVector DirectResultIndices = + getAArch64GoDirectResultIndices( + MF.getFunction().getAttributes(), + Layout.Results.size() - llvm::count_if(Layout.Results, + [](const auto &R) { + return !R.InRegs && + R.Size != 0; + }), + llvm::count_if(Layout.Results, [](const auto &R) { + return !R.InRegs && R.Size != 0; + })); SmallVector MemOps; SmallVector, 8> RetRegs; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid AArch64 Go direct result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9328,14 +9427,26 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoCallArgTypes(CLI.getArgs(), LayoutMap); - SmallVector ResultTys; - goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); + AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); + SmallVector ResultTys = + getAArch64GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); goabi::CallLayout Layout = goabi::computeCallLayout( ArgTys, ResultTys, DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); + SmallVector DirectResultTys; + goabi::getReturnTypes(CLI.RetTy, + CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), + DirectResultTys); + SmallVector IsByRef; + for (const TargetLowering::ArgListEntry &Arg : CLI.getArgs()) + IsByRef.push_back(Arg.IsByRef); + SmallVector MemoryResultMap = getAArch64GoMemoryResultMap( + CallAttrs, CLI.getArgs().size(), IsByRef, DirectResultTys.size()); + SmallVector DirectResultIndices = + getAArch64GoDirectResultIndices( + CallAttrs, DirectResultTys.size(), + llvm::count_if(CLI.getArgs(), + [](const auto &Arg) { return Arg.IsByRef; })); unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); // Layout.TotalStackSize rounds the logical Go argument area to the target @@ -9353,12 +9464,28 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector MemOpChains; SmallVector, 8> RegsToPass; + SmallVector MemoryResultDests(Layout.Results.size()); MachineFunction::CallSiteInfo CSInfo; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { if (Group.Index >= CLI.getArgs().size()) continue; + if (CLI.getArgs()[Group.Index].IsByRef) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Outs[Group.Start].Flags.isByRef()) + report_fatal_error("invalid AArch64 Go memory result call carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Outs[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + report_fatal_error( + "AArch64 Go memory result call carrier disagrees with Go ABI " + "layout"); + MemoryResultDests[ResultIndex] = OutVals[Group.Start]; + continue; + } + if (CLI.getArgs()[Group.Index].IsNest) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); RegsToPass.emplace_back(AArch64::X26, OutVals[Group.Start]); @@ -9366,6 +9493,28 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, } const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutMap[Group.Index]]; + bool IsByVal = CLI.getArgs()[Group.Index].IsByVal; + bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isByVal() != IsByVal || IsByVal != MustBeByVal) + report_fatal_error( + "AArch64 Go call argument carrier disagrees with Go ABI layout"); + if (IsByVal) { + if (Group.End != Group.Start + 1 || + Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid AArch64 Go byval call argument"); + assert(StackPtr && "missing Go call stack pointer"); + uint64_t Offset = StackBias + ArgLayout.StackOffset; + SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, + DAG.getIntPtrConstant(Offset, DL)); + Align Alignment = Outs[Group.Start].Flags.getNonZeroByValAlign(); + MemOpChains.push_back(DAG.getMemcpy( + Chain, DL, Addr, OutVals[Group.Start], + DAG.getConstant(ArgLayout.Size, DL, PtrVT), Alignment, Alignment, + /*isVol=*/false, /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt, + MachinePointerInfo::getStack(MF, Offset), MachinePointerInfo())); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9454,7 +9603,10 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, SmallVector ResultVals(Ins.size()); for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid AArch64 Go direct call result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (!ResultLayout.InRegs) continue; unsigned IntPiece = 0; @@ -9488,10 +9640,8 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, // Keep the call sequence active while reading stack results. The load chain // is after the call, so the physical SP is the current post-growth Go stack; // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + bool HasStackResults = llvm::any_of( + Layout.Results, [](const auto &Result) { return !Result.InRegs; }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP, PtrVT, InGlue); @@ -9499,7 +9649,8 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, InGlue = ResultStackPtr.getValue(2); } for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (ResultLayout.InRegs) continue; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -9517,6 +9668,25 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, } } + SmallVector MemoryResultCopies; + for (unsigned I = 0; I != MemoryResultDests.size(); ++I) { + if (!MemoryResultDests[I].getNode()) + continue; + const goabi::ValueLayout &ResultLayout = Layout.Results[I]; + SDValue Src = DAG.getNode( + ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(StackBias + ResultLayout.StackOffset, DL)); + MemoryResultCopies.push_back(DAG.getMemcpy( + Chain, DL, MemoryResultDests[I], Src, + DAG.getConstant(ResultLayout.Size, DL, PtrVT), ResultLayout.Alignment, + ResultLayout.Alignment, /*isVol=*/false, /*AlwaysInline=*/false, + /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), + MachinePointerInfo::getStack(MF, + StackBias + ResultLayout.StackOffset))); + } + if (!MemoryResultCopies.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemoryResultCopies); + Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); @@ -9534,10 +9704,13 @@ AArch64TargetLowering::getArgumentCopyElisionFrameInfo( const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const AArch64FunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) - if (Home.ArgNo == Arg.getArgNo() && + if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) + // Stack-assigned values use typed byval and are already bound directly + // to their fixed incoming objects. Only register parameters still need + // their frontend store redirected into the canonical spill home. return ArgumentCopyElisionFrameInfo{Home.FrameIndex, - Home.valueAlreadyInFrame()}; + /*ValueAlreadyInFrame=*/false}; return std::nullopt; } @@ -9553,8 +9726,7 @@ int AArch64TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); const AArch64Subtarget &Subtarget = MF.getSubtarget(); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F.getReturnType(), F.getAttributes()), - MF.getDataLayout(), + ArgTys, getAArch64GoReturnTypes(F), MF.getDataLayout(), getAArch64GoABIConfig(*this, Subtarget, F.getCallingConv())); if (Layout.ArgSize == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 0532218514007..5dc1bbeb88c7c 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -11498,6 +11498,21 @@ let Predicates = [HasMOPS], Defs = [NZCV], Size = 12, mayStore = 1 in { [], "$Rd = $Rd_wb,$Rn = $Rn_wb,@earlyclobber $Rn_wb">, Sched<[]>; } } + +// Populate a Go call's outgoing stack-argument area without introducing a +// libcall. The AsmPrinter expands this to a baseline-AArch64 vector loop with +// scalar tails. All three GPR operands are tied because the expansion advances +// them; Q31 and NZCV are explicit clobbers so register allocation preserves any +// live values. +let Defs = [Q31, NZCV], Size = 104, mayLoad = 1, mayStore = 1 in { + def GoMemoryCopyPseudo + : Pseudo<(outs GPR64common:$Rd_wb, GPR64common:$Rs_wb, + GPR64:$Rn_wb), + (ins GPR64common:$Rd, GPR64common:$Rs, GPR64:$Rn), [], + "$Rd = $Rd_wb,$Rs = $Rs_wb,$Rn = $Rn_wb">, + Sched<[]>; +} + let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, mayStore = 1 in { def MOPSMemorySetTaggingPseudo : Pseudo<(outs GPR64common:$Rd_wb, GPR64:$Rn_wb), (ins GPR64common:$Rd, GPR64:$Rn, GPR64:$Rm), diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp index 286e5638fc088..56e4946eebd90 100644 --- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp @@ -12,6 +12,7 @@ #include "AArch64SelectionDAGInfo.h" #include "AArch64MachineFunctionInfo.h" +#include "llvm/CodeGen/GoCallingConv.h" #define GET_SDNODE_DESC #include "AArch64GenSDNodeInfo.inc" @@ -234,6 +235,16 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( Size, DstAlign, SrcAlign, isVolatile, DstPtrInfo, SrcPtrInfo); + // A Go call cannot use a C memcpy libcall to populate its outgoing argument + // area: the copy is part of the call sequence and must not introduce another + // call or safepoint. Small constant copies have already been expanded by + // SelectionDAG. Keep large copies compact by expanding this pseudo to a + // baseline-AArch64 loop in the AsmPrinter. + if (goabi::isGoCallingConv( + DAG.getMachineFunction().getFunction().getCallingConv())) + return EmitMOPS(AArch64::GoMemoryCopyPseudo, DAG, DL, Chain, Dst, Src, Size, + DstAlign, SrcAlign, isVolatile, DstPtrInfo, SrcPtrInfo); + auto *AFI = DAG.getMachineFunction().getInfo(); SMEAttrs Attrs = AFI->getSMEFnAttrs(); if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index f48a98b2437e2..2b44820996125 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -153,10 +153,10 @@ getX86GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr()) + if (Arg.hasNestAttr() || Arg.hasAttribute(Attribute::ByRef)) continue; LayoutMap[Index] = ArgTys.size(); - ArgTys.push_back(Arg.getType()); + ArgTys.push_back(goabi::getParameterType(Arg)); } return ArgTys; } @@ -167,21 +167,65 @@ getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest) + if (Args[I].IsNest || Args[I].IsByRef) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].OrigTy); + ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); } return ArgTys; } -static SmallVector getX86GoReturnTypes(Type *RetTy, - const AttributeList &Attrs) { +static SmallVector getX86GoReturnTypes(const Function &F) { + SmallVector MemoryResultTys; + for (const Argument &Arg : F.args()) + if (Arg.hasAttribute(Attribute::ByRef)) + MemoryResultTys.push_back(Arg.getParamByRefType()); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), ResultTys); + goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, + ResultTys); + return ResultTys; +} + +static SmallVector +getX86GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, + const TargetLowering::ArgListTy &Args) { + SmallVector MemoryResultTys; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsByRef) + MemoryResultTys.push_back(Arg.IndirectType); + SmallVector ResultTys; + goabi::getReturnTypes(RetTy, Attrs, MemoryResultTys, ResultTys); return ResultTys; } +static SmallVector +getX86GoDirectResultIndices(const AttributeList &Attrs, + unsigned DirectResultCount, + unsigned MemoryResultCount) { + SmallVector MemoryIndices = goabi::getMemoryResultIndices( + Attrs, DirectResultCount, MemoryResultCount); + SmallVector DirectIndices; + for (unsigned I = 0, E = DirectResultCount + MemoryResultCount; I != E; ++I) + if (!llvm::is_contained(MemoryIndices, I)) + DirectIndices.push_back(I); + return DirectIndices; +} + +static SmallVector getX86GoMemoryResultMap(const AttributeList &Attrs, + unsigned ParamCount, + ArrayRef IsByRef, + unsigned DirectResultCount) { + unsigned MemoryResultCount = llvm::count(IsByRef, true); + SmallVector MemoryIndices = goabi::getMemoryResultIndices( + Attrs, DirectResultCount, MemoryResultCount); + SmallVector Map(ParamCount, -1); + unsigned Next = 0; + for (unsigned I = 0; I != ParamCount; ++I) + if (IsByRef[I]) + Map[I] = MemoryIndices[Next++]; + return Map; +} + static SDValue lowerX86GoFormalArguments( const X86TargetLowering &TLI, SDValue Chain, MachineFunction &MF, const SmallVectorImpl &Ins, const SDLoc &DL, @@ -202,8 +246,7 @@ static SDValue lowerX86GoFormalArguments( goabi::ABIConfig ABIConfig = getX86GoABIConfig(Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), - DAG.getDataLayout(), ABIConfig); + ArgTys, getX86GoReturnTypes(F), DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( ArgTys, Layout, DAG.getDataLayout(), ABIConfig); @@ -226,6 +269,15 @@ static SDValue lowerX86GoFormalArguments( FuncInfo->clearGoArgHomes(); FuncInfo->clearGoArgPointerSlots(); + SmallVector DirectResultTys; + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + DirectResultTys); + SmallVector IsByRef; + for (const Argument &Arg : F.args()) + IsByRef.push_back(Arg.hasAttribute(Attribute::ByRef)); + SmallVector MemoryResultMap = getX86GoMemoryResultMap( + F.getAttributes(), F.arg_size(), IsByRef, DirectResultTys.size()); + auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { uint64_t PointerSize = EntryArgs.PointerSize; @@ -264,6 +316,23 @@ static SDValue lowerX86GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); + if (Arg->hasAttribute(Attribute::ByRef)) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Ins[Group.Start].Flags.isByRef()) + report_fatal_error("invalid X86 Go memory result carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Ins[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + report_fatal_error( + "X86 Go memory result carrier disagrees with Go ABI layout"); + int FI = + MFI.CreateFixedObject(ResultLayout.Size, ResultLayout.StackOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + InVals.push_back(DAG.getFrameIndex(FI, PtrVT)); + continue; + } if (Arg->hasNestAttr()) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); Register VReg = @@ -279,15 +348,25 @@ static SDValue lowerX86GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; + bool IsByVal = Ins[Group.Start].Flags.isByVal(); + bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasByValAttr() != IsByVal || IsByVal != MustBeByVal) + report_fatal_error( + "X86 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = ArgLayout.InRegs ? ArgSpillOffsets[LayoutIndex] : ArgLayout.StackOffset; int64_t FixedHomeOffset = LogicalHomeOffset; - int HomeFI = - ArgLayout.InRegs - ? MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/false) - : MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, - /*IsImmutable=*/true); + int HomeFI; + if (ArgLayout.InRegs) + HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false); + else if (IsByVal) + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/false, + /*isAliased=*/true); + else + HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, + /*IsImmutable=*/true); X86MachineFunctionInfo::GoArgHome &Home = FuncInfo->addGoArgHome(Group.Index, HomeFI); // LLVM may replace an unused incoming pointer with poison at every call @@ -299,6 +378,14 @@ static SDValue lowerX86GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); + if (IsByVal) { + if (Group.End != Group.Start + 1 || + Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid X86 Go byval formal argument"); + InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -355,17 +442,25 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, MachineFrameInfo &MFI = MF.getFrameInfo(); SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(MF.getFunction(), LayoutMap); - SmallVector ResultTys = - getX86GoReturnTypes(MF.getFunction().getReturnType(), - MF.getFunction().getAttributes()); + SmallVector ResultTys = getX86GoReturnTypes(MF.getFunction()); goabi::CallLayout Layout = goabi::computeCallLayout( ArgTys, ResultTys, DAG.getDataLayout(), getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); + unsigned MemoryResultCount = + llvm::count_if(Layout.Results, [](const auto &Result) { + return !Result.InRegs && Result.Size != 0; + }); + SmallVector DirectResultIndices = getX86GoDirectResultIndices( + MF.getFunction().getAttributes(), + Layout.Results.size() - MemoryResultCount, MemoryResultCount); SmallVector MemOps; SmallVector, 8> RetRegs; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid X86 Go direct result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -440,15 +535,27 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, report_fatal_error("Go calling convention does not support varargs"); SmallVector ArgLayoutMap; - SmallVector ArgTys = getX86GoCallArgTypes(CLI.getArgs(), ArgLayoutMap); - SmallVector ResultTys; + SmallVector ArgTys = + getX86GoCallArgTypes(CLI.getArgs(), ArgLayoutMap); + AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); + SmallVector ResultTys = + getX86GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); + goabi::CallLayout Layout = + goabi::computeCallLayout(ArgTys, ResultTys, DAG.getDataLayout(), + getX86GoABIConfig(Subtarget, CLI.CallConv)); + SmallVector DirectResultTys; goabi::getReturnTypes(CLI.RetTy, - goabi::isGoCallingConv(CLI.CallConv) && CLI.CB && - goabi::hasTupleResultsAttr(*CLI.CB), - ResultTys); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getX86GoABIConfig(Subtarget, CLI.CallConv)); + CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), + DirectResultTys); + SmallVector IsByRef; + for (const TargetLowering::ArgListEntry &Arg : CLI.getArgs()) + IsByRef.push_back(Arg.IsByRef); + SmallVector MemoryResultMap = getX86GoMemoryResultMap( + CallAttrs, CLI.getArgs().size(), IsByRef, DirectResultTys.size()); + unsigned MemoryResultCount = llvm::count_if( + CLI.getArgs(), [](const auto &Arg) { return Arg.IsByRef; }); + SmallVector DirectResultIndices = getX86GoDirectResultIndices( + CallAttrs, DirectResultTys.size(), MemoryResultCount); unsigned NumBytes = Layout.TotalStackSize; Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); @@ -459,19 +566,58 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, SmallVector MemOpChains; SmallVector, 8> RegsToPass; + SmallVector MemoryResultDests(Layout.Results.size()); MachineFunction::CallSiteInfo CSInfo; for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Outs))) { if (Group.Index >= CLI.getArgs().size()) continue; + if (CLI.getArgs()[Group.Index].IsByRef) { + int ResultIndex = MemoryResultMap[Group.Index]; + if (ResultIndex < 0 || Group.End != Group.Start + 1 || + !Outs[Group.Start].Flags.isByRef()) + report_fatal_error("invalid X86 Go memory result call carrier"); + const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; + if (ResultLayout.InRegs || ResultLayout.Size == 0 || + Outs[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + report_fatal_error( + "X86 Go memory result call carrier disagrees with Go ABI layout"); + MemoryResultDests[ResultIndex] = OutVals[Group.Start]; + continue; + } + if (CLI.getArgs()[Group.Index].IsNest) { assert(Group.End == Group.Start + 1 && "unexpected split nest arg"); RegsToPass.emplace_back(X86::RDX, OutVals[Group.Start]); continue; } - const goabi::ValueLayout &ArgLayout = Layout.Args[ArgLayoutMap[Group.Index]]; + const goabi::ValueLayout &ArgLayout = + Layout.Args[ArgLayoutMap[Group.Index]]; + bool IsByVal = CLI.getArgs()[Group.Index].IsByVal; + bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isByVal() != IsByVal || IsByVal != MustBeByVal) + report_fatal_error( + "X86 Go call argument carrier disagrees with Go ABI layout"); + if (IsByVal) { + if (Group.End != Group.Start + 1 || + Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) + report_fatal_error("invalid X86 Go byval call argument"); + assert(StackPtr && "missing Go call stack pointer"); + SDValue Addr = + DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, + DAG.getIntPtrConstant(ArgLayout.StackOffset, DL)); + Align Alignment = Outs[Group.Start].Flags.getNonZeroByValAlign(); + MemOpChains.push_back(DAG.getMemcpy( + Chain, DL, Addr, OutVals[Group.Start], + DAG.getConstant(ArgLayout.Size, DL, PtrVT), Alignment, Alignment, + /*isVol=*/false, /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt, + MachinePointerInfo::getStack(MF, ArgLayout.StackOffset), + MachinePointerInfo())); + continue; + } + unsigned IntPiece = 0; unsigned FPPiece = 0; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -541,7 +687,10 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, SmallVector ResultVals(Ins.size()); for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + if (Group.Index >= DirectResultIndices.size()) + report_fatal_error("invalid X86 Go direct call result index"); + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (!ResultLayout.InRegs) continue; unsigned IntPiece = 0; @@ -575,10 +724,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, // Keep the call sequence active while reading stack results. The load chain // is after the call, so the physical SP is the current post-growth Go stack; // CALLSEQ_END cannot release the outgoing frame until every result is read. - bool HasStackResults = - llvm::any_of(groupGoArgs(ArrayRef(Ins)), [&](const auto &Group) { - return !Layout.Results[Group.Index].InRegs; - }); + bool HasStackResults = llvm::any_of( + Layout.Results, [](const auto &Result) { return !Result.InRegs; }); SDValue ResultStackPtr; if (HasStackResults) { ResultStackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), @@ -587,7 +734,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, InGlue = ResultStackPtr.getValue(2); } for (const GoArgGroup &Group : groupGoArgs(ArrayRef(Ins))) { - const goabi::ValueLayout &ResultLayout = Layout.Results[Group.Index]; + const goabi::ValueLayout &ResultLayout = + Layout.Results[DirectResultIndices[Group.Index]]; if (ResultLayout.InRegs) continue; for (unsigned I = Group.Start; I != Group.End; ++I) { @@ -605,6 +753,24 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, } } + SmallVector MemoryResultCopies; + for (unsigned I = 0; I != MemoryResultDests.size(); ++I) { + if (!MemoryResultDests[I].getNode()) + continue; + const goabi::ValueLayout &ResultLayout = Layout.Results[I]; + SDValue Src = + DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(ResultLayout.StackOffset, DL)); + MemoryResultCopies.push_back(DAG.getMemcpy( + Chain, DL, MemoryResultDests[I], Src, + DAG.getConstant(ResultLayout.Size, DL, PtrVT), ResultLayout.Alignment, + ResultLayout.Alignment, /*isVol=*/false, /*AlwaysInline=*/false, + /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), + MachinePointerInfo::getStack(MF, ResultLayout.StackOffset))); + } + if (!MemoryResultCopies.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemoryResultCopies); + Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, HasStackResults ? SDValue() : InGlue, DL); @@ -623,10 +789,13 @@ X86TargetLowering::getArgumentCopyElisionFrameInfo(const Argument &Arg, uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const X86MachineFunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) - if (Home.ArgNo == Arg.getArgNo() && + if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) + // Stack-assigned values use typed byval and are already bound directly + // to their fixed incoming objects. Only register parameters still need + // their frontend store redirected into the canonical spill home. return ArgumentCopyElisionFrameInfo{Home.FrameIndex, - Home.valueAlreadyInFrame()}; + /*ValueAlreadyInFrame=*/false}; return std::nullopt; } @@ -641,8 +810,7 @@ int X86TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F.getReturnType(), F.getAttributes()), - MF.getDataLayout(), + ArgTys, getX86GoReturnTypes(F), MF.getDataLayout(), getX86GoABIConfig(MF.getSubtarget(), F.getCallingConv())); if (Layout.ArgSize == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp index 4736dca6ba57d..cc59c317d5c54 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp @@ -400,6 +400,8 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy( MachinePointerInfo SrcPtrInfo) const { const X86Subtarget &Subtarget = DAG.getMachineFunction().getSubtarget(); + const bool IsGo = goabi::isGoCallingConv( + DAG.getMachineFunction().getFunction().getCallingConv()); // If to a segment-relative address space, use the default lowering. if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256) @@ -427,7 +429,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy( return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Size.getValueType(), Alignment, isVolatile, - AlwaysInline, DstPtrInfo, SrcPtrInfo); + AlwaysInline || IsGo, DstPtrInfo, SrcPtrInfo); } return SDValue(); diff --git a/llvm/test/CodeGen/AArch64/go-argument-homes.ll b/llvm/test/CodeGen/AArch64/go-argument-homes.ll new file mode 100644 index 0000000000000..e6d0015a42ff2 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-argument-homes.ll @@ -0,0 +1,27 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 < %s | \ +; RUN: FileCheck %s --check-prefix=ASM + +%pair = type { i64, i64 } + +; Exhaust AArch64's integer register budget so the pair arrives in one typed +; stack home at the target's Go stack bias. The carrier is bound directly to +; that home and needs no frontend alloca or copy. +define goabiinternal ptr @aarch64_stack_pair_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, + i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, + ptr byval(%pair) align 8 %value.home) { +entry: + ret ptr %value.home +} + +; MIR-LABEL: name: aarch64_stack_pair_home +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 8, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +; MIR-NOT: STRXui {{.*}}%fixed-stack.0 +; MIR: ADDXri %fixed-stack.0 +; ASM-LABEL: aarch64_stack_pair_home: +; ASM: add x0, sp, #8 diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index d41426c27aa01..e4d75e7c22326 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -53,13 +53,16 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @abi0_second_int(i64 %a, i64 %b) { +define goabi0 i64 @abi0_second_int( + ptr byval(i64) align 8 %a.home, + ptr byval(i64) align 8 %b.home) { ; A64-LABEL: abi0_second_int: ; Go's arm64 stack ABI reserves 0(SP) for the return PC. ; A64: ldr x[[REG:[0-9]+]], [sp, #16] ; A64: str x[[REG]], [sp, #24] ; A64: ret entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } @@ -71,10 +74,16 @@ define goabi0 i64 @abi0_call_second_int() { ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] -; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. -; A64: str x[[RET]], [sp, #56] +; The 64-byte caller frame leaves this function's ABI0 result at entry SP+8. +; A64: str x[[RET]], [sp, #72] entry: - %ret = call goabi0 i64 @abi0_second_int(i64 11, i64 22) + %a.home = alloca i64, align 8 + %b.home = alloca i64, align 8 + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @abi0_second_int( + ptr byval(i64) align 8 %a.home, + ptr byval(i64) align 8 %b.home) ret i64 %ret } @@ -111,30 +120,35 @@ define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { + ptr byval(%pair) align 8 %value.home) { ; A64-LABEL: stack_pair: ; A64: ldr x0, [sp, #16] ; A64: ret entry: + %value = load %pair, ptr %value.home, align 8 %right = extractvalue %pair %value, 1 ret i64 %right } define goabiinternal i64 @call_stack_pair() { ; A64-LABEL: call_stack_pair: -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: str x{{[0-9]+}}, [x{{[0-9]+}}, #16] +; A64: ldr q[[VALUE:[0-9]+]], [sp, #{{[0-9]+}}] +; A64: mov x[[BASE:[0-9]+]], sp +; A64: stur q[[VALUE]], [x[[BASE]], #8] ; A64: bl stack_pair entry: + %value.home = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, - %pair { i64 13, i64 17 }) + ptr byval(%pair) align 8 %value.home) ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr byval([8 x i8]) align 1 %value.home) { ; A64-LABEL: stack_bytes: ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #8] ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #15] @@ -142,13 +156,15 @@ define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { ; A64-DAG: strb w{{[0-9]+}}, [sp, #23] ; A64: ret entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; A64-LABEL: call_stack_bytes: -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #8] -; A64-DAG: strb w{{[0-9]+}}, [x{{[0-9]+}}, #15] +; A64: ldr x[[VALUE:[0-9]+]], [sp, #{{[0-9]+}}] +; A64: mov x[[BASE:[0-9]+]], sp +; A64: str x[[VALUE]], [x[[BASE]], #8] ; A64: bl stack_bytes ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #16] ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #23] @@ -158,8 +174,11 @@ define goabiinternal i16 @call_stack_bytes() { ; A64-O2: ldrb w{{[0-9]+}}, [sp, #16] ; A64-O2: add sp, sp, #{{[0-9]+}} entry: + %value.home = alloca [8 x i8], align 1 + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr byval([8 x i8]) align 1 %value.home) %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/AArch64/go-stack-byval.ll b/llvm/test/CodeGen/AArch64/go-stack-byval.ll new file mode 100644 index 0000000000000..ccfe6ae7c00aa --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-stack-byval.ll @@ -0,0 +1,54 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] + +declare goabiinternal void @consume(ptr byval(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +; CHECK: bl consume +entry: + call goabiinternal void @consume(ptr byval(%large) align 8 %source) + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr byval(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: ldr x0, [sp, #8] +; CHECK-NEXT: ret +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr byref(%large) align 8 %result) #0 { +; CHECK-LABEL: write_memory_result: +; CHECK: str x{{[0-9]+}}, [sp, #8] +; CHECK-NEXT: ret +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: bl write_memory_result +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr byref(%large) align 8 %result) #0 + %value = load i64, ptr %result, align 8 + ret i64 %value +} + +attributes #0 = { "go_memory_results"="0" } diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll index 00961b87513ae..42ce2adeed12e 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll @@ -11,8 +11,9 @@ define goabiinternal ptr addrspace(1) @scalar_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -27,8 +28,9 @@ define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + ptr byval(%aggregate) align 8 %value.home) gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -50,8 +52,11 @@ define goabiinternal ptr addrspace(1) @merged_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr byval(ptr addrspace(1)) align 8 %p16.home, + ptr byval(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -69,8 +74,11 @@ define goabiinternal ptr addrspace(1) @relocated_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + ptr byval(ptr addrspace(1)) align 8 %p16.home, + ptr byval(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -115,21 +123,34 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK: fixedStack: ; CHECK: - { id: 0, type: default, offset: 8, size: 8, ; CHECK: isImmutable: false -; CHECK: stack: [] +; A scalar loaded from the typed incoming home is a distinct SSA value. Keep +; it in one ordinary relocation slot; do not copy the complete argument home. +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: = LDRXui %fixed-stack.0, 0 +; CHECK: STRXui killed {{.*}}, %stack.0, 0 ; CHECK: STATEPOINT 1, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.0, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0) +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 +; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.0 ; CHECK-LABEL: name: aggregate_stack_arg -; CHECK: stack: [] +; The aggregate remains one typed fixed incoming object. Only its two live +; pointer fields need scalar relocation slots. +; CHECK: fixedStack: +; CHECK: - { id: 0, type: default, offset: 8, size: 24, +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: - { id: 1, name: '', type: default, offset: 0, size: 8, +; CHECK: = LDRXui %fixed-stack.0, 0 +; CHECK: = LDRXui %fixed-stack.0, 2 ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1) +; CHECK-SAME: 2, 2, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0), +; CHECK-SAME: (volatile load store (s64) on %stack.1) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.1 ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: @@ -140,10 +161,11 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: relocated_stack_arg -; CHECK: stack: [] +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, ; CHECK: STATEPOINT 4, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME:[0-9]+]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME:[0-9]+]], 0, ; CHECK: STATEPOINT 5, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, ; CHECK: STATEPOINT 6, -; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, diff --git a/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll new file mode 100644 index 0000000000000..08eba10dcab80 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll @@ -0,0 +1,37 @@ +; REQUIRES: aarch64-registered-target +; RUN: llc -filetype=obj < %s -o %t.o +; RUN: %python %S/../../MC/GoObj/Inputs/dump-goobj.py %t.o | FileCheck %s + +; Optimized Go IR can contain private lookup tables whose bytes are identical +; before fixups but whose relocations name different symbols. Their Go content +; hashes must include the relocation target identity, matching cmd/internal/obj, +; so the linker cannot fold the tables together. + +target triple = "aarch64-apple-darwin-goobj" + +@runtime.targetA = global i8 1, section ".rodata", !goobj.symbol.nonpackage !2 +@runtime.targetB = global i8 2, section ".rodata", !goobj.symbol.nonpackage !2 +@table.a = private constant [1 x ptr] [ptr @runtime.targetA], align 8 +@table.b = private constant [1 x ptr] [ptr @runtime.targetB], align 8 +@llvm.compiler.used = appending global [2 x ptr] [ptr @table.a, ptr @table.b], section "llvm.metadata" + +define ptr @p.lookup(i1 %which) !goobj.symbol.index !3 { +entry: + %table = select i1 %which, ptr @table.a, ptr @table.b + ret ptr %table +} + +!goobj.config = !{!0} +!0 = !{!"goallc.goobj", !"darwin", !"arm64", !"go1.27", !"GOARM64", !"v8.0", !"", !"p", !"0", !"0", !1} +!1 = !{!"regabiwrappers", !"regabiargs"} +!2 = !{i1 true} +!3 = !{i32 0} + +; CHECK: hasheddef 0: .Ltable.b abi=0 type=3 size=8 align=8 +; CHECK-NEXT: hasheddef 1: .Ltable.a abi=0 type=3 size=8 align=8 +; These are cmd/internal/obj's native SHA-256-based hashes for size 8, +; default rodata, one R_ADDR relocation, and each non-package target name. +; CHECK: hash 0: 0f368520f5468968253a6a8c2a0e2f05 +; CHECK-NEXT: hash 1: 05c6989a6d8dc052c2e96991d2f58be6 +; CHECK: reloc 2.{{[0-9]+}}: off=0 size=8 type=1 add=0 target=runtime.targetB +; CHECK: reloc 3.{{[0-9]+}}: off=0 size=8 type=1 add=0 target=runtime.targetA diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index 50ee4a27d6226..5a4a8803f6dac 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -11,16 +11,16 @@ entry: ret void } -define goabiinternal i64 @large_home_offset([4096 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_offset( + ptr byval([4096 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg } -define goabiinternal i64 @large_home_boundary([4094 x i64] %stackarg, - i64 %regarg) +define goabiinternal i64 @large_home_boundary( + ptr byval([4094 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll index 1bec5369e70a4..695e2007eaf78 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll @@ -32,8 +32,9 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %p16) "frame-pointer"="non-leaf" { + ptr byval(ptr) align 8 %p16.home) "frame-pointer"="non-leaf" { entry: + %p16 = load ptr, ptr %p16.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 diff --git a/llvm/test/CodeGen/Generic/go-abi0-frame.ll b/llvm/test/CodeGen/Generic/go-abi0-frame.ll index fd7d23c8cbe9d..b8c86ceff4e4c 100644 --- a/llvm/test/CodeGen/Generic/go-abi0-frame.ll +++ b/llvm/test/CodeGen/Generic/go-abi0-frame.ll @@ -12,7 +12,8 @@ declare ptr @llvm.go.abi0.frame() ; The intrinsic denotes one mutable, aliased object spanning the two arguments ; and one result slot. It deliberately overlaps the per-value fixed objects. -define goabi0 ptr @abi0_frame(i64 %a, i64 %b) noinline { +define goabi0 ptr @abi0_frame( + ptr byval(i64) align 8 %a, ptr byval(i64) align 8 %b) noinline { entry: %frame = call ptr @llvm.go.abi0.frame() ret ptr %frame diff --git a/llvm/test/CodeGen/Generic/go-argument-homes.ll b/llvm/test/CodeGen/Generic/go-argument-homes.ll index 67e7696baafdd..b66cfa5c6831d 100644 --- a/llvm/test/CodeGen/Generic/go-argument-homes.ll +++ b/llvm/test/CodeGen/Generic/go-argument-homes.ll @@ -9,7 +9,6 @@ ; RUN: FileCheck %s --check-prefix=A64-ASM %padded = type { i8, i64 } -%pair = type { i64, i64 } ; A register argument's canonical alloca is remapped to its ABI spill home and ; the IR store initializes that home. @@ -66,48 +65,3 @@ entry: ; A64-ASM-DAG: strb w0, [sp, #8] ; A64-ASM-DAG: str x1, [sp, #16] ; A64-ASM: add x0, sp, #8 - -; Exhaust X86's integer register budget so the pair arrives in one stack home. -define goabiinternal ptr @x86_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; X86-LABEL: name: x86_stack_pair_home -; X86: fixedStack: -; X86-NEXT: - { id: 0, type: default, offset: 0, size: 16 -; X86-NEXT: isImmutable: false, isAliased: true -; X86: MOV64rm %fixed-stack.0 -; X86: MOV64rm %fixed-stack.0{{.*}}8 -; X86-NOT: MOV64mr %fixed-stack.0 -; X86: LEA64r %fixed-stack.0 -; X86-ASM-LABEL: x86_stack_pair_home: -; X86-ASM-DAG: movq 8(%rsp), %rax -; X86-ASM-DAG: movq 16(%rsp), %rax -; X86-ASM: leaq 8(%rsp), %rax - -; Exhaust AArch64's integer register budget so the pair arrives in one stack -; home at the target's Go stack bias. -define goabiinternal ptr @aarch64_stack_pair_home( - i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %pair %value) { -entry: - %home = alloca %pair, align 8 - store %pair %value, ptr %home, align 8 - ret ptr %home -} - -; A64-LABEL: name: aarch64_stack_pair_home -; A64: fixedStack: -; A64-NEXT: - { id: 0, type: default, offset: 8, size: 16 -; A64-NEXT: isImmutable: false, isAliased: true -; A64-NOT: STRXui {{.*}}%fixed-stack.0 -; A64: ADDXri %fixed-stack.0 -; A64-ASM-LABEL: aarch64_stack_pair_home: -; A64-ASM: add x0, sp, #8 diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll index 428cdd6055b65..b711fa91d3352 100644 --- a/llvm/test/CodeGen/Generic/goobj-nosplit.ll +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -7,7 +7,7 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s declare goabiinternal void @callee(ptr) -declare goabi0 void @"callee.abi0"(ptr) +declare goabi0 void @"callee.abi0"(ptr byval(ptr) align 8) define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { entry: @@ -30,9 +30,12 @@ entry: %closure = alloca [3 x ptr], align 8 %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 + %argument = getelementptr [3 x ptr], ptr %closure, i64 0, i64 2 store volatile ptr @callee, ptr %code, align 8 store volatile ptr %pointer, ptr %context, align 8 - call goabi0 void @"callee.abi0"(ptr %closure) + store ptr %closure, ptr %argument, align 8 + call goabi0 void @"callee.abi0"( + ptr byval(ptr) align 8 %argument) ret void } diff --git a/llvm/test/CodeGen/X86/go-argument-homes.ll b/llvm/test/CodeGen/X86/go-argument-homes.ll new file mode 100644 index 0000000000000..b646be45257ea --- /dev/null +++ b/llvm/test/CodeGen/X86/go-argument-homes.ll @@ -0,0 +1,25 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 -verify-machineinstrs \ +; RUN: -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=MIR +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 < %s | \ +; RUN: FileCheck %s --check-prefix=ASM + +%pair = type { i64, i64 } + +; Exhaust X86's integer register budget so the pair arrives in one typed stack +; home. The byval carrier is the home itself; no frontend alloca or copy exists. +define goabiinternal ptr @x86_stack_pair_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr byval(%pair) align 8 %value.home) { +entry: + ret ptr %value.home +} + +; MIR-LABEL: name: x86_stack_pair_home +; MIR: fixedStack: +; MIR-NEXT: - { id: 0, type: default, offset: 0, size: 16 +; MIR-NEXT: isImmutable: false, isAliased: true +; MIR-NOT: MOV64mr %fixed-stack.0 +; MIR: LEA64r %fixed-stack.0 +; ASM-LABEL: x86_stack_pair_home: +; ASM: leaq 8(%rsp), %rax diff --git a/llvm/test/CodeGen/X86/go-callconv.ll b/llvm/test/CodeGen/X86/go-callconv.ll index 226f7007968ad..6a215166fe6b7 100644 --- a/llvm/test/CodeGen/X86/go-callconv.ll +++ b/llvm/test/CodeGen/X86/go-callconv.ll @@ -61,25 +61,39 @@ entry: ret { i64, %go.empty.carrier, i64 } %r1 } -define goabi0 i64 @"abi0_second_int"(i64 %a, i64 %b) { +define goabi0 i64 @"abi0_second_int"( + ptr byval(i64) align 8 %a.home, + ptr byval(i64) align 8 %b.home) { ; X86-LABEL: "abi0_second_int": -; X86: movq 16(%rsp), %rax -; X86: movq %rax, 24(%rsp) +; X86: leaq 16(%rsp), %[[BHOME:r[a-z0-9]+]] +; X86: movq (%[[BHOME]]), %[[B:r[a-z0-9]+]] +; X86: movq %[[B]], 24(%rsp) ; X86: retq entry: + %b = load i64, ptr %b.home, align 8 ret i64 %b } define goabi0 i64 @"abi0_call_second_int"() { ; X86-LABEL: "abi0_call_second_int": +; X86-DAG: movq $11, [[ASRC:[0-9]+]](%rsp) +; X86-DAG: movq $22, [[BSRC:[0-9]+]](%rsp) +; X86: movq [[BSRC]](%rsp), %[[B:r[a-z0-9]+]] ; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86: movq $22, 8(%[[BASE]]) -; X86: movq $11, (%[[BASE]]) +; X86: movq %[[B]], 8(%[[BASE]]) +; X86: movq [[ASRC]](%rsp), %[[A:r[a-z0-9]+]] +; X86: movq %[[A]], (%[[BASE]]) ; X86: callq "abi0_second_int" ; X86: movq %rsp, %[[RELOAD:r[a-z0-9]+]] ; X86: movq 16(%[[RELOAD]]), %rax entry: - %ret = call goabi0 i64 @"abi0_second_int"(i64 11, i64 22) + %a.home = alloca i64, align 8 + %b.home = alloca i64, align 8 + store i64 11, ptr %a.home, align 8 + store i64 22, ptr %b.home, align 8 + %ret = call goabi0 i64 @"abi0_second_int"( + ptr byval(i64) align 8 %a.home, + ptr byval(i64) align 8 %b.home) ret i64 %ret } @@ -125,9 +139,16 @@ define goabiinternal i64 @call_method_results(ptr %callee, ptr %recv, ; X86-O2: addq 56(%rsp), %rax ; X86-O2: addq ${{[0-9]+}}, %rsp entry: + %integers.home = alloca [2 x i64], align 8 + %floats.home = alloca [2 x double], align 8 + store [2 x i64] [i64 456, i64 789], ptr %integers.home, align 8 + store [2 x double] [double 3.4, double 5.6], ptr %floats.home, align 8 %result = call goabiinternal %method.results %callee( - ptr %recv, i64 123, [2 x i64] [i64 456, i64 789], - double 1.2, [2 x double] [double 3.4, double 5.6], ptr nest %ctxt) #0 + ptr %recv, i64 123, + ptr byval([2 x i64]) align 8 %integers.home, + double 1.2, + ptr byval([2 x double]) align 8 %floats.home, + ptr nest %ctxt) #0 %s = extractvalue %method.results %result, 0 %a = extractvalue %method.results %result, 1 %x = extractvalue %method.results %result, 2 @@ -148,13 +169,16 @@ entry: define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, %pair %value) { + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr byval(%pair) align 8 %value.home) { ; X86-LABEL: stack_pair: -; X86-DAG: movq 8(%rsp), %[[LEFT:r[a-z0-9]+]] -; X86-DAG: movq 16(%rsp), %[[RIGHT:r[a-z0-9]+]] +; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] +; X86-DAG: movq (%[[HOME]]), %[[LEFT:r[a-z0-9]+]] +; X86-DAG: movq 8(%[[HOME]]), %[[RIGHT:r[a-z0-9]+]] ; X86: addq %[[RIGHT]], %[[LEFT]] ; X86: retq entry: + %value = load %pair, ptr %value.home, align 8 %left = extractvalue %pair %value, 0 %right = extractvalue %pair %value, 1 %sum = add i64 %left, %right @@ -163,39 +187,55 @@ entry: define goabiinternal i64 @call_stack_pair() { ; X86-LABEL: call_stack_pair: -; X86-DAG: movq $13, (%[[BASE:r[a-z0-9]+]]) -; X86-DAG: movq $17, 8(%[[BASE]]) +; X86-DAG: movq $13, [[LEFTSRC:[0-9]+]](%rsp) +; X86-DAG: movq $17, [[RIGHTSRC:[0-9]+]](%rsp) +; X86-DAG: movq [[LEFTSRC]](%rsp), %[[LEFT:r[a-z0-9]+]] +; X86-DAG: movq [[RIGHTSRC]](%rsp), %[[RIGHT:r[a-z0-9]+]] +; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] +; X86-DAG: movq %[[LEFT]], (%[[BASE]]) +; X86-DAG: movq %[[RIGHT]], 8(%[[BASE]]) ; X86: callq stack_pair entry: + %value.home = alloca %pair, align 8 + store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, - %pair { i64 13, i64 17 }) + ptr byval(%pair) align 8 %value.home) ret i64 %result } -define goabiinternal [8 x i8] @stack_bytes([8 x i8] %value) { +define goabiinternal [8 x i8] @stack_bytes( + ptr byval([8 x i8]) align 1 %value.home) { ; X86-LABEL: stack_bytes: -; X86-DAG: movb 8(%rsp), %{{[a-z0-9]+}} -; X86-DAG: movb 15(%rsp), %{{[a-z0-9]+}} +; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] +; X86-DAG: movb (%[[HOME]]), %{{[a-z0-9]+}} +; X86-DAG: movb 7(%[[HOME]]), %{{[a-z0-9]+}} ; X86-DAG: movb %{{[a-z0-9]+}}, 16(%rsp) ; X86-DAG: movb %{{[a-z0-9]+}}, 23(%rsp) ; X86: retq entry: + %value = load [8 x i8], ptr %value.home, align 1 ret [8 x i8] %value } define goabiinternal i16 @call_stack_bytes() { ; X86-LABEL: call_stack_bytes: -; X86-DAG: movb $1, (%[[BASE:r[a-z0-9]+]]) -; X86-DAG: movb $8, 7(%[[BASE]]) +; X86-DAG: movb $1, [[SRC:[0-9]+]](%rsp) +; X86-DAG: movb $8, [[SRCEND:[0-9]+]](%rsp) +; X86: movq [[SRC]](%rsp), %[[VALUE:r[a-z0-9]+]] +; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] +; X86: movq %[[VALUE]], (%[[BASE]]) ; X86: callq stack_bytes ; X86: movq %rsp, %[[RESULT_BASE:r[a-z0-9]+]] ; X86-DAG: movzbl 8(%[[RESULT_BASE]]), %{{[a-z0-9]+}} ; X86-DAG: movzbl 15(%[[RESULT_BASE]]), %{{[a-z0-9]+}} entry: + %value.home = alloca [8 x i8], align 1 + store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], + ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8]) + ptr byval([8 x i8]) align 1 %value.home) %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/X86/go-stack-byval.ll b/llvm/test/CodeGen/X86/go-stack-byval.ll new file mode 100644 index 0000000000000..4d1683f4ce273 --- /dev/null +++ b/llvm/test/CodeGen/X86/go-stack-byval.ll @@ -0,0 +1,50 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] + +declare goabiinternal void @consume(ptr byval(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: rep;movsq +; CHECK: callq consume +entry: + call goabiinternal void @consume(ptr byval(%large) align 8 %source) + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr byval(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: movq 8(%rsp), %rax +; CHECK-NEXT: retq +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr byref(%large) align 8 %result) #0 { +; CHECK-LABEL: write_memory_result: +; CHECK: movq $42, 8(%rsp) +; CHECK-NEXT: retq +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: callq write_memory_result +; CHECK: rep;movsq +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr byref(%large) align 8 %result) #0 + %value = load i64, ptr %result, align 8 + ret i64 %value +} + +attributes #0 = { "go_memory_results"="0" } diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index 0547d1a782189..d1c8d8017b4fa 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -73,18 +73,16 @@ entry: ; An aggregate assigned wholly to the stack reuses its caller-populated slot; ; the lifetime marker must not make SelectionDAG allocate and copy a local. -define goabiinternal void @inactive_stack_aggregate(%stack_aggregate %value) +define goabiinternal void @inactive_stack_aggregate( + ptr byval(%stack_aggregate) align 8 %value.home) gc "statepoint-example" { entry: - %home = alloca %stack_aggregate, align 8 - call void @llvm.lifetime.start.p0(i64 16, ptr %home) - store %stack_aggregate %value, ptr %home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 3, i32 0, ptr elementtype(void ()) @safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, - i64 1095520067, i64 11, ptr %home, i64 0, i64 16, + i64 1095520067, i64 11, ptr %value.home, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 15) ] ret void diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index e5f7a24173352..48993e59b4ba5 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -35,10 +35,24 @@ entry: } declare goabiinternal void @many_stack_args( - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64, - i64, i64, i64, i64, i64, i64, i64, i64) + i64, i64, i64, i64, i64, i64, i64, i64, i64, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8, ptr byval(i64) align 8, + ptr byval(i64) align 8) +@many_stack_arg_values = private constant [23 x i64] [ + i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, + i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, + i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, + i64 27, i64 28, i64 29, i64 30, i64 31] @condition = external global i1 ; X86 lowers the stack arguments below to push sequences instead of reserving @@ -51,10 +65,30 @@ entry: call: call goabiinternal void @many_stack_args( - i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, - i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, - i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, - i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31) + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 0), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 1), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 2), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 3), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 4), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 5), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 6), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 7), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 8), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 9), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 10), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 11), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 12), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 13), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 14), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 15), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 16), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 17), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 18), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 19), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 20), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 21), + ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 22)) br label %join join: diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth.ll b/llvm/test/CodeGen/X86/goobj-stack-growth.ll index 691d27e10d43e..636511a49efe9 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth.ll @@ -23,9 +23,15 @@ entry: ret i64 %value } -define goabi0 void @"abi0_pointer_arguments"(ptr %first, ptr %second, ptr %third) +define goabi0 void @"abi0_pointer_arguments"( + ptr byval(ptr) align 8 %first.home, + ptr byval(ptr) align 8 %second.home, + ptr byval(ptr) align 8 %third.home) "frame-pointer"="non-leaf" { entry: + %first = load ptr, ptr %first.home, align 8 + %second = load ptr, ptr %second.home, align 8 + %third = load ptr, ptr %third.home, align 8 call goabiinternal void @use_three_pointers( ptr %first, ptr %second, ptr %third) ret void @@ -52,10 +58,14 @@ entry: define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) { + i64 %a6, i64 %a7, i64 %a8, + ptr byval(i64) align 8 %a9, ptr byval(i64) align 8 %a10, + ptr byval(i64) align 8 %a11, ptr byval(i64) align 8 %a12, + ptr byval(i64) align 8 %a13, ptr byval(i64) align 8 %a14, + ptr byval(i64) align 8 %a15, + ptr byval(ptr) align 8 %pointer.home) { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1 @@ -64,10 +74,14 @@ entry: define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, %pointer.aggregate %value) + i64 %a6, i64 %a7, i64 %a8, + ptr byval(i64) align 8 %a9, ptr byval(i64) align 8 %a10, + ptr byval(i64) align 8 %a11, ptr byval(i64) align 8 %a12, + ptr byval(i64) align 8 %a13, + ptr byval(%pointer.aggregate) align 8 %value.home) "go_results_tuple" { entry: + %value = load %pointer.aggregate, ptr %value.home, align 8 %buf = alloca [5000 x i8], align 8 %slot = getelementptr inbounds [5000 x i8], ptr %buf, i64 0, i64 4999 store volatile i8 1, ptr %slot, align 1 From 2290b28d24067859b020a9de9b55b96e6dc8e663 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 18:24:40 +0800 Subject: [PATCH 2/6] [Go] Add explicit goret result carriers --- llvm/docs/LangRef.md | 23 ++ llvm/include/llvm/Bitcode/LLVMBitCodes.h | 5 +- llvm/include/llvm/CodeGen/GoCallingConv.h | 16 +- llvm/include/llvm/CodeGen/TargetCallingConv.h | 35 +- llvm/include/llvm/CodeGen/TargetLowering.h | 13 +- llvm/include/llvm/IR/Argument.h | 12 +- llvm/include/llvm/IR/Attributes.h | 8 + llvm/include/llvm/IR/Attributes.td | 5 + llvm/include/llvm/IR/Function.h | 5 + llvm/include/llvm/IR/InstrTypes.h | 9 + llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 28 +- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 + llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 2 + llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 11 +- llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 11 +- llvm/lib/CodeGen/GoCallingConv.cpp | 68 ++-- .../SelectionDAG/SelectionDAGBuilder.cpp | 26 +- .../SelectionDAG/StatepointLowering.cpp | 2 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 252 +++++++------- llvm/lib/FuzzMutate/IRMutator.cpp | 20 +- llvm/lib/IR/Attributes.cpp | 47 +-- llvm/lib/IR/Function.cpp | 19 +- llvm/lib/IR/Value.cpp | 4 +- llvm/lib/IR/Verifier.cpp | 116 ++++++- .../Target/AArch64/AArch64ISelLowering.cpp | 115 +++---- llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 324 ++++++++++-------- llvm/lib/Transforms/Utils/CodeExtractor.cpp | 11 +- llvm/test/Assembler/goret-attribute.ll | 27 ++ llvm/test/CodeGen/AArch64/go-stack-byval.ll | 6 +- llvm/test/CodeGen/X86/go-stack-byval.ll | 6 +- llvm/test/Verifier/goret.ll | 54 +++ 31 files changed, 804 insertions(+), 478 deletions(-) create mode 100644 llvm/test/Assembler/goret-attribute.ll create mode 100644 llvm/test/Verifier/goret.ll diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md index f555ca543ffdc..f91ebc74c0d9d 100644 --- a/llvm/docs/LangRef.md +++ b/llvm/docs/LangRef.md @@ -1408,6 +1408,29 @@ Currently, only the following parameter attributes are defined: This is intended for representing ABI constraints, and is not intended to be inferred for optimization use. +(attr_goret)= + +`goret()` and `"goretindex"=""` +: The `goret` argument attribute marks a pointer parameter as the logical + destination of a Go ABI result whose in-memory type is ``. The paired + `"goretindex"=""` string attribute gives that result's zero-based index + in the complete Go result sequence. Both attributes are required on the + same parameter. + + These attributes are valid only with the `goabiinternal` and `goabi0` + calling conventions. The `goret` parameter is an IR-level carrier and is + not passed as a machine pointer argument. For a callee, the target binds it + to the result's Go ABI stack home. For a call, the target copies the value + from that physical result home to the pointer operand after the call. + Consequently, the pointer is dereferenceable for the storage size of + `` and the callee may write that storage as its result. + + `goretindex` values must be unique and increasing in parameter order, and + each value must be smaller than the number of direct LLVM results plus the + number of `goret` results. The type and index are ABI-affecting and must be + present consistently on the function and its call sites. Neither attribute + is valid for return values. + (attr_preallocated)= `preallocated()` diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 358f9a65a80af..e1cbbf80c923b 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -826,6 +826,7 @@ enum AttributeKindCodes { ATTR_KIND_NOOUTLINE = 107, ATTR_KIND_FLATTEN = 108, ATTR_KIND_NOIPA = 109, + ATTR_KIND_GORET = 110, }; enum ComdatSelectionKindCodes { @@ -844,7 +845,7 @@ enum SymtabCodes { SYMTAB_BLOB = 1, }; -} // End bitc namespace -} // End llvm namespace +} // namespace bitc +} // namespace llvm #endif diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index e6429d463ee2f..0236136c672fa 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -32,7 +32,7 @@ class Type; namespace goabi { inline constexpr StringLiteral TupleResultsAttr = "go_results_tuple"; -inline constexpr StringLiteral MemoryResultsAttr = "go_memory_results"; +inline constexpr StringLiteral GoRetIndexAttr = "goretindex"; inline constexpr StringLiteral PadTypeName = "go.abi.pad"; // Target frame lowering must not synthesize a morestack edge for such a // function. GoObj Go functions otherwise use the native Go default: emit a @@ -91,6 +91,11 @@ struct CallLayout { uint64_t TotalStackSize = 0; }; +struct MemoryResult { + unsigned Index = 0; + Type *Ty = nullptr; +}; + struct EntryArgsInfo { uint32_t PointerSize = 0; uint64_t ArgSize = 0; @@ -101,6 +106,8 @@ struct EntryArgsInfo { bool hasTupleResultsAttr(const AttributeList &Attrs); bool hasTupleResultsAttr(const Function &F); bool hasTupleResultsAttr(const CallBase &CB); +unsigned getGoRetIndex(const Argument &Arg); +unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo); /// Return the logical Go ABI type of an IR parameter. Non-empty values that /// the Go frontend assigned wholly to memory use a typed byval pointer as the /// LLVM carrier; direct parameters retain their logical type. @@ -110,11 +117,8 @@ SmallBitVector getPaddingPieces(Type *Ty); void getReturnTypes(Type *ReturnType, bool TupleResults, SmallVectorImpl &ResultTys); -SmallVector getMemoryResultIndices(const AttributeList &Attrs, - unsigned DirectResultCount, - unsigned MemoryResultCount); -void getReturnTypes(Type *ReturnType, const AttributeList &Attrs, - ArrayRef MemoryResultTys, +void getReturnTypes(Type *ReturnType, bool TupleResults, + ArrayRef MemoryResults, SmallVectorImpl &ResultTys); CallLayout computeCallLayout(ArrayRef ArgTys, diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h index f197c7f1645ec..f968033b8a472 100644 --- a/llvm/include/llvm/CodeGen/TargetCallingConv.h +++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -33,6 +33,7 @@ namespace ISD { unsigned IsSRet : 1; ///< Hidden struct-ret ptr unsigned IsByVal : 1; ///< Struct passed by value unsigned IsByRef : 1; ///< Passed in memory + unsigned IsGoRet : 1; ///< Go result written to its ABI result home unsigned IsNest : 1; ///< Nested fn static chain unsigned IsReturned : 1; ///< Always returned unsigned IsSplit : 1; @@ -57,17 +58,17 @@ namespace ISD { /// Whether this is part of a variable argument list (non-fixed). unsigned IsVarArg : 1; - unsigned ByValOrByRefSize = 0; ///< Byval or byref struct size + unsigned ByValOrIndirectSize = 0; ///< Byval, byref, or goret size unsigned PointerAddrSpace = 0; ///< Address space of pointer argument public: ArgFlagsTy() : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsSRet(0), IsByVal(0), - IsByRef(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), - IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0), - IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0), - IsSecArgPass(0), MemAlign(0), OrigAlign(0), + IsByRef(0), IsGoRet(0), IsNest(0), IsReturned(0), IsSplit(0), + IsInAlloca(0), IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), + IsSwiftAsync(0), IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), + IsHvaStart(0), IsSecArgPass(0), MemAlign(0), OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0), IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) { static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big"); @@ -94,6 +95,9 @@ namespace ISD { bool isByRef() const { return IsByRef; } void setByRef() { IsByRef = 1; } + bool isGoRet() const { return IsGoRet; } + void setGoRet() { IsGoRet = 1; } + bool isInAlloca() const { return IsInAlloca; } void setInAlloca() { IsInAlloca = 1; } @@ -176,21 +180,30 @@ namespace ISD { } unsigned getByValSize() const { - assert(isByVal() && !isByRef()); - return ByValOrByRefSize; + assert(isByVal() && !isByRef() && !isGoRet()); + return ByValOrIndirectSize; } void setByValSize(unsigned S) { - assert(isByVal() && !isByRef()); - ByValOrByRefSize = S; + assert(isByVal() && !isByRef() && !isGoRet()); + ByValOrIndirectSize = S; } unsigned getByRefSize() const { assert(!isByVal() && isByRef()); - return ByValOrByRefSize; + return ByValOrIndirectSize; } void setByRefSize(unsigned S) { assert(!isByVal() && isByRef()); - ByValOrByRefSize = S; + ByValOrIndirectSize = S; + } + + unsigned getGoRetSize() const { + assert(!isByVal() && !isByRef() && isGoRet()); + return ByValOrIndirectSize; + } + void setGoRetSize(unsigned S) { + assert(!isByVal() && !isByRef() && isGoRet()); + ByValOrIndirectSize = S; } unsigned getPointerAddrSpace() const { return PointerAddrSpace; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 125b73e59b218..ee15d0904d28d 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -332,6 +332,7 @@ class LLVM_ABI TargetLoweringBase { bool IsNest : 1; bool IsByVal : 1; bool IsByRef : 1; + bool IsGoRet : 1; bool IsInAlloca : 1; bool IsPreallocated : 1; bool IsReturned : 1; @@ -341,13 +342,15 @@ class LLVM_ABI TargetLoweringBase { bool IsCFGuardTarget : 1; MaybeAlign Alignment = std::nullopt; Type *IndirectType = nullptr; + unsigned GoRetIndex = 0; ArgListEntry(Value *Val, SDValue Node, Type *Ty) : Val(Val), Node(Node), OrigTy(Ty), Ty(Ty), IsSExt(false), IsZExt(false), IsNoExt(false), IsInReg(false), IsSRet(false), - IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false), - IsPreallocated(false), IsReturned(false), IsSwiftSelf(false), - IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {} + IsNest(false), IsByVal(false), IsByRef(false), IsGoRet(false), + IsInAlloca(false), IsPreallocated(false), IsReturned(false), + IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false), + IsCFGuardTarget(false) {} explicit ArgListEntry(Value *Val, SDValue Node = SDValue()) : ArgListEntry(Val, Node, Val->getType()) {} @@ -1689,8 +1692,8 @@ class LLVM_ABI TargetLoweringBase { /// extending virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const { return false; } - // Returns true if Extend can be folded into the index of a masked gathers/scatters - // on this target. + // Returns true if Extend can be folded into the index of a masked + // gathers/scatters on this target. virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const { return false; } diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h index 05fd7ff4df326..be7b0c34294ed 100644 --- a/llvm/include/llvm/IR/Argument.h +++ b/llvm/include/llvm/IR/Argument.h @@ -84,6 +84,9 @@ class Argument final : public Value { /// Return true if this argument has the byref attribute. LLVM_ABI bool hasByRefAttr() const; + /// Return true if this argument has the goret attribute. + LLVM_ABI bool hasGoRetAttr() const; + /// Return true if this argument has the swiftself attribute. LLVM_ABI bool hasSwiftSelfAttr() const; @@ -100,8 +103,8 @@ class Argument final : public Value { LLVM_ABI uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const; /// Return true if this argument has the byval, sret, inalloca, preallocated, - /// or byref attribute. These attributes represent arguments being passed by - /// value (which may or may not involve a stack copy) + /// byref, or goret attribute. These attributes represent values with an + /// associated in-memory ABI type. LLVM_ABI bool hasPointeeInMemoryValueAttr() const; /// If hasPointeeInMemoryValueAttr returns true, the in-memory ABI type is @@ -122,6 +125,9 @@ class Argument final : public Value { /// If this is a byref argument, return its type. LLVM_ABI Type *getParamByRefType() const; + /// If this is a goret argument, return its logical result type. + LLVM_ABI Type *getParamGoRetType() const; + /// If this is an inalloca argument, return its type. LLVM_ABI Type *getParamInAllocaType() const; @@ -188,6 +194,6 @@ class Argument final : public Value { } }; -} // End llvm namespace +} // namespace llvm #endif diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h index 049f699b2980b..fb7c25e0e19f1 100644 --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -205,6 +205,7 @@ class Attribute { LLVM_ABI static Attribute getWithStructRetType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithByRefType(LLVMContext &Context, Type *Ty); + LLVM_ABI static Attribute getWithGoRetType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty); LLVM_ABI static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty); @@ -497,6 +498,7 @@ class AttributeSet { LLVM_ABI Type *getByValType() const; LLVM_ABI Type *getStructRetType() const; LLVM_ABI Type *getByRefType() const; + LLVM_ABI Type *getGoRetType() const; LLVM_ABI Type *getPreallocatedType() const; LLVM_ABI Type *getInAllocaType() const; LLVM_ABI Type *getElementType() const; @@ -975,6 +977,9 @@ class AttributeList { /// Return the byref type for the specified function parameter. LLVM_ABI Type *getParamByRefType(unsigned ArgNo) const; + /// Return the goret type for the specified function parameter. + LLVM_ABI Type *getParamGoRetType(unsigned ArgNo) const; + /// Return the preallocated type for the specified function parameter. LLVM_ABI Type *getParamPreallocatedType(unsigned ArgNo) const; @@ -1225,6 +1230,9 @@ class AttrBuilder { /// Retrieve the byref type. Type *getByRefType() const { return getTypeAttr(Attribute::ByRef); } + /// Retrieve the goret type. + Type *getGoRetType() const { return getTypeAttr(Attribute::GoRet); } + /// Retrieve the preallocated type. Type *getPreallocatedType() const { return getTypeAttr(Attribute::Preallocated); diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 4e45100b54d38..9575e79a80e9c 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -108,6 +108,11 @@ def ByVal : TypeAttr<"byval", IntersectPreserve, [ParamAttr]>; /// Mark in-memory ABI type. def ByRef : TypeAttr<"byref", IntersectPreserve, [ParamAttr]>; +/// Go ABI result written through the caller's logical result home. The +/// companion "goretindex" string attribute identifies this result in the +/// complete Go result sequence. +def GoRet : TypeAttr<"goret", IntersectPreserve, [ParamAttr]>; + /// Parameter or return value may not contain uninitialized or poison bits. def NoUndef : EnumAttr<"noundef", IntersectAnd, [ParamAttr, RetAttr]>; diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index bd28f0d9902da..97a2d55263f35 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -488,6 +488,11 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node { return AttributeSets.getParamByRefType(ArgNo); } + /// Extract the goret type for a parameter. + Type *getParamGoRetType(unsigned ArgNo) const { + return AttributeSets.getParamGoRetType(ArgNo); + } + /// Extract the preallocated type for a parameter. Type *getParamPreallocatedType(unsigned ArgNo) const { return AttributeSets.getParamPreallocatedType(ArgNo); diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 5f7df6a4eb6f8..5b9ed55426b01 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1861,6 +1861,15 @@ class CallBase : public Instruction { return nullptr; } + /// Extract the goret type for a call or parameter. + Type *getParamGoRetType(unsigned ArgNo) const { + if (auto *Ty = Attrs.getParamGoRetType(ArgNo)) + return Ty; + if (const Function *F = getCalledFunction()) + return F->getAttributes().getParamGoRetType(ArgNo); + return nullptr; + } + /// Extract the byval type for a call or parameter. Type *getParamByValType(unsigned ArgNo) const { if (auto *Ty = Attrs.getParamByValType(ArgNo)) diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index c57323ed41525..771bd24eb0f3c 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2302,6 +2302,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoUndef; case bitc::ATTR_KIND_BYREF: return Attribute::ByRef; + case bitc::ATTR_KIND_GORET: + return Attribute::GoRet; case bitc::ATTR_KIND_MUSTPROGRESS: return Attribute::MustProgress; case bitc::ATTR_KIND_HOT: @@ -3658,8 +3660,8 @@ Error BitcodeReader::parseConstants() { {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2]}); break; } - case bitc::CST_CODE_CE_EXTRACTELT - : { // CE_EXTRACTELT: [opty, opval, opty, opval] + case bitc::CST_CODE_CE_EXTRACTELT: { // CE_EXTRACTELT: [opty, opval, opty, + // opval] if (Record.size() < 3) return error("Invalid extractelement constexpr record"); unsigned OpTyID = Record[0]; @@ -3682,8 +3684,8 @@ Error BitcodeReader::parseConstants() { {(unsigned)Record[1], IdxRecord}); break; } - case bitc::CST_CODE_CE_INSERTELT - : { // CE_INSERTELT: [opval, opval, opty, opval] + case bitc::CST_CODE_CE_INSERTELT: { // CE_INSERTELT: [opval, opval, opty, + // opval] VectorType *OpTy = dyn_cast(CurTy); if (Record.size() < 3 || !OpTy) return error("Invalid insertelement constexpr record"); @@ -5566,7 +5568,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_VSELECT: {// VSELECT: [ty,opval,opval,predty,pred] + case bitc::FUNC_CODE_INST_VSELECT: { // VSELECT: + // [ty,opval,opval,predty,pred] // new form of select // handles select i1 or select [N x i1] unsigned OpNum = 0; @@ -5636,7 +5639,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_SHUFFLEVEC: {// SHUFFLEVEC: [opval,ty,opval,opval] + case bitc::FUNC_CODE_INST_SHUFFLEVEC: { // SHUFFLEVEC: + // [opval,ty,opval,opval] unsigned OpNum = 0; Value *Vec1, *Vec2, *Mask; unsigned Vec1TypeID; @@ -5659,10 +5663,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } - case bitc::FUNC_CODE_INST_CMP: // CMP: [opty, opval, opval, pred] - // Old form of ICmp/FCmp returning bool - // Existed to differentiate between icmp/fcmp and vicmp/vfcmp which were - // both legal on vectors but had different behaviour. + case bitc::FUNC_CODE_INST_CMP: // CMP: [opty, opval, opval, pred] + // Old form of ICmp/FCmp returning bool + // Existed to differentiate between icmp/fcmp + // and vicmp/vfcmp which were both legal on + // vectors but had different behaviour. case bitc::FUNC_CODE_INST_CMP2: { // CMP2: [opty, opval, opval, pred] // FCmp/ICmp returning bool or vector of bool @@ -6499,7 +6504,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_STORE: - case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, vol] + case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, + // vol] unsigned OpNum = 0; Value *Val, *Ptr; unsigned PtrTypeID, ValTypeID; diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 492240515b489..f4cab1bd67c7c 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -984,6 +984,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NOUNDEF; case Attribute::ByRef: return bitc::ATTR_KIND_BYREF; + case Attribute::GoRet: + return bitc::ATTR_KIND_GORET; case Attribute::MustProgress: return bitc::ATTR_KIND_MUSTPROGRESS; case Attribute::PresplitCoroutine: diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index 2a1ad62c2804c..1e2f7a3e2d299 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -1084,6 +1084,8 @@ void ValueEnumerator::incorporateFunction(const Function &F) { EnumerateType(I.getParamStructRetType()); else if (I.hasAttribute(Attribute::ByRef)) EnumerateType(I.getParamByRefType()); + else if (I.hasAttribute(Attribute::GoRet)) + EnumerateType(I.getAttribute(Attribute::GoRet).getValueAsType()); } FirstFuncConstantID = Values.size(); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 71d360a21590f..f517e6e3e3915 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -265,16 +265,17 @@ static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL, } SmallVector ArgTys; - SmallVector MemoryResultTys; + SmallVector MemoryResults; for (const Argument &Arg : F.args()) - if (Arg.hasAttribute(Attribute::ByRef)) - MemoryResultTys.push_back(Arg.getParamByRefType()); + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); else if (!Arg.hasNestAttr()) ArgTys.push_back(goabi::getParameterType(Arg)); SmallVector ResultTys; - goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, - ResultTys); + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + MemoryResults, ResultTys); uint64_t Size = goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize; if (Size > std::numeric_limits::max()) diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index a580b78462453..3bfc0a95bc936 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -51,6 +51,8 @@ static void addFlagsFromAttrSet(ISD::ArgFlagsTy &Flags, AttributeSet Attrs) { Flags.setByVal(); if (Attrs.hasAttribute(Attribute::ByRef)) Flags.setByRef(); + if (Attrs.hasAttribute(Attribute::GoRet)) + Flags.setGoRet(); if (Attrs.hasAttribute(Attribute::InAlloca)) { Flags.setInAlloca(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -250,23 +252,28 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, Align MemAlign = DL.getABITypeAlign(Arg.Ty); if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || - Flags.isByRef()) { + Flags.isByRef() || Flags.isGoRet()) { assert(OpIdx >= AttributeList::FirstArgIndex); unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamByRefType(ParamIdx); + if (!ElementTy) + ElementTy = FuncInfo.getParamGoRetType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); if (!ElementTy) ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); - assert(ElementTy && "Must have byval, inalloca or preallocated type"); + assert(ElementTy && + "Must have byval, byref, goret, inalloca or preallocated type"); uint64_t MemSize = DL.getTypeAllocSize(ElementTy); if (Flags.isByRef()) Flags.setByRefSize(MemSize); + else if (Flags.isGoRet()) + Flags.setGoRetSize(MemSize); else Flags.setByValSize(MemSize); diff --git a/llvm/lib/CodeGen/GoCallingConv.cpp b/llvm/lib/CodeGen/GoCallingConv.cpp index 50166dfafd555..139f3a69985b0 100644 --- a/llvm/lib/CodeGen/GoCallingConv.cpp +++ b/llvm/lib/CodeGen/GoCallingConv.cpp @@ -222,6 +222,22 @@ bool hasTupleResultsAttr(const Function &F) { return F.hasFnAttribute(TupleResultsAttr); } +static unsigned getGoRetIndex(Attribute Attr) { + unsigned Index; + if (!Attr.isStringAttribute() || + Attr.getValueAsString().getAsInteger(10, Index)) + report_fatal_error("invalid or missing goretindex parameter attribute"); + return Index; +} + +unsigned getGoRetIndex(const Argument &Arg) { + return getGoRetIndex(Arg.getAttributes().getAttribute(GoRetIndexAttr)); +} + +unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo) { + return getGoRetIndex(CB.getParamAttr(ArgNo, GoRetIndexAttr)); +} + bool hasTupleResultsAttr(const CallBase &CB) { if (CB.hasFnAttr(TupleResultsAttr)) return true; @@ -249,52 +265,26 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, ResultTys.push_back(ReturnType); } -SmallVector getMemoryResultIndices(const AttributeList &Attrs, - unsigned DirectResultCount, - unsigned MemoryResultCount) { - Attribute Attr = Attrs.getFnAttr(MemoryResultsAttr); - if (!Attr.isValid()) { - if (MemoryResultCount != 0) - report_fatal_error("Go memory result carriers have no index attribute"); - return {}; - } - if (!Attr.isStringAttribute() || Attr.getValueAsString().empty()) - report_fatal_error("invalid Go memory result index attribute"); - - unsigned TotalResultCount = DirectResultCount + MemoryResultCount; - SmallVector Indices; - SmallVector Fields; - Attr.getValueAsString().split(Fields, ',', /*MaxSplit=*/-1, - /*KeepEmpty=*/true); - for (StringRef Field : Fields) { - unsigned Index; - if (Field.empty() || Field.getAsInteger(10, Index) || - Index >= TotalResultCount || - (!Indices.empty() && Index <= Indices.back())) - report_fatal_error("invalid Go memory result index attribute"); - Indices.push_back(Index); - } - if (Indices.size() != MemoryResultCount) - report_fatal_error("Go memory result index count mismatch"); - return Indices; -} - -void getReturnTypes(Type *ReturnType, const AttributeList &Attrs, - ArrayRef MemoryResultTys, +void getReturnTypes(Type *ReturnType, bool TupleResults, + ArrayRef MemoryResults, SmallVectorImpl &ResultTys) { SmallVector DirectResultTys; - getReturnTypes(ReturnType, hasTupleResultsAttr(Attrs), DirectResultTys); - SmallVector MemoryIndices = getMemoryResultIndices( - Attrs, DirectResultTys.size(), MemoryResultTys.size()); - if (MemoryIndices.empty()) { + getReturnTypes(ReturnType, TupleResults, DirectResultTys); + if (MemoryResults.empty()) { ResultTys.append(DirectResultTys.begin(), DirectResultTys.end()); return; } - unsigned TotalResultCount = DirectResultTys.size() + MemoryResultTys.size(); + unsigned TotalResultCount = DirectResultTys.size() + MemoryResults.size(); ResultTys.assign(TotalResultCount, nullptr); - for (auto [Index, Ty] : llvm::zip_equal(MemoryIndices, MemoryResultTys)) - ResultTys[Index] = Ty; + std::optional Previous; + for (const MemoryResult &Result : MemoryResults) { + if (!Result.Ty || Result.Index >= TotalResultCount || + ResultTys[Result.Index] || (Previous && *Previous >= Result.Index)) + report_fatal_error("invalid Go memory result carrier"); + ResultTys[Result.Index] = Result.Ty; + Previous = Result.Index; + } auto Direct = DirectResultTys.begin(); for (Type *&Ty : ResultTys) if (!Ty) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 7b325dfa6a6e8..6e8f07428a873 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -11742,6 +11742,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { Flags.setByVal(); if (Args[i].IsByRef) Flags.setByRef(); + if (Args[i].IsGoRet) + Flags.setGoRet(); if (Args[i].IsPreallocated) { Flags.setPreallocated(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -11777,6 +11779,13 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { MemAlign = *MA; else MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); + } else if (Args[i].IsGoRet) { + unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); + Flags.setGoRetSize(FrameSize); + if (auto MA = Args[i].Alignment) + MemAlign = *MA; + else + MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL); } else if (auto MA = Args[i].Alignment) { MemAlign = *MA; } else { @@ -11799,8 +11808,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { else if (Args[i].IsZExt) ExtendKind = ISD::ZERO_EXTEND; - // Conservatively only handle 'returned' on non-vectors that can be lowered, - // for now. + // Conservatively only handle 'returned' on non-vectors that can be + // lowered, for now. if (Args[i].IsReturned && !Op.getValueType().isVector() && CanLowerReturn) { assert((CLI.RetTy == Args[i].Ty || @@ -12313,6 +12322,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { Flags.setByVal(); if (Arg.hasAttribute(Attribute::ByRef)) Flags.setByRef(); + if (Arg.hasAttribute(Attribute::GoRet)) + Flags.setGoRet(); if (Arg.hasAttribute(Attribute::InAlloca)) { Flags.setInAlloca(); // Set the byval flag for CCAssignFn callbacks that don't know about @@ -12342,7 +12353,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) { Align MemAlign; Type *ArgMemTy = nullptr; if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || - Flags.isByRef()) { + Flags.isByRef() || Flags.isGoRet()) { if (!ArgMemTy) ArgMemTy = Arg.getPointeeInMemoryValueType(); @@ -12359,6 +12370,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL); if (Flags.isByRef()) Flags.setByRefSize(MemSize); + else if (Flags.isGoRet()) + Flags.setGoRetSize(MemSize); else Flags.setByValSize(MemSize); } else if (auto ParamAlign = Arg.getParamStackAlign()) { @@ -12484,8 +12497,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { ArrayRef(&InVals[i], NumParts), *TLI, ArgHasUses); } - // If this argument is unused then remember its value. It is used to generate - // debugging information. + // If this argument is unused then remember its value. It is used to + // generate debugging information. bool isSwiftErrorArg = TLI->supportSwiftError() && Arg.hasAttribute(Attribute::SwiftError); @@ -12893,7 +12906,8 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } else { Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); CurMF->insert(BBI, Fallthrough); - // Put Cond in a virtual register to make it available from the new blocks. + // Put Cond in a virtual register to make it available from the new + // blocks. ExportFromCurrentBlock(Cond); } UnhandledProbs -= I->Prob; diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index eda649ae8be0a..2137d05cb72bf 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -1019,7 +1019,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( (isa(V) || (isa(V) && (cast(V)->hasByValAttr() || - cast(V)->hasAttribute(Attribute::ByRef)) && + cast(V)->hasGoRetAttr()) && FuncInfo.getArgumentFrameIndex(cast(V)) == cast(SDV)->getIndex()))) { Record.type = RecordType::FrameIndexRemat; diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 3a46fbb9d7611..01fe54938a9a4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -17,6 +17,7 @@ #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/CodeGenCommonISel.h" +#include "llvm/CodeGen/GoCallingConv.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" @@ -127,6 +128,7 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest); IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal); IsByRef = Call->paramHasAttr(ArgIdx, Attribute::ByRef); + IsGoRet = Call->paramHasAttr(ArgIdx, Attribute::GoRet); IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated); IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); @@ -135,7 +137,8 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); Alignment = Call->getParamStackAlign(ArgIdx); IndirectType = nullptr; - assert(IsByVal + IsByRef + IsPreallocated + IsInAlloca + IsSRet <= 1 && + assert(IsByVal + IsByRef + IsGoRet + IsPreallocated + IsInAlloca + IsSRet <= + 1 && "multiple ABI attributes?"); if (IsByVal) { IndirectType = Call->getParamByValType(ArgIdx); @@ -147,6 +150,12 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, if (!Alignment) Alignment = Call->getParamAlign(ArgIdx); } + if (IsGoRet) { + IndirectType = Call->getParamGoRetType(ArgIdx); + GoRetIndex = goabi::getGoRetIndex(*Call, ArgIdx); + if (!Alignment) + Alignment = Call->getParamAlign(ArgIdx); + } if (IsPreallocated) IndirectType = Call->getParamPreallocatedType(ArgIdx); if (IsInAlloca) @@ -1067,11 +1076,12 @@ static SDValue combineShiftToAVG(SDValue Op, } return false; }; - bool IsCeil = - (ExtOpA.getOpcode() == ISD::ADD && - MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) || - (ExtOpB.getOpcode() == ISD::ADD && - MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB)); + bool IsCeil = (ExtOpA.getOpcode() == ISD::ADD && + MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), + ExtOpB, ExtOpA)) || + (ExtOpB.getOpcode() == ISD::ADD && + MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), + ExtOpA, ExtOpB)); // If the shift is signed (sra): // - Needs >= 2 sign bit for both operands. @@ -2345,12 +2355,14 @@ bool TargetLowering::SimplifyDemandedBits( // See if we don't demand either half of the rotated bits. if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) { - Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType()); + Op1 = + TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType()); return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); } if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) { - Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType()); + Op1 = + TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType()); return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); } } @@ -2464,8 +2476,8 @@ bool TargetLowering::SimplifyDemandedBits( // op legalization. // FIXME: Limit to scalars for now. if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector()) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, - Op.getOperand(0))); + return TLO.CombineTo( + Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, Op.getOperand(0))); Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); break; @@ -2618,7 +2630,8 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); unsigned InBits = SrcVT.getScalarSizeInBits(); - unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; + unsigned InElts = + SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; APInt InDemandedElts = DemandedElts.zext(InElts); @@ -2681,7 +2694,8 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); unsigned InBits = SrcVT.getScalarSizeInBits(); - unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; + unsigned InElts = + SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; // If we only need the bottom element then we can just bitcast. @@ -4110,10 +4124,10 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( return SDValue(); } -SDValue -TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, - SDValue N1, MutableArrayRef Mask, - SelectionDAG &DAG) const { +SDValue TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, + SDValue N0, SDValue N1, + MutableArrayRef Mask, + SelectionDAG &DAG) const { bool LegalMask = isShuffleMaskLegal(Mask, VT); if (!LegalMask) { std::swap(N0, N1); @@ -4621,7 +4635,8 @@ static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, // FIXME: Add vector support? Need to be careful with setcc result type below. SDValue CTPOP = N0; if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && - N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits())) + N0.getScalarValueSizeInBits() > + Log2_32(N0.getOperand(0).getScalarValueSizeInBits())) CTPOP = N0.getOperand(0); if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse()) @@ -4917,8 +4932,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // (zext x) == C --> x == (trunc C) // (sext x) == C --> x == (trunc C) - if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && - DCI.isBeforeLegalize() && N0->hasOneUse()) { + if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && DCI.isBeforeLegalize() && + N0->hasOneUse()) { unsigned MinBits = N0.getValueSizeInBits(); SDValue PreExt; bool Signed = false; @@ -4954,9 +4969,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits(); // Make sure we're not losing bits from the constant. - if (MinBits > 0 && - MinBits < C1.getBitWidth() && - MinBits >= ReqdBits) { + if (MinBits > 0 && MinBits < C1.getBitWidth() && MinBits >= ReqdBits) { EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits); if (isTypeDesirableForOp(ISD::SETCC, MinVT)) { // Will get folded away. @@ -4996,8 +5009,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, cast(TopSetCC.getOperand(2))->get(), TopSetCC.getOperand(0).getValueType()); return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), - TopSetCC.getOperand(1), - InvCond); + TopSetCC.getOperand(1), InvCond); } } } @@ -5005,10 +5017,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // If the LHS is '(and load, const)', the RHS is 0, the test is for // equality or unsigned, and all 1 bits of the const are in the same // partial word, see if we can shorten the load. - if (DCI.isBeforeLegalize() && - !ISD::isSignedIntSetCC(Cond) && - N0.getOpcode() == ISD::AND && C1 == 0 && - N0.getNode()->hasOneUse() && + if (DCI.isBeforeLegalize() && !ISD::isSignedIntSetCC(Cond) && + N0.getOpcode() == ISD::AND && C1 == 0 && N0.getNode()->hasOneUse() && isa(N0.getOperand(0)) && N0.getOperand(0).getNode()->hasOneUse() && isa(N0.getOperand(1))) { @@ -5122,8 +5132,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT); SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT); - SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0), - NewConst, Cond); + SDValue NewSetCC = + DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0), NewConst, Cond); return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); } break; @@ -5186,9 +5196,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We // can only do this if the top bits are known zero. unsigned BitWidth = N0.getValueSizeInBits(); - if (DAG.MaskedValueIsZero(N0, - APInt::getHighBitsSet(BitWidth, - BitWidth-1))) { + if (DAG.MaskedValueIsZero( + N0, APInt::getHighBitsSet(BitWidth, BitWidth - 1))) { // Okay, get the un-inverted input value. SDValue Val; if (N0.getOpcode() == ISD::XOR) { @@ -5198,8 +5207,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, N0.getOperand(0).getOpcode() == ISD::XOR); // ((X^1)&1)^1 -> X & 1 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), - N0.getOperand(0).getOperand(0), - N0.getOperand(1)); + N0.getOperand(0).getOperand(0), N0.getOperand(1)); } return DAG.getSetCC(dl, VT, Val, N1, @@ -5229,13 +5237,15 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) { // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0. if (Op0.getValueType().bitsGT(VT)) - Op0 = DAG.getNode(ISD::AND, dl, VT, - DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), - DAG.getConstant(1, dl, VT)); + Op0 = DAG.getNode( + ISD::AND, dl, VT, + DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), + DAG.getConstant(1, dl, VT)); else if (Op0.getValueType().bitsLT(VT)) - Op0 = DAG.getNode(ISD::AND, dl, VT, - DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), - DAG.getConstant(1, dl, VT)); + Op0 = DAG.getNode( + ISD::AND, dl, VT, + DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), + DAG.getConstant(1, dl, VT)); return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, dl, Op0.getValueType()), @@ -5322,8 +5332,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, (!N1C->isOpaque() || (C.getBitWidth() <= 64 && isLegalICmpImmediate(C.getSExtValue())))) { return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(C, dl, N1.getValueType()), - NewCC); + DAG.getConstant(C, dl, N1.getValueType()), NewCC); } } } @@ -5342,8 +5351,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, (!N1C->isOpaque() || (C.getBitWidth() <= 64 && isLegalICmpImmediate(C.getSExtValue())))) { return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(C, dl, N1.getValueType()), - NewCC); + DAG.getConstant(C, dl, N1.getValueType()), NewCC); } } } @@ -5461,9 +5469,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // SETUGE X, SINTMIN -> SETLT X, 0 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) || (Cond == ISD::SETUGE && C1.isMinSignedValue())) - return DAG.getSetCC(dl, VT, N0, - DAG.getConstant(0, dl, N1.getValueType()), - ISD::SETLT); + return DAG.getSetCC( + dl, VT, N0, DAG.getConstant(0, dl, N1.getValueType()), ISD::SETLT); // SETULT X, SINTMIN -> SETGT X, -1 // SETULE X, SINTMAX -> SETGT X, -1 @@ -5625,11 +5632,20 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, bool IsNegInf = CFP->getValueAPF().isNegative(); ISD::CondCode NewCond = ISD::SETCC_INVALID; switch (Cond) { - case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break; - case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break; - case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break; - case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break; - default: break; + case ISD::SETOEQ: + NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; + break; + case ISD::SETUEQ: + NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; + break; + case ISD::SETUNE: + NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; + break; + case ISD::SETONE: + NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; + break; + default: + break; } if (NewCond != ISD::SETCC_INVALID && isCondCodeLegal(NewCond, N0.getSimpleValueType())) @@ -5653,8 +5669,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO // if it is not already. ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; - if (NewCond != Cond && - (DCI.isBeforeLegalizeOps() || + if (NewCond != Cond && (DCI.isBeforeLegalizeOps() || isCondCodeLegal(NewCond, N0.getSimpleValueType()))) return DAG.getSetCC(dl, VT, N0, N1, NewCond); } @@ -5776,7 +5791,8 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) { SDValue Temp; switch (Cond) { - default: llvm_unreachable("Unknown integer setcc!"); + default: + llvm_unreachable("Unknown integer setcc!"); case ISD::SETEQ: // X == Y -> ~(X^Y) Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); N0 = DAG.getNOT(dl, Temp, OpVT); @@ -5893,7 +5909,8 @@ TargetLowering::getConstraintType(StringRef Constraint) const { if (S == 1) { switch (Constraint[0]) { - default: break; + default: + break; case 'r': return C_RegisterClass; case 'm': // memory @@ -5960,7 +5977,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, char ConstraintLetter = Constraint[0]; switch (ConstraintLetter) { - default: break; + default: + break; case 'X': // Allows any operand case 'i': // Simple Integer or Relocatable Constant case 'n': // Simple Integer @@ -6170,7 +6188,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, if (!OpTy->isSingleValueType() && OpTy->isSized()) { unsigned BitSize = DL.getTypeSizeInBits(OpTy); switch (BitSize) { - default: break; + default: + break; case 1: case 8: case 16: @@ -6308,8 +6327,8 @@ static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) { /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight - TargetLowering::getMultipleConstraintMatchWeight( - AsmOperandInfo &info, int maIndex) const { +TargetLowering::getMultipleConstraintMatchWeight(AsmOperandInfo &info, + int maIndex) const { InlineAsm::ConstraintCodeVector *rCodes; if (maIndex >= (int)info.multipleAlternatives.size()) rCodes = &info.Codes; @@ -6332,8 +6351,8 @@ TargetLowering::ConstraintWeight /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight - TargetLowering::getSingleConstraintMatchWeight( - AsmOperandInfo &info, const char *constraint) const { +TargetLowering::getSingleConstraintMatchWeight(AsmOperandInfo &info, + const char *constraint) const { ConstraintWeight weight = CW_Invalid; Value *CallOperandVal = info.CallOperandVal; // If we don't have a value, we can't do a match, @@ -6450,8 +6469,7 @@ static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, /// Determines the constraint code and constraint type to use for the specific /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType. -void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, - SDValue Op, +void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG) const { assert(!OpInfo.Codes.empty() && "Must have at least one constraint"); @@ -6629,7 +6647,8 @@ static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N, return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); } -SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, +SDValue +TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl &Created) const { AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes(); @@ -9950,8 +9969,7 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, } bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, - SDValue &Chain, - SelectionDAG &DAG) const { + SDValue &Chain, SelectionDAG &DAG) const { SDLoc dl(SDValue(Node, 0)); unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; SDValue Src = Node->getOperand(OpNo); @@ -9964,8 +9982,8 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT); // Only expand vector types if we have the appropriate vector bit operations. - unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : - ISD::FP_TO_SINT; + unsigned SIntOpcode = + Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) || !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT))) return false; @@ -9996,8 +10014,8 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, SDValue Sel; if (Node->isStrictFPOpcode()) { - Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, - Node->getOperand(0), /*IsSignaling*/ true); + Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, Node->getOperand(0), + /*IsSignaling*/ true); Chain = Sel.getValue(1); } else { Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); @@ -10015,12 +10033,12 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, // Result = fp_to_sint(Src - FltOfs) ^ IntOfs // TODO: Should any fast-math-flags be set for the FSUB? - SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel, - DAG.getConstantFP(0.0, dl, SrcVT), Cst); + SDValue FltOfs = + DAG.getSelect(dl, SrcVT, Sel, DAG.getConstantFP(0.0, dl, SrcVT), Cst); Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); - SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel, - DAG.getConstant(0, dl, DstVT), - DAG.getConstant(SignMask, dl, DstVT)); + SDValue IntOfs = + DAG.getSelect(dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT), + DAG.getConstant(SignMask, dl, DstVT)); SDValue SInt; if (Node->isStrictFPOpcode()) { SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, @@ -10156,12 +10174,12 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node, // Insert canonicalizes if it's possible we need to quiet to get correct // sNaN behavior. if (!DAG.isKnownNeverSNaN(Quiet0)) { - Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, - Node->getFlags()); + Quiet0 = + DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, Node->getFlags()); } if (!DAG.isKnownNeverSNaN(Quiet1)) { - Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, - Node->getFlags()); + Quiet1 = + DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, Node->getFlags()); } } @@ -11019,12 +11037,11 @@ SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG, if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON) return ExtLoad; - EVT SetCCVT = - getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); + EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); SDValue Zero = DAG.getConstant(0, DL, VT); SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ); - return DAG.getSelect(DL, VT, SrcIsZero, - DAG.getConstant(BitWidth, DL, VT), ExtLoad); + return DAG.getSelect(DL, VT, SrcIsZero, DAG.getConstant(BitWidth, DL, VT), + ExtLoad); } SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const { @@ -11548,8 +11565,7 @@ SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { return DAG.getNode(ISD::XOR, dl, VT, Tmp, Rotr8); } Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); - Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, - DAG.getConstant(0xFF00, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT)); Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); @@ -11559,8 +11575,8 @@ SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); case MVT::i64: Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); - Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, - DAG.getConstant(255ULL<<8, dl, VT)); + Tmp7 = + DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(255ULL << 8, dl, VT)); Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, DAG.getConstant(255ULL<<16, dl, VT)); @@ -11788,8 +11804,7 @@ SDValue TargetLowering::expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const { } std::pair -TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, - SelectionDAG &DAG) const { +TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const { SDLoc SL(LD); SDValue Chain = LD->getChain(); SDValue BasePTR = LD->getBasePtr(); @@ -11965,20 +11980,19 @@ TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const { if (VT.isFloatingPoint() || VT.isVector()) { EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) { - if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && - LoadedVT.isVector()) { + if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && LoadedVT.isVector()) { // Scalarize the load and let the individual components be handled. return scalarizeVectorLoad(LD, DAG); } // Expand to a (misaligned) integer load of the same size, // then bitconvert to floating point or vector. - SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, - LD->getMemOperand()); + SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getMemOperand()); SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); if (LoadedVT != VT) - Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : - ISD::ANY_EXTEND, dl, VT, Result); + Result = + DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND, + dl, VT, Result); return std::make_pair(Result, newLoad.getValue(1)); } @@ -12231,11 +12245,10 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, return Result; } -SDValue -TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, - const SDLoc &DL, EVT DataVT, - SelectionDAG &DAG, - bool IsCompressedMemory) const { +SDValue TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, + const SDLoc &DL, EVT DataVT, + SelectionDAG &DAG, + bool IsCompressedMemory) const { SDValue Increment; EVT AddrVT = Addr.getValueType(); EVT MaskVT = Mask.getValueType(); @@ -12259,8 +12272,8 @@ TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg); } // Scale is an element size in bytes. - SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL, - AddrVT); + SDValue Scale = + DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL, AddrVT); Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT); Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale); } else @@ -12557,7 +12570,8 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { unsigned BitWidth = LHS.getScalarValueSizeInBits(); EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); - SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); + SDValue Result = + DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); SDValue SumDiff = Result.getValue(0); SDValue Overflow = Result.getValue(1); SDValue Zero = DAG.getConstant(0, dl, VT); @@ -12954,8 +12968,8 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { // Saturated to SatMin if wide product is negative, and SatMax if wide // product is positive ... SDValue Zero = DAG.getConstant(0, dl, VT); - SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax, - ISD::SETLT); + SDValue ResultIfOverflow = + DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax, ISD::SETLT); // ... but only if we overflowed. return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); } @@ -12964,22 +12978,21 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { // Saturate to max if ((Hi >> (Scale - 1)) > 0), // which is the same as if (Hi > (1 << (Scale - 1)) - 1) - SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), - dl, VT); + SDValue LowMask = + DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), dl, VT); Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); // Saturate to min if (Hi >> (Scale - 1)) < -1), // which is the same as if (HI < (-1 << (Scale - 1)) - SDValue HighMask = - DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), - dl, VT); + SDValue HighMask = DAG.getConstant( + APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), dl, VT); Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); return Result; } -SDValue -TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, - SDValue LHS, SDValue RHS, - unsigned Scale, SelectionDAG &DAG) const { +SDValue TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, + SDValue LHS, SDValue RHS, + unsigned Scale, + SelectionDAG &DAG) const { assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && "Expected a fixed point division opcode"); @@ -13217,8 +13230,8 @@ bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result, SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); } else { - Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, - DAG.getConstant(0, dl, VT), ISD::SETNE); + Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, DAG.getConstant(0, dl, VT), + ISD::SETNE); } // Truncate the result if SetCC returns a larger type than needed. @@ -13313,7 +13326,8 @@ SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const { return Res; } -SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const { +SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, + SelectionDAG &DAG) const { SDLoc dl(Node); SDValue AccOp = Node->getOperand(0); SDValue VecOp = Node->getOperand(1); diff --git a/llvm/lib/FuzzMutate/IRMutator.cpp b/llvm/lib/FuzzMutate/IRMutator.cpp index b56f3704d4305..5dd216dc5c845 100644 --- a/llvm/lib/FuzzMutate/IRMutator.cpp +++ b/llvm/lib/FuzzMutate/IRMutator.cpp @@ -415,13 +415,19 @@ static bool isUnsupportedFunction(Function *F) { // behavior may be undefined. // We don't call those functions for now to prevent UB from happening. auto IsABIAttribute = [](AttributeSet A) { - static const Attribute::AttrKind ABIAttrs[] = { - Attribute::StructRet, Attribute::ByVal, - Attribute::InAlloca, Attribute::InReg, - Attribute::StackAlignment, Attribute::SwiftSelf, - Attribute::SwiftAsync, Attribute::SwiftError, - Attribute::Preallocated, Attribute::ByRef, - Attribute::ZExt, Attribute::SExt}; + static const Attribute::AttrKind ABIAttrs[] = {Attribute::StructRet, + Attribute::ByVal, + Attribute::InAlloca, + Attribute::InReg, + Attribute::StackAlignment, + Attribute::SwiftSelf, + Attribute::SwiftAsync, + Attribute::SwiftError, + Attribute::Preallocated, + Attribute::ByRef, + Attribute::GoRet, + Attribute::ZExt, + Attribute::SExt}; return llvm::any_of(ABIAttrs, [&](Attribute::AttrKind kind) { return A.hasAttribute(kind); diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 4087b25951a1c..2347125a62576 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -265,6 +265,10 @@ Attribute Attribute::getWithByRefType(LLVMContext &Context, Type *Ty) { return get(Context, ByRef, Ty); } +Attribute Attribute::getWithGoRetType(LLVMContext &Context, Type *Ty) { + return get(Context, GoRet, Ty); +} + Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) { return get(Context, Preallocated, Ty); } @@ -1216,6 +1220,10 @@ Type *AttributeSet::getByRefType() const { return SetNode ? SetNode->getAttributeType(Attribute::ByRef) : nullptr; } +Type *AttributeSet::getGoRetType() const { + return SetNode ? SetNode->getAttributeType(Attribute::GoRet) : nullptr; +} + Type *AttributeSet::getByValType() const { return SetNode ? SetNode->getAttributeType(Attribute::ByVal) : nullptr; } @@ -1489,9 +1497,7 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const { /// Map from AttributeList index to the internal array index. Adding one happens /// to work, because -1 wraps around to 0. -static unsigned attrIdxToArrayIdx(unsigned Index) { - return Index + 1; -} +static unsigned attrIdxToArrayIdx(unsigned Index) { return Index + 1; } AttributeListImpl::AttributeListImpl(ArrayRef Sets) : NumAttrSets(Sets.size()) { @@ -1539,7 +1545,6 @@ bool AttributeListImpl::hasAttrSomewhere(Attribute::AttrKind Kind, return true; } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void AttributeListImpl::dump() const { AttributeList(const_cast(this)).dump(); @@ -1596,7 +1601,8 @@ AttributeList::get(LLVMContext &C, // list. SmallVector, 8> AttrPairVec; for (ArrayRef>::iterator I = Attrs.begin(), - E = Attrs.end(); I != E; ) { + E = Attrs.end(); + I != E;) { unsigned Index = I->first; SmallVector AttrVec; while (I != E && I->first == Index) { @@ -2003,6 +2009,10 @@ Type *AttributeList::getParamByRefType(unsigned Index) const { return getAttributes(Index + FirstArgIndex).getByRefType(); } +Type *AttributeList::getParamGoRetType(unsigned Index) const { + return getAttributes(Index + FirstArgIndex).getGoRetType(); +} + Type *AttributeList::getParamPreallocatedType(unsigned Index) const { return getAttributes(Index + FirstArgIndex).getPreallocatedType(); } @@ -2256,7 +2266,8 @@ AttrBuilder &AttrBuilder::addStackAlignmentAttr(MaybeAlign Align) { } AttrBuilder &AttrBuilder::addDereferenceableAttr(uint64_t Bytes) { - if (Bytes == 0) return *this; + if (Bytes == 0) + return *this; return addRawIntAttr(Attribute::Dereferenceable, Bytes); } @@ -2680,8 +2691,8 @@ static void adjustCallerStackProbes(Function &Caller, const Function &Callee) { /// If the inlined function defines the size of guard region /// on the stack, then ensure that the calling function defines a guard region /// that is no larger. -static void -adjustCallerStackProbeSize(Function &Caller, const Function &Callee) { +static void adjustCallerStackProbeSize(Function &Caller, + const Function &Callee) { Attribute CalleeAttr = Callee.getFnAttribute("stack-probe-size"); if (CalleeAttr.isValid()) { Attribute CallerAttr = Caller.getFnAttribute("stack-probe-size"); @@ -2708,8 +2719,8 @@ adjustCallerStackProbeSize(Function &Caller, const Function &Callee) { /// to merge the attribute this way. Heuristics that would use /// min-legal-vector-width to determine inline compatibility would need to be /// handled as part of inline cost analysis. -static void -adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) { +static void adjustMinLegalVectorWidth(Function &Caller, + const Function &Callee) { Attribute CallerAttr = Caller.getFnAttribute("min-legal-vector-width"); if (CallerAttr.isValid()) { Attribute CalleeAttr = Callee.getFnAttribute("min-legal-vector-width"); @@ -2729,21 +2740,19 @@ adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) { /// If the inlined function has null_pointer_is_valid attribute, /// set this attribute in the caller post inlining. -static void -adjustNullPointerValidAttr(Function &Caller, const Function &Callee) { +static void adjustNullPointerValidAttr(Function &Caller, + const Function &Callee) { if (Callee.nullPointerIsDefined() && !Caller.nullPointerIsDefined()) { Caller.addFnAttr(Attribute::NullPointerIsValid); } } struct EnumAttr { - static bool isSet(const Function &Fn, - Attribute::AttrKind Kind) { + static bool isSet(const Function &Fn, Attribute::AttrKind Kind) { return Fn.hasFnAttribute(Kind); } - static void set(Function &Fn, - Attribute::AttrKind Kind, bool Val) { + static void set(Function &Fn, Attribute::AttrKind Kind, bool Val) { if (Val) Fn.addFnAttr(Kind); else @@ -2752,14 +2761,12 @@ struct EnumAttr { }; struct StrBoolAttr { - static bool isSet(const Function &Fn, - StringRef Kind) { + static bool isSet(const Function &Fn, StringRef Kind) { auto A = Fn.getFnAttribute(Kind); return A.getValueAsString() == "true"; } - static void set(Function &Fn, - StringRef Kind, bool Val) { + static void set(Function &Fn, StringRef Kind, bool Val) { Fn.addFnAttr(Kind, Val ? "true" : "false"); } }; diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 567949d194726..7eefb7d08f725 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -140,6 +140,12 @@ bool Argument::hasByRefAttr() const { return hasAttribute(Attribute::ByRef); } +bool Argument::hasGoRetAttr() const { + if (!getType()->isPointerTy()) + return false; + return hasAttribute(Attribute::GoRet); +} + bool Argument::hasSwiftSelfAttr() const { return getParent()->hasParamAttribute(getArgNo(), Attribute::SwiftSelf); } @@ -175,11 +181,11 @@ bool Argument::hasPointeeInMemoryValueAttr() const { Attrs.hasAttribute(Attribute::StructRet) || Attrs.hasAttribute(Attribute::InAlloca) || Attrs.hasAttribute(Attribute::Preallocated) || - Attrs.hasAttribute(Attribute::ByRef); + Attrs.hasAttribute(Attribute::ByRef) || + Attrs.hasAttribute(Attribute::GoRet); } -/// For a byval, sret, inalloca, or preallocated parameter, get the in-memory -/// parameter type. +/// For a parameter with an in-memory ABI type, get that type. static Type *getMemoryParamAllocType(AttributeSet ParamAttrs) { // FIXME: All the type carrying attributes are mutually exclusive, so there // should be a single query to get the stored type that handles any of them. @@ -187,6 +193,8 @@ static Type *getMemoryParamAllocType(AttributeSet ParamAttrs) { return ByValTy; if (Type *ByRefTy = ParamAttrs.getByRefType()) return ByRefTy; + if (Type *GoRetTy = ParamAttrs.getGoRetType()) + return GoRetTy; if (Type *PreAllocTy = ParamAttrs.getPreallocatedType()) return PreAllocTy; if (Type *InAllocaTy = ParamAttrs.getInAllocaType()) @@ -231,6 +239,11 @@ Type *Argument::getParamByRefType() const { return getParent()->getParamByRefType(getArgNo()); } +Type *Argument::getParamGoRetType() const { + assert(getType()->isPointerTy() && "Only pointers have goret types"); + return getParent()->getParamGoRetType(getArgNo()); +} + Type *Argument::getParamInAllocaType() const { assert(getType()->isPointerTy() && "Only pointers have inalloca types"); return getParent()->getParamInAllocaType(getArgNo()); diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index 6d0170dd23e75..a13e923bd4c11 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -843,7 +843,7 @@ bool Value::canBeFreed() const { if (isa(this)) return false; - // Handle byval/byref/sret/inalloca/preallocated arguments. The storage + // Handle byval/byref/goret/sret/inalloca/preallocated arguments. The storage // lifetime is guaranteed to be longer than the callee's lifetime. if (auto *A = dyn_cast(this)) { if (A->hasPointeeInMemoryValueAttr()) @@ -917,7 +917,7 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, if (const Argument *A = dyn_cast(this)) { DerefBytes = A->getDereferenceableBytes(); if (DerefBytes == 0) { - // Handle byval/byref/inalloca/preallocated arguments + // Handle byval/byref/goret/inalloca/preallocated arguments. if (Type *ArgMemTy = A->getPointeeInMemoryValueType()) { if (ArgMemTy->isSized()) { // FIXME: Why isn't this the type alloc size? diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index e35f0d5dcca4a..bf6d9aa565bfd 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -55,6 +55,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -459,6 +460,8 @@ class Verifier : public InstVisitor, VerifierSupport { bool verifyAttributeCount(AttributeList Attrs, unsigned Params); void verifyAttributeTypes(AttributeSet Attrs, const Value *V); void verifyParameterAttrs(AttributeSet Attrs, Type *Ty, const Value *V); + void verifyGoRetAttrs(FunctionType *FT, AttributeList Attrs, + CallingConv::ID CC, const Value *V); void checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr, const Value *V); void verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, @@ -548,7 +551,8 @@ void Verifier::visit(Instruction &I) { InstVisitor::visit(I); } -// Helper to iterate over indirect users. By returning false, the callback can ask to stop traversing further. +// Helper to iterate over indirect users. By returning false, the callback can +// ask to stop traversing further. static void forEachUser(const Value *User, SmallPtrSet &Visited, llvm::function_ref Callback) { @@ -764,8 +768,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { } } - if (GV.hasName() && (GV.getName() == "llvm.used" || - GV.getName() == "llvm.compiler.used")) { + if (GV.hasName() && + (GV.getName() == "llvm.used" || GV.getName() == "llvm.compiler.used")) { Check(!GV.hasInitializer() || GV.hasAppendingLinkage(), "invalid linkage for intrinsic global variable", &GV); Check(GV.materialized_use_empty(), @@ -2098,6 +2102,16 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, "'byref', and 'sret' are incompatible!", V); + if (Attrs.hasAttribute(Attribute::GoRet)) + Check(AttrCount == 0, + "Attribute 'goret' is incompatible with other ABI parameter " + "attributes!", + V); + + Check(Attrs.hasAttribute(Attribute::GoRet) == + Attrs.hasAttribute("goretindex"), + "Attributes 'goret' and 'goretindex' must be used together!", V); + Check(!(Attrs.hasAttribute(Attribute::InAlloca) && Attrs.hasAttribute(Attribute::ReadOnly)), "Attributes " @@ -2184,6 +2198,16 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, (1ULL << 32), "huge 'byref' arguments are unsupported", V); } + if (Attrs.hasAttribute(Attribute::GoRet)) { + Type *GoRetTy = Attrs.getAttribute(Attribute::GoRet).getValueAsType(); + SmallPtrSet Visited; + Check(GoRetTy->isSized(&Visited), + "Attribute 'goret' does not support unsized types!", V); + Check(!GoRetTy->containsNonLocalTargetExtType(), + "'goret' argument has illegal target extension type", V); + Check(DL.getTypeAllocSize(GoRetTy).getKnownMinValue() < (1ULL << 32), + "huge 'goret' arguments are unsupported", V); + } if (Attrs.hasAttribute(Attribute::InAlloca)) { SmallPtrSet Visited; Check(Attrs.getInAllocaType()->isSized(&Visited), @@ -2226,6 +2250,55 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, } } +void Verifier::verifyGoRetAttrs(FunctionType *FT, AttributeList Attrs, + CallingConv::ID CC, const Value *V) { + unsigned MemoryResultCount = 0; + for (unsigned I = 0; I != FT->getNumParams(); ++I) + MemoryResultCount += Attrs.hasParamAttr(I, Attribute::GoRet); + if (MemoryResultCount == 0) + return; + + Check(CC == CallingConv::GoABIInternal || CC == CallingConv::GoABI0, + "'goret' is only valid with a Go calling convention", V); + + unsigned DirectResultCount = 0; + if (!FT->getReturnType()->isVoidTy()) { + if (Attrs.hasFnAttr("go_results_tuple")) { + auto *ST = dyn_cast(FT->getReturnType()); + Check(ST, "'go_results_tuple' requires a struct return type", V); + if (!ST) + return; + DirectResultCount = ST->getNumElements(); + } else { + DirectResultCount = 1; + } + } + + unsigned TotalResultCount = DirectResultCount + MemoryResultCount; + SmallBitVector Seen(TotalResultCount); + std::optional Previous; + for (unsigned I = 0; I != FT->getNumParams(); ++I) { + AttributeSet ParamAttrs = Attrs.getParamAttrs(I); + if (!ParamAttrs.hasAttribute(Attribute::GoRet)) + continue; + StringRef IndexValue = + ParamAttrs.getAttribute("goretindex").getValueAsString(); + uint64_t Index; + Check(!IndexValue.getAsInteger(10, Index), + "'goretindex' must be an unsigned decimal integer", V); + if (IndexValue.getAsInteger(10, Index)) + continue; + Check(Index < TotalResultCount, "'goretindex' is out of range", V); + if (Index >= TotalResultCount) + continue; + Check(!Seen.test(Index), "duplicate 'goretindex'", V); + Check(!Previous || *Previous < Index, + "'goretindex' values must be in increasing parameter order", V); + Seen.set(Index); + Previous = Index; + } +} + void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr, const Value *V) { if (Attrs.hasFnAttr(Attr)) { @@ -3056,6 +3129,7 @@ void Verifier::visitFunction(const Function &F) { // Check function attributes. verifyFunctionAttrs(FT, Attrs, &F, IsIntrinsic, /* IsInlineAsm */ false); + verifyGoRetAttrs(FT, Attrs, F.getCallingConv(), &F); // On function declarations/definitions, we do not support the builtin // attribute. We do not check this in VerifyFunctionAttrs since that is @@ -3413,8 +3487,7 @@ void Verifier::visitBasicBlock(BasicBlock &BB) { } // Check that all instructions have their parent pointers set up correctly. - for (auto &I : BB) - { + for (auto &I : BB) { Check(I.getParent() == &BB, "Instruction has bogus parent pointer!"); } @@ -3889,6 +3962,7 @@ void Verifier::visitCallBase(CallBase &Call) { // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic, Call.isInlineAsm()); + verifyGoRetAttrs(FTy, Attrs, Call.getCallingConv(), &Call); // Conservatively check the inalloca argument. // We have a bug if we can find that there is an underlying alloca without @@ -3906,7 +3980,8 @@ void Verifier::visitCallBase(CallBase &Call) { for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) { if (Call.paramHasAttr(i, Attribute::SwiftError)) { Value *SwiftErrorArg = Call.getArgOperand(i); - if (auto AI = dyn_cast(SwiftErrorArg->stripInBoundsOffsets())) { + if (auto AI = + dyn_cast(SwiftErrorArg->stripInBoundsOffsets())) { Check(AI->isSwiftError(), "swifterror argument for call has mismatched alloca", AI, Call); continue; @@ -4137,25 +4212,34 @@ void Verifier::verifyTailCCMustTailAttrs(const AttrBuilder &Attrs, Twine("preallocated attribute not allowed in ") + Context); Check(!Attrs.contains(Attribute::ByRef), Twine("byref attribute not allowed in ") + Context); + Check(!Attrs.contains(Attribute::GoRet), + Twine("goret attribute not allowed in ") + Context); + Check(!Attrs.contains("goretindex"), + Twine("goretindex attribute not allowed in ") + Context); } -static AttrBuilder getParameterABIAttributes(LLVMContext& C, unsigned I, AttributeList Attrs) { +static AttrBuilder getParameterABIAttributes(LLVMContext &C, unsigned I, + AttributeList Attrs) { static const Attribute::AttrKind ABIAttrs[] = { Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, Attribute::InReg, Attribute::StackAlignment, Attribute::SwiftSelf, Attribute::SwiftAsync, Attribute::SwiftError, Attribute::Preallocated, - Attribute::ByRef}; + Attribute::ByRef, Attribute::GoRet}; AttrBuilder Copy(C); for (auto AK : ABIAttrs) { Attribute Attr = Attrs.getParamAttrs(I).getAttribute(AK); if (Attr.isValid()) Copy.addAttribute(Attr); } + Attribute GoRetIndex = Attrs.getParamAttrs(I).getAttribute("goretindex"); + if (GoRetIndex.isValid()) + Copy.addAttribute(GoRetIndex); - // `align` is ABI-affecting only in combination with `byval` or `byref`. + // `align` is ABI-affecting only in combination with an indirect ABI value. if (Attrs.hasParamAttr(I, Attribute::Alignment) && (Attrs.hasParamAttr(I, Attribute::ByVal) || - Attrs.hasParamAttr(I, Attribute::ByRef))) + Attrs.hasParamAttr(I, Attribute::ByRef) || + Attrs.hasParamAttr(I, Attribute::GoRet))) Copy.addAlignmentAttr(Attrs.getParamAlignment(I)); return Copy; } @@ -4196,12 +4280,14 @@ void Verifier::verifyMustTailCall(CallInst &CI) { // - Only sret, byval, swiftself, and swiftasync ABI-impacting attributes // are allowed in swifttailcc call for (unsigned I = 0, E = CallerTy->getNumParams(); I != E; ++I) { - AttrBuilder ABIAttrs = getParameterABIAttributes(F->getContext(), I, CallerAttrs); + AttrBuilder ABIAttrs = + getParameterABIAttributes(F->getContext(), I, CallerAttrs); SmallString<32> Context{CCName, StringRef(" musttail caller")}; verifyTailCCMustTailAttrs(ABIAttrs, Context); } for (unsigned I = 0, E = CalleeTy->getNumParams(); I != E; ++I) { - AttrBuilder ABIAttrs = getParameterABIAttributes(F->getContext(), I, CalleeAttrs); + AttrBuilder ABIAttrs = + getParameterABIAttributes(F->getContext(), I, CalleeAttrs); SmallString<32> Context{CCName, StringRef(" musttail callee")}; verifyTailCCMustTailAttrs(ABIAttrs, Context); } @@ -4225,8 +4311,10 @@ void Verifier::verifyMustTailCall(CallInst &CI) { // - All ABI-impacting function attributes, such as sret, byval, inreg, // returned, preallocated, and inalloca, must match. for (unsigned I = 0, E = CallerTy->getNumParams(); I != E; ++I) { - AttrBuilder CallerABIAttrs = getParameterABIAttributes(F->getContext(), I, CallerAttrs); - AttrBuilder CalleeABIAttrs = getParameterABIAttributes(F->getContext(), I, CalleeAttrs); + AttrBuilder CallerABIAttrs = + getParameterABIAttributes(F->getContext(), I, CallerAttrs); + AttrBuilder CalleeABIAttrs = + getParameterABIAttributes(F->getContext(), I, CalleeAttrs); Check(CallerABIAttrs == CalleeABIAttrs, "cannot guarantee tail call due to mismatched ABI impacting " "function attributes", diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 707976695a018..26cc8ba199e79 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9058,7 +9058,7 @@ getAArch64GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr() || Arg.hasAttribute(Attribute::ByRef)) + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) continue; LayoutMap[Index] = ArgTys.size(); ArgTys.push_back(goabi::getParameterType(Arg)); @@ -9072,7 +9072,7 @@ getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest || Args[I].IsByRef) + if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); @@ -9081,54 +9081,58 @@ getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, } static SmallVector getAArch64GoReturnTypes(const Function &F) { - SmallVector MemoryResultTys; + SmallVector MemoryResults; for (const Argument &Arg : F.args()) - if (Arg.hasAttribute(Attribute::ByRef)) - MemoryResultTys.push_back(Arg.getParamByRefType()); + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); SmallVector ResultTys; - goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, - ResultTys); + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + MemoryResults, ResultTys); return ResultTys; } static SmallVector getAArch64GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, const TargetLowering::ArgListTy &Args) { - SmallVector MemoryResultTys; + SmallVector MemoryResults; for (const TargetLowering::ArgListEntry &Arg : Args) - if (Arg.IsByRef) - MemoryResultTys.push_back(Arg.IndirectType); + if (Arg.IsGoRet) + MemoryResults.push_back({Arg.GoRetIndex, Arg.IndirectType}); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, Attrs, MemoryResultTys, ResultTys); + goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), MemoryResults, + ResultTys); return ResultTys; } static SmallVector -getAArch64GoDirectResultIndices(const AttributeList &Attrs, - unsigned DirectResultCount, - unsigned MemoryResultCount) { - SmallVector MemoryIndices = goabi::getMemoryResultIndices( - Attrs, DirectResultCount, MemoryResultCount); +getAArch64GoDirectResultIndices(ArrayRef MemoryResults, + unsigned DirectResultCount) { SmallVector DirectIndices; - for (unsigned I = 0, E = DirectResultCount + MemoryResultCount; I != E; ++I) - if (!llvm::is_contained(MemoryIndices, I)) + for (unsigned I = 0, E = DirectResultCount + MemoryResults.size(); I != E; + ++I) + if (!llvm::any_of(MemoryResults, + [I](const auto &Result) { return Result.Index == I; })) DirectIndices.push_back(I); return DirectIndices; } -static SmallVector -getAArch64GoMemoryResultMap(const AttributeList &Attrs, unsigned ParamCount, - ArrayRef IsByRef, - unsigned DirectResultCount) { - unsigned MemoryResultCount = llvm::count(IsByRef, true); - SmallVector MemoryIndices = goabi::getMemoryResultIndices( - Attrs, DirectResultCount, MemoryResultCount); - SmallVector Map(ParamCount, -1); - unsigned Next = 0; - for (unsigned I = 0; I != ParamCount; ++I) - if (IsByRef[I]) - Map[I] = MemoryIndices[Next++]; - return Map; +static SmallVector +getAArch64GoMemoryResults(const Function &F) { + SmallVector Results; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + Results.push_back({goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + return Results; +} + +static SmallVector +getAArch64GoMemoryResults(const TargetLowering::ArgListTy &Args) { + SmallVector Results; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + Results.push_back({Arg.GoRetIndex, Arg.IndirectType}); + return Results; } static SDValue lowerAArch64GoFormalArguments( @@ -9171,11 +9175,10 @@ static SDValue lowerAArch64GoFormalArguments( SmallVector DirectResultTys; goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), DirectResultTys); - SmallVector IsByRef; + SmallVector MemoryResultMap(F.arg_size(), -1); for (const Argument &Arg : F.args()) - IsByRef.push_back(Arg.hasAttribute(Attribute::ByRef)); - SmallVector MemoryResultMap = getAArch64GoMemoryResultMap( - F.getAttributes(), F.arg_size(), IsByRef, DirectResultTys.size()); + if (Arg.hasGoRetAttr()) + MemoryResultMap[Arg.getArgNo()] = goabi::getGoRetIndex(Arg); auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { @@ -9205,14 +9208,14 @@ static SDValue lowerAArch64GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); - if (Arg->hasAttribute(Attribute::ByRef)) { + if (Arg->hasGoRetAttr()) { int ResultIndex = MemoryResultMap[Group.Index]; if (ResultIndex < 0 || Group.End != Group.Start + 1 || - !Ins[Group.Start].Flags.isByRef()) + !Ins[Group.Start].Flags.isGoRet()) report_fatal_error("invalid AArch64 Go memory result carrier"); const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs || ResultLayout.Size == 0 || - Ins[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + Ins[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) report_fatal_error( "AArch64 Go memory result carrier disagrees with Go ABI layout"); int FI = MFI.CreateFixedObject(ResultLayout.Size, @@ -9334,17 +9337,11 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, ArgTys, ResultTys, DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, MF.getFunction().getCallingConv())); unsigned StackBias = getAArch64GoStackBias(MF.getFunction().getCallingConv()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(MF.getFunction()); SmallVector DirectResultIndices = - getAArch64GoDirectResultIndices( - MF.getFunction().getAttributes(), - Layout.Results.size() - llvm::count_if(Layout.Results, - [](const auto &R) { - return !R.InRegs && - R.Size != 0; - }), - llvm::count_if(Layout.Results, [](const auto &R) { - return !R.InRegs && R.Size != 0; - })); + getAArch64GoDirectResultIndices(MemoryResults, Layout.Results.size() - + MemoryResults.size()); SmallVector MemOps; SmallVector, 8> RetRegs; @@ -9437,16 +9434,14 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, goabi::getReturnTypes(CLI.RetTy, CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), DirectResultTys); - SmallVector IsByRef; - for (const TargetLowering::ArgListEntry &Arg : CLI.getArgs()) - IsByRef.push_back(Arg.IsByRef); - SmallVector MemoryResultMap = getAArch64GoMemoryResultMap( - CallAttrs, CLI.getArgs().size(), IsByRef, DirectResultTys.size()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(CLI.getArgs()); + SmallVector MemoryResultMap(CLI.getArgs().size(), -1); + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) + if (CLI.getArgs()[I].IsGoRet) + MemoryResultMap[I] = CLI.getArgs()[I].GoRetIndex; SmallVector DirectResultIndices = - getAArch64GoDirectResultIndices( - CallAttrs, DirectResultTys.size(), - llvm::count_if(CLI.getArgs(), - [](const auto &Arg) { return Arg.IsByRef; })); + getAArch64GoDirectResultIndices(MemoryResults, DirectResultTys.size()); unsigned StackBias = getAArch64GoStackBias(CLI.CallConv); // Layout.TotalStackSize rounds the logical Go argument area to the target @@ -9471,14 +9466,14 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, if (Group.Index >= CLI.getArgs().size()) continue; - if (CLI.getArgs()[Group.Index].IsByRef) { + if (CLI.getArgs()[Group.Index].IsGoRet) { int ResultIndex = MemoryResultMap[Group.Index]; if (ResultIndex < 0 || Group.End != Group.Start + 1 || - !Outs[Group.Start].Flags.isByRef()) + !Outs[Group.Start].Flags.isGoRet()) report_fatal_error("invalid AArch64 Go memory result call carrier"); const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs || ResultLayout.Size == 0 || - Outs[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + Outs[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) report_fatal_error( "AArch64 Go memory result call carrier disagrees with Go ABI " "layout"); diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 2b44820996125..6037d572be0d4 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -153,7 +153,7 @@ getX86GoArgTypes(const Function &F, SmallVectorImpl &LayoutMap) { LayoutMap.assign(F.arg_size(), -1); for (const Argument &Arg : F.args()) { unsigned Index = Arg.getArgNo(); - if (Arg.hasNestAttr() || Arg.hasAttribute(Attribute::ByRef)) + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) continue; LayoutMap[Index] = ArgTys.size(); ArgTys.push_back(goabi::getParameterType(Arg)); @@ -167,7 +167,7 @@ getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, SmallVector ArgTys; LayoutMap.assign(Args.size(), -1); for (unsigned I = 0; I != Args.size(); ++I) { - if (Args[I].IsNest || Args[I].IsByRef) + if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); @@ -176,54 +176,58 @@ getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, } static SmallVector getX86GoReturnTypes(const Function &F) { - SmallVector MemoryResultTys; + SmallVector MemoryResults; for (const Argument &Arg : F.args()) - if (Arg.hasAttribute(Attribute::ByRef)) - MemoryResultTys.push_back(Arg.getParamByRefType()); + if (Arg.hasGoRetAttr()) + MemoryResults.push_back( + {goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); SmallVector ResultTys; - goabi::getReturnTypes(F.getReturnType(), F.getAttributes(), MemoryResultTys, - ResultTys); + goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), + MemoryResults, ResultTys); return ResultTys; } static SmallVector getX86GoCallReturnTypes(Type *RetTy, const AttributeList &Attrs, const TargetLowering::ArgListTy &Args) { - SmallVector MemoryResultTys; + SmallVector MemoryResults; for (const TargetLowering::ArgListEntry &Arg : Args) - if (Arg.IsByRef) - MemoryResultTys.push_back(Arg.IndirectType); + if (Arg.IsGoRet) + MemoryResults.push_back({Arg.GoRetIndex, Arg.IndirectType}); SmallVector ResultTys; - goabi::getReturnTypes(RetTy, Attrs, MemoryResultTys, ResultTys); + goabi::getReturnTypes(RetTy, goabi::hasTupleResultsAttr(Attrs), MemoryResults, + ResultTys); return ResultTys; } static SmallVector -getX86GoDirectResultIndices(const AttributeList &Attrs, - unsigned DirectResultCount, - unsigned MemoryResultCount) { - SmallVector MemoryIndices = goabi::getMemoryResultIndices( - Attrs, DirectResultCount, MemoryResultCount); +getX86GoDirectResultIndices(ArrayRef MemoryResults, + unsigned DirectResultCount) { SmallVector DirectIndices; - for (unsigned I = 0, E = DirectResultCount + MemoryResultCount; I != E; ++I) - if (!llvm::is_contained(MemoryIndices, I)) + for (unsigned I = 0, E = DirectResultCount + MemoryResults.size(); I != E; + ++I) + if (!llvm::any_of(MemoryResults, + [I](const auto &Result) { return Result.Index == I; })) DirectIndices.push_back(I); return DirectIndices; } -static SmallVector getX86GoMemoryResultMap(const AttributeList &Attrs, - unsigned ParamCount, - ArrayRef IsByRef, - unsigned DirectResultCount) { - unsigned MemoryResultCount = llvm::count(IsByRef, true); - SmallVector MemoryIndices = goabi::getMemoryResultIndices( - Attrs, DirectResultCount, MemoryResultCount); - SmallVector Map(ParamCount, -1); - unsigned Next = 0; - for (unsigned I = 0; I != ParamCount; ++I) - if (IsByRef[I]) - Map[I] = MemoryIndices[Next++]; - return Map; +static SmallVector +getX86GoMemoryResults(const Function &F) { + SmallVector Results; + for (const Argument &Arg : F.args()) + if (Arg.hasGoRetAttr()) + Results.push_back({goabi::getGoRetIndex(Arg), Arg.getParamGoRetType()}); + return Results; +} + +static SmallVector +getX86GoMemoryResults(const TargetLowering::ArgListTy &Args) { + SmallVector Results; + for (const TargetLowering::ArgListEntry &Arg : Args) + if (Arg.IsGoRet) + Results.push_back({Arg.GoRetIndex, Arg.IndirectType}); + return Results; } static SDValue lowerX86GoFormalArguments( @@ -272,11 +276,10 @@ static SDValue lowerX86GoFormalArguments( SmallVector DirectResultTys; goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), DirectResultTys); - SmallVector IsByRef; + SmallVector MemoryResultMap(F.arg_size(), -1); for (const Argument &Arg : F.args()) - IsByRef.push_back(Arg.hasAttribute(Attribute::ByRef)); - SmallVector MemoryResultMap = getX86GoMemoryResultMap( - F.getAttributes(), F.arg_size(), IsByRef, DirectResultTys.size()); + if (Arg.hasGoRetAttr()) + MemoryResultMap[Arg.getArgNo()] = goabi::getGoRetIndex(Arg); auto RecordPointerSlots = [&](int FI, uint64_t ArgOffset, uint64_t Size, bool IsLiveAtEntry) { @@ -316,14 +319,14 @@ static SDValue lowerX86GoFormalArguments( continue; const Argument *Arg = F.getArg(Group.Index); - if (Arg->hasAttribute(Attribute::ByRef)) { + if (Arg->hasGoRetAttr()) { int ResultIndex = MemoryResultMap[Group.Index]; if (ResultIndex < 0 || Group.End != Group.Start + 1 || - !Ins[Group.Start].Flags.isByRef()) + !Ins[Group.Start].Flags.isGoRet()) report_fatal_error("invalid X86 Go memory result carrier"); const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs || ResultLayout.Size == 0 || - Ins[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + Ins[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) report_fatal_error( "X86 Go memory result carrier disagrees with Go ABI layout"); int FI = @@ -446,13 +449,10 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, goabi::CallLayout Layout = goabi::computeCallLayout( ArgTys, ResultTys, DAG.getDataLayout(), getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); - unsigned MemoryResultCount = - llvm::count_if(Layout.Results, [](const auto &Result) { - return !Result.InRegs && Result.Size != 0; - }); + SmallVector MemoryResults = + getX86GoMemoryResults(MF.getFunction()); SmallVector DirectResultIndices = getX86GoDirectResultIndices( - MF.getFunction().getAttributes(), - Layout.Results.size() - MemoryResultCount, MemoryResultCount); + MemoryResults, Layout.Results.size() - MemoryResults.size()); SmallVector MemOps; SmallVector, 8> RetRegs; @@ -470,9 +470,9 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, MVT CopyVT = getX86GoCopyVT(Val.getSimpleValueType()); if (Val.getSimpleValueType() != CopyVT) Val = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Val); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); + MCPhysReg PReg = getX86GoPhysReg(Out.VT, Out.OrigTy, + ResultLayout.IntRegStart + IntPiece, + ResultLayout.FPRegStart + FPPiece); if (isX86GoFloatPiece(Out.OrigTy)) ++FPPiece; else @@ -488,10 +488,11 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, /*IsImmutable=*/false); if (MF.getInfo()->hasGoABI0FrameIndex()) MFI.setIsAliasedObjectIndex(FI, true); - SDValue Addr = DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); - MemOps.push_back(storeX86GoStackPiece( - DAG, Chain, DL, Val, Out.ArgVT, Addr, - MachinePointerInfo::getFixedStack(MF, FI))); + SDValue Addr = + DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout())); + MemOps.push_back( + storeX86GoStackPiece(DAG, Chain, DL, Val, Out.ArgVT, Addr, + MachinePointerInfo::getFixedStack(MF, FI))); } } @@ -547,22 +548,22 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, goabi::getReturnTypes(CLI.RetTy, CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), DirectResultTys); - SmallVector IsByRef; - for (const TargetLowering::ArgListEntry &Arg : CLI.getArgs()) - IsByRef.push_back(Arg.IsByRef); - SmallVector MemoryResultMap = getX86GoMemoryResultMap( - CallAttrs, CLI.getArgs().size(), IsByRef, DirectResultTys.size()); - unsigned MemoryResultCount = llvm::count_if( - CLI.getArgs(), [](const auto &Arg) { return Arg.IsByRef; }); - SmallVector DirectResultIndices = getX86GoDirectResultIndices( - CallAttrs, DirectResultTys.size(), MemoryResultCount); + SmallVector MemoryResults = + getX86GoMemoryResults(CLI.getArgs()); + SmallVector MemoryResultMap(CLI.getArgs().size(), -1); + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) + if (CLI.getArgs()[I].IsGoRet) + MemoryResultMap[I] = CLI.getArgs()[I].GoRetIndex; + SmallVector DirectResultIndices = + getX86GoDirectResultIndices(MemoryResults, DirectResultTys.size()); unsigned NumBytes = Layout.TotalStackSize; Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; if (NumBytes != 0) - StackPtr = DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), PtrVT); + StackPtr = + DAG.getCopyFromReg(Chain, DL, RegInfo->getStackRegister(), PtrVT); SmallVector MemOpChains; SmallVector, 8> RegsToPass; @@ -573,14 +574,14 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, if (Group.Index >= CLI.getArgs().size()) continue; - if (CLI.getArgs()[Group.Index].IsByRef) { + if (CLI.getArgs()[Group.Index].IsGoRet) { int ResultIndex = MemoryResultMap[Group.Index]; if (ResultIndex < 0 || Group.End != Group.Start + 1 || - !Outs[Group.Start].Flags.isByRef()) + !Outs[Group.Start].Flags.isGoRet()) report_fatal_error("invalid X86 Go memory result call carrier"); const goabi::ValueLayout &ResultLayout = Layout.Results[ResultIndex]; if (ResultLayout.InRegs || ResultLayout.Size == 0 || - Outs[Group.Start].Flags.getByRefSize() != ResultLayout.Size) + Outs[Group.Start].Flags.getGoRetSize() != ResultLayout.Size) report_fatal_error( "X86 Go memory result call carrier disagrees with Go ABI layout"); MemoryResultDests[ResultIndex] = OutVals[Group.Start]; @@ -627,9 +628,9 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, MVT CopyVT = getX86GoCopyVT(Arg.getSimpleValueType()); if (Arg.getSimpleValueType() != CopyVT) Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, CopyVT, Arg); - MCPhysReg PReg = getX86GoPhysReg( - Out.VT, Out.OrigTy, ArgLayout.IntRegStart + IntPiece, - ArgLayout.FPRegStart + FPPiece); + MCPhysReg PReg = getX86GoPhysReg(Out.VT, Out.OrigTy, + ArgLayout.IntRegStart + IntPiece, + ArgLayout.FPRegStart + FPPiece); if (isX86GoFloatPiece(Out.OrigTy)) ++FPPiece; else @@ -641,14 +642,13 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, } assert(StackPtr && "missing Go call stack pointer"); - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant(ArgLayout.StackOffset + Out.PartOffset, - DL)); + SDValue Addr = DAG.getNode( + ISD::ADD, DL, PtrVT, StackPtr, + DAG.getIntPtrConstant(ArgLayout.StackOffset + Out.PartOffset, DL)); MemOpChains.push_back(storeX86GoStackPiece( DAG, Chain, DL, Arg, Out.ArgVT, Addr, - MachinePointerInfo::getStack(MF, - ArgLayout.StackOffset + Out.PartOffset))); + MachinePointerInfo::getStack(MF, ArgLayout.StackOffset + + Out.PartOffset))); } } @@ -675,7 +675,8 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, Ops.push_back(Callee); for (const auto &[Reg, Val] : RegsToPass) Ops.push_back(DAG.getRegister(Reg, Val.getValueType())); - Ops.push_back(DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); + Ops.push_back( + DAG.getRegisterMask(RegInfo->getCallPreservedMask(MF, CLI.CallConv))); if (InGlue.getNode()) Ops.push_back(InGlue); @@ -698,9 +699,9 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, for (unsigned I = Group.Start; I != Group.End; ++I) { const ISD::InputArg &In = Ins[I]; MVT CopyVT = getX86GoCopyVT(In.VT); - MCPhysReg PReg = getX86GoPhysReg( - In.VT, In.OrigTy, ResultLayout.IntRegStart + IntPiece, - ResultLayout.FPRegStart + FPPiece); + MCPhysReg PReg = + getX86GoPhysReg(In.VT, In.OrigTy, ResultLayout.IntRegStart + IntPiece, + ResultLayout.FPRegStart + FPPiece); SDValue Val = DAG.getCopyFromReg(Chain, DL, PReg, CopyVT, InGlue); Chain = Val.getValue(1); InGlue = Val.getValue(2); @@ -740,14 +741,13 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, continue; for (unsigned I = Group.Start; I != Group.End; ++I) { const ISD::InputArg &In = Ins[I]; - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, ResultStackPtr, - DAG.getIntPtrConstant(ResultLayout.StackOffset + In.PartOffset, - DL)); + SDValue Addr = DAG.getNode( + ISD::ADD, DL, PtrVT, ResultStackPtr, + DAG.getIntPtrConstant(ResultLayout.StackOffset + In.PartOffset, DL)); SDValue Load = loadX86GoStackPiece( DAG, Chain, DL, In.VT, In.ArgVT, Addr, - MachinePointerInfo::getStack( - MF, ResultLayout.StackOffset + In.PartOffset)); + MachinePointerInfo::getStack(MF, ResultLayout.StackOffset + + In.PartOffset)); Chain = Load.getValue(1); ResultVals[I] = Load; } @@ -991,13 +991,12 @@ unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv( if (VT.isVectorOf(MVT::bf16) && isTypeLegal(MVT::f16)) VT = VT.changeVectorElementType(Context, MVT::f16); - return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT, - NumIntermediates, RegisterVT); + return TargetLowering::getVectorTypeBreakdownForCallingConv( + Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); } EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL, - LLVMContext& Context, - EVT VT) const { + LLVMContext &Context, EVT VT) const { if (!VT.isVector()) return MVT::i8; @@ -1289,9 +1288,8 @@ SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, /// This returns the relocation base for the given PIC jumptable, /// the same as getPICJumpTableRelocBase, but as an MCExpr. -const MCExpr *X86TargetLowering:: -getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, - MCContext &Ctx) const { +const MCExpr *X86TargetLowering::getPICJumpTableRelocBaseExpr( + const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { // X86-64 uses RIP relative addressing based on the jump table label. if (Subtarget.isPICStyleRIPRel() || (Subtarget.is64Bit() && @@ -1310,19 +1308,35 @@ X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, switch (VT.SimpleTy) { default: return TargetLowering::findRepresentativeClass(TRI, VT); - case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: + case MVT::i8: + case MVT::i16: + case MVT::i32: + case MVT::i64: RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass; break; case MVT::x86mmx: RRC = &X86::VR64RegClass; break; - case MVT::f32: case MVT::f64: - case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: - case MVT::v4f32: case MVT::v2f64: - case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64: - case MVT::v8f32: case MVT::v4f64: - case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64: - case MVT::v16f32: case MVT::v8f64: + case MVT::f32: + case MVT::f64: + case MVT::v16i8: + case MVT::v8i16: + case MVT::v4i32: + case MVT::v2i64: + case MVT::v4f32: + case MVT::v2f64: + case MVT::v32i8: + case MVT::v16i16: + case MVT::v8i32: + case MVT::v4i64: + case MVT::v8f32: + case MVT::v4f64: + case MVT::v64i8: + case MVT::v32i16: + case MVT::v16i32: + case MVT::v8i64: + case MVT::v16f32: + case MVT::v8f64: RRC = &X86::VR128XRegClass; break; } @@ -1341,8 +1355,8 @@ static bool hasStackGuardSlotTLS(const Triple &TargetTriple) { TargetTriple.isOSFuchsia() || TargetTriple.isAndroid(); } -static Constant* SegmentOffset(IRBuilderBase &IRB, - int Offset, unsigned AddressSpace) { +static Constant *SegmentOffset(IRBuilderBase &IRB, int Offset, + unsigned AddressSpace) { return ConstantExpr::getIntToPtr( ConstantInt::getSigned(Type::getInt32Ty(IRB.getContext()), Offset), IRB.getPtrTy(AddressSpace)); @@ -1662,8 +1676,8 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, if (ValVT == MVT::x86mmx) { if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy); - ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, - ValToCopy); + ValToCopy = + DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, ValToCopy); // If we don't have SSE2 available, convert to v4f32 so the generated // register is legal. if (!Subtarget.hasSSE2()) @@ -1742,9 +1756,9 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg, getPointerTy(MF.getDataLayout())); - Register RetValReg - = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? - X86::RAX : X86::EAX; + Register RetValReg = + (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? X86::RAX + : X86::EAX; Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Glue); Glue = Chain.getValue(1); @@ -1794,7 +1808,8 @@ bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { if (Copy->getOpcode() == ISD::CopyToReg) { // If the copy has a glue operand, we conservatively assume it isn't safe to // perform a tail call. - if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) + if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == + MVT::Glue) return false; TCChain = Copy->getOperand(0); } else if (Copy->getOpcode() != ISD::FP_EXTEND) @@ -2160,7 +2175,8 @@ X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv, // could be overwritten by lowering of arguments in case of a tail call. if (Flags.isByVal()) { unsigned Bytes = Flags.getByValSize(); - if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects. + if (Bytes == 0) + Bytes = 1; // Don't create zero-sized stack objects. // FIXME: For now, all byval parameter objects are marked as aliasing. This // can be improved with deeper analysis. @@ -2252,15 +2268,13 @@ static ArrayRef get64BitArgumentGPRs(CallingConv::ID CallConv, assert(Subtarget.is64Bit()); if (Subtarget.isCallingConvWin64(CallConv)) { - static const MCPhysReg GPR64ArgRegsWin64[] = { - X86::RCX, X86::RDX, X86::R8, X86::R9 - }; + static const MCPhysReg GPR64ArgRegsWin64[] = {X86::RCX, X86::RDX, X86::R8, + X86::R9}; return GPR64ArgRegsWin64; } - static const MCPhysReg GPR64ArgRegs64Bit[] = { - X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 - }; + static const MCPhysReg GPR64ArgRegs64Bit[] = {X86::RDI, X86::RSI, X86::RDX, + X86::RCX, X86::R8, X86::R9}; return GPR64ArgRegs64Bit; } @@ -2283,10 +2297,9 @@ static ArrayRef get64BitArgumentXMMs(MachineFunction &MF, // registers. return {}; - static const MCPhysReg XMMArgRegs64Bit[] = { - X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, - X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 - }; + static const MCPhysReg XMMArgRegs64Bit[] = {X86::XMM0, X86::XMM1, X86::XMM2, + X86::XMM3, X86::XMM4, X86::XMM5, + X86::XMM6, X86::XMM7}; return XMMArgRegs64Bit; } @@ -2511,8 +2524,8 @@ SDValue X86TargetLowering::LowerFormalArguments( const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { if (goabi::isGoCallingConv(CallConv)) - return lowerX86GoFormalArguments(*this, Chain, DAG.getMachineFunction(), Ins, - dl, DAG, InVals); + return lowerX86GoFormalArguments(*this, Chain, DAG.getMachineFunction(), + Ins, dl, DAG, InVals); MachineFunction &MF = DAG.getMachineFunction(); X86MachineFunctionInfo *FuncInfo = MF.getInfo(); @@ -2819,11 +2832,11 @@ static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl) { // Store the return address to the appropriate stack slot. - if (!FPDiff) return Chain; + if (!FPDiff) + return Chain; // Calculate the new stack slot for the return address. - int NewReturnAddrFI = - MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize, - false); + int NewReturnAddrFI = MF.getFrameInfo().CreateFixedObject( + SlotSize, (int64_t)FPDiff - SlotSize, false); SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT); Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, MachinePointerInfo::getFixedStack( @@ -2886,9 +2899,8 @@ X86TargetLowering::ByValCopyKind X86TargetLowering::ByValNeedsCopyForTailCall( return CopyViaTemp; } -SDValue -X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, - SmallVectorImpl &InVals) const { +SDValue X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, + SmallVectorImpl &InVals) const { if (goabi::isGoCallingConv(CLI.CallConv)) return lowerX86GoCall(*this, CLI, InVals); SelectionDAG &DAG = CLI.DAG; @@ -2909,8 +2921,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool ShouldGuaranteeTCO = shouldGuaranteeTCO( CallConv, MF.getTarget().Options.GuaranteedTailCallOpt); X86MachineFunctionInfo *X86Info = MF.getInfo(); - bool HasNCSR = (CB && isa(CB) && - CB->hasFnAttr("no_caller_saved_registers")); + bool HasNCSR = + (CB && isa(CB) && CB->hasFnAttr("no_caller_saved_registers")); bool IsIndirectCall = (CB && isa(CB) && CB->isIndirectCall()); bool IsCFICall = IsIndirectCall && CLI.CFIType; const Module *M = MF.getFunction().getParent(); @@ -3144,8 +3156,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // Promote the value if needed. switch (VA.getLocInfo()) { - default: llvm_unreachable("Unknown loc info!"); - case CCValAssign::Full: break; + default: + llvm_unreachable("Unknown loc info!"); + case CCValAssign::Full: + break; case CCValAssign::SExt: Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); break; @@ -3210,10 +3224,18 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // shadow reg if callee is a varargs function. Register ShadowReg; switch (VA.getLocReg()) { - case X86::XMM0: ShadowReg = X86::RCX; break; - case X86::XMM1: ShadowReg = X86::RDX; break; - case X86::XMM2: ShadowReg = X86::R8; break; - case X86::XMM3: ShadowReg = X86::R9; break; + case X86::XMM0: + ShadowReg = X86::RCX; + break; + case X86::XMM1: + ShadowReg = X86::RDX; + break; + case X86::XMM2: + ShadowReg = X86::R8; + break; + case X86::XMM3: + ShadowReg = X86::R9; + break; } if (ShadowReg) RegsToPass.push_back(std::make_pair(ShadowReg, Arg)); @@ -3223,8 +3245,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (!StackPtr.getNode()) StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(), getPointerTy(DAG.getDataLayout())); - MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, - dl, DAG, VA, Flags, isByVal)); + MemOpChains.push_back( + LowerMemOpCallTo(Chain, StackPtr, Arg, dl, DAG, VA, Flags, isByVal)); } } @@ -3240,9 +3262,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // go through the PLT and do not need EBX to hold the GOT base. if ((Callee->getOpcode() == ISD::GlobalAddress || Callee->getOpcode() == ISD::ExternalSymbol)) - RegsToPass.push_back(std::make_pair( - Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), - getPointerTy(DAG.getDataLayout())))); + RegsToPass.push_back( + std::make_pair(Register(X86::EBX), + DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), + getPointerTy(DAG.getDataLayout())))); } else { // If we are tail calling and generating PIC/GOT style code load the // address of the callee into ECX. The value in ecx is used as target of @@ -3273,16 +3296,14 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // registers used and is in the range 0 - 8 inclusive. // Count the number of XMM registers allocated. - static const MCPhysReg XMMArgRegs[] = { - X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, - X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 - }; + static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, + X86::XMM3, X86::XMM4, X86::XMM5, + X86::XMM6, X86::XMM7}; unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs); - assert((Subtarget.hasSSE1() || !NumXMMRegs) - && "SSE registers cannot be used when SSE is disabled"); - RegsToPass.push_back(std::make_pair(Register(X86::AL), - DAG.getConstant(NumXMMRegs, dl, - MVT::i8))); + assert((Subtarget.hasSSE1() || !NumXMMRegs) && + "SSE registers cannot be used when SSE is disabled"); + RegsToPass.push_back(std::make_pair( + Register(X86::AL), DAG.getConstant(NumXMMRegs, dl, MVT::i8))); } if (isVarArg && IsMustTail) { @@ -3952,8 +3973,11 @@ bool X86TargetLowering::isEligibleForSiblingCallOpt( continue; Register Reg = VA.getLocReg(); switch (Reg) { - default: break; - case X86::EAX: case X86::EDX: case X86::ECX: + default: + break; + case X86::EAX: + case X86::EDX: + case X86::ECX: if (++NumInRegs == MaxInRegs) return false; break; diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 4c33848dacf51..c4d35831ccaef 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -285,7 +285,8 @@ static bool definedInRegion(const SetVector &Blocks, Value *V) { /// function being code extracted, but not in the region being extracted. /// These values must be passed in as live-ins to the function. static bool definedInCaller(const SetVector &Blocks, Value *V) { - if (isa(V)) return true; + if (isa(V)) + return true; if (Instruction *I = dyn_cast(V)) if (!Blocks.count(I->getParent())) return true; @@ -725,7 +726,8 @@ void CodeExtractor::severSplitPHINodesOfEntry(BasicBlock *&Header) { if (Header != &Header->getParent()->getEntryBlock()) { PHINode *PN = dyn_cast(Header->begin()); - if (!PN) return; // No PHI nodes. + if (!PN) + return; // No PHI nodes. // If the header node contains any PHI nodes, check to see if there is more // than one entry from outside the region. If so, we need to sever the @@ -766,8 +768,8 @@ void CodeExtractor::severSplitPHINodesOfEntry(BasicBlock *&Header) { TI->replaceUsesOfWith(OldPred, NewBB); } - // Okay, everything within the region is now branching to the right block, we - // just have to update the PHI nodes now, inserting PHI nodes into NewBB. + // Okay, everything within the region is now branching to the right block, + // we just have to update the PHI nodes now, inserting PHI nodes into NewBB. BasicBlock::iterator AfterPHIs; for (AfterPHIs = OldPred->begin(); isa(AfterPHIs); ++AfterPHIs) { PHINode *PN = cast(AfterPHIs); @@ -1046,6 +1048,7 @@ Function *CodeExtractor::constructFunctionDeclaration( case Attribute::ZExt: case Attribute::ImmArg: case Attribute::ByRef: + case Attribute::GoRet: case Attribute::WriteOnly: case Attribute::Writable: case Attribute::DeadOnUnwind: diff --git a/llvm/test/Assembler/goret-attribute.ll b/llvm/test/Assembler/goret-attribute.ll new file mode 100644 index 0000000000000..94fb1e18f48a6 --- /dev/null +++ b/llvm/test/Assembler/goret-attribute.ll @@ -0,0 +1,27 @@ +; RUN: llvm-as < %s | llvm-dis | FileCheck %s + +%stack.result = type { ptr, i64 } + +; CHECK: define goabiinternal { i64, i64 } @roundtrip( +; CHECK-SAME: ptr goret(%stack.result) align 8 "goretindex"="1" %result1, +; CHECK-SAME: ptr goret(i32) align 4 "goretindex"="3" %result3) +define goabiinternal { i64, i64 } @roundtrip( + ptr goret(%stack.result) "goretindex"="1" align 8 %result1, + ptr goret(i32) "goretindex"="3" align 4 %result3) #0 { + ret { i64, i64 } zeroinitializer +} + +; CHECK: call goabiinternal { i64, i64 } @roundtrip( +; CHECK-SAME: ptr goret(%stack.result) align 8 "goretindex"="1" %result1, +; CHECK-SAME: ptr goret(i32) align 4 "goretindex"="3" %result3) +define goabiinternal void @roundtrip_callsite() { +entry: + %result1 = alloca %stack.result, align 8 + %result3 = alloca i32, align 4 + %direct = call goabiinternal { i64, i64 } @roundtrip( + ptr goret(%stack.result) "goretindex"="1" align 8 %result1, + ptr goret(i32) "goretindex"="3" align 4 %result3) #0 + ret void +} + +attributes #0 = { "go_results_tuple" } diff --git a/llvm/test/CodeGen/AArch64/go-stack-byval.ll b/llvm/test/CodeGen/AArch64/go-stack-byval.ll index ccfe6ae7c00aa..4a49971c83b5a 100644 --- a/llvm/test/CodeGen/AArch64/go-stack-byval.ll +++ b/llvm/test/CodeGen/AArch64/go-stack-byval.ll @@ -27,7 +27,7 @@ entry: } define goabiinternal void @write_memory_result( - ptr byref(%large) align 8 %result) #0 { + ptr goret(%large) "goretindex"="0" align 8 %result) { ; CHECK-LABEL: write_memory_result: ; CHECK: str x{{[0-9]+}}, [sp, #8] ; CHECK-NEXT: ret @@ -46,9 +46,7 @@ define goabiinternal i64 @copy_large_memory_result() { entry: %result = alloca %large, align 8 call goabiinternal void @write_memory_result( - ptr byref(%large) align 8 %result) #0 + ptr goret(%large) "goretindex"="0" align 8 %result) %value = load i64, ptr %result, align 8 ret i64 %value } - -attributes #0 = { "go_memory_results"="0" } diff --git a/llvm/test/CodeGen/X86/go-stack-byval.ll b/llvm/test/CodeGen/X86/go-stack-byval.ll index 4d1683f4ce273..b4316e3e658c2 100644 --- a/llvm/test/CodeGen/X86/go-stack-byval.ll +++ b/llvm/test/CodeGen/X86/go-stack-byval.ll @@ -25,7 +25,7 @@ entry: } define goabiinternal void @write_memory_result( - ptr byref(%large) align 8 %result) #0 { + ptr goret(%large) "goretindex"="0" align 8 %result) { ; CHECK-LABEL: write_memory_result: ; CHECK: movq $42, 8(%rsp) ; CHECK-NEXT: retq @@ -42,9 +42,7 @@ define goabiinternal i64 @copy_large_memory_result() { entry: %result = alloca %large, align 8 call goabiinternal void @write_memory_result( - ptr byref(%large) align 8 %result) #0 + ptr goret(%large) "goretindex"="0" align 8 %result) %value = load i64, ptr %result, align 8 ret i64 %value } - -attributes #0 = { "go_memory_results"="0" } diff --git a/llvm/test/Verifier/goret.ll b/llvm/test/Verifier/goret.ll new file mode 100644 index 0000000000000..ac2482002b045 --- /dev/null +++ b/llvm/test/Verifier/goret.ll @@ -0,0 +1,54 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s + +%opaque = type opaque + +; CHECK: Attributes 'goret' and 'goretindex' must be used together! +define goabiinternal void @missing_index(ptr goret(i64) %result) { + ret void +} + +; CHECK: Attributes 'goret' and 'goretindex' must be used together! +define goabiinternal void @missing_type(ptr "goretindex"="0" %result) { + ret void +} + +; CHECK: 'goret' is only valid with a Go calling convention +define void @wrong_cc(ptr goret(i64) "goretindex"="0" %result) { + ret void +} + +; CHECK: duplicate 'goretindex' +define goabiinternal void @duplicate( + ptr goret(i64) "goretindex"="0" %result0, + ptr goret(i64) "goretindex"="0" %result1) { + ret void +} + +; CHECK: 'goretindex' values must be in increasing parameter order +define goabiinternal void @unordered( + ptr goret(i64) "goretindex"="1" %result1, + ptr goret(i64) "goretindex"="0" %result0) { + ret void +} + +; CHECK: 'goretindex' is out of range +define goabiinternal void @out_of_range(ptr goret(i64) "goretindex"="1" %result) { + ret void +} + +; CHECK: Attribute 'goret' does not support unsized types! +define goabiinternal void @unsized(ptr goret(%opaque) "goretindex"="0" %result) { + ret void +} + +; CHECK: Attribute 'goret' is incompatible with other ABI parameter attributes! +define goabiinternal void @incompatible( + ptr byval(i64) goret(i64) "goretindex"="0" %result) { + ret void +} + +; CHECK: 'goretindex' must be an unsigned decimal integer +define goabiinternal void @invalid_index( + ptr goret(i64) "goretindex"="not-a-number" %result) { + ret void +} From 102aaa49461f8327821070989533dbcf5a1b1e5d Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 22:06:46 +0800 Subject: [PATCH 3/6] [Go] Lower stack inputs through preallocated homes --- llvm/include/llvm/CodeGen/GoCallingConv.h | 16 ++- llvm/include/llvm/IR/Intrinsics.td | 2 +- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 6 +- llvm/lib/CodeGen/GoCallingConv.cpp | 55 ++++++-- .../SelectionDAG/StatepointLowering.cpp | 118 +++++++++++++++- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 33 +++++ .../Target/AArch64/AArch64ISelLowering.cpp | 133 +++++++++++++----- .../AArch64/AArch64MachineFunctionInfo.h | 31 ++++ llvm/lib/Target/X86/X86ISelLowering.cpp | 24 ++-- llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 108 +++++++++----- llvm/lib/Target/X86/X86MachineFunctionInfo.h | 9 ++ .../test/CodeGen/AArch64/go-argument-homes.ll | 2 +- llvm/test/CodeGen/AArch64/go-callconv.ll | 60 +++++--- llvm/test/CodeGen/AArch64/go-stack-byval.ll | 52 ------- .../CodeGen/AArch64/go-stack-preallocated.ll | 100 +++++++++++++ .../AArch64/go-statepoint-stack-args.ll | 86 ++++++----- .../AArch64/go-statepoint-stack-results.ll | 31 ++++ llvm/test/CodeGen/AArch64/goobj-abi.ll | 6 +- .../AArch64/goobj-private-reloc-constants.ll | 2 +- .../AArch64/goobj-register-argument-homes.ll | 4 +- .../CodeGen/AArch64/goobj-stack-growth.ll | 2 +- llvm/test/CodeGen/X86/go-argument-homes.ll | 5 +- llvm/test/CodeGen/X86/go-callconv.ll | 79 ++++++----- llvm/test/CodeGen/X86/go-memset-inline.ll | 16 ++- llvm/test/CodeGen/X86/go-stack-byval.ll | 48 ------- .../test/CodeGen/X86/go-stack-preallocated.ll | 96 +++++++++++++ .../CodeGen/X86/go-statepoint-stack-args.ll | 95 ++++++++++--- .../X86/goobj-alloca-argument-homes.ll | 2 +- .../X86/goobj-stack-growth-metadata.ll | 66 +++------ llvm/test/CodeGen/X86/goobj-stack-growth.ll | 29 ++-- .../SimplifyCFG/preallocated-nomerge.ll | 38 +++++ 31 files changed, 975 insertions(+), 379 deletions(-) delete mode 100644 llvm/test/CodeGen/AArch64/go-stack-byval.ll create mode 100644 llvm/test/CodeGen/AArch64/go-stack-preallocated.ll delete mode 100644 llvm/test/CodeGen/X86/go-stack-byval.ll create mode 100644 llvm/test/CodeGen/X86/go-stack-preallocated.ll create mode 100644 llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll diff --git a/llvm/include/llvm/CodeGen/GoCallingConv.h b/llvm/include/llvm/CodeGen/GoCallingConv.h index 0236136c672fa..16dce37182465 100644 --- a/llvm/include/llvm/CodeGen/GoCallingConv.h +++ b/llvm/include/llvm/CodeGen/GoCallingConv.h @@ -109,9 +109,13 @@ bool hasTupleResultsAttr(const CallBase &CB); unsigned getGoRetIndex(const Argument &Arg); unsigned getGoRetIndex(const CallBase &CB, unsigned ArgNo); /// Return the logical Go ABI type of an IR parameter. Non-empty values that -/// the Go frontend assigned wholly to memory use a typed byval pointer as the -/// LLVM carrier; direct parameters retain their logical type. +/// the Go frontend assigned wholly to memory use a typed preallocated pointer +/// as the LLVM carrier; direct parameters retain their logical type. Type *getParameterType(const Argument &Arg); +/// Return one bit per logical Go input parameter. A set bit means that the Go +/// frontend assigned the complete value to memory and represented it with a +/// typed preallocated carrier. Nest and goret carrier parameters are omitted. +SmallBitVector getMemoryArgMask(const Function &F); // Mirrors ComputeValueTypes and marks leaves originating in %go.abi.pad. SmallBitVector getPaddingPieces(Type *Ty); @@ -120,10 +124,14 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, void getReturnTypes(Type *ReturnType, bool TupleResults, ArrayRef MemoryResults, SmallVectorImpl &ResultTys); +SmallBitVector getMemoryResultMask(unsigned NumResults, + ArrayRef MemoryResults); CallLayout computeCallLayout(ArrayRef ArgTys, - ArrayRef ResultTys, const DataLayout &DL, - const ABIConfig &Config); + ArrayRef ResultTys, + const SmallBitVector &MemoryArgs, + const SmallBitVector &MemoryResults, + const DataLayout &DL, const ABIConfig &Config); EntryArgsInfo computeEntryArgsInfo(ArrayRef ArgTys, const CallLayout &Layout, diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 900dc31967845..e9dad4488fee3 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1118,7 +1118,7 @@ def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty], - [ImmArg>]>; + [ImmArg>, IntrNoMerge]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty], [ImmArg>]>; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index f517e6e3e3915..1d60e032cc2f9 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -277,7 +277,11 @@ static uint32_t getGoObjArgSize(const Function &F, const DataLayout &DL, goabi::getReturnTypes(F.getReturnType(), goabi::hasTupleResultsAttr(F), MemoryResults, ResultTys); uint64_t Size = - goabi::computeCallLayout(ArgTys, ResultTys, DL, Config).ArgSize; + goabi::computeCallLayout( + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), DL, + Config) + .ArgSize; if (Size > std::numeric_limits::max()) report_fatal_error("GoObj function argument size exceeds uint32 limit"); return static_cast(Size); diff --git a/llvm/lib/CodeGen/GoCallingConv.cpp b/llvm/lib/CodeGen/GoCallingConv.cpp index 139f3a69985b0..10b97e2368967 100644 --- a/llvm/lib/CodeGen/GoCallingConv.cpp +++ b/llvm/lib/CodeGen/GoCallingConv.cpp @@ -180,7 +180,8 @@ static uint64_t alignToValue(uint64_t Value, Align Alignment) { } static ValueLayout computeValueLayout(Type *Ty, const DataLayout &DL, - const ABIConfig &Config, unsigned &IntReg, + const ABIConfig &Config, + bool MustUseMemory, unsigned &IntReg, unsigned &FPReg) { ValueLayout Layout; Layout.Ty = Ty; @@ -197,7 +198,7 @@ static ValueLayout computeValueLayout(Type *Ty, const DataLayout &DL, unsigned IntAfter = IntReg; unsigned FPAfter = FPReg; - if (classifyType(Ty, DL, Config, IntAfter, FPAfter)) { + if (!MustUseMemory && classifyType(Ty, DL, Config, IntAfter, FPAfter)) { Layout.InRegs = true; Layout.IntRegCount = IntAfter - IntReg; Layout.FPRegCount = FPAfter - FPReg; @@ -247,7 +248,19 @@ bool hasTupleResultsAttr(const CallBase &CB) { } Type *getParameterType(const Argument &Arg) { - return Arg.hasByValAttr() ? Arg.getParamByValType() : Arg.getType(); + if (Arg.hasPreallocatedAttr()) + return Arg.getAttributes().getPreallocatedType(); + return Arg.getType(); +} + +SmallBitVector getMemoryArgMask(const Function &F) { + SmallBitVector MemoryArgs; + for (const Argument &Arg : F.args()) { + if (Arg.hasNestAttr() || Arg.hasGoRetAttr()) + continue; + MemoryArgs.push_back(Arg.hasPreallocatedAttr()); + } + return MemoryArgs; } void getReturnTypes(Type *ReturnType, bool TupleResults, @@ -293,9 +306,29 @@ void getReturnTypes(Type *ReturnType, bool TupleResults, report_fatal_error("Go direct result index count mismatch"); } +SmallBitVector getMemoryResultMask(unsigned NumResults, + ArrayRef MemoryResults) { + SmallBitVector Mask(NumResults); + std::optional Previous; + for (const MemoryResult &Result : MemoryResults) { + if (Result.Index >= NumResults || Mask.test(Result.Index) || + (Previous && *Previous >= Result.Index)) + report_fatal_error("invalid Go memory result carrier"); + Mask.set(Result.Index); + Previous = Result.Index; + } + return Mask; +} + CallLayout computeCallLayout(ArrayRef ArgTys, - ArrayRef ResultTys, const DataLayout &DL, - const ABIConfig &Config) { + ArrayRef ResultTys, + const SmallBitVector &MemoryArgs, + const SmallBitVector &MemoryResults, + const DataLayout &DL, const ABIConfig &Config) { + if (MemoryArgs.size() != ArgTys.size() || + MemoryResults.size() != ResultTys.size()) + report_fatal_error("Go ABI memory constraints do not match value counts"); + CallLayout Layout; Layout.Args.reserve(ArgTys.size()); Layout.Results.reserve(ResultTys.size()); @@ -303,9 +336,9 @@ CallLayout computeCallLayout(ArrayRef ArgTys, unsigned NextInt = 0; unsigned NextFP = 0; uint64_t StackArgsEnd = 0; - for (Type *ArgTy : ArgTys) { - ValueLayout ArgLayout = - computeValueLayout(ArgTy, DL, Config, NextInt, NextFP); + for (auto [Index, ArgTy] : llvm::enumerate(ArgTys)) { + ValueLayout ArgLayout = computeValueLayout( + ArgTy, DL, Config, MemoryArgs.test(Index), NextInt, NextFP); if (!ArgLayout.InRegs) StackArgsEnd = layoutStackValue(StackArgsEnd, ArgLayout); Layout.Args.push_back(ArgLayout); @@ -316,9 +349,9 @@ CallLayout computeCallLayout(ArrayRef ArgTys, NextFP = 0; uint64_t StackResultsEnd = alignToValue(StackArgsEnd, Config.PtrAlign); uint64_t StackResultsStart = StackResultsEnd; - for (Type *ResultTy : ResultTys) { - ValueLayout ResultLayout = - computeValueLayout(ResultTy, DL, Config, NextInt, NextFP); + for (auto [Index, ResultTy] : llvm::enumerate(ResultTys)) { + ValueLayout ResultLayout = computeValueLayout( + ResultTy, DL, Config, MemoryResults.test(Index), NextInt, NextFP); if (!ResultLayout.InRegs) StackResultsEnd = layoutStackValue(StackResultsEnd, ResultLayout); Layout.Results.push_back(ResultLayout); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 2137d05cb72bf..059e9cc45f408 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallBitVector.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -78,7 +79,45 @@ static cl::opt MaxRegistersForGCPointers( typedef FunctionLoweringInfo::StatepointRelocationRecord RecordType; -static std::optional> +struct ArgumentValueOffset { + const Argument *Arg; + uint64_t Offset; +}; + +/// Return true when the pointer carrier for an incoming memory argument is +/// only used to read that argument. In that case the loaded SSA values remain +/// available in their fixed incoming homes across a Go stack-growing call. +/// +/// Preallocated itself does not imply immutability. Reject stores, captures, +/// and all pointer transformations other than constant-addressing operations; +/// values from a mutable or escaped home must use an ordinary relocation slot. +static bool isReadOnlyArgumentHome(const Argument *Arg) { + SmallVector Worklist(1, Arg); + SmallPtrSet Seen; + while (!Worklist.empty()) { + const Value *Pointer = Worklist.pop_back_val(); + if (!Seen.insert(Pointer).second) + continue; + + for (const User *U : Pointer->users()) { + if (const auto *Load = dyn_cast(U)) { + if (Load->getPointerOperand() != Pointer || Load->isVolatile() || + Load->isAtomic()) + return false; + continue; + } + if (isa(U) || isa(U) || + isa(U)) { + Worklist.push_back(cast(U)); + continue; + } + return false; + } + } + return true; +} + +static std::optional getArgumentValueOffset(const Value *V, const DataLayout &DL) { uint64_t Offset = 0; while (const auto *Extract = dyn_cast(V)) { @@ -99,10 +138,30 @@ getArgumentValueOffset(const Value *V, const DataLayout &DL) { V = Extract->getAggregateOperand(); } + // A first-class parameter exposes its pointer pieces directly. A typed + // preallocated parameter instead exposes the same logical value through a + // load from its incoming home. Recover that carrier and any constant byte + // offset so both IR forms can use the same fixed-home optimization. + if (const auto *Load = dyn_cast(V)) { + if (Load->isVolatile() || Load->isAtomic()) + return std::nullopt; + const Value *Pointer = Load->getPointerOperand(); + APInt PointerOffset(DL.getIndexTypeSizeInBits(Pointer->getType()), 0); + V = Pointer->stripAndAccumulateConstantOffsets(DL, PointerOffset, + /*AllowNonInbounds=*/true); + if (PointerOffset.isNegative() || PointerOffset.getActiveBits() > 64) + return std::nullopt; + auto NewOffset = checkedAddUnsigned(Offset, PointerOffset.getZExtValue()); + if (!NewOffset) + return std::nullopt; + Offset = *NewOffset; + } + const auto *Arg = dyn_cast(V); - if (!Arg) + if (!Arg || ((Arg->hasPreallocatedAttr() || Arg->hasGoRetAttr()) && + !isReadOnlyArgumentHome(Arg))) return std::nullopt; - return std::pair(Arg, Offset); + return ArgumentValueOffset{Arg, Offset}; } static void pushStackMapConstant(SmallVectorImpl& Ops, @@ -367,6 +426,30 @@ static SDNode *peelCallResultChain(SDNode *Node) { Node = Mem->getChain().getNode(); continue; } + + // A target may represent a large memory-result copy with a custom node + // rather than a MemSDNode. Peel any non-glued node with exactly one chain + // input and one chain result. Stop at glued target call nodes. + bool HasGlueResult = false; + unsigned ChainResults = 0; + for (unsigned I = 0; I != Node->getNumValues(); ++I) { + HasGlueResult |= Node->getValueType(I) == MVT::Glue; + ChainResults += Node->getValueType(I) == MVT::Other; + } + if (Node->getOpcode() != ISD::TokenFactor && !HasGlueResult) { + SDNode *ChainInput = nullptr; + unsigned ChainInputs = 0; + for (SDValue Operand : Node->ops()) { + if (Operand.getValueType() != MVT::Other) + continue; + ChainInput = Operand.getNode(); + ++ChainInputs; + } + if (ChainInputs == 1 && ChainResults == 1) { + Node = ChainInput; + continue; + } + } break; } @@ -653,8 +736,31 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, if (auto ArgValue = getArgumentValueOffset(V, Builder.DAG.getDataLayout())) { uint64_t Size = PtrSD.getValueType().getStoreSize().getKnownMinValue(); - int FI = Builder.FuncInfo.getArgumentValueHome(ArgValue->first, - ArgValue->second, Size); + int FI = Builder.FuncInfo.getArgumentValueHome(ArgValue->Arg, + ArgValue->Offset, Size); + if (FI == INT_MAX && ArgValue->Arg->hasPreallocatedAttr()) { + int HomeFI = Builder.FuncInfo.getArgumentFrameIndex(ArgValue->Arg); + MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo(); + if (HomeFI != INT_MAX && MFI.isFixedObjectIndex(HomeFI) && + ArgValue->Offset <= uint64_t(MFI.getObjectSize(HomeFI)) && + Size <= uint64_t(MFI.getObjectSize(HomeFI)) - ArgValue->Offset) { + FI = HomeFI; + if (ArgValue->Offset != 0 || + MFI.getObjectSize(HomeFI) != int64_t(Size)) { + // Stack maps name one exact frame object. Make a narrow aliasing + // view for this pointer field while retaining the complete typed + // preallocated object for ordinary parameter accesses. + int64_t ViewOffset = MFI.getObjectOffset(HomeFI) + ArgValue->Offset; + MFI.setIsImmutableObjectIndex(HomeFI, false); + MFI.setIsAliasedObjectIndex(HomeFI, true); + FI = MFI.CreateFixedObject(Size, ViewOffset, + /*IsImmutable=*/false, + /*IsAliased=*/true); + } + Builder.FuncInfo.addArgumentValueHome(ArgValue->Arg, ArgValue->Offset, + Size, FI); + } + } if (FI != INT_MAX) { MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo(); MFI.setIsImmutableObjectIndex(FI, false); @@ -1018,7 +1124,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( isa(SDV) && (isa(V) || (isa(V) && - (cast(V)->hasByValAttr() || + (cast(V)->hasPreallocatedAttr() || cast(V)->hasGoRetAttr()) && FuncInfo.getArgumentFrameIndex(cast(V)) == cast(SDV)->getIndex()))) { diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 3f23e34d54c55..293200e6784e0 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -5108,6 +5108,39 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { default: break; + case ISD::PREALLOCATED_SETUP: { + auto *MFI = CurDAG->getMachineFunction().getInfo(); + auto CallId = MFI->getPreallocatedIdForCallSite( + cast(Node->getOperand(1))->getValue()); + SDValue Chain = Node->getOperand(0); + SDValue CallIdValue = + CurDAG->getTargetConstant(CallId, SDLoc(Node), MVT::i32); + MachineSDNode *New = + CurDAG->getMachineNode(TargetOpcode::PREALLOCATED_SETUP, SDLoc(Node), + MVT::Other, CallIdValue, Chain); + ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); + CurDAG->RemoveDeadNode(Node); + return; + } + + case ISD::PREALLOCATED_ARG: { + auto *MFI = CurDAG->getMachineFunction().getInfo(); + auto CallId = MFI->getPreallocatedIdForCallSite( + cast(Node->getOperand(1))->getValue()); + SDValue Ops[] = {CurDAG->getTargetConstant(CallId, SDLoc(Node), MVT::i32), + Node->getOperand(2), Node->getOperand(0)}; + MachineSDNode *New = CurDAG->getMachineNode( + TargetOpcode::PREALLOCATED_ARG, SDLoc(Node), + CurDAG->getVTList(CurDAG->getTargetLoweringInfo().getPointerTy( + CurDAG->getDataLayout()), + MVT::Other), + Ops); + ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); + ReplaceUses(SDValue(Node, 1), SDValue(New, 1)); + CurDAG->RemoveDeadNode(Node); + return; + } + case ISD::ATOMIC_CMP_SWAP: if (SelectCMP_SWAP(Node)) return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 26cc8ba199e79..564bdeba73d3b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -13,6 +13,7 @@ #include "AArch64ISelLowering.h" #include "AArch64CallingConvention.h" #include "AArch64ExpandImm.h" +#include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" #include "AArch64PerfectShuffle.h" #include "AArch64RegisterInfo.h" @@ -3604,6 +3605,30 @@ MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter( MI.dump(); #endif llvm_unreachable("Unexpected instruction for custom inserter!"); + case TargetOpcode::PREALLOCATED_SETUP: { + auto *FuncInfo = MI.getMF()->getInfo(); + int64_t PreallocatedId = MI.getOperand(0).getImm(); + if (!FuncInfo->preallocatedUsesReservedCallFrame(PreallocatedId)) + report_fatal_error( + "AArch64 preallocated call requires a reserved Go call frame"); + // Go reserves its maximum outgoing argument area in the function frame. + // The preallocated setup token therefore owns that existing area and must + // not adjust SP a second time. + MI.eraseFromParent(); + return BB; + } + case TargetOpcode::PREALLOCATED_ARG: { + int64_t PreallocatedId = MI.getOperand(1).getImm(); + int64_t ArgIndex = MI.getOperand(2).getImm(); + auto *FuncInfo = MI.getMF()->getInfo(); + size_t ArgOffset = + FuncInfo->getPreallocatedArgOffsets(PreallocatedId)[ArgIndex]; + emitFrameOffset(*BB, MI, MI.getDebugLoc(), MI.getOperand(0).getReg(), + AArch64::SP, StackOffset::getFixed(ArgOffset), + Subtarget->getInstrInfo()); + MI.eraseFromParent(); + return BB; + } case AArch64::EntryPStateSM: return EmitEntryPStateSM(MI, BB); case AArch64::F128CSEL: @@ -9075,11 +9100,23 @@ getAArch64GoCallArgTypes(const TargetLowering::ArgListTy &Args, if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); + ArgTys.push_back(Args[I].IsPreallocated ? Args[I].IndirectType + : Args[I].OrigTy); } return ArgTys; } +static SmallBitVector +getAArch64GoCallMemoryArgMask(const TargetLowering::ArgListTy &Args) { + SmallBitVector MemoryArgs; + for (const TargetLowering::ArgListEntry &Arg : Args) { + if (Arg.IsNest || Arg.IsGoRet) + continue; + MemoryArgs.push_back(Arg.IsPreallocated); + } + return MemoryArgs; +} + static SmallVector getAArch64GoReturnTypes(const Function &F) { SmallVector MemoryResults; for (const Argument &Arg : F.args()) @@ -9147,10 +9184,15 @@ static SDValue lowerAArch64GoFormalArguments( SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getAArch64GoReturnTypes(F); + SmallVector MemoryResults = + getAArch64GoMemoryResults(F); goabi::ABIConfig ABIConfig = getAArch64GoABIConfig(TLI, Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F), DAG.getDataLayout(), ABIConfig); + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( ArgTys, Layout, DAG.getDataLayout(), ABIConfig); @@ -9237,9 +9279,10 @@ static SDValue lowerAArch64GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; - bool IsByVal = Ins[Group.Start].Flags.isByVal(); - bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; - if (Arg->hasByValAttr() != IsByVal || IsByVal != MustBeByVal) + bool IsPreallocated = Ins[Group.Start].Flags.isPreallocated(); + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasPreallocatedAttr() != IsPreallocated || + IsPreallocated != MustBePreallocated) report_fatal_error( "AArch64 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = @@ -9249,7 +9292,7 @@ static SDValue lowerAArch64GoFormalArguments( if (ArgLayout.InRegs) HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, /*IsImmutable=*/false); - else if (IsByVal) + else if (IsPreallocated) HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, /*IsImmutable=*/false, /*isAliased=*/true); @@ -9267,10 +9310,10 @@ static SDValue lowerAArch64GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); - if (IsByVal) { + if (IsPreallocated) { if (Group.End != Group.Start + 1 || Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) - report_fatal_error("invalid AArch64 Go byval formal argument"); + report_fatal_error("invalid AArch64 Go preallocated formal argument"); InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); continue; } @@ -9333,12 +9376,14 @@ static SDValue lowerAArch64GoReturn(const AArch64TargetLowering &TLI, SmallVector ArgTys = getAArch64GoArgTypes(MF.getFunction(), LayoutMap); SmallVector ResultTys = getAArch64GoReturnTypes(MF.getFunction()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(MF.getFunction()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), + ArgTys, ResultTys, goabi::getMemoryArgMask(MF.getFunction()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, MF.getFunction().getCallingConv())); unsigned StackBias = getAArch64GoStackBias(MF.getFunction().getCallingConv()); - SmallVector MemoryResults = - getAArch64GoMemoryResults(MF.getFunction()); SmallVector DirectResultIndices = getAArch64GoDirectResultIndices(MemoryResults, Layout.Results.size() - MemoryResults.size()); @@ -9427,15 +9472,16 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); SmallVector ResultTys = getAArch64GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); + SmallVector MemoryResults = + getAArch64GoMemoryResults(CLI.getArgs()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); + ArgTys, ResultTys, getAArch64GoCallMemoryArgMask(CLI.getArgs()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getAArch64GoABIConfig(TLI, Subtarget, CLI.CallConv)); SmallVector DirectResultTys; goabi::getReturnTypes(CLI.RetTy, CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), DirectResultTys); - SmallVector MemoryResults = - getAArch64GoMemoryResults(CLI.getArgs()); SmallVector MemoryResultMap(CLI.getArgs().size(), -1); for (unsigned I = 0; I != CLI.getArgs().size(); ++I) if (CLI.getArgs()[I].IsGoRet) @@ -9451,6 +9497,26 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, // The caller frame itself remains stack-aligned; reserve only the bytes // through the last physical argument home here. unsigned NumBytes = Layout.ArgSize + StackBias; + SmallVector PreallocatedOffsets; + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) { + if (!CLI.getArgs()[I].IsPreallocated) + continue; + int LayoutIndex = LayoutMap[I]; + if (LayoutIndex < 0 || Layout.Args[LayoutIndex].InRegs) + report_fatal_error( + "AArch64 Go preallocated call argument has no stack home"); + PreallocatedOffsets.push_back(StackBias + + Layout.Args[LayoutIndex].StackOffset); + } + if (CLI.IsPreallocated != !PreallocatedOffsets.empty()) + report_fatal_error( + "AArch64 Go preallocated call bundle disagrees with its arguments"); + if (CLI.IsPreallocated) { + auto *FuncInfo = MF.getInfo(); + size_t PreallocatedId = FuncInfo->getPreallocatedIdForCallSite(CLI.CB); + FuncInfo->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets); + FuncInfo->setPreallocatedUsesReservedCallFrame(PreallocatedId); + } Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; @@ -9488,25 +9554,18 @@ static SDValue lowerAArch64GoCall(const AArch64TargetLowering &TLI, } const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutMap[Group.Index]]; - bool IsByVal = CLI.getArgs()[Group.Index].IsByVal; - bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; - if (Outs[Group.Start].Flags.isByVal() != IsByVal || IsByVal != MustBeByVal) + bool IsPreallocated = CLI.getArgs()[Group.Index].IsPreallocated; + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isPreallocated() != IsPreallocated || + IsPreallocated != MustBePreallocated) report_fatal_error( "AArch64 Go call argument carrier disagrees with Go ABI layout"); - if (IsByVal) { + if (IsPreallocated) { if (Group.End != Group.Start + 1 || Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) - report_fatal_error("invalid AArch64 Go byval call argument"); - assert(StackPtr && "missing Go call stack pointer"); - uint64_t Offset = StackBias + ArgLayout.StackOffset; - SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant(Offset, DL)); - Align Alignment = Outs[Group.Start].Flags.getNonZeroByValAlign(); - MemOpChains.push_back(DAG.getMemcpy( - Chain, DL, Addr, OutVals[Group.Start], - DAG.getConstant(ArgLayout.Size, DL, PtrVT), Alignment, Alignment, - /*isVol=*/false, /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt, - MachinePointerInfo::getStack(MF, Offset), MachinePointerInfo())); + report_fatal_error("invalid AArch64 Go preallocated call argument"); + // llvm.call.preallocated.arg exposes the final outgoing home. The + // frontend has already initialized it before this call. continue; } @@ -9696,14 +9755,15 @@ AArch64TargetLowering::getArgumentCopyElisionFrameInfo( const Argument &Arg, MachineFunction &MF) const { if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) return std::nullopt; + // Memory carriers already denote their canonical fixed ABI homes. This + // hook only redirects the frontend spill of a direct register parameter. + if (Arg.hasPreallocatedAttr() || Arg.hasGoRetAttr()) + return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const AArch64FunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) - // Stack-assigned values use typed byval and are already bound directly - // to their fixed incoming objects. Only register parameters still need - // their frontend store redirected into the canonical spill home. return ArgumentCopyElisionFrameInfo{Home.FrameIndex, /*ValueAlreadyInFrame=*/false}; return std::nullopt; @@ -9720,8 +9780,13 @@ int AArch64TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector LayoutMap; SmallVector ArgTys = getAArch64GoArgTypes(F, LayoutMap); const AArch64Subtarget &Subtarget = MF.getSubtarget(); + SmallVector ResultTys = getAArch64GoReturnTypes(F); + SmallVector MemoryResults = + getAArch64GoMemoryResults(F); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getAArch64GoReturnTypes(F), MF.getDataLayout(), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + MF.getDataLayout(), getAArch64GoABIConfig(*this, Subtarget, F.getCallingConv())); if (Layout.ArgSize == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h index c7d02a93e090f..2e64791d852b6 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -16,6 +16,7 @@ #include "AArch64SMEAttributes.h" #include "AArch64Subtarget.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" @@ -84,6 +85,12 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { int GoABI0FrameIndex = 0; bool HasGoABI0FrameIndex = false; + /// Preallocated call-frame layouts, keyed by the consuming IR call. These + /// fields are used only during instruction selection. + DenseMap PreallocatedIds; + SmallVector, 0> PreallocatedArgOffsets; + SmallVector PreallocatedUsesReservedCallFrame; + /// Number of bytes of arguments this function has on the stack. If the callee /// is expected to restore the argument stack this should be a multiple of 16, /// all usable during a tail call. @@ -329,6 +336,30 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { HasGoABI0FrameIndex = true; } + size_t getPreallocatedIdForCallSite(const Value *Call) { + auto Insert = PreallocatedIds.insert({Call, PreallocatedIds.size()}); + if (Insert.second) { + PreallocatedArgOffsets.emplace_back(); + PreallocatedUsesReservedCallFrame.push_back(false); + } + return Insert.first->second; + } + + void setPreallocatedArgOffsets(size_t Id, ArrayRef Offsets) { + PreallocatedArgOffsets[Id].assign(Offsets.begin(), Offsets.end()); + } + ArrayRef getPreallocatedArgOffsets(size_t Id) const { + assert(!PreallocatedArgOffsets[Id].empty() && "arg offsets not set"); + return PreallocatedArgOffsets[Id]; + } + + void setPreallocatedUsesReservedCallFrame(size_t Id) { + PreallocatedUsesReservedCallFrame[Id] = true; + } + bool preallocatedUsesReservedCallFrame(size_t Id) const { + return PreallocatedUsesReservedCallFrame[Id]; + } + void clearGoArgPointerSlots() { GoArgPointerSlots.clear(); } void addGoArgPointerSlot(int FrameIndex, uint32_t OffsetWithinObject, uint32_t ArgWord) { diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index d3fff2d4b54e2..9c63ae2fa4445 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -38838,22 +38838,28 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } case TargetOpcode::PREALLOCATED_SETUP: { - assert(Subtarget.is32Bit() && "preallocated only used in 32-bit"); auto *MFI = MF->getInfo(); - MFI->setHasPreallocatedCall(true); int64_t PreallocatedId = MI.getOperand(0).getImm(); + if (MFI->preallocatedUsesReservedCallFrame(PreallocatedId)) { + // Go's outgoing argument area is already part of the fixed function + // frame. The setup token owns that area without moving SP again. + MI.eraseFromParent(); + return BB; + } + MFI->setHasPreallocatedCall(true); size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId); assert(StackAdjustment != 0 && "0 stack adjustment"); LLVM_DEBUG(dbgs() << "PREALLOCATED_SETUP stack adjustment " << StackAdjustment << "\n"); - BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP) - .addReg(X86::ESP) + Register StackPtr = Subtarget.is64Bit() ? X86::RSP : X86::ESP; + unsigned SubOpcode = Subtarget.is64Bit() ? X86::SUB64ri32 : X86::SUB32ri; + BuildMI(*BB, MI, MIMD, TII->get(SubOpcode), StackPtr) + .addReg(StackPtr) .addImm(StackAdjustment); MI.eraseFromParent(); return BB; } case TargetOpcode::PREALLOCATED_ARG: { - assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit"); int64_t PreallocatedId = MI.getOperand(1).getImm(); int64_t ArgIdx = MI.getOperand(2).getImm(); auto *MFI = MF->getInfo(); @@ -38861,9 +38867,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx << ", arg offset " << ArgOffset << "\n"); // stack pointer + offset - addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r), - MI.getOperand(0).getReg()), - X86::ESP, false, ArgOffset); + Register StackPtr = Subtarget.is64Bit() ? X86::RSP : X86::ESP; + unsigned LeaOpcode = Subtarget.is64Bit() ? X86::LEA64r : X86::LEA32r; + addRegOffset( + BuildMI(*BB, MI, MIMD, TII->get(LeaOpcode), MI.getOperand(0).getReg()), + StackPtr, false, ArgOffset); MI.eraseFromParent(); return BB; } diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 6037d572be0d4..dc1eca74f611b 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -170,11 +170,23 @@ getX86GoCallArgTypes(const TargetLowering::ArgListTy &Args, if (Args[I].IsNest || Args[I].IsGoRet) continue; LayoutMap[I] = ArgTys.size(); - ArgTys.push_back(Args[I].IsByVal ? Args[I].IndirectType : Args[I].OrigTy); + ArgTys.push_back(Args[I].IsPreallocated ? Args[I].IndirectType + : Args[I].OrigTy); } return ArgTys; } +static SmallBitVector +getX86GoCallMemoryArgMask(const TargetLowering::ArgListTy &Args) { + SmallBitVector MemoryArgs; + for (const TargetLowering::ArgListEntry &Arg : Args) { + if (Arg.IsNest || Arg.IsGoRet) + continue; + MemoryArgs.push_back(Arg.IsPreallocated); + } + return MemoryArgs; +} + static SmallVector getX86GoReturnTypes(const Function &F) { SmallVector MemoryResults; for (const Argument &Arg : F.args()) @@ -247,10 +259,14 @@ static SDValue lowerX86GoFormalArguments( SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getX86GoReturnTypes(F); + SmallVector MemoryResults = getX86GoMemoryResults(F); goabi::ABIConfig ABIConfig = getX86GoABIConfig(Subtarget, F.getCallingConv()); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F), DAG.getDataLayout(), ABIConfig); + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), ABIConfig); goabi::EntryArgsInfo EntryArgs = goabi::computeEntryArgsInfo( ArgTys, Layout, DAG.getDataLayout(), ABIConfig); @@ -351,9 +367,10 @@ static SDValue lowerX86GoFormalArguments( unsigned LayoutIndex = LayoutMap[Group.Index]; const goabi::ValueLayout &ArgLayout = Layout.Args[LayoutIndex]; - bool IsByVal = Ins[Group.Start].Flags.isByVal(); - bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; - if (Arg->hasByValAttr() != IsByVal || IsByVal != MustBeByVal) + bool IsPreallocated = Ins[Group.Start].Flags.isPreallocated(); + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Arg->hasPreallocatedAttr() != IsPreallocated || + IsPreallocated != MustBePreallocated) report_fatal_error( "X86 Go formal argument carrier disagrees with Go ABI layout"); uint64_t LogicalHomeOffset = @@ -363,7 +380,7 @@ static SDValue lowerX86GoFormalArguments( if (ArgLayout.InRegs) HomeFI = MFI.CreateFixedSpillStackObject(ArgLayout.Size, FixedHomeOffset, /*IsImmutable=*/false); - else if (IsByVal) + else if (IsPreallocated) HomeFI = MFI.CreateFixedObject(ArgLayout.Size, FixedHomeOffset, /*IsImmutable=*/false, /*isAliased=*/true); @@ -381,10 +398,10 @@ static SDValue lowerX86GoFormalArguments( RecordPointerSlots(HomeFI, LogicalHomeOffset, ArgLayout.Size, IsLiveAtEntry); - if (IsByVal) { + if (IsPreallocated) { if (Group.End != Group.Start + 1 || Ins[Group.Start].Flags.getByValSize() != ArgLayout.Size) - report_fatal_error("invalid X86 Go byval formal argument"); + report_fatal_error("invalid X86 Go preallocated formal argument"); InVals.push_back(DAG.getFrameIndex(HomeFI, PtrVT)); continue; } @@ -446,11 +463,13 @@ static SDValue lowerX86GoReturn(const X86TargetLowering &TLI, SDValue Chain, SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(MF.getFunction(), LayoutMap); SmallVector ResultTys = getX86GoReturnTypes(MF.getFunction()); - goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, ResultTys, DAG.getDataLayout(), - getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); SmallVector MemoryResults = getX86GoMemoryResults(MF.getFunction()); + goabi::CallLayout Layout = goabi::computeCallLayout( + ArgTys, ResultTys, goabi::getMemoryArgMask(MF.getFunction()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), + getX86GoABIConfig(Subtarget, MF.getFunction().getCallingConv())); SmallVector DirectResultIndices = getX86GoDirectResultIndices( MemoryResults, Layout.Results.size() - MemoryResults.size()); @@ -541,15 +560,16 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, AttributeList CallAttrs = CLI.CB ? CLI.CB->getAttributes() : AttributeList(); SmallVector ResultTys = getX86GoCallReturnTypes(CLI.RetTy, CallAttrs, CLI.getArgs()); - goabi::CallLayout Layout = - goabi::computeCallLayout(ArgTys, ResultTys, DAG.getDataLayout(), - getX86GoABIConfig(Subtarget, CLI.CallConv)); + SmallVector MemoryResults = + getX86GoMemoryResults(CLI.getArgs()); + goabi::CallLayout Layout = goabi::computeCallLayout( + ArgTys, ResultTys, getX86GoCallMemoryArgMask(CLI.getArgs()), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + DAG.getDataLayout(), getX86GoABIConfig(Subtarget, CLI.CallConv)); SmallVector DirectResultTys; goabi::getReturnTypes(CLI.RetTy, CLI.CB && goabi::hasTupleResultsAttr(*CLI.CB), DirectResultTys); - SmallVector MemoryResults = - getX86GoMemoryResults(CLI.getArgs()); SmallVector MemoryResultMap(CLI.getArgs().size(), -1); for (unsigned I = 0; I != CLI.getArgs().size(); ++I) if (CLI.getArgs()[I].IsGoRet) @@ -558,6 +578,25 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, getX86GoDirectResultIndices(MemoryResults, DirectResultTys.size()); unsigned NumBytes = Layout.TotalStackSize; + SmallVector PreallocatedOffsets; + for (unsigned I = 0; I != CLI.getArgs().size(); ++I) { + if (!CLI.getArgs()[I].IsPreallocated) + continue; + int LayoutIndex = ArgLayoutMap[I]; + if (LayoutIndex < 0 || Layout.Args[LayoutIndex].InRegs) + report_fatal_error("X86 Go preallocated call argument has no stack home"); + PreallocatedOffsets.push_back(Layout.Args[LayoutIndex].StackOffset); + } + if (CLI.IsPreallocated != !PreallocatedOffsets.empty()) + report_fatal_error( + "X86 Go preallocated call bundle disagrees with its arguments"); + if (CLI.IsPreallocated) { + auto *FuncInfo = MF.getInfo(); + size_t PreallocatedId = FuncInfo->getPreallocatedIdForCallSite(CLI.CB); + FuncInfo->setPreallocatedStackSize(PreallocatedId, NumBytes); + FuncInfo->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets); + FuncInfo->setPreallocatedUsesReservedCallFrame(PreallocatedId); + } Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); SDValue StackPtr; @@ -596,26 +635,18 @@ static SDValue lowerX86GoCall(const X86TargetLowering &TLI, const goabi::ValueLayout &ArgLayout = Layout.Args[ArgLayoutMap[Group.Index]]; - bool IsByVal = CLI.getArgs()[Group.Index].IsByVal; - bool MustBeByVal = !ArgLayout.InRegs && ArgLayout.Size != 0; - if (Outs[Group.Start].Flags.isByVal() != IsByVal || IsByVal != MustBeByVal) + bool IsPreallocated = CLI.getArgs()[Group.Index].IsPreallocated; + bool MustBePreallocated = !ArgLayout.InRegs && ArgLayout.Size != 0; + if (Outs[Group.Start].Flags.isPreallocated() != IsPreallocated || + IsPreallocated != MustBePreallocated) report_fatal_error( "X86 Go call argument carrier disagrees with Go ABI layout"); - if (IsByVal) { + if (IsPreallocated) { if (Group.End != Group.Start + 1 || Outs[Group.Start].Flags.getByValSize() != ArgLayout.Size) - report_fatal_error("invalid X86 Go byval call argument"); - assert(StackPtr && "missing Go call stack pointer"); - SDValue Addr = - DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, - DAG.getIntPtrConstant(ArgLayout.StackOffset, DL)); - Align Alignment = Outs[Group.Start].Flags.getNonZeroByValAlign(); - MemOpChains.push_back(DAG.getMemcpy( - Chain, DL, Addr, OutVals[Group.Start], - DAG.getConstant(ArgLayout.Size, DL, PtrVT), Alignment, Alignment, - /*isVol=*/false, /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt, - MachinePointerInfo::getStack(MF, ArgLayout.StackOffset), - MachinePointerInfo())); + report_fatal_error("invalid X86 Go preallocated call argument"); + // llvm.call.preallocated.arg exposes the final outgoing home. The + // frontend has already initialized it before this call. continue; } @@ -785,15 +816,16 @@ X86TargetLowering::getArgumentCopyElisionFrameInfo(const Argument &Arg, MachineFunction &MF) const { if (!goabi::isGoCallingConv(MF.getFunction().getCallingConv())) return std::nullopt; + // Memory carriers already denote their canonical fixed ABI homes. This + // hook only redirects the frontend spill of a direct register parameter. + if (Arg.hasPreallocatedAttr() || Arg.hasGoRetAttr()) + return std::nullopt; const auto *FuncInfo = MF.getInfo(); uint64_t ArgSize = MF.getDataLayout().getTypeAllocSize(Arg.getType()); for (const X86MachineFunctionInfo::GoArgHome &Home : FuncInfo->getGoArgHomes()) if (!Home.valueAlreadyInFrame() && Home.ArgNo == Arg.getArgNo() && MF.getFrameInfo().getObjectSize(Home.FrameIndex) == int64_t(ArgSize)) - // Stack-assigned values use typed byval and are already bound directly - // to their fixed incoming objects. Only register parameters still need - // their frontend store redirected into the canonical spill home. return ArgumentCopyElisionFrameInfo{Home.FrameIndex, /*ValueAlreadyInFrame=*/false}; return std::nullopt; @@ -809,8 +841,12 @@ int X86TargetLowering::getGoABI0FrameIndex(MachineFunction &MF) const { SmallVector LayoutMap; SmallVector ArgTys = getX86GoArgTypes(F, LayoutMap); + SmallVector ResultTys = getX86GoReturnTypes(F); + SmallVector MemoryResults = getX86GoMemoryResults(F); goabi::CallLayout Layout = goabi::computeCallLayout( - ArgTys, getX86GoReturnTypes(F), MF.getDataLayout(), + ArgTys, ResultTys, goabi::getMemoryArgMask(F), + goabi::getMemoryResultMask(ResultTys.size(), MemoryResults), + MF.getDataLayout(), getX86GoABIConfig(MF.getSubtarget(), F.getCallingConv())); if (Layout.ArgSize == 0) report_fatal_error("llvm.go.abi0.frame requires a non-empty ABI0 frame"); diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h index a5d182aba7f8b..380f363410696 100644 --- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h +++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h @@ -203,6 +203,7 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { DenseMap PreallocatedIds; SmallVector PreallocatedStackSizes; SmallVector, 0> PreallocatedArgOffsets; + SmallVector PreallocatedUsesReservedCallFrame; // True if a function clobbers FP/BP according to its calling convention. bool FPClobberedByCall = false; @@ -366,6 +367,7 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { if (Insert.second) { PreallocatedStackSizes.push_back(0); PreallocatedArgOffsets.emplace_back(); + PreallocatedUsesReservedCallFrame.push_back(false); } return Insert.first->second; } @@ -388,6 +390,13 @@ class X86MachineFunctionInfo : public MachineFunctionInfo { return PreallocatedArgOffsets[Id]; } + void setPreallocatedUsesReservedCallFrame(size_t Id) { + PreallocatedUsesReservedCallFrame[Id] = true; + } + bool preallocatedUsesReservedCallFrame(size_t Id) const { + return PreallocatedUsesReservedCallFrame[Id]; + } + bool getFPClobberedByCall() const { return FPClobberedByCall; } void setFPClobberedByCall(bool C) { FPClobberedByCall = C; } diff --git a/llvm/test/CodeGen/AArch64/go-argument-homes.ll b/llvm/test/CodeGen/AArch64/go-argument-homes.ll index e6d0015a42ff2..797be356f46fc 100644 --- a/llvm/test/CodeGen/AArch64/go-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/go-argument-homes.ll @@ -12,7 +12,7 @@ define goabiinternal ptr @aarch64_stack_pair_home( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - ptr byval(%pair) align 8 %value.home) { + ptr preallocated(%pair) align 8 %value.home) { entry: ret ptr %value.home } diff --git a/llvm/test/CodeGen/AArch64/go-callconv.ll b/llvm/test/CodeGen/AArch64/go-callconv.ll index e4d75e7c22326..1fa6bd7a6b1fd 100644 --- a/llvm/test/CodeGen/AArch64/go-callconv.ll +++ b/llvm/test/CodeGen/AArch64/go-callconv.ll @@ -1,6 +1,9 @@ ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O0 < %s | FileCheck %s --check-prefix=A64 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 < %s | FileCheck %s --check-prefix=A64-O2 +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) + define goabiinternal i64 @second_int(i64 %a, i64 %b) { ; A64-LABEL: second_int: ; A64: mov x0, x1 @@ -54,8 +57,8 @@ entry: } define goabi0 i64 @abi0_second_int( - ptr byval(i64) align 8 %a.home, - ptr byval(i64) align 8 %b.home) { + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) { ; A64-LABEL: abi0_second_int: ; Go's arm64 stack ABI reserves 0(SP) for the return PC. ; A64: ldr x[[REG:[0-9]+]], [sp, #16] @@ -68,22 +71,27 @@ entry: define goabi0 i64 @abi0_call_second_int() { ; A64-LABEL: abi0_call_second_int: -; A64: mov x[[BASE:[0-9]+]], sp -; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #8] -; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #16] +; A64-DAG: add x[[AHOME:[0-9]+]], sp, #8 +; A64-DAG: add x[[BHOME:[0-9]+]], sp, #16 +; A64-DAG: str x{{[0-9]+}}, [x[[AHOME]]] +; A64-DAG: str x{{[0-9]+}}, [x[[BHOME]]] ; A64: bl abi0_second_int ; A64: mov x[[BASE_RELOAD:[0-9]+]], sp ; A64: ldr x[[RET:[0-9]+]], [x[[BASE_RELOAD]], #24] -; The 64-byte caller frame leaves this function's ABI0 result at entry SP+8. -; A64: str x[[RET]], [sp, #72] +; The 48-byte caller frame leaves this function's ABI0 result at entry SP+8. +; A64: str x[[RET]], [sp, #56] entry: - %a.home = alloca i64, align 8 - %b.home = alloca i64, align 8 + %setup = call token @llvm.call.preallocated.setup(i32 2) + %a.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(i64) + %b.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated(i64) store i64 11, ptr %a.home, align 8 store i64 22, ptr %b.home, align 8 %ret = call goabi0 i64 @abi0_second_int( - ptr byval(i64) align 8 %a.home, - ptr byval(i64) align 8 %b.home) + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) + ["preallocated"(token %setup)] ret i64 %ret } @@ -120,7 +128,7 @@ define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - ptr byval(%pair) align 8 %value.home) { + ptr preallocated(%pair) align 8 %value.home) { ; A64-LABEL: stack_pair: ; A64: ldr x0, [sp, #16] ; A64: ret @@ -132,23 +140,26 @@ entry: define goabiinternal i64 @call_stack_pair() { ; A64-LABEL: call_stack_pair: -; A64: ldr q[[VALUE:[0-9]+]], [sp, #{{[0-9]+}}] -; A64: mov x[[BASE:[0-9]+]], sp -; A64: stur q[[VALUE]], [x[[BASE]], #8] +; A64: add x[[BASE:[0-9]+]], sp, #8 +; A64-DAG: str x{{[0-9]+}}, [x[[BASE]]] +; A64-DAG: str x{{[0-9]+}}, [x[[BASE]], #8] ; A64: bl stack_pair entry: - %value.home = alloca %pair, align 8 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%pair) store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, - ptr byval(%pair) align 8 %value.home) + ptr preallocated(%pair) align 8 %value.home) + ["preallocated"(token %setup)] ret i64 %result } define goabiinternal [8 x i8] @stack_bytes( - ptr byval([8 x i8]) align 1 %value.home) { + ptr preallocated([8 x i8]) align 1 %value.home) { ; A64-LABEL: stack_bytes: ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #8] ; A64-DAG: ldrb w{{[0-9]+}}, [sp, #15] @@ -162,9 +173,9 @@ entry: define goabiinternal i16 @call_stack_bytes() { ; A64-LABEL: call_stack_bytes: -; A64: ldr x[[VALUE:[0-9]+]], [sp, #{{[0-9]+}}] -; A64: mov x[[BASE:[0-9]+]], sp -; A64: str x[[VALUE]], [x[[BASE]], #8] +; A64: add x[[BASE:[0-9]+]], sp, #8 +; A64-DAG: strb w{{[0-9]+}}, [x[[BASE]]] +; A64-DAG: strb w{{[0-9]+}}, [x[[BASE]], #7] ; A64: bl stack_bytes ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #16] ; A64-DAG: ldrb w{{[0-9]+}}, [x{{[0-9]+}}, #23] @@ -174,11 +185,14 @@ define goabiinternal i16 @call_stack_bytes() { ; A64-O2: ldrb w{{[0-9]+}}, [sp, #16] ; A64-O2: add sp, sp, #{{[0-9]+}} entry: - %value.home = alloca [8 x i8], align 1 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([8 x i8]) store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - ptr byval([8 x i8]) align 1 %value.home) + ptr preallocated([8 x i8]) align 1 %value.home) + ["preallocated"(token %setup)] %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/AArch64/go-stack-byval.ll b/llvm/test/CodeGen/AArch64/go-stack-byval.ll deleted file mode 100644 index 4a49971c83b5a..0000000000000 --- a/llvm/test/CodeGen/AArch64/go-stack-byval.ll +++ /dev/null @@ -1,52 +0,0 @@ -; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s - -%large = type [40000 x i64] - -declare goabiinternal void @consume(ptr byval(%large) align 8) - -define goabiinternal void @copy_large_stack_argument(ptr %source) { -; CHECK-LABEL: copy_large_stack_argument: -; CHECK-NOT: memcpy -; CHECK: ldr q31, [x{{[0-9]+}}] -; CHECK-NEXT: str q31, [x{{[0-9]+}}] -; CHECK: b.hs -; CHECK: bl consume -entry: - call goabiinternal void @consume(ptr byval(%large) align 8 %source) - ret void -} - -define goabiinternal i64 @read_incoming_stack_argument( - ptr byval(%large) align 8 %value) { -; CHECK-LABEL: read_incoming_stack_argument: -; CHECK: ldr x0, [sp, #8] -; CHECK-NEXT: ret -entry: - %result = load i64, ptr %value, align 8 - ret i64 %result -} - -define goabiinternal void @write_memory_result( - ptr goret(%large) "goretindex"="0" align 8 %result) { -; CHECK-LABEL: write_memory_result: -; CHECK: str x{{[0-9]+}}, [sp, #8] -; CHECK-NEXT: ret -entry: - store i64 42, ptr %result, align 8 - ret void -} - -define goabiinternal i64 @copy_large_memory_result() { -; CHECK-LABEL: copy_large_memory_result: -; CHECK-NOT: memcpy -; CHECK: bl write_memory_result -; CHECK: ldr q31, [x{{[0-9]+}}] -; CHECK-NEXT: str q31, [x{{[0-9]+}}] -; CHECK: b.hs -entry: - %result = alloca %large, align 8 - call goabiinternal void @write_memory_result( - ptr goret(%large) "goretindex"="0" align 8 %result) - %value = load i64, ptr %result, align 8 - ret i64 %value -} diff --git a/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll b/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll new file mode 100644 index 0000000000000..2a166dee09efb --- /dev/null +++ b/llvm/test/CodeGen/AArch64/go-stack-preallocated.ll @@ -0,0 +1,100 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] +%go.abi.pad = type { i8 } +%go_memory = type { i64, [0 x i64], [2 x [0 x i64]], %go.abi.pad } + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1 immarg) +declare goabiinternal void @consume(ptr preallocated(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +; CHECK: bl consume +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%large) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 320000, i1 false) + call goabiinternal void @consume(ptr preallocated(%large) align 8 %home) + ["preallocated"(token %setup)] + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr preallocated(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: ldr x0, [sp, #8] +; CHECK-NEXT: ret +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +; The Go ABI rejects arrays with length greater than one from register +; decomposition even when their element type is zero-sized. The preallocated +; carrier is authoritative; reconstructing the ABI from the remaining LLVM +; leaves alone would incorrectly assign this value to x1. +define goabiinternal i64 @read_go_assigned_memory_argument( + i8 %head, ptr preallocated(%go_memory) align 8 %value, i64 %tail) { +; CHECK-LABEL: read_go_assigned_memory_argument: +; CHECK: ldr x8, [sp, #8] +; CHECK-NEXT: add x0, x8, x1 +; CHECK-NEXT: ret +entry: + %word = load i64, ptr %value, align 8 + %result = add i64 %word, %tail + ret i64 %result +} + +declare goabiinternal i64 @consume_go_assigned_memory_argument( + i8, ptr preallocated(%go_memory) align 8, i64) + +define goabiinternal i64 @pass_go_assigned_memory_argument(ptr %source) { +; CHECK-LABEL: pass_go_assigned_memory_argument: +; CHECK: add x8, sp, #8 +; CHECK: mov w0, #7 +; CHECK: mov w1, #11 +; CHECK: str q0, [x8] +; CHECK: bl consume_go_assigned_memory_argument +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%go_memory) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 16, i1 false) + %result = call goabiinternal i64 @consume_go_assigned_memory_argument( + i8 7, ptr preallocated(%go_memory) align 8 %home, i64 11) + ["preallocated"(token %setup)] + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) { +; CHECK-LABEL: write_memory_result: +; CHECK: str x{{[0-9]+}}, [sp, #8] +; CHECK-NEXT: ret +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: bl write_memory_result +; CHECK: ldr q31, [x{{[0-9]+}}] +; CHECK-NEXT: str q31, [x{{[0-9]+}}] +; CHECK: b.hs +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) + %value = load i64, ptr %result, align 8 + ret i64 %value +} diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll index 42ce2adeed12e..83c5813205152 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-args.ll @@ -11,7 +11,7 @@ define goabiinternal ptr addrspace(1) @scalar_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr byval(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home) gc "statepoint-example" { entry: %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -28,7 +28,7 @@ define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - ptr byval(%aggregate) align 8 %value.home) gc "statepoint-example" { + ptr preallocated(%aggregate) align 8 %value.home) gc "statepoint-example" { entry: %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 @@ -47,13 +47,35 @@ entry: ret ptr addrspace(1) %result } +; A preallocated carrier is writable. Once the home is modified, the earlier +; loaded SSA value must use a distinct relocation slot. +define goabiinternal ptr addrspace(1) @mutable_stack_arg( + ptr addrspace(1) %replacement, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, + i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, + i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %value.home) + gc "statepoint-example" { +entry: + %original = load ptr addrspace(1), ptr %value.home, align 8 + store ptr addrspace(1) %replacement, ptr %value.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %original) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + define goabiinternal ptr addrspace(1) @merged_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr byval(ptr addrspace(1)) align 8 %p16.home, - ptr byval(i1) align 1 %condition.home) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %condition = load i1, ptr %condition.home, align 1 @@ -74,8 +96,8 @@ define goabiinternal ptr addrspace(1) @relocated_stack_arg( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr byval(ptr addrspace(1)) align 8 %p16.home, - ptr byval(i1) align 1 %condition.home) gc "statepoint-example" { + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %condition = load i1, ptr %condition.home, align 1 @@ -123,34 +145,35 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK: fixedStack: ; CHECK: - { id: 0, type: default, offset: 8, size: 8, ; CHECK: isImmutable: false -; A scalar loaded from the typed incoming home is a distinct SSA value. Keep -; it in one ordinary relocation slot; do not copy the complete argument home. -; CHECK: stack: -; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, -; CHECK: = LDRXui %fixed-stack.0, 0 -; CHECK: STRXui killed {{.*}}, %stack.0, 0 +; A read-only scalar remains in its exact typed incoming home. +; CHECK: stack: [] ; CHECK: STATEPOINT 1, -; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, -; CHECK-SAME: (volatile load store (s64) on %stack.0) +; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.0 +; CHECK-NEXT: [[SCALAR_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 ; CHECK-LABEL: name: aggregate_stack_arg -; The aggregate remains one typed fixed incoming object. Only its two live -; pointer fields need scalar relocation slots. +; The aggregate remains one typed fixed incoming object. Its two read-only +; pointer fields use narrow fixed views of that object. ; CHECK: fixedStack: -; CHECK: - { id: 0, type: default, offset: 8, size: 24, -; CHECK: stack: -; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, -; CHECK: - { id: 1, name: '', type: default, offset: 0, size: 8, -; CHECK: = LDRXui %fixed-stack.0, 0 -; CHECK: = LDRXui %fixed-stack.0, 2 +; CHECK: - { id: 2, type: default, offset: 8, size: 24, +; CHECK: stack: [] ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %stack.0), -; CHECK-SAME: (volatile load store (s64) on %stack.1) +; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.1, 0, 1, 8, %fixed-stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1), +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0) ; CHECK-NEXT: ADJCALLSTACKUP -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0 + +; CHECK-LABEL: name: mutable_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: STRXui {{.*}}, %fixed-stack.0, 0 +; CHECK: STRXui {{.*}}, %stack.0, 0 +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: @@ -161,11 +184,10 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: relocated_stack_arg -; CHECK: stack: -; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: stack: [] ; CHECK: STATEPOINT 4, -; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME:[0-9]+]], 0, +; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME:[0-9]+]], 0, ; CHECK: STATEPOINT 5, -; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, ; CHECK: STATEPOINT 6, -; CHECK-SAME: 2, 1, 1, 8, %stack.[[HOME]], 0, +; CHECK-SAME: 2, 1, 1, 8, %fixed-stack.[[HOME]], 0, diff --git a/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll b/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll index 0c68fc30db9ea..4ad54d0586df5 100644 --- a/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll +++ b/llvm/test/CodeGen/AArch64/go-statepoint-stack-results.ll @@ -6,8 +6,12 @@ i64, i64, i64, i64, i64, i64, i64, ptr, ptr, ptr } +%memory_result = type [722 x i64] declare goabiinternal %results @overflow_results() #0 +declare goabiinternal void @two_memory_results( + ptr goret(%memory_result) "goretindex"="0" align 8, + ptr goret(%memory_result) "goretindex"="1" align 8) define void @statepoint_with_register_and_stack_results() gc "statepoint-example" { @@ -41,6 +45,33 @@ entry: ; CHECK-NEXT: bl _use declare void @use(ptr) +declare void @use_i64(i64) + +define goabiinternal void @statepoint_with_two_memory_results() + gc "statepoint-example" { +entry: + %first = alloca %memory_result, align 8 + %second = alloca %memory_result, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, + ptr elementtype(void (ptr, ptr)) @two_memory_results, + i32 2, i32 0, + ptr goret(%memory_result) "goretindex"="0" align 8 %first, + ptr goret(%memory_result) "goretindex"="1" align 8 %second, + i32 0, i32 0) + %first_value = load i64, ptr %first, align 8 + %second_value = load i64, ptr %second, align 8 + call void @use_i64(i64 %first_value) + call void @use_i64(i64 %second_value) + ret void +} + +; CHECK-LABEL: _statepoint_with_two_memory_results: +; CHECK: bl _two_memory_results +; CHECK: bl _use_i64 +; CHECK: bl _use_i64 + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare %results @llvm.experimental.gc.result.results(token) diff --git a/llvm/test/CodeGen/AArch64/goobj-abi.ll b/llvm/test/CodeGen/AArch64/goobj-abi.ll index 12803449afa58..6cc665717b7ec 100644 --- a/llvm/test/CodeGen/AArch64/goobj-abi.ll +++ b/llvm/test/CodeGen/AArch64/goobj-abi.ll @@ -11,8 +11,12 @@ entry: ret i64 %sum } -define goabi0 i64 @"stackadd"(i64 %a, i64 %b) #0 { +define goabi0 i64 @"stackadd"( + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) #0 { entry: + %a = load i64, ptr %a.home, align 8 + %b = load i64, ptr %b.home, align 8 %sum = add i64 %a, %b ret i64 %sum } diff --git a/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll index 08eba10dcab80..2f3f24ed45683 100644 --- a/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll +++ b/llvm/test/CodeGen/AArch64/goobj-private-reloc-constants.ll @@ -22,7 +22,7 @@ entry: } !goobj.config = !{!0} -!0 = !{!"goallc.goobj", !"darwin", !"arm64", !"go1.27", !"GOARM64", !"v8.0", !"", !"p", !"0", !"0", !1} +!0 = !{!"goallc.goobj", !"darwin", !"arm64", !"go1.27", !"GOARM64", !"v8.0", !"", !"p", !"0", !"0", !"0", !1} !1 = !{!"regabiwrappers", !"regabiargs"} !2 = !{i1 true} !3 = !{i32 0} diff --git a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll index 5a4a8803f6dac..3a735546dba92 100644 --- a/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll +++ b/llvm/test/CodeGen/AArch64/goobj-register-argument-homes.ll @@ -12,7 +12,7 @@ entry: } define goabiinternal i64 @large_home_offset( - ptr byval([4096 x i64]) align 8 %stackarg, i64 %regarg) + ptr preallocated([4096 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() @@ -20,7 +20,7 @@ entry: } define goabiinternal i64 @large_home_boundary( - ptr byval([4094 x i64]) align 8 %stackarg, i64 %regarg) + ptr preallocated([4094 x i64]) align 8 %stackarg, i64 %regarg) "frame-pointer"="non-leaf" { entry: call goabiinternal void @"runtime.GC"() diff --git a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll index 695e2007eaf78..1a230bb5faa18 100644 --- a/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/AArch64/goobj-stack-growth.ll @@ -32,7 +32,7 @@ define goabiinternal ptr @mixed_register_and_stack_pointer_args( i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr byval(ptr) align 8 %p16.home) "frame-pointer"="non-leaf" { + ptr preallocated(ptr) align 8 %p16.home) "frame-pointer"="non-leaf" { entry: %p16 = load ptr, ptr %p16.home, align 8 %buf = alloca [8192 x i8], align 16 diff --git a/llvm/test/CodeGen/X86/go-argument-homes.ll b/llvm/test/CodeGen/X86/go-argument-homes.ll index b646be45257ea..d7abbc70b8951 100644 --- a/llvm/test/CodeGen/X86/go-argument-homes.ll +++ b/llvm/test/CodeGen/X86/go-argument-homes.ll @@ -6,11 +6,12 @@ %pair = type { i64, i64 } ; Exhaust X86's integer register budget so the pair arrives in one typed stack -; home. The byval carrier is the home itself; no frontend alloca or copy exists. +; home. The preallocated carrier is the home itself; no frontend alloca or copy +; exists. define goabiinternal ptr @x86_stack_pair_home( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, - ptr byval(%pair) align 8 %value.home) { + ptr preallocated(%pair) align 8 %value.home) { entry: ret ptr %value.home } diff --git a/llvm/test/CodeGen/X86/go-callconv.ll b/llvm/test/CodeGen/X86/go-callconv.ll index 6a215166fe6b7..237e112017122 100644 --- a/llvm/test/CodeGen/X86/go-callconv.ll +++ b/llvm/test/CodeGen/X86/go-callconv.ll @@ -2,6 +2,9 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 < %s | FileCheck %s --check-prefix=X86-O2 ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -O0 -filetype=null < %s +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) + define goabiinternal i64 @second_int(i64 %a, i64 %b) { ; X86-LABEL: second_int: ; X86: movq %rbx, %rax @@ -62,8 +65,8 @@ entry: } define goabi0 i64 @"abi0_second_int"( - ptr byval(i64) align 8 %a.home, - ptr byval(i64) align 8 %b.home) { + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) { ; X86-LABEL: "abi0_second_int": ; X86: leaq 16(%rsp), %[[BHOME:r[a-z0-9]+]] ; X86: movq (%[[BHOME]]), %[[B:r[a-z0-9]+]] @@ -76,24 +79,25 @@ entry: define goabi0 i64 @"abi0_call_second_int"() { ; X86-LABEL: "abi0_call_second_int": -; X86-DAG: movq $11, [[ASRC:[0-9]+]](%rsp) -; X86-DAG: movq $22, [[BSRC:[0-9]+]](%rsp) -; X86: movq [[BSRC]](%rsp), %[[B:r[a-z0-9]+]] -; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86: movq %[[B]], 8(%[[BASE]]) -; X86: movq [[ASRC]](%rsp), %[[A:r[a-z0-9]+]] -; X86: movq %[[A]], (%[[BASE]]) +; X86-DAG: leaq (%rsp), %[[AHOME:r[a-z0-9]+]] +; X86-DAG: leaq 8(%rsp), %[[BHOME:r[a-z0-9]+]] +; X86-DAG: movq $11, (%[[AHOME]]) +; X86-DAG: movq $22, (%[[BHOME]]) ; X86: callq "abi0_second_int" ; X86: movq %rsp, %[[RELOAD:r[a-z0-9]+]] ; X86: movq 16(%[[RELOAD]]), %rax entry: - %a.home = alloca i64, align 8 - %b.home = alloca i64, align 8 + %setup = call token @llvm.call.preallocated.setup(i32 2) + %a.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(i64) + %b.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated(i64) store i64 11, ptr %a.home, align 8 store i64 22, ptr %b.home, align 8 %ret = call goabi0 i64 @"abi0_second_int"( - ptr byval(i64) align 8 %a.home, - ptr byval(i64) align 8 %b.home) + ptr preallocated(i64) align 8 %a.home, + ptr preallocated(i64) align 8 %b.home) + ["preallocated"(token %setup)] ret i64 %ret } @@ -139,16 +143,19 @@ define goabiinternal i64 @call_method_results(ptr %callee, ptr %recv, ; X86-O2: addq 56(%rsp), %rax ; X86-O2: addq ${{[0-9]+}}, %rsp entry: - %integers.home = alloca [2 x i64], align 8 - %floats.home = alloca [2 x double], align 8 + %setup = call token @llvm.call.preallocated.setup(i32 2) + %integers.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([2 x i64]) + %floats.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 1) + preallocated([2 x double]) store [2 x i64] [i64 456, i64 789], ptr %integers.home, align 8 store [2 x double] [double 3.4, double 5.6], ptr %floats.home, align 8 %result = call goabiinternal %method.results %callee( ptr %recv, i64 123, - ptr byval([2 x i64]) align 8 %integers.home, + ptr preallocated([2 x i64]) align 8 %integers.home, double 1.2, - ptr byval([2 x double]) align 8 %floats.home, - ptr nest %ctxt) #0 + ptr preallocated([2 x double]) align 8 %floats.home, + ptr nest %ctxt) #0 ["preallocated"(token %setup)] %s = extractvalue %method.results %result, 0 %a = extractvalue %method.results %result, 1 %x = extractvalue %method.results %result, 2 @@ -170,7 +177,7 @@ entry: define goabiinternal i64 @stack_pair( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, - ptr byval(%pair) align 8 %value.home) { + ptr preallocated(%pair) align 8 %value.home) { ; X86-LABEL: stack_pair: ; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] ; X86-DAG: movq (%[[HOME]]), %[[LEFT:r[a-z0-9]+]] @@ -187,26 +194,25 @@ entry: define goabiinternal i64 @call_stack_pair() { ; X86-LABEL: call_stack_pair: -; X86-DAG: movq $13, [[LEFTSRC:[0-9]+]](%rsp) -; X86-DAG: movq $17, [[RIGHTSRC:[0-9]+]](%rsp) -; X86-DAG: movq [[LEFTSRC]](%rsp), %[[LEFT:r[a-z0-9]+]] -; X86-DAG: movq [[RIGHTSRC]](%rsp), %[[RIGHT:r[a-z0-9]+]] -; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86-DAG: movq %[[LEFT]], (%[[BASE]]) -; X86-DAG: movq %[[RIGHT]], 8(%[[BASE]]) +; X86: leaq (%rsp), %[[BASE:r[a-z0-9]+]] +; X86-DAG: movq $13, (%[[BASE]]) +; X86-DAG: movq $17, 8(%[[BASE]]) ; X86: callq stack_pair entry: - %value.home = alloca %pair, align 8 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%pair) store %pair { i64 13, i64 17 }, ptr %value.home, align 8 %result = call goabiinternal i64 @stack_pair( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, - ptr byval(%pair) align 8 %value.home) + ptr preallocated(%pair) align 8 %value.home) + ["preallocated"(token %setup)] ret i64 %result } define goabiinternal [8 x i8] @stack_bytes( - ptr byval([8 x i8]) align 1 %value.home) { + ptr preallocated([8 x i8]) align 1 %value.home) { ; X86-LABEL: stack_bytes: ; X86: leaq 8(%rsp), %[[HOME:r[a-z0-9]+]] ; X86-DAG: movb (%[[HOME]]), %{{[a-z0-9]+}} @@ -221,21 +227,22 @@ entry: define goabiinternal i16 @call_stack_bytes() { ; X86-LABEL: call_stack_bytes: -; X86-DAG: movb $1, [[SRC:[0-9]+]](%rsp) -; X86-DAG: movb $8, [[SRCEND:[0-9]+]](%rsp) -; X86: movq [[SRC]](%rsp), %[[VALUE:r[a-z0-9]+]] -; X86: movq %rsp, %[[BASE:r[a-z0-9]+]] -; X86: movq %[[VALUE]], (%[[BASE]]) +; X86: leaq (%rsp), %[[BASE:r[a-z0-9]+]] +; X86-DAG: movb $1, (%[[BASE]]) +; X86-DAG: movb $8, 7(%[[BASE]]) ; X86: callq stack_bytes ; X86: movq %rsp, %[[RESULT_BASE:r[a-z0-9]+]] ; X86-DAG: movzbl 8(%[[RESULT_BASE]]), %{{[a-z0-9]+}} ; X86-DAG: movzbl 15(%[[RESULT_BASE]]), %{{[a-z0-9]+}} entry: - %value.home = alloca [8 x i8], align 1 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %value.home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated([8 x i8]) store [8 x i8] [i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8], ptr %value.home, align 1 %result = call goabiinternal [8 x i8] @stack_bytes( - ptr byval([8 x i8]) align 1 %value.home) + ptr preallocated([8 x i8]) align 1 %value.home) + ["preallocated"(token %setup)] %first = extractvalue [8 x i8] %result, 0 %last = extractvalue [8 x i8] %result, 7 %first.ext = zext i8 %first to i16 diff --git a/llvm/test/CodeGen/X86/go-memset-inline.ll b/llvm/test/CodeGen/X86/go-memset-inline.ll index 378aec5f18ece..821e6c9cd6d64 100644 --- a/llvm/test/CodeGen/X86/go-memset-inline.ll +++ b/llvm/test/CodeGen/X86/go-memset-inline.ll @@ -4,20 +4,26 @@ declare void @llvm.memset.p0.i64(ptr writeonly, i8, i64, i1 immarg) define goabiinternal void @constant_memset(ptr %dst) { ; CHECK-LABEL: constant_memset: -; CHECK: rep -; CHECK-SAME: stos ; CHECK-NOT: callq memset +; CHECK: movb $0, +; CHECK: cmpq $800, +; CHECK: jb ; CHECK: retq call void @llvm.memset.p0.i64(ptr align 8 %dst, i8 0, i64 800, i1 false) ret void } -define goabi0 void @"dynamic_memset"(ptr %dst, i64 %size) { +define goabi0 void @"dynamic_memset"( + ptr preallocated(ptr) align 8 %dst.home, + ptr preallocated(i64) align 8 %size.home) { ; CHECK-LABEL: "dynamic_memset": -; CHECK: rep -; CHECK-SAME: stosb ; CHECK-NOT: callq memset +; CHECK: movb $0, +; CHECK: cmpq +; CHECK: jb ; CHECK: retq + %dst = load ptr, ptr %dst.home, align 8 + %size = load i64, ptr %size.home, align 8 call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 0, i64 %size, i1 false) ret void } diff --git a/llvm/test/CodeGen/X86/go-stack-byval.ll b/llvm/test/CodeGen/X86/go-stack-byval.ll deleted file mode 100644 index b4316e3e658c2..0000000000000 --- a/llvm/test/CodeGen/X86/go-stack-byval.ll +++ /dev/null @@ -1,48 +0,0 @@ -; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s - -%large = type [40000 x i64] - -declare goabiinternal void @consume(ptr byval(%large) align 8) - -define goabiinternal void @copy_large_stack_argument(ptr %source) { -; CHECK-LABEL: copy_large_stack_argument: -; CHECK-NOT: memcpy -; CHECK: rep;movsq -; CHECK: callq consume -entry: - call goabiinternal void @consume(ptr byval(%large) align 8 %source) - ret void -} - -define goabiinternal i64 @read_incoming_stack_argument( - ptr byval(%large) align 8 %value) { -; CHECK-LABEL: read_incoming_stack_argument: -; CHECK: movq 8(%rsp), %rax -; CHECK-NEXT: retq -entry: - %result = load i64, ptr %value, align 8 - ret i64 %result -} - -define goabiinternal void @write_memory_result( - ptr goret(%large) "goretindex"="0" align 8 %result) { -; CHECK-LABEL: write_memory_result: -; CHECK: movq $42, 8(%rsp) -; CHECK-NEXT: retq -entry: - store i64 42, ptr %result, align 8 - ret void -} - -define goabiinternal i64 @copy_large_memory_result() { -; CHECK-LABEL: copy_large_memory_result: -; CHECK-NOT: memcpy -; CHECK: callq write_memory_result -; CHECK: rep;movsq -entry: - %result = alloca %large, align 8 - call goabiinternal void @write_memory_result( - ptr goret(%large) "goretindex"="0" align 8 %result) - %value = load i64, ptr %result, align 8 - ret i64 %value -} diff --git a/llvm/test/CodeGen/X86/go-stack-preallocated.ll b/llvm/test/CodeGen/X86/go-stack-preallocated.ll new file mode 100644 index 0000000000000..c204f28ca96fc --- /dev/null +++ b/llvm/test/CodeGen/X86/go-stack-preallocated.ll @@ -0,0 +1,96 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O2 -verify-machineinstrs < %s | FileCheck %s + +%large = type [40000 x i64] +%go.abi.pad = type { i8 } +%go_memory = type { i64, [0 x i64], [2 x [0 x i64]], %go.abi.pad } + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1 immarg) +declare goabiinternal void @consume(ptr preallocated(%large) align 8) + +define goabiinternal void @copy_large_stack_argument(ptr %source) { +; CHECK-LABEL: copy_large_stack_argument: +; CHECK-NOT: memcpy +; CHECK: rep;movsq +; CHECK: callq consume +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%large) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 320000, i1 false) + call goabiinternal void @consume(ptr preallocated(%large) align 8 %home) + ["preallocated"(token %setup)] + ret void +} + +define goabiinternal i64 @read_incoming_stack_argument( + ptr preallocated(%large) align 8 %value) { +; CHECK-LABEL: read_incoming_stack_argument: +; CHECK: movq 8(%rsp), %rax +; CHECK-NEXT: retq +entry: + %result = load i64, ptr %value, align 8 + ret i64 %result +} + +; The Go ABI rejects arrays with length greater than one from register +; decomposition even when their element type is zero-sized. The preallocated +; carrier is authoritative; reconstructing the ABI from the remaining LLVM +; leaves alone would incorrectly assign this value to RBX. +define goabiinternal i64 @read_go_assigned_memory_argument( + i8 %head, ptr preallocated(%go_memory) align 8 %value, i64 %tail) { +; CHECK-LABEL: read_go_assigned_memory_argument: +; CHECK: movq %rbx, %rax +; CHECK-NEXT: addq 8(%rsp), %rax +; CHECK-NEXT: retq +entry: + %word = load i64, ptr %value, align 8 + %result = add i64 %word, %tail + ret i64 %result +} + +declare goabiinternal i64 @consume_go_assigned_memory_argument( + i8, ptr preallocated(%go_memory) align 8, i64) + +define goabiinternal i64 @pass_go_assigned_memory_argument(ptr %source) { +; CHECK-LABEL: pass_go_assigned_memory_argument: +; CHECK: leaq (%rsp), %rcx +; CHECK: movq {{.*}}, (%rcx) +; CHECK: movl $11, %ebx +; CHECK: movb $7, %al +; CHECK: callq consume_go_assigned_memory_argument +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%go_memory) + call void @llvm.memcpy.p0.p0.i64(ptr align 8 %home, ptr align 8 %source, + i64 16, i1 false) + %result = call goabiinternal i64 @consume_go_assigned_memory_argument( + i8 7, ptr preallocated(%go_memory) align 8 %home, i64 11) + ["preallocated"(token %setup)] + ret i64 %result +} + +define goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) { +; CHECK-LABEL: write_memory_result: +; CHECK: movq $42, 8(%rsp) +; CHECK-NEXT: retq +entry: + store i64 42, ptr %result, align 8 + ret void +} + +define goabiinternal i64 @copy_large_memory_result() { +; CHECK-LABEL: copy_large_memory_result: +; CHECK-NOT: memcpy +; CHECK: callq write_memory_result +; CHECK: rep;movsq +entry: + %result = alloca %large, align 8 + call goabiinternal void @write_memory_result( + ptr goret(%large) "goretindex"="0" align 8 %result) + %value = load i64, ptr %result, align 8 + ret i64 %value +} diff --git a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll index 17a4bcf33b224..4fa58509c74b3 100644 --- a/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll +++ b/llvm/test/CodeGen/X86/go-statepoint-stack-args.ll @@ -9,10 +9,18 @@ declare goabiinternal void @safepoint() define goabiinternal ptr addrspace(1) @scalar_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home) + gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 1, i32 0, ptr elementtype(void ()) @safepoint, @@ -25,10 +33,17 @@ entry: define goabiinternal ptr addrspace(1) @aggregate_stack_arg( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, - i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, - i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, - %aggregate %value) gc "statepoint-example" { + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(%aggregate) align 8 %value.home) + gc "statepoint-example" { entry: + %value = load %aggregate, ptr %value.home, align 8 %first = extractvalue %aggregate %value, 0 %second = extractvalue %aggregate %value, 2 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -45,13 +60,43 @@ entry: ret ptr addrspace(1) %result } +; A preallocated carrier is writable. Once the home is modified, the earlier +; loaded SSA value must use a distinct relocation slot. +define goabiinternal ptr addrspace(1) @mutable_stack_arg( + ptr addrspace(1) %replacement, + i64 %a1, i64 %a2, i64 %a3, i64 %a4, + i64 %a5, i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(ptr addrspace(1)) align 8 %value.home) + gc "statepoint-example" { +entry: + %original = load ptr addrspace(1), ptr %value.home, align 8 + store ptr addrspace(1) %replacement, ptr %value.home, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 7, i32 0, ptr elementtype(void ()) @safepoint, + i32 0, i32 0, i32 0, i32 0) + [ "gc-live"(ptr addrspace(1) %original) ] + %relocated = call ptr addrspace(1) @llvm.experimental.gc.relocate.p1( + token %token, i32 0, i32 0) + ret ptr addrspace(1) %relocated +} + define goabiinternal ptr addrspace(1) @merged_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %merged = select i1 %condition, ptr addrspace(1) %p0, ptr addrspace(1) %p16 %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @@ -67,10 +112,19 @@ entry: define goabiinternal ptr addrspace(1) @relocated_stack_arg( ptr addrspace(1) %p0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, - i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, - i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr addrspace(1) %p16, i1 %condition) gc "statepoint-example" { + i64 %a6, i64 %a7, i64 %a8, + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr addrspace(1)) align 8 %p16.home, + ptr preallocated(i1) align 1 %condition.home) gc "statepoint-example" { entry: + %p16 = load ptr addrspace(1), ptr %p16.home, align 8 + %condition = load i1, ptr %condition.home, align 1 %token1 = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0( i64 4, i32 0, ptr elementtype(void ()) @safepoint, @@ -125,11 +179,20 @@ declare ptr addrspace(1) @llvm.experimental.gc.relocate.p1( ; CHECK-LABEL: name: aggregate_stack_arg ; CHECK: stack: [] ; CHECK: STATEPOINT 2, -; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.0, 0, 1, 8, %fixed-stack.1, 0, -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0{{[^)]*}}), -; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1{{[^)]*}}) +; CHECK-SAME: 2, 2, 1, 8, %fixed-stack.1, 0, 1, 8, %fixed-stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.1{{[^)]*}}), +; CHECK-SAME: (volatile load store (s64) on %fixed-stack.0{{[^)]*}}) ; CHECK-NEXT: ADJCALLSTACKUP64 -; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.1 +; CHECK-NEXT: [[AGGREGATE_RELOC:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0 + +; CHECK-LABEL: name: mutable_stack_arg +; CHECK: stack: +; CHECK-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8, +; CHECK: MOV64mr %fixed-stack.0, +; CHECK: MOV64mr %stack.0, +; CHECK: STATEPOINT 7, +; CHECK-SAME: 2, 1, 1, 8, %stack.0, 0, +; CHECK-SAME: (volatile load store (s64) on %stack.0) ; CHECK-LABEL: name: merged_stack_arg ; CHECK: stack: diff --git a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll index d1c8d8017b4fa..d40120808f559 100644 --- a/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll +++ b/llvm/test/CodeGen/X86/goobj-alloca-argument-homes.ll @@ -74,7 +74,7 @@ entry: ; An aggregate assigned wholly to the stack reuses its caller-populated slot; ; the lifetime marker must not make SelectionDAG allocate and copy a local. define goabiinternal void @inactive_stack_aggregate( - ptr byval(%stack_aggregate) align 8 %value.home) + ptr preallocated(%stack_aggregate) align 8 %value.home) gc "statepoint-example" { entry: %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll index 48993e59b4ba5..1b912f4fefdd0 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth-metadata.ll @@ -34,25 +34,13 @@ entry: ret i64 %sum } +%stack.args = type [23 x i64] + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) declare goabiinternal void @many_stack_args( i64, i64, i64, i64, i64, i64, i64, i64, i64, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8, ptr byval(i64) align 8, - ptr byval(i64) align 8) -@many_stack_arg_values = private constant [23 x i64] [ - i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, - i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, - i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, - i64 27, i64 28, i64 29, i64 30, i64 31] + ptr preallocated(%stack.args) align 8) @condition = external global i1 ; X86 lowers the stack arguments below to push sequences instead of reserving @@ -64,31 +52,18 @@ entry: br i1 %cond, label %call, label %join call: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(%stack.args) + store %stack.args [ + i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, + i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, + i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, + i64 27, i64 28, i64 29, i64 30, i64 31], ptr %home, align 8 call goabiinternal void @many_stack_args( i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 0), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 1), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 2), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 3), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 4), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 5), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 6), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 7), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 8), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 9), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 10), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 11), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 12), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 13), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 14), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 15), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 16), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 17), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 18), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 19), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 20), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 21), - ptr byval(i64) align 8 getelementptr inbounds ([23 x i64], ptr @many_stack_arg_values, i64 0, i64 22)) + ptr preallocated(%stack.args) align 8 %home) + ["preallocated"(token %setup)] br label %join join: @@ -106,10 +81,10 @@ join: ; CHECK-NEXT: aux 0.{{[0-9]+}}: type=pcdata target= pc=[0-{{[0-9]+}}:-1] ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack_noctxt ; CHECK: reloc {{[0-9]+}}.{{[0-9]+}}: off={{[0-9]+}} size=4 type=7 add=0 target=runtime.morestack -; The outgoing frame is 72 bytes of alignment/register-argument space plus 23 -; 8-byte PUSHes. PCSP must reach 256 bytes at the call, return to zero before -; the CFG join, and thereby let funcMaxSPDelta size a grown Go stack correctly. -; CHECK: aux 2.{{[0-9]+}}: type=pcsp target= pc=[{{.*}}:256,{{[0-9]+}}-{{[0-9]+}}:0] +; The outgoing frame is 72 bytes of register-argument homes plus one 184-byte +; preallocated value. Go reserves it in the containing frame, so PCSP is stable +; across the call while funcMaxSPDelta still sees the full 256-byte frame. +; CHECK: aux 2.{{[0-9]+}}: type=pcsp target= pc=[0-[[PROLOG:[0-9]+]]:0,[[PROLOG]]-[[EPILOG:[0-9]+]]:256,[[EPILOG]]-{{[0-9]+}}:0] ; ASM-LABEL: big_frame: ; Match Go's single stack-check loop: the hot entry executes the check @@ -156,7 +131,8 @@ join: ; PEI-SAME: implicit $rsp, implicit $ssp, implicit $rdx ; PEI-LABEL: name: large_outgoing_frame -; PEI: stackSize: 0 +; PEI: stackSize: 256 +; PEI-NEXT: offsetAdjustment: -256 ; PEI: maxCallFrameSize: 256 ; PEI: $r12 = LEA64r $rsp, 1, $noreg, -128, $noreg ; PEI: CMP64rm $r12, $r14, 1, $noreg, 16, $noreg diff --git a/llvm/test/CodeGen/X86/goobj-stack-growth.ll b/llvm/test/CodeGen/X86/goobj-stack-growth.ll index 636511a49efe9..8b4b7e9e4ff52 100644 --- a/llvm/test/CodeGen/X86/goobj-stack-growth.ll +++ b/llvm/test/CodeGen/X86/goobj-stack-growth.ll @@ -24,9 +24,9 @@ entry: } define goabi0 void @"abi0_pointer_arguments"( - ptr byval(ptr) align 8 %first.home, - ptr byval(ptr) align 8 %second.home, - ptr byval(ptr) align 8 %third.home) + ptr preallocated(ptr) align 8 %first.home, + ptr preallocated(ptr) align 8 %second.home, + ptr preallocated(ptr) align 8 %third.home) "frame-pointer"="non-leaf" { entry: %first = load ptr, ptr %first.home, align 8 @@ -59,11 +59,14 @@ entry: define goabiinternal ptr @scalar_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, - ptr byval(i64) align 8 %a9, ptr byval(i64) align 8 %a10, - ptr byval(i64) align 8 %a11, ptr byval(i64) align 8 %a12, - ptr byval(i64) align 8 %a13, ptr byval(i64) align 8 %a14, - ptr byval(i64) align 8 %a15, - ptr byval(ptr) align 8 %pointer.home) { + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(i64) align 8 %a14, + ptr preallocated(i64) align 8 %a15, + ptr preallocated(ptr) align 8 %pointer.home) { entry: %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [5000 x i8], align 8 @@ -75,10 +78,12 @@ entry: define goabiinternal { ptr, ptr } @aggregate_stack_argument( i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, - ptr byval(i64) align 8 %a9, ptr byval(i64) align 8 %a10, - ptr byval(i64) align 8 %a11, ptr byval(i64) align 8 %a12, - ptr byval(i64) align 8 %a13, - ptr byval(%pointer.aggregate) align 8 %value.home) + ptr preallocated(i64) align 8 %a9, + ptr preallocated(i64) align 8 %a10, + ptr preallocated(i64) align 8 %a11, + ptr preallocated(i64) align 8 %a12, + ptr preallocated(i64) align 8 %a13, + ptr preallocated(%pointer.aggregate) align 8 %value.home) "go_results_tuple" { entry: %value = load %pointer.aggregate, ptr %value.home, align 8 diff --git a/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll b/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll new file mode 100644 index 0000000000000..3be5f938c7b75 --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/preallocated-nomerge.ll @@ -0,0 +1,38 @@ +; RUN: opt -passes='default' -S %s | FileCheck %s + +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare void @left_callee(ptr preallocated(i32)) +declare void @right_callee(ptr preallocated(i32)) + +; Each setup owns one physical outgoing call frame. SimplifyCFG must not merge +; identical setup/arg pairs from mutually exclusive branches and then let two +; calls consume the same token. +define void @branch_local_call_frames(i1 %condition) { +; CHECK-LABEL: define void @branch_local_call_frames( +; CHECK: %left.setup = {{(tail )?}}call token @llvm.call.preallocated.setup(i32 1) +; CHECK: call void @left_callee +; CHECK: %right.setup = {{(tail )?}}call token @llvm.call.preallocated.setup(i32 1) +; CHECK: call void @right_callee +entry: + br i1 %condition, label %left, label %right + +left: + %left.setup = call token @llvm.call.preallocated.setup(i32 1) + %left.home = call ptr @llvm.call.preallocated.arg(token %left.setup, i32 0) preallocated(i32) + store i32 1, ptr %left.home, align 4 + call void @left_callee(ptr preallocated(i32) %left.home) + ["preallocated"(token %left.setup)] + br label %exit + +right: + %right.setup = call token @llvm.call.preallocated.setup(i32 1) + %right.home = call ptr @llvm.call.preallocated.arg(token %right.setup, i32 0) preallocated(i32) + store i32 2, ptr %right.home, align 4 + call void @right_callee(ptr preallocated(i32) %right.home) + ["preallocated"(token %right.setup)] + br label %exit + +exit: + ret void +} From 26f67e9119953d28fdf1cb36df5af41b994950e6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 22:21:45 +0800 Subject: [PATCH 4/6] [Go] Use preallocated carriers in ABI0 frame test --- llvm/test/CodeGen/Generic/go-abi0-frame.ll | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/Generic/go-abi0-frame.ll b/llvm/test/CodeGen/Generic/go-abi0-frame.ll index b8c86ceff4e4c..777556f787fc9 100644 --- a/llvm/test/CodeGen/Generic/go-abi0-frame.ll +++ b/llvm/test/CodeGen/Generic/go-abi0-frame.ll @@ -13,7 +13,8 @@ declare ptr @llvm.go.abi0.frame() ; The intrinsic denotes one mutable, aliased object spanning the two arguments ; and one result slot. It deliberately overlaps the per-value fixed objects. define goabi0 ptr @abi0_frame( - ptr byval(i64) align 8 %a, ptr byval(i64) align 8 %b) noinline { + ptr preallocated(i64) align 8 %a, + ptr preallocated(i64) align 8 %b) noinline { entry: %frame = call ptr @llvm.go.abi0.frame() ret ptr %frame From b20331e1ff6e5a5e666a3d7d5581e7b7cae894b6 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 23:27:55 +0800 Subject: [PATCH 5/6] [Statepoints] Recover calls through target result copies --- .../SelectionDAG/StatepointLowering.cpp | 35 +++++++++++++++---- .../X86/go-statepoint-stack-results.ll | 32 +++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 059e9cc45f408..fbd3dbcda1b6f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -418,6 +418,16 @@ static void reservePreviousStackSlotForValue(const Value *IncomingValue, /// reference lowered call result static SDNode *peelCallResultChain(SDNode *Node) { while (true) { + if (Node->getOpcode() == ISD::CALLSEQ_START || + Node->getOpcode() == ISD::CALLSEQ_END) + break; + // Target call nodes carry the preserved-register mask. They may also have + // explicit memory operands and no glue result, so recognize this semantic + // boundary before considering target memory-copy nodes below. + if (llvm::any_of(Node->ops(), [](SDValue Operand) { + return isa(Operand.getNode()); + })) + break; if (Node->getOpcode() == ISD::CopyFromReg) { Node = Node->getOperand(0).getNode(); continue; @@ -428,20 +438,22 @@ static SDNode *peelCallResultChain(SDNode *Node) { } // A target may represent a large memory-result copy with a custom node - // rather than a MemSDNode. Peel any non-glued node with exactly one chain - // input and one chain result. Stop at glued target call nodes. - bool HasGlueResult = false; + // rather than a MemSDNode. Peel a single chain input/result after the + // register-mask check above has established that this is not the call. unsigned ChainResults = 0; for (unsigned I = 0; I != Node->getNumValues(); ++I) { - HasGlueResult |= Node->getValueType(I) == MVT::Glue; ChainResults += Node->getValueType(I) == MVT::Other; } - if (Node->getOpcode() != ISD::TokenFactor && !HasGlueResult) { + if (Node->getOpcode() != ISD::TokenFactor) { SDNode *ChainInput = nullptr; unsigned ChainInputs = 0; for (SDValue Operand : Node->ops()) { if (Operand.getValueType() != MVT::Other) continue; + // Target nodes may use an MVT::Other VTSDNode as an immediate type + // descriptor (for example X86ISD::REP_MOVS). It is not a chain edge. + if (isa(Operand.getNode())) + continue; ChainInput = Operand.getNode(); ++ChainInputs; } @@ -951,6 +963,17 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Glue is always last operand Glue = CallNode->getOperand(CallNode->getNumOperands() - 1); } + unsigned FixedCallOperands = CallHasIncomingGlue ? 4 : 3; + if (CallNode->getNumOperands() < FixedCallOperands) + report_fatal_error( + Twine("statepoint call chain did not resolve to a call in ") + + DAG.getMachineFunction().getName()); + unsigned RegMaskOperand = + CallNode->getNumOperands() - (CallHasIncomingGlue ? 2 : 1); + if (!isa(CallNode->getOperand(RegMaskOperand).getNode())) + report_fatal_error( + Twine("statepoint call chain did not resolve to a call in ") + + DAG.getMachineFunction().getName()); // Build the GC_TRANSITION_START node if necessary. // @@ -1001,7 +1024,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Calculate and push starting position of vmstate arguments // Get number of arguments incoming directly into call node unsigned NumCallRegArgs = - CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3); + CallNode->getNumOperands() - FixedCallOperands; Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32)); // Add call target diff --git a/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll b/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll index 0b383ba9a4858..793e45632bbb9 100644 --- a/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll +++ b/llvm/test/CodeGen/X86/go-statepoint-stack-results.ll @@ -5,8 +5,12 @@ ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr, ptr } +%memory_result = type [722 x i64] declare goabiinternal %results @overflow_results() #0 +declare goabiinternal void @two_memory_results( + ptr goret(%memory_result) "goretindex"="0" align 8, + ptr goret(%memory_result) "goretindex"="1" align 8) define void @statepoint_with_register_and_stack_results() gc "statepoint-example" { @@ -33,6 +37,34 @@ entry: ; CHECK-DAG: movq 8(%rsp), [[STACK1:%r[a-z0-9]+]] declare void @use(ptr) +declare void @use_i64(i64) + +define goabiinternal void @statepoint_with_two_memory_results() + gc "statepoint-example" { +entry: + %first = alloca %memory_result, align 8 + %second = alloca %memory_result, align 8 + %token = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, + ptr elementtype(void (ptr, ptr)) @two_memory_results, + i32 2, i32 0, + ptr goret(%memory_result) "goretindex"="0" align 8 %first, + ptr goret(%memory_result) "goretindex"="1" align 8 %second, + i32 0, i32 0) + %first_value = load i64, ptr %first, align 8 + %second_value = load i64, ptr %second, align 8 + call void @use_i64(i64 %first_value) + call void @use_i64(i64 %second_value) + ret void +} + +; CHECK-LABEL: statepoint_with_two_memory_results: +; CHECK: callq two_memory_results +; CHECK-COUNT-2: rep;movsq +; CHECK: callq use_i64 +; CHECK: callq use_i64 + declare token @llvm.experimental.gc.statepoint.p0( i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) declare %results @llvm.experimental.gc.result.results(token) From 5f9499711cfd85c7d0fb64e9cf4b75fb1573682d Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 23:27:55 +0800 Subject: [PATCH 6/6] [Go] Use preallocated carrier in nosplit ABI test --- llvm/test/CodeGen/Generic/goobj-nosplit.ll | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/llvm/test/CodeGen/Generic/goobj-nosplit.ll b/llvm/test/CodeGen/Generic/goobj-nosplit.ll index b711fa91d3352..010502bdce050 100644 --- a/llvm/test/CodeGen/Generic/goobj-nosplit.ll +++ b/llvm/test/CodeGen/Generic/goobj-nosplit.ll @@ -7,7 +7,10 @@ ; RUN: llc -mtriple=x86_64-unknown-linux-goobj -filetype=obj -o %t.x86.o %s declare goabiinternal void @callee(ptr) -declare goabi0 void @"callee.abi0"(ptr byval(ptr) align 8) +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare goabi0 void @"callee.abi0"( + ptr preallocated(ptr) align 8) define goabiinternal void @nosplit(ptr %pointer) "go-nosplit" { entry: @@ -30,12 +33,15 @@ entry: %closure = alloca [3 x ptr], align 8 %code = getelementptr [3 x ptr], ptr %closure, i64 0, i64 0 %context = getelementptr [3 x ptr], ptr %closure, i64 0, i64 1 - %argument = getelementptr [3 x ptr], ptr %closure, i64 0, i64 2 store volatile ptr @callee, ptr %code, align 8 store volatile ptr %pointer, ptr %context, align 8 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %argument = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) + preallocated(ptr) store ptr %closure, ptr %argument, align 8 call goabi0 void @"callee.abi0"( - ptr byval(ptr) align 8 %argument) + ptr preallocated(ptr) align 8 %argument) + ["preallocated"(token %setup)] ret void }