From cc8d2dd7657b4d7662098282c169425b24c5e0ad Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 1 Aug 2026 12:27:22 +0800 Subject: [PATCH] cmd/llvmplugin: describe pointer allocas as memory roots --- src/cmd/internal/testdir/llvm_alloca_test.go | 50 ++- src/cmd/llvmplugin/CMakeLists.txt | 35 ++ src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp | 30 +- src/cmd/llvmplugin/GoALLCStatepoints.cpp | 192 ++++++--- src/cmd/llvmplugin/README.md | 78 ++-- .../alloca-pointer-nonentry-unsupported.ll | 19 + .../testdata/alloca-pointer-roots.ll | 95 ++++- .../alloca-pointer-select-unsupported.ll | 14 + .../testdata/check-alloca-pointer-roots.py | 392 +++++++++++------- .../testdata/check-alloca-ptrmap-malformed.py | 110 +++++ test/abi/llvm_alloca_statepoint.go | 6 +- test/llvm_alloca_statepoint_gc.go | 40 ++ 12 files changed, 806 insertions(+), 255 deletions(-) create mode 100644 src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll create mode 100644 src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll create mode 100644 src/cmd/llvmplugin/testdata/check-alloca-ptrmap-malformed.py diff --git a/src/cmd/internal/testdir/llvm_alloca_test.go b/src/cmd/internal/testdir/llvm_alloca_test.go index 03d5fdbae6cf2d..e5566501bbed0b 100644 --- a/src/cmd/internal/testdir/llvm_alloca_test.go +++ b/src/cmd/internal/testdir/llvm_alloca_test.go @@ -67,20 +67,29 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { "-filetype=null", "-o", "-", goallcIR) rewrittenFunction := llvmAllocaIRFunction(t, rewrittenIR, "p.localAcrossSafepoints") - // Four static ordinary call sites each keep all four pointer leaves live. - // The canonical loads are marked so SelectionDAG must use the corresponding - // alloca subslot rather than silently allocating a separate spill. - if got, want := bytes.Count(rewrittenFunction, - []byte("!llvm.statepoint.fixed_stack_home")), 16; got != want { - t.Fatalf("fixed-stack-home loads=%d, want %d\n%s", got, want, rewrittenFunction) + // Four ordinary call sites describe the same 40-byte alloca as memory in a + // deopt suffix. Its four pointer leaves are bits 0, 2, 3, and 4 (0b11101). + // They must not become SSA roots or be written back after a mutating callee. + if bytes.Contains(rewrittenFunction, []byte("llvm.statepoint.fixed_stack_home")) { + t.Fatalf("obsolete fixed-stack-home metadata survived\n%s", rewrittenFunction) } if got, want := bytes.Count(rewrittenFunction, []byte("@llvm.experimental.gc.statepoint")), 4; got != want { t.Fatalf("ordinary statepoints=%d, want %d\n%s", got, want, rewrittenFunction) } + if got, want := bytes.Count(rewrittenFunction, []byte(`"deopt"(`)), 4; got != want { + t.Fatalf("alloca deopt records=%d, want %d\n%s", got, want, rewrittenFunction) + } if got, want := bytes.Count(rewrittenFunction, - []byte("@llvm.experimental.gc.relocate")), 16; got != want { - t.Fatalf("alloca relocates=%d, want %d\n%s", got, want, rewrittenFunction) + []byte("i64 40, i64 8, i64 8, i64 5, i64 64, i64 1, i64 29")), 4; got != want { + t.Fatalf("alloca bitmap payloads=%d, want %d\n%s", got, want, rewrittenFunction) + } + if bytes.Contains(rewrittenFunction, []byte(`"gc-live"`)) || + bytes.Contains(rewrittenFunction, []byte("@llvm.experimental.gc.relocate")) { + t.Fatalf("alloca leaves became SSA roots\n%s", rewrittenFunction) + } + if got, want := bytes.Count(rewrittenFunction, []byte("store ptr null")), 4; got != want { + t.Fatalf("alloca pointer initializers=%d, want %d\n%s", got, want, rewrittenFunction) } runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") @@ -99,6 +108,11 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { got, want, machineFunction) } for _, statepoint := range ordinaryStatepoints { + for _, constant := range []string{"1195461697", "1347703373", "40", "29", "1095519299"} { + if !bytes.Contains(statepoint, []byte(constant)) { + t.Fatalf("STATEPOINT lost alloca contract constant %s: %s", constant, statepoint) + } + } stackObject := regexp.MustCompile(`(%stack\.[^,\s]+)`).FindSubmatch(statepoint) if len(stackObject) != 2 { t.Fatalf("STATEPOINT has no alloca frame index: %s", statepoint) @@ -109,14 +123,24 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { object, stackObject[1], statepoint) } } - for _, offset := range []string{"0", "16", "24", "32"} { - pattern := regexp.QuoteMeta(string(stackObject[1])) + `,\s+` + offset + `(?:\D|$)` - if !regexp.MustCompile(pattern).Match(statepoint) { - t.Fatalf("STATEPOINT does not reuse alloca offset %s: %s", offset, statepoint) - } + pattern := `0,\s+` + regexp.QuoteMeta(string(stackObject[1])) + `,\s+0` + if !regexp.MustCompile(pattern).Match(statepoint) { + t.Fatalf("STATEPOINT does not carry the direct alloca frame base: %s", statepoint) } } + goallcAssembly := runLLVMABICommand(t, nil, llc, + "-load-pass-plugin="+plugin, "-verify-machineinstrs", + "-filetype=asm", "-o", "-", goallcIR) + betweenCalls := regexp.MustCompile(`(?s)\bbl\s+p\.mutateLocal\n(.*?)\bbl\s+p\.safepoint`). + FindSubmatch(goallcAssembly) + if len(betweenCalls) != 2 { + t.Fatalf("GoALLC assembly has no adjacent mutateLocal/safepoint calls\n%s", goallcAssembly) + } + if regexp.MustCompile(`(?m)^\s*(?:str|stp)\b`).Match(betweenCalls[1]) { + t.Fatalf("GoALLC restored an alloca field after mutateLocal:\n%s", betweenCalls[1]) + } + runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, "-filetype=obj", goallcIR, "-o", goallcObject) symbol := findLLVMABISymbol(t, readLLVMABIObject(t, goallcObject), diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 1e6e425f761339..5e0c6254bb612e 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -370,6 +370,41 @@ if(BUILD_TESTING) "${CMAKE_CURRENT_SOURCE_DIR}/testdata/alloca-pointer-roots.ll" ) + add_test( + NAME GoALLCStatepoints.MalformedAllocaPointerMapsFail + COMMAND + "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/check-alloca-ptrmap-malformed.py" + --llc "${GOALLC_LLC_EXECUTABLE}" + --plugin "$" + ) + + add_test( + NAME GoALLCStatepoints.PointerAllocaSelectFails + COMMAND + "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/check-statepoint-failure.py" + --llc "${GOALLC_LLC_EXECUTABLE}" + --plugin "$" + --input + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/alloca-pointer-select-unsupported.ll" + --contains + "require pointer-containing alloca PHI/select addresses to resolve to one zero-offset alloca" + ) + + add_test( + NAME GoALLCStatepoints.NonEntryPointerAllocaFails + COMMAND + "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/check-statepoint-failure.py" + --llc "${GOALLC_LLC_EXECUTABLE}" + --plugin "$" + --input + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/alloca-pointer-nonentry-unsupported.ll" + --contains + "require a single fixed entry-block pointer-containing alloca" + ) + add_test( NAME GoALLCStatepoints.DynamicPointerAllocaFails COMMAND diff --git a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp b/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp index 0df2828578c7fb..d8e88ffc55df66 100644 --- a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp +++ b/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp @@ -10,6 +10,9 @@ #include "llvm/MC/MCContext.h" #include "llvm/Support/ErrorHandling.h" +#include +#include + using namespace llvm; namespace { @@ -82,17 +85,30 @@ bool GoALLCStackMapPrinter::emitStackMaps(StackMaps &SM, AsmPrinter &AP) { getNonnegativeConstant(CSI.Locations[2], "deopt count"); if (NumDeopts > CSI.Locations.size() - 3) report_fatal_error("malformed GoALLC statepoint deopt operands"); - size_t FirstGCLocation = 3 + static_cast(NumDeopts); + if (NumDeopts > std::numeric_limits::max()) + report_fatal_error("GoALLC statepoint has too many deopt operands"); MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, CSI.ID, CSI.IsIndirectCall, Info.StackSize, - PointerSize, {}}; - Entry.Locations.reserve(CSI.Locations.size() - FirstGCLocation); + PointerSize, + static_cast(NumDeopts), + {}}; + Entry.Locations.reserve(CSI.Locations.size() - 3); for (const StackMaps::Location &Location : - ArrayRef(CSI.Locations).drop_front(FirstGCLocation)) { - Entry.Locations.push_back({convertLocationType(Location.Type), - Location.Size, Location.Reg, - Location.Offset}); + ArrayRef(CSI.Locations).drop_front(3)) { + auto Type = convertLocationType(Location.Type); + int64_t Offset = Location.Offset; + if (Location.Type == StackMaps::Location::Constant || + Location.Type == StackMaps::Location::ConstantIndex) { + std::optional Constant = SM.getConstantValue(Location); + if (!Constant) + report_fatal_error( + "GoALLC statepoint contains an invalid constant-pool index"); + Type = MCContext::GoObjStackMapLocation::Constant; + Offset = *Constant; + } + Entry.Locations.push_back( + {Type, Location.Size, Location.Reg, Offset}); } AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry)); } diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 072f5cb4212254..35be200bd4b564 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -10,6 +10,7 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/BinaryFormat/GoObj.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" @@ -37,7 +38,6 @@ namespace { constexpr StringLiteral GoALLCGCName = "goallc"; constexpr StringLiteral GCLeafAttr = "gc-leaf-function"; constexpr StringLiteral GoResultsTupleAttr = "go_results_tuple"; -constexpr StringLiteral FixedStackHomeMD = "llvm.statepoint.fixed_stack_home"; constexpr StringLiteral GoNilCheckMD = "goallc.nilcheck"; // This strategy exists for statepoint verification and lowering. GoALLC owns @@ -93,6 +93,14 @@ struct PointerAllocaLeaf { PointerType *Type; }; +struct PointerAllocaRecord { + AllocaInst *Alloca; + uint64_t ByteSize; + uint64_t Alignment; + uint64_t BitCount; + SmallVector BitmapWords; +}; + enum class LivenessKind { PointerAggregates, ScalarPointers, @@ -117,6 +125,11 @@ bool containsPointer(Type *Ty) { bool isStaticAllocaAddress(const Value *V) { if (!V->getType()->isPointerTy()) return false; + // findAllocaForValue understands PHI/select cycles and proves that every + // path is the same zero-offset alloca. Keep the ordinary underlying-object + // path as well so constant nonzero GEPs remain frame addresses. + if (const AllocaInst *Alloca = findAllocaForValue(V, /*OffsetZero=*/true)) + return Alloca->isStaticAlloca(); const Value *Object = getUnderlyingObject(V); const auto *Alloca = dyn_cast(Object); return Alloca && Alloca->isStaticAlloca(); @@ -509,8 +522,9 @@ Error validatePointerAllocaAccesses(AllocaInst &Alloca, Function &F) { if (auto *Intrinsic = dyn_cast(&I); Intrinsic && Intrinsic->isLifetimeStartOrEnd() && Intrinsic->arg_size() != 0 && - getUnderlyingObject( - Intrinsic->getArgOperand(Intrinsic->arg_size() - 1)) == &Alloca) + findAllocaForValue( + Intrinsic->getArgOperand(Intrinsic->arg_size() - 1), + /*OffsetZero=*/false) == &Alloca) return createStringError( std::errc::not_supported, "GoALLC statepoints do not support lifetime markers on " @@ -537,7 +551,8 @@ Error validatePointerAllocaAccesses(AllocaInst &Alloca, Function &F) { Address = CmpXchg->getPointerOperand(); UnsupportedAccess = true; } - if (Address && UnsupportedAccess && getUnderlyingObject(Address) == &Alloca) + if (Address && UnsupportedAccess && + findAllocaForValue(Address, /*OffsetZero=*/false) == &Alloca) return createStringError( std::errc::not_supported, "GoALLC statepoints do not support volatile or atomic access to " @@ -546,7 +561,8 @@ Error validatePointerAllocaAccesses(AllocaInst &Alloca, Function &F) { return Error::success(); } -Error normalizePointerAllocas(Function &F) { +Error collectPointerAllocas(Function &F, + SmallVectorImpl &Records) { bool HasSafepoint = llvm::any_of(instructions(F), [](Instruction &I) { auto *Call = dyn_cast(&I); return Call && !isa(Call) && !isLeafCall(*Call); @@ -554,20 +570,6 @@ Error normalizePointerAllocas(Function &F) { if (!HasSafepoint) return Error::success(); - SmallVector Calls; - for (Instruction &I : instructions(F)) { - auto *Call = dyn_cast(&I); - if (!Call || isa(Call) || isLeafCall(*Call)) - continue; - auto *OrdinaryCall = dyn_cast(Call); - if (!OrdinaryCall) - continue; - if (OrdinaryCall->isMustTailCall()) - return createStringError(std::errc::not_supported, - "GoALLC statepoints do not support musttail"); - Calls.push_back(OrdinaryCall); - } - const DataLayout &DL = F.getDataLayout(); SmallVector>, 8> PointerAllocas; @@ -576,7 +578,9 @@ Error normalizePointerAllocas(Function &F) { if (!Alloca || !containsPointer(Alloca->getAllocatedType())) continue; auto *ArraySize = dyn_cast(Alloca->getArraySize()); - if (!Alloca->isStaticAlloca() || !ArraySize || !ArraySize->isOne()) + if (!Alloca->isStaticAlloca() || + Alloca->getParent() != &F.getEntryBlock() || !ArraySize || + !ArraySize->isOne()) return createStringError( std::errc::not_supported, "GoALLC statepoints require a single fixed entry-block " @@ -601,10 +605,60 @@ Error normalizePointerAllocas(Function &F) { if (Error Err = enumeratePointerAllocaLeaves(Alloca->getAllocatedType(), DL, Path, 0, Leaves)) return std::move(Err); + uint64_t ByteSize = AllocationSize->getFixedValue(); + uint64_t PointerSize = DL.getPointerSize(0); + if (!PointerSize || ByteSize == 0 || ByteSize % PointerSize != 0 || + Alloca->getAlign() < DL.getABITypeAlign(Alloca->getAllocatedType())) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require pointer-aligned fixed alloca layouts"); + uint64_t BitCount = ByteSize / PointerSize; + SmallVector BitmapWords((BitCount + 63) / 64, 0); + for (const PointerAllocaLeaf &Leaf : Leaves) { + if (Leaf.Offset % PointerSize != 0 || Leaf.Offset >= ByteSize) + return createStringError( + std::errc::not_supported, + "GoALLC statepoint alloca pointer slot is not pointer-aligned"); + uint64_t Bit = Leaf.Offset / PointerSize; + uint64_t Mask = uint64_t(1) << (Bit % 64); + if (BitmapWords[Bit / 64] & Mask) + return createStringError( + std::errc::invalid_argument, + "GoALLC statepoint alloca pointer slots overlap"); + BitmapWords[Bit / 64] |= Mask; + } + Records.push_back({Alloca, ByteSize, Alloca->getAlign().value(), BitCount, + std::move(BitmapWords)}); PointerAllocas.push_back({Alloca, std::move(Leaves)}); } - MDNode *FixedHome = MDNode::get(F.getContext(), {}); + auto IsEligiblePointerAlloca = [&](const AllocaInst *Candidate) { + return llvm::any_of(PointerAllocas, [&](const auto &Entry) { + return Entry.first == Candidate; + }); + }; + for (Instruction &I : instructions(F)) { + if (!I.getType()->isPointerTy() || + (!isa(I) && !isa(I))) + continue; + bool TouchesPointerAlloca = llvm::any_of(I.operands(), [&](Value *Input) { + const AllocaInst *Alloca = + findAllocaForValue(Input, /*OffsetZero=*/false); + return Alloca && IsEligiblePointerAlloca(Alloca); + }); + if (!TouchesPointerAlloca) + continue; + const AllocaInst *Alloca = findAllocaForValue(&I, /*OffsetZero=*/true); + if (!Alloca || !IsEligiblePointerAlloca(Alloca)) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require pointer-containing alloca PHI/select " + "addresses to resolve to one zero-offset alloca"); + } + + // Pointer-containing Go stack objects must start with zero pointer words. + // This is object initialization, not statepoint root spilling: there is no + // per-call load, relocation, or write-back in the alloca ptrmap model. for (auto &[Alloca, Leaves] : PointerAllocas) { IRBuilder<> InitBuilder(Alloca->getNextNode()); InitBuilder.SetCurrentDebugLocation(Alloca->getDebugLoc()); @@ -618,34 +672,6 @@ Error normalizePointerAllocas(Function &F) { } } - for (CallInst *Call : Calls) { - IRBuilder<> Before(Call); - Before.SetCurrentDebugLocation(Call->getDebugLoc()); - Instruction *InsertBefore = Call->getNextNode(); - IRBuilder<> After(InsertBefore); - After.SetCurrentDebugLocation(Call->getDebugLoc()); - for (auto &[Alloca, Leaves] : PointerAllocas) { - for (const PointerAllocaLeaf &Leaf : Leaves) { - std::string Name = allocaLeafName(*Alloca, Leaf); - Align Alignment = commonAlignment(Alloca->getAlign(), Leaf.Offset); - Value *BeforeAddress = - pointerAllocaLeafAddress(Before, *Alloca, Leaf, Name + ".pre.addr"); - LoadInst *Root = Before.CreateAlignedLoad(Leaf.Type, BeforeAddress, - Alignment, Name + ".root"); - // Keep each memory root as a distinct SelectionDAG value. A plain load - // can be forwarded from an earlier store or folded with another leaf - // (notably when zero initialization makes both values null), losing - // the one-to-one mapping between the root and its fixed stack home. - // Volatile preserves that identity without introducing another spill - // slot; LowerStatepoint still records the original alloca FI+offset. - Root->setVolatile(true); - Root->setMetadata(FixedStackHomeMD, FixedHome); - Value *AfterAddress = - pointerAllocaLeafAddress(After, *Alloca, Leaf, Name + ".post.addr"); - After.CreateAlignedStore(Root, AfterAddress, Alignment); - } - } - } return Error::success(); } @@ -658,17 +684,21 @@ Error validateSafepoint(const SafepointRecord &Record) { if (Call.isMustTailCall()) return createStringError(std::errc::not_supported, "GoALLC statepoints do not support musttail"); - if (Call.getNumOperandBundles() != 0) + if (Call.getNumOperandBundles() != 0 && + (Call.getNumOperandBundles() != 1 || + !Call.getOperandBundle(LLVMContext::OB_deopt))) return createStringError( std::errc::not_supported, - "GoALLC statepoints do not yet support call operand bundles"); + "GoALLC statepoints only support a single deopt call operand bundle"); for (unsigned I = 0; I != Call.arg_size(); ++I) { for (Attribute Attr : Call.getAttributes().getParamAttrs(I)) { - if (!Attr.hasAttribute(Attribute::Nest)) + if (!Attr.hasAttribute(Attribute::Nest) && + !Attr.hasAttribute(Attribute::Captures) && + !Attr.hasAttribute(Attribute::ReadOnly)) return createStringError( std::errc::not_supported, - "GoALLC statepoints only support the nest call parameter " - "attribute"); + "GoALLC statepoints only support nest, captures, and readonly " + "call parameter attributes"); } } for (Value *V : Record.Live) { @@ -680,17 +710,62 @@ Error validateSafepoint(const SafepointRecord &Record) { return Error::success(); } -Error rewriteCall(SafepointRecord &Record) { +void appendAllocaPtrMapDeoptOperands( + IRBuilder<> &Builder, ArrayRef Allocas, + SmallVectorImpl &Deopt) { + if (Allocas.empty()) + return; + // ProtocolLength covers BEGIN through END, but not the trailing duplicate + // length. The envelope itself therefore contributes BEGIN, length, + // record-count, and END. + uint64_t ProtocolLength = 4; + for (const PointerAllocaRecord &Alloca : Allocas) + ProtocolLength += 10 + Alloca.BitmapWords.size(); + + auto AppendConstant = [&](uint64_t Value) { + Deopt.push_back(ConstantInt::get(Builder.getInt64Ty(), Value)); + }; + AppendConstant(GoObj::AllocaPtrMapBeginMagic); + AppendConstant(ProtocolLength); + AppendConstant(Allocas.size()); + for (const PointerAllocaRecord &Alloca : Allocas) { + AppendConstant(GoObj::AllocaPtrMapRecordTag); + AppendConstant(10 + Alloca.BitmapWords.size()); + Deopt.push_back(Alloca.Alloca); + AppendConstant(0); // First contract version describes the whole alloca. + AppendConstant(Alloca.ByteSize); + AppendConstant(Alloca.Alignment); + AppendConstant(Alloca.Alloca->getDataLayout().getPointerSize(0)); + AppendConstant(Alloca.BitCount); + AppendConstant(GoObj::AllocaPtrMapBitmapWordBits); + AppendConstant(Alloca.BitmapWords.size()); + for (uint64_t Word : Alloca.BitmapWords) + AppendConstant(Word); + } + AppendConstant(GoObj::AllocaPtrMapEndMagic); + AppendConstant(ProtocolLength); +} + +Error rewriteCall(SafepointRecord &Record, + ArrayRef PointerAllocas) { CallInst *Call = Record.Call; SmallVector CallArgs(Call->args()); SmallVector GCLive(Record.Live.begin(), Record.Live.end()); + SmallVector Deopt; + if (auto Bundle = Call->getOperandBundle(LLVMContext::OB_deopt)) + for (const Use &Input : Bundle->Inputs) + Deopt.push_back(Input.get()); FunctionCallee Callee(Call->getFunctionType(), Call->getCalledOperand()); IRBuilder<> Builder(Call); Builder.SetCurrentDebugLocation(Call->getDebugLoc()); + appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, Deopt); Record.Statepoint = Builder.CreateGCStatepointCall( - Record.ID, 0, Callee, CallArgs, std::nullopt, GCLive, "statepoint_token"); + Record.ID, 0, Callee, CallArgs, + Deopt.empty() ? std::nullopt + : std::optional>(ArrayRef(Deopt)), + GCLive, "statepoint_token"); Record.Statepoint->setCallingConv(Call->getCallingConv()); if (Call->hasFnAttr(GoResultsTupleAttr)) Record.Statepoint->addFnAttr( @@ -830,7 +905,8 @@ Error rewriteFunction(Function &F) { return Error::success(); } - if (Error Err = normalizePointerAllocas(F)) + SmallVector PointerAllocas; + if (Error Err = collectPointerAllocas(F, PointerAllocas)) return Err; if (Error Err = scalarizeLivePointerAggregates(F)) return Err; @@ -857,7 +933,7 @@ Error rewriteFunction(Function &F) { if (Error Err = validateSafepoint(Record)) return Err; for (SafepointRecord &Record : llvm::reverse(Records)) - if (Error Err = rewriteCall(Record)) + if (Error Err = rewriteCall(Record, PointerAllocas)) return Err; repairRelocationSSA(F, DT, Records); return Error::success(); diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index af74cdc0bf8b18..7d285328847a21 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -47,7 +47,7 @@ The current SSA value and CFG rewrite support matrix is: | Value or control-flow shape | Status | Current contract | | --- | --- | --- | | Pointer arguments | AArch64 GoObj qualified; SelectionDAG home reuse also tested on X86 | Values live after a call use caller statepoints; exact stack inputs stay in their fixed ABI homes, while register inputs and transformed values use normal statepoint spills. Call-only arguments are described by the callee's type-derived entry map. | -| Static `alloca` addresses | Supported | Proven static-allocation addresses and constant GEP/cast forms are frame addresses and do not enter `gc-live`; pointers loaded from memory remain tracked. | +| Static `alloca` addresses | Supported | Proven static-allocation addresses and constant GEP/cast forms are frame addresses and do not enter `gc-live`; a PHI/select is accepted only when every path resolves to the same zero-offset alloca. Pointers loaded from memory remain tracked. | | `select`, GEP, and pointer casts | Supported | Each resulting pointer SSA value is tracked conservatively. | | Pointer-valued call results | Supported | `gc.result` replaces the ordinary result and later safepoints relocate it. | | Multiple ordinary calls | Supported | Stable IDs and live sets remain per call; the next statepoint consumes the current relocated SSA value. | @@ -55,11 +55,11 @@ The current SSA value and CFG rewrite support matrix is: | Loops and irreducible CFG | Supported | Relocation definitions are propagated through backedge and multi-entry PHIs without a shape-specific algorithm. | | Fixed struct/array SSA aggregates | Supported | Pointer leaves are scalarized before liveness and reconstructed from the current relocated SSA leaves. | | Aggregate arguments and call results | Supported for IR rewriting | The wrapped call keeps its real aggregate ABI type. Only leaves live after the call enter caller `gc-live`; supported fixed formal layouts also contribute pointer words to AArch64 entry maps. | -| Aggregate load results and store operands | Supported | First-class SSA values use aggregate normalization; fixed pointer-containing allocas additionally keep every pointer leaf synchronized around safepoints. | -| Pointer-containing `alloca` storage | Supported for IR rewriting | Every pointer leaf in a single fixed entry-block alloca is zero-initialized, loaded before each safepoint, relocated as a scalar root, and written back. The canonical load requires exact fixed-home lowering and must never fall back to a separate spill. | +| Aggregate load results and store operands | Supported | First-class SSA values use aggregate normalization. Pointer leaves stored in a fixed alloca remain memory roots and are not converted to SSA roots. | +| Pointer-containing `alloca` storage | GoObj qualified for fixed layouts | Pointer slots in a single fixed entry-block alloca are zero-initialized once. Every safepoint carries the alloca address and a 64-bit-word pointer bitmap in a self-describing deopt suffix; no leaf preload, `gc-live`, `gc.relocate`, or post-call write-back is generated. | | Fixed and scalable vectors | Unsupported | Vector lane and scalable-count semantics require a separate design; fails closed. | | General moving-GC base/derived analysis | Unsupported | Base and derived indexes are identical in the current non-moving-heap phase. | -| `invoke`, `callbr`, operand bundles, non-`nest` parameter attributes, and `musttail` | Unsupported | Fails closed rather than widening the call contract. | +| `invoke`, `callbr`, non-deopt operand bundles, unsupported parameter attributes, and `musttail` | Unsupported | One ordinary deopt bundle is preserved before the alloca suffix. `nest`, `captures`, and `readonly` parameter attributes are preserved; other shapes fail closed. | This matrix describes verified IR rewriting, not full runtime qualification. The Darwin/arm64 Go execution whitelist additionally covers one unconditional @@ -103,24 +103,38 @@ an aggregate call result is projected first and its pointer leaves become roots at later safepoints. An aggregate loaded from memory can be normalized as an independent SSA value -and an aggregate store can consume a rebuilt value. For a fixed -pointer-containing alloca, a separate memory-root normalization enumerates its -pointer leaf offsets. It first zeroes those slots, then inserts a canonical load -before every ordinary call and a matching write-back after it. The existing -whole-function relocation SSA rewrite turns the stored pre-call definition -into the corresponding `gc.relocate` result on the returning edge. The -canonical load is volatile so SelectionDAG cannot fold distinct field homes -together, and carries `!llvm.statepoint.fixed_stack_home`; SelectionDAG must map -it to the original alloca frame index and byte offset or fail closed. -Falling back to an extra spill would be incorrect when the callee updates the -address-passed object. +and an aggregate store can consume a rebuilt value. A fixed pointer-containing +alloca follows a different memory-root contract. The plugin enumerates its +pointer offsets, zeroes those pointer slots once, and appends one record per +alloca to every ordinary statepoint's deopt operands. The record carries the +alloca address, whole-object size and alignment, pointer size, valid bitmap bit +count, and 64-bit bitmap words. The envelope and every record carry explicit +lengths, and a trailing duplicate envelope length makes the suffix recoverable +after ordinary deopt operands. There is deliberately no contract version in +the first grammar. + +LowerStatepoint lowers the deopt alloca address through its normal direct +FrameIndex path and preserves the adjacent constants in Machine StackMaps. It +does not create a root spill. The Go-owned StackMaps bridge retains the deopt +prefix and resolves both inline constants and `ConstantIndex` values. The +GoObj writer strictly parses the suffix, maps each direct frame location plus +bitmap bit to `LocalsPointerMaps`, and fails closed on malformed lengths, +non-direct bases, inconsistent layout, nonzero padding, duplicates, overlaps, +or locations outside the GC locals range. Runtime and the Go linker see only +ordinary locals pointer maps and `PCDATA_StackMapIndex`; the deopt grammar does +not escape the object writer. + +The alloca memory remains the source of truth while the call executes. A +callee may clear or replace fields through an address argument, and the caller +observes those changes because no relocated pre-call SSA value is written back. This first implementation conservatively records every pointer leaf at every safepoint for the lifetime of the frame. It may therefore retain otherwise dead heap objects longer than native Go's reachability-sensitive stack-object metadata, but its locals pointer map is functionally sufficient for scanning -and relocation. Constants are not roots in the general SSA model; canonical -alloca roots are loads even when the slot currently contains null. +and relocation. Constants are not roots in the general SSA model. Alloca +records describe the memory layout even when a pointer slot currently contains +null. Address passing to a callee is supported. A volatile byte load carrying the compiler-owned empty `!goallc.nilcheck` marker is recognized as an SSA `OpNilCheck` and remains in place for its faulting semantics; this does not @@ -137,14 +151,15 @@ narrow Go ABI contract above. The final combined order is: -1. **Fixed alloca memory-root normalization.** Enumerate fixed pointer leaves, - zero them before the first safepoint, and insert canonical load/write-back - pairs around every ordinary call. +1. **Fixed alloca memory-root description.** Enumerate fixed pointer leaves, + zero them at object initialization, and construct a whole-alloca bitmap + record without inserting per-call memory traffic. 2. **Aggregate normalization.** Use aggregate-only liveness to find and decompose supported live first-class struct/array values, then rebuild aggregates immediately before their uses. 3. **Scalar statepoint insertion.** Compute liveness, build scalar-only - `gc-live` bundles, and emit `gc.result` and `gc.relocate`. + `gc-live` bundles, append the alloca records to deopt, and emit `gc.result` + and `gc.relocate` only for ordinary SSA roots. 4. **Whole-function relocation SSA.** Model the original scalar definition and every relocate as stores to temporary promotable allocas, rewrite old uses through loads, and call the public `PromoteMemToReg` utility. This constructs @@ -171,9 +186,11 @@ notice; BSD-only Go source files should not silently absorb copied code. `GoALLCStackMapPrinter.cpp` is the Go-owned boundary between LLVM Machine StackMaps and GoObj. It uses the standard -`AsmPrinter -> GCMetadataPrinter::emitStackMaps` hook and copies only raw -machine locations into `MCContext`. LLVM's generic `StackMaps.cpp` has no -GoALLC or GoObj branch. LLVM records GoObj statepoint callsites at the CALL +`AsmPrinter -> GCMetadataPrinter::emitStackMaps` hook, retains deopt and GC +locations in `MCContext`, and resolves StackMaps constant-pool indexes without +adding a parallel serializer. LLVM's generic `StackMaps.cpp` only exposes the +constant resolver; it has no GoALLC grammar or GoObj policy. LLVM records GoObj +statepoint callsites at the CALL start, matching Go's `PCDATA_StackMapIndex` convention without a command-line mode. The frontend's stack-growth attribute asks LLVM to express the late-generated `runtime.morestack` call as a physical @@ -196,9 +213,10 @@ The GoObj writer interprets locations after final layout. An `Indirect [SP+offset]` location in the current frame contributes a locals pointer bit; one in the post-prologue caller-owned argument/result area contributes an args pointer bit. A `Direct SP+offset` stack address contributes -to neither bitmap because the address itself, rather than the slot contents, -is the pointer. Static alloca addresses are excluded from `gc-live` before this -point; only their loaded pointer leaves request indirect fixed-home locations. +to neither bitmap as an ordinary GC root because the address itself, rather +than the slot contents, is the pointer. Inside a validated alloca deopt record, +the same direct location is instead the frame base whose bitmap selects memory +slots. Static alloca addresses are excluded from `gc-live` before this point. Ordinary stack inputs use the same statepoint path. SelectionDAG formal lowering records a value home only when a Go ABI pointer part is a direct, @@ -240,7 +258,11 @@ This phase fails closed for unsupported formal aggregate layouts; dynamic or realigned frames; raw register roots; pointer vectors or ABI layouts whose pointer words do not map to fixed homes; and ordinary statepoint stack locations outside either the GC locals range or the adjusted caller-owned -argument/result range. +argument/result range. Pointer-containing allocas additionally fail closed for +dynamic or multiple-element allocation, scalable or realigned layout, +non-default-address-space pointers, pointer vectors, lifetime markers, +volatile or atomic access, and any record that does not resolve to one unique +direct frame object at offset zero. `FUNCDATA_StackObjects` and `PCDATA_ArgLiveIndex` are not implemented. Pointer-containing fixed allocas are represented conservatively in `LocalsPointerMaps`; the alloca address itself is never treated as the root. diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll new file mode 100644 index 00000000000000..8e4711f0e797ae --- /dev/null +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll @@ -0,0 +1,19 @@ +target triple = "x86_64-unknown-linux-goobj" + +declare goabiinternal void @safepoint() + +define goabiinternal void @nonentry_pointer_alloca( + i1 %allocate) "go-stack-growth-statepoint" gc "goallc" { +entry: + call goabiinternal void @safepoint() + br i1 %allocate, label %allocate.block, label %exit + +allocate.block: + %slot = alloca ptr, align 8 + store ptr null, ptr %slot, align 8 + call goabiinternal void @safepoint() + br label %exit + +exit: + ret void +} diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index a8dade6e0790da..0c7d3dbe86103a 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -2,16 +2,22 @@ target triple = "x86_64-unknown-linux-goobj" %nested = type { ptr, i64, [2 x { i32, ptr }] } %pointer_field = type { i64, ptr } +%high_bitmap = type { [63 x i64], ptr } declare goabiinternal void @safepoint() declare goabiinternal void @mutate_pointer_slot(ptr) +declare goabiinternal void @mutate_nocapture(ptr captures(none)) +declare goabiinternal void @escape_pointer_slot(ptr) +declare goabiinternal void @unknown_writing() +declare goabiinternal i64 @readonly_pointer_slot(ptr readonly) memory(read) +declare goabiinternal i64 @readnone_callee() memory(none) define goabiinternal ptr @pointer_slot(ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 %nilcheck = load volatile i8, ptr %slot, align 1, !goallc.nilcheck !0 - call goabiinternal void @safepoint() + call goabiinternal void @safepoint() [ "deopt"(i64 7) ] %result = load ptr, ptr %slot, align 8 ret ptr %result } @@ -103,4 +109,91 @@ entry: ret void } +define goabiinternal ptr @alloca_high_bitmap_word( + ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca %high_bitmap, align 8 + %field = getelementptr inbounds %high_bitmap, ptr %slot, i32 0, i32 1 + store ptr %pointer, ptr %field, align 8 + call goabiinternal void @safepoint() + %result = load ptr, ptr %field, align 8 + ret ptr %result +} + +define goabiinternal ptr @alloca_multiple_records( + ptr %first, ptr %second) "go-stack-growth-statepoint" gc "goallc" { +entry: + %left = alloca ptr, align 8 + %right = alloca ptr, align 8 + store ptr %first, ptr %left, align 8 + store ptr %second, ptr %right, align 8 + call goabiinternal void @safepoint() + %result = load ptr, ptr %left, align 8 + ret ptr %result +} + +define goabiinternal ptr @alloca_select_same_base( + i1 %choose, ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca ptr, align 8 + %same = getelementptr inbounds i8, ptr %slot, i64 0 + %selected = select i1 %choose, ptr %slot, ptr %same + store ptr %pointer, ptr %selected, align 8 + call goabiinternal void @safepoint() + %result = load ptr, ptr %selected, align 8 + ret ptr %result +} + +define goabiinternal ptr @alloca_nocapture_writable( + ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca ptr, align 8 + store ptr %pointer, ptr %slot, align 8 + call goabiinternal void @mutate_nocapture(ptr captures(none) %slot) + %result = load ptr, ptr %slot, align 8 + ret ptr %result +} + +define goabiinternal ptr @alloca_escaped_before_unknown_write( + ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca ptr, align 8 + store ptr %pointer, ptr %slot, align 8 + call goabiinternal void @escape_pointer_slot(ptr %slot) + call goabiinternal void @unknown_writing() + %result = load ptr, ptr %slot, align 8 + ret ptr %result +} + +define goabiinternal i64 @alloca_readonly_and_readnone( + ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca ptr, align 8 + store ptr %pointer, ptr %slot, align 8 + %read = call goabiinternal i64 @readonly_pointer_slot(ptr readonly %slot) + %pure = call goabiinternal i64 @readnone_callee() + %result = load ptr, ptr %slot, align 8 + %bits = ptrtoint ptr %result to i64 + %sum.0 = add i64 %read, %pure + %sum.1 = add i64 %sum.0, %bits + ret i64 %sum.1 +} + +@llvm.used = appending global [14 x ptr] [ + ptr @pointer_slot, + ptr @nested_whole_aggregate, + ptr @alloca_call_skip, + ptr @alloca_multiple_calls, + ptr @alloca_loop, + ptr @alloca_gep_address_across_call, + ptr @alloca_address_passed_to_callee, + ptr @alloca_uninitialized_at_safepoint, + ptr @alloca_high_bitmap_word, + ptr @alloca_multiple_records, + ptr @alloca_select_same_base, + ptr @alloca_nocapture_writable, + ptr @alloca_escaped_before_unknown_write, + ptr @alloca_readonly_and_readnone +], section "llvm.metadata" + !0 = !{} diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll new file mode 100644 index 00000000000000..c66e005c806494 --- /dev/null +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll @@ -0,0 +1,14 @@ +target triple = "x86_64-unknown-linux-goobj" + +declare goabiinternal void @safepoint() + +define goabiinternal void @select_different_pointer_allocas( + i1 %choose) "go-stack-growth-statepoint" gc "goallc" { +entry: + %left = alloca ptr, align 8 + %right = alloca ptr, align 8 + %selected = select i1 %choose, ptr %left, ptr %right + store ptr null, ptr %selected, align 8 + call goabiinternal void @safepoint() + ret void +} diff --git a/src/cmd/llvmplugin/testdata/check-alloca-pointer-roots.py b/src/cmd/llvmplugin/testdata/check-alloca-pointer-roots.py index 6a97c4045dc28a..08597ee9d6921c 100644 --- a/src/cmd/llvmplugin/testdata/check-alloca-pointer-roots.py +++ b/src/cmd/llvmplugin/testdata/check-alloca-pointer-roots.py @@ -4,13 +4,32 @@ import re import subprocess import sys +import tempfile + + +BEGIN = 1195461697 +TAG = 1347703373 +END = 1095519299 +WORD_BITS = 64 def fail(message): - print(f"alloca pointer root check failed: {message}", file=sys.stderr) + print(f"alloca pointer-map check failed: {message}", file=sys.stderr) raise SystemExit(1) +def run(command, *, input_text=None): + result = subprocess.run( + command, input=input_text, capture_output=True, text=True + ) + if result.returncode != 0: + fail( + f"command failed ({' '.join(command)}):\n" + f"{result.stdout}{result.stderr}" + ) + return result.stdout + result.stderr + + def function_body(ir, name): match = re.search( rf"define goabiinternal [^@]+@{name}\b.*?^}}", @@ -22,9 +41,169 @@ def function_body(ir, name): return match.group(0) -def require(text, pattern, description): - if not re.search(pattern, text, re.MULTILINE | re.DOTALL): - fail(f"missing {description}: /{pattern}/") +def deopt_bundles(function): + return re.findall(r'"deopt"\((.*?)\)', function) + + +def parse_i64(token, description): + match = re.fullmatch(r"i64 (-?[0-9]+)", token) + if not match: + fail(f"{description} is not an i64 constant: {token}") + return int(match.group(1)) + + +def parse_protocol(bundle): + tokens = [token.strip() for token in bundle.split(",")] + if len(tokens) < 5: + fail(f"truncated protocol: {bundle}") + protocol_length = parse_i64(tokens[-1], "trailing protocol length") + protocol_start = len(tokens) - protocol_length - 1 + if protocol_start < 0: + fail(f"protocol length exceeds deopt bundle: {bundle}") + if parse_i64(tokens[protocol_start], "begin magic") != BEGIN: + fail(f"wrong begin magic: {bundle}") + if parse_i64(tokens[protocol_start + 1], "protocol length") != protocol_length: + fail(f"protocol length copies disagree: {bundle}") + if parse_i64(tokens[-2], "end magic") != END: + fail(f"wrong end magic: {bundle}") + + record_count = parse_i64(tokens[protocol_start + 2], "record count") + cursor = protocol_start + 3 + records = [] + for record_index in range(record_count): + if cursor + 10 > len(tokens) - 1: + fail(f"truncated record {record_index}: {bundle}") + if parse_i64(tokens[cursor], "record tag") != TAG: + fail(f"wrong record tag {record_index}: {bundle}") + record_length = parse_i64(tokens[cursor + 1], "record length") + base = tokens[cursor + 2] + if not re.fullmatch(r"ptr %[A-Za-z0-9_.]+", base): + fail(f"record {record_index} has non-alloca base: {base}") + byte_offset = parse_i64(tokens[cursor + 3], "byte offset") + byte_size = parse_i64(tokens[cursor + 4], "byte size") + alignment = parse_i64(tokens[cursor + 5], "alignment") + pointer_size = parse_i64(tokens[cursor + 6], "pointer size") + bit_count = parse_i64(tokens[cursor + 7], "bit count") + word_bits = parse_i64(tokens[cursor + 8], "word width") + word_count = parse_i64(tokens[cursor + 9], "word count") + if record_length != 10 + word_count: + fail(f"record {record_index} length is inconsistent: {bundle}") + if cursor + record_length > len(tokens) - 2: + fail(f"record {record_index} overruns protocol: {bundle}") + words = [ + parse_i64(tokens[cursor + 10 + word], "bitmap word") + & ((1 << WORD_BITS) - 1) + for word in range(word_count) + ] + records.append( + ( + base.removeprefix("ptr %"), + byte_offset, + byte_size, + alignment, + pointer_size, + bit_count, + word_bits, + tuple(words), + ) + ) + cursor += record_length + if cursor != len(tokens) - 2: + fail(f"records do not cover protocol payload: {bundle}") + return tokens[:protocol_start], protocol_length, records + + +def expect_records(ir, function_name, expected, expected_prefixes=None): + function = function_body(ir, function_name) + bundles = deopt_bundles(function) + if len(bundles) != len(expected): + fail( + f"{function_name} has {len(bundles)} deopt bundles, " + f"want {len(expected)}" + ) + if expected_prefixes is None: + expected_prefixes = [[] for _ in expected] + for bundle, want_records, want_prefix in zip( + bundles, expected, expected_prefixes + ): + got_prefix, _, got_records = parse_protocol(bundle) + if got_prefix != want_prefix: + fail( + f"{function_name} ordinary deopt prefix={got_prefix}, " + f"want {want_prefix}" + ) + if got_records != want_records: + fail( + f"{function_name} records={got_records}, " + f"want {want_records}" + ) + return function + + +def record(base, size, bits, words): + return (base, 0, size, 8, 8, bits, WORD_BITS, tuple(words)) + + +def check_rewritten_ir(ir): + if "llvm.statepoint.fixed_stack_home" in ir: + fail("obsolete fixed-stack-home metadata survived") + if re.search(r"\.gc\.leaf(?:\.[0-9]+)*\.root", ir): + fail("alloca pointer leaf was preloaded as an SSA root") + + statepoints = re.findall(r"@llvm\.experimental\.gc\.statepoint", ir) + # Seventeen calls plus the intrinsic declaration. + if len(statepoints) != 18: + fail(f"found {len(statepoints) - 1} statepoints, want 17") + if len(re.findall(r'"deopt"\(', ir)) != 17: + fail("not every ordinary safepoint carries an alloca record") + + null_initializers = re.findall(r"^\s*store ptr null, ptr ", ir, re.MULTILINE) + if len(null_initializers) != 17: + fail(f"found {len(null_initializers)} null initializers, want 17") + + one = [record("slot", 8, 1, [1])] + expect_records(ir, "pointer_slot", [one], [["i64 7"]]) + expect_records(ir, "nested_whole_aggregate", [ + [record("slot", 48, 6, [0b101001])] + ]) + expect_records(ir, "alloca_call_skip", [one]) + expect_records(ir, "alloca_multiple_calls", [one, one]) + expect_records(ir, "alloca_loop", [one]) + expect_records(ir, "alloca_gep_address_across_call", [ + [record("slot", 16, 2, [0b10])] + ]) + escaped = expect_records(ir, "alloca_address_passed_to_callee", [one]) + expect_records(ir, "alloca_uninitialized_at_safepoint", [one]) + expect_records(ir, "alloca_high_bitmap_word", [[ + record("slot", 512, 64, [1 << 63]) + ]]) + expect_records(ir, "alloca_multiple_records", [[ + record("left", 8, 1, [1]), + record("right", 8, 1, [1]), + ]]) + selected = expect_records(ir, "alloca_select_same_base", [one]) + if '"gc-live"(ptr %selected)' in selected or "%selected.relocated" in selected: + fail("proven same-base alloca select was treated as a heap root") + expect_records(ir, "alloca_nocapture_writable", [one]) + expect_records(ir, "alloca_escaped_before_unknown_write", [one, one]) + expect_records(ir, "alloca_readonly_and_readnone", [one, one]) + + if "gc.relocate" in escaped or ".relocated" in escaped: + fail("callee-writable alloca received a relocated write-back") + statepoint_end = escaped.index("@llvm.experimental.gc.statepoint") + if re.search(r"store ptr .*ptr %slot", escaped[statepoint_end:]): + fail("callee-writable alloca is stored after the call") + + relocates = re.findall( + r"= call coldcc ptr @llvm\.experimental\.gc\.relocate", ir + ) + if len(relocates) != 1: + fail(f"found {len(relocates)} scalar relocates, want 1") + uninitialized = function_body(ir, "alloca_uninitialized_at_safepoint") + if '"gc-live"(ptr %pointer)' not in uninitialized: + fail("ordinary scalar SSA pointer is missing from gc-live") + if "%pointer.relocated" not in uninitialized: + fail("ordinary scalar SSA pointer was not relocated") def main(): @@ -35,7 +214,7 @@ def main(): parser.add_argument("--input", required=True) args = parser.parse_args() - rewrite = subprocess.run( + rewritten = run( [ args.llc, f"-load-pass-plugin={args.plugin}", @@ -44,15 +223,10 @@ def main(): "-o", "-", args.input, - ], - capture_output=True, - text=True, + ] ) - if rewrite.returncode != 0: - fail(f"llc failed:\n{rewrite.stdout}{rewrite.stderr}") - ir = rewrite.stdout + rewrite.stderr - - verify = subprocess.run( + check_rewritten_ir(rewritten) + run( [ args.opt, f"-load-pass-plugin={args.plugin}", @@ -60,144 +234,72 @@ def main(): "-disable-output", "-", ], - input=ir, - capture_output=True, - text=True, + input_text=rewritten, ) - if verify.returncode != 0: - fail(f"opt verifier failed:\n{verify.stdout}{verify.stderr}") - roots = re.findall( - r"^\s*%[\w.]+ = load volatile ptr, ptr .*" - r"!llvm\.statepoint\.fixed_stack_home !\d+", - ir, - re.MULTILINE, - ) - if len(roots) != 11: - fail(f"found {len(roots)} canonical roots, want 11") - - live_lines = [line for line in ir.splitlines() if '"gc-live"' in line] - if len(live_lines) != 9: - fail(f"found {len(live_lines)} gc-live bundles, want 9") - for line in live_lines: - bundle = line.split('"gc-live"', 1)[1] - if ".gc.leaf." not in bundle or ".root" not in bundle: - fail(f"alloca root missing from gc-live: {line.strip()}") - if re.search(r"\bptr %slot(?=[,)])", bundle): - fail(f"static alloca address survived in gc-live: {line.strip()}") - if any(marker in bundle for marker in ("%nested", "{", "[")): - fail(f"aggregate survived in gc-live: {line.strip()}") - if re.search(r"%slot\.relocated\d* = call coldcc ptr", ir): - fail("static alloca address received a gc.relocate") + with tempfile.TemporaryDirectory() as directory: + optimized = run( + [ + args.opt, + f"-load-pass-plugin={args.plugin}", + "-passes=default,verify", + "-S", + "-o", + "-", + "-", + ], + input_text=rewritten, + ) + if len(re.findall(r'"deopt"\(', optimized)) != 17: + fail("default did not preserve all alloca records") + if "i64 -9223372036854775808" not in optimized: + fail("default did not preserve the high bitmap word") - null_initializers = re.findall(r"^\s*store ptr null, ptr ", ir, re.MULTILINE) - if len(null_initializers) != 10: - fail(f"found {len(null_initializers)} null leaf initializers, want 10") - - pointer_slot = function_body(ir, "pointer_slot") - require( - pointer_slot, - r"%nilcheck = load volatile i8, ptr %slot, align 1, " - r"!goallc\.nilcheck !\d+", - "frontend-marked alloca nil check", - ) - require( - pointer_slot, - r"%slot\.gc\.leaf\.root\.relocated = call coldcc ptr " - r"@llvm\.experimental\.gc\.relocate", - "scalar alloca leaf relocation", - ) - require( - pointer_slot, - r"store ptr %slot\.gc\.leaf\.root\.relocated, " - r"ptr %slot", - "scalar alloca leaf write-back", - ) + machine_ir = run( + [ + args.llc, + f"-load-pass-plugin={args.plugin}", + "-verify-machineinstrs", + "-stop-after=finalize-isel", + "-o", + "-", + args.input, + ] + ) + statepoint_lines = re.findall(r"(?m)^.*STATEPOINT.*$", machine_ir) + if len(statepoint_lines) != 17: + fail(f"MIR has {len(statepoint_lines)} statepoints, want 17") + for statepoint in statepoint_lines: + if str(BEGIN) not in statepoint or str(TAG) not in statepoint: + fail(f"MIR statepoint lost the alloca record: {statepoint}") + if "%stack." not in statepoint: + fail(f"MIR record has no direct frame index: {statepoint}") - nested = function_body(ir, "nested_whole_aggregate") - for path in ("0", "2.0.1", "2.1.1"): - require( - nested, - rf"%slot\.gc\.leaf\.{path}\.root = load volatile ptr, ptr " - rf"%slot\.gc\.leaf\.{path}\.pre\.addr.*" - rf"!llvm\.statepoint\.fixed_stack_home", - f"nested canonical root {path}", + output = f"{directory}/alloca-pointer-roots.goobj" + run( + [ + args.llc, + f"-load-pass-plugin={args.plugin}", + "-verify-machineinstrs", + "-filetype=obj", + "-o", + output, + args.input, + ] ) - require( - nested, - rf"store ptr %slot\.gc\.leaf\.{path}\.root\.relocated, " - rf"ptr %slot\.gc\.leaf\.{path}\.post\.addr", - f"nested relocated write-back {path}", + optimized_output = f"{directory}/alloca-pointer-roots-o2.goobj" + run( + [ + args.llc, + f"-load-pass-plugin={args.plugin}", + "-verify-machineinstrs", + "-filetype=obj", + "-o", + optimized_output, + "-", + ], + input_text=optimized, ) - require( - nested, - r"%reloaded = load %nested, ptr %slot", - "whole aggregate reload from the fixed alloca", - ) - - call_skip = function_body(ir, "alloca_call_skip") - if re.search(r"%slot.* = phi ptr", call_skip): - fail("call/skip formed a relocation PHI for a static alloca address") - - multiple = function_body(ir, "alloca_multiple_calls") - if multiple.count("!llvm.statepoint.fixed_stack_home") != 2: - fail("multiple-call function did not reload its canonical home twice") - if len(re.findall(r"\.root\d*\.relocated = call coldcc ptr", multiple)) != 2: - fail("multiple-call function did not relocate both canonical roots") - - loop = function_body(ir, "alloca_loop") - require( - loop, - r"%slot\.gc\.leaf\.root\.relocated = call coldcc ptr " - r"@llvm\.experimental\.gc\.relocate", - "loop canonical root relocation", - ) - require( - loop, - r"store ptr %slot\.gc\.leaf\.root\.relocated, ptr %slot", - "loop canonical root write-back before the backedge", - ) - require(loop, r"br i1 %again, label %loop, label %exit", "loop backedge") - - gep = function_body(ir, "alloca_gep_address_across_call") - gep_live = next( - (line for line in gep.splitlines() if '"gc-live"' in line), - None, - ) - if not gep_live: - fail("GEP-address function is missing gc-live") - if re.search(r"\bptr %field(?=[,)])", gep_live): - fail("static alloca GEP address survived in gc-live") - if "%field.relocated" in gep: - fail("static alloca GEP address received a gc.relocate") - require( - gep, - r"%slot\.gc\.leaf\.1\.root\.relocated = call coldcc ptr " - r"@llvm\.experimental\.gc\.relocate", - "GEP-address alloca pointer leaf relocation", - ) - - escaped = function_body(ir, "alloca_address_passed_to_callee") - require( - escaped, - r"@mutate_pointer_slot, i32 1, i32 0, ptr %slot,.*" - r'"gc-live"\(ptr %slot\.gc\.leaf\.root\)', - "address-passed alloca statepoint root", - ) - require( - escaped, - r"store ptr %slot\.gc\.leaf\.root\.relocated, " - r"ptr %slot", - "address-passed alloca relocated write-back", - ) - - uninitialized = function_body(ir, "alloca_uninitialized_at_safepoint") - require( - uninitialized, - r"store ptr null, ptr %slot.*?" - r"%slot\.gc\.leaf\.root = load volatile ptr, ptr %slot", - "null initialization before the first safepoint root load", - ) if __name__ == "__main__": diff --git a/src/cmd/llvmplugin/testdata/check-alloca-ptrmap-malformed.py b/src/cmd/llvmplugin/testdata/check-alloca-ptrmap-malformed.py new file mode 100644 index 00000000000000..bc6883e28b7ced --- /dev/null +++ b/src/cmd/llvmplugin/testdata/check-alloca-ptrmap-malformed.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + + +MODULE = r''' +target triple = "x86_64-unknown-linux-goobj" + +declare goabiinternal void @callee() +declare token @llvm.experimental.gc.statepoint.p0( + i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) + +define goabiinternal void @test() #0 gc "goallc" { +entry: + %slot = alloca [2 x ptr], align 8 + store [2 x ptr] zeroinitializer, ptr %slot, align 8 + %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) + @llvm.experimental.gc.statepoint.p0( + i64 1, i32 0, ptr elementtype(void ()) @callee, + i32 0, i32 0, i32 0, i32 0) [ "deopt"(__DEOPT__) ] + ret void +} + +attributes #0 = { "go-stack-growth-statepoint" } +''' + + +CASES = { + "truncated": ( + "i64 1195461697, i64 15, i64 1, i64 1347703373", + "protocol is truncated", + ), + "bad_length": ( + "i64 1195461697, i64 14, i64 1, i64 1347703373, i64 11, " + "ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, " + "i64 1, i64 3, i64 1095519299, i64 15", + "protocol envelope is malformed", + ), + "duplicate": ( + "i64 1195461697, i64 26, i64 2, " + "i64 1347703373, i64 11, ptr %slot, i64 0, i64 16, i64 8, " + "i64 8, i64 2, i64 64, i64 1, i64 3, " + "i64 1347703373, i64 11, ptr %slot, i64 0, i64 16, i64 8, " + "i64 8, i64 2, i64 64, i64 1, i64 3, " + "i64 1095519299, i64 26", + "duplicate frame record", + ), + "overlap": ( + "i64 1195461697, i64 26, i64 2, " + "i64 1347703373, i64 11, ptr %slot, i64 0, i64 8, i64 8, " + "i64 8, i64 1, i64 64, i64 1, i64 1, " + "i64 1347703373, i64 11, ptr %slot, i64 0, i64 16, i64 8, " + "i64 8, i64 2, i64 64, i64 1, i64 3, " + "i64 1095519299, i64 26", + "records overlap", + ), + "padding": ( + "i64 1195461697, i64 15, i64 1, i64 1347703373, i64 11, " + "ptr %slot, i64 0, i64 8, i64 8, i64 8, i64 1, i64 64, " + "i64 1, i64 3, i64 1095519299, i64 15", + "padding bits are nonzero", + ), + "non_direct": ( + "i64 1195461697, i64 15, i64 1, i64 1347703373, i64 11, " + "i64 0, i64 0, i64 8, i64 8, i64 8, i64 1, i64 64, " + "i64 1, i64 1, i64 1095519299, i64 15", + "base is not a direct frame location", + ), +} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--llc", required=True) + parser.add_argument("--plugin", required=True) + args = parser.parse_args() + + for name, (deopt, diagnostic) in CASES.items(): + module = MODULE.replace("__DEOPT__", deopt) + result = subprocess.run( + [ + args.llc, + f"-load-pass-plugin={args.plugin}", + "-verify-machineinstrs", + "-filetype=obj", + "-o", + os.devnull, + "-", + ], + input=module, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + if result.returncode == 0: + print(f"{name}: malformed contract unexpectedly succeeded", file=sys.stderr) + raise SystemExit(1) + if diagnostic not in output: + print( + f"{name}: missing expected diagnostic {diagnostic!r}:\n{output}", + file=sys.stderr, + ) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/test/abi/llvm_alloca_statepoint.go b/test/abi/llvm_alloca_statepoint.go index 6d09abeee82daf..8193aa30db0b5e 100644 --- a/test/abi/llvm_alloca_statepoint.go +++ b/test/abi/llvm_alloca_statepoint.go @@ -26,8 +26,8 @@ var ( func safepoint() // mutateLocal is intentionally opaque to both compilers. A real callee may -// update the pointer fields while the call is a safepoint, so GoALLC must use -// the original alloca slots as the relocation homes. +// update the pointer fields while the call is a safepoint, so GoALLC must scan +// the original alloca memory rather than write back pre-call SSA values. // //go:noescape func mutateLocal(value *pointerLocal, branch bool) @@ -35,7 +35,7 @@ func mutateLocal(value *pointerLocal, branch bool) // localAcrossSafepoints deliberately takes the address of a pointer-containing // local. Its pointer leaves must remain rooted in that original stack object // across both safepoint and mutateLocal. The latter may update the object, so a -// relocated value from a separate spill slot must not be stored over its write. +// relocated pre-call value must not be stored over its write. // //go:noinline func localAcrossSafepoints(branch bool, rounds int) uintptr { diff --git a/test/llvm_alloca_statepoint_gc.go b/test/llvm_alloca_statepoint_gc.go index fac8c58a23baa9..a8b9106ab1da92 100644 --- a/test/llvm_alloca_statepoint_gc.go +++ b/test/llvm_alloca_statepoint_gc.go @@ -15,6 +15,11 @@ type pointerLocal struct { tail [2]*int } +type pointerPair struct { + first *int + second *int +} + //go:noinline func newValue(value int) *int { pointer := new(int) @@ -39,6 +44,16 @@ func mutateLocal(value *pointerLocal, first, second *int, branch bool) { } } +//go:noinline +func movePair(destination *pointerPair, source pointerPair) { + *destination = source +} + +//go:noinline +func zeroPair(destination *pointerPair) { + *destination = pointerPair{} +} + // grow keeps the caller's address-taken local live while recursion repeatedly // grows and copies the stack. The pointer-free padding makes stack growth // deterministic without adding unrelated pointer-map bits. @@ -93,7 +108,32 @@ func exercise(branch bool) int { return got + localSum(&value) } +// exerciseWriteBarrierMutation covers the aggregate replacement and clearing +// semantics used by wbMove/wbZero lowering in callees that receive the address +// of the caller's pointer-containing local. A statepoint must not restore the +// pre-call contents over either mutation. +// +//go:noinline +func exerciseWriteBarrierMutation() { + first := newValue(29) + second := newValue(31) + var value pointerPair + movePair(&value, pointerPair{first: first, second: second}) + first, second = nil, nil + runtime.GC() + if value.first == nil || value.second == nil || + *value.first != 29 || *value.second != 31 { + panic("wbMove mutation of address-taken local was lost") + } + zeroPair(&value) + runtime.GC() + if value.first != nil || value.second != nil { + panic("wbZero mutation of address-taken local was overwritten") + } +} + func main() { + exerciseWriteBarrierMutation() // The recursive helper adds one for every odd depth. const recursiveAdjustment = 600 if got, want := exercise(true), 2*(13+23+17+13+17)+recursiveAdjustment; got != want {