From 4bda010d575d4b85434c4c58e871d4fd40693a92 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 16:40:47 +0800 Subject: [PATCH 1/8] cmd/compile: support building runtime with LLVM --- src/cmd/compile/internal/ssa/llvmdata.go | 56 ++- src/cmd/compile/internal/ssa/llvmdebug.go | 20 +- src/cmd/compile/internal/ssa/ssa2llvm.go | 362 +++++++++++++++--- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 184 +++++++++ src/cmd/compile/internal/ssagen/nowb.go | 22 ++ src/cmd/compile/internal/ssagen/pgen.go | 1 + src/cmd/internal/testdir/llvm_abi_test.go | 106 +++-- src/cmd/internal/testdir/llvm_alloca_test.go | 14 +- src/cmd/internal/testdir/llvm_test.go | 110 ++++++ src/cmd/llvmplugin/GoALLCInlineAnchors.cpp | 85 +++- src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp | 37 +- src/cmd/llvmplugin/testdata/debug-inline.ll | 34 +- src/cmd/llvmtoolexec/main.go | 58 +++ src/cmd/llvmtoolexec/main_test.go | 21 + .../vendor/github.com/goallc/go-llvm/ir.go | 3 + src/runtime/export_test.go | 14 + test/codegen/_cgo_llvm_unsafe_args.go | 14 +- test/codegen/llvm_opendefer.go | 31 +- 18 files changed, 1061 insertions(+), 111 deletions(-) diff --git a/src/cmd/compile/internal/ssa/llvmdata.go b/src/cmd/compile/internal/ssa/llvmdata.go index 955e741ff88eba..27d61f6b32edfa 100644 --- a/src/cmd/compile/internal/ssa/llvmdata.go +++ b/src/cmd/compile/internal/ssa/llvmdata.go @@ -193,6 +193,15 @@ func LowerGoObjData() { setGoObjKeepMetadata(g, s) setGoObjGotypeMetadata(g, s) setGoObjMarkerRelocMetadata(g, s) + // Every compiler LSym is an object-format definition even when no + // ordinary LLVM instruction refers to it. In particular, assembly + // FUNCDATA directives name the args_stackmap and arginfo symbols that + // the compiler emits for bodyless assembly functions. Those references + // live in a different archive member and are therefore invisible to + // LLVM's optimizer. Keep the definitions distinct and present through + // GlobalDCE and ConstantMerge; Go linker reachability still decides + // whether the resulting GoObj symbols survive in the final binary. + preserveGoObjMetadataValues(g) } emitGoObjCompilerUsed() } @@ -243,6 +252,14 @@ func setGoObjPackageSymbolIndexMetadata(value llvm.Value, s *obj.LSym) { if value.IsNil() || s == nil || s.PkgIdx != goobj.PkgIdxSelf || !s.Indexed() || s.SymIdx < 0 { base.Fatalf("invalid LLVM GoObj package symbol index") } + // Calls emitted before obj.NumberSyms may initially classify a runtime + // builtin as an undefined PkgIdxBuiltin reference. When compiling runtime, + // that same LLVM GlobalValue later becomes this package's definition. + // Definitions are addressed by their package symbol index, so remove the + // now-stale undefined-reference attachment before handing the module to the + // GoObj AsmPrinter. + value.EraseGlobalMetadata(GlobalCtxt.MDKindID("goobj.builtin")) + value.EraseGlobalMetadata(GlobalCtxt.MDKindID("goobj.import")) value.SetGlobalMetadata(GlobalCtxt.MDKindID(goObjSymbolIndexMD), GlobalCtxt.MDNode([]llvm.Metadata{ llvm.ConstInt(GlobalCtxt.Int32Type(), uint64(s.SymIdx), false).ConstantAsMetadata(), })) @@ -275,7 +292,11 @@ func llvmGoDataRef(s *obj.LSym) llvm.Value { if s == nil { base.Fatalf("nil Go data symbol in LLVM lowering") } - if s.Type == objabi.STEXT || s.Type == objabi.STEXTFIPS || s.ABI() == obj.ABIInternal { + // FuncPCABI0 carries an ABI0 LSym through OpAddr, but bodyless assembly + // functions still have the unresolved Sxxx kind here. Recover the semantic + // function identity from the front end before choosing an LLVM GlobalValue; + // ABI alone is insufficient because ordinary data symbols also use ABI0. + if llvmGoFunctionSymbol(s) { data := map[*obj.LSym]bool(nil) if currentLLVMDataLowerer != nil { data = currentLLVMDataLowerer.data @@ -325,6 +346,28 @@ func llvmGoDataRef(s *obj.LSym) llvm.Value { return g } +func llvmGoFunctionSymbol(s *obj.LSym) bool { + if s.Type == objabi.STEXT || s.Type == objabi.STEXTFIPS || s.ABI() == obj.ABIInternal { + return true + } + // Bodyless assembly declarations are initialized without setupTextLSym, so + // their LSym remains Sxxx. typecheck.Target.Funcs is the authoritative list + // of current-package function declarations and includes generated ABI + // wrappers before LLVM module initialization. + if typecheck.Target == nil { + return false + } + for _, fn := range typecheck.Target.Funcs { + if fn == nil || fn.Nname == nil || fn.Sym() == nil || fn.Sym().Name == "_" { + continue + } + if fn.LinksymABI(fn.ABI) == s { + return true + } + } + return false +} + func (l *llvmDataLowerer) globalName(s *obj.LSym) string { if s.Name != "" { return s.Name @@ -459,7 +502,7 @@ func llvmExternalDataRef(s *obj.LSym, data map[*obj.LSym]bool) llvm.Value { // at this point. Their ABI nevertheless identifies them as functions (for // example runtime.memequal64 in an equality closure), so do not rely on // STEXT alone here. - if s.Type == objabi.STEXT || s.Type == objabi.STEXTFIPS || s.ABI() == obj.ABIInternal { + if llvmGoFunctionSymbol(s) { storageName := llvmFunctionStorageName(s.Name, llvmCallConv(s.ABI())) if f := CurrentModule.NamedFunction(storageName); !f.IsNil() { attachGoObjSymbolRef(f, s) @@ -564,9 +607,18 @@ func setGoObjFunctionFlags(fn llvm.Value, s *obj.LSym) { if s.ReflectMethod() { flag |= goobj.SymFlagReflectMethod } + if s.NoSplit() { + flag |= goobj.SymFlagNoSplit + } + if s.IsPkgInit() { + flag2 |= goobj.SymFlagPkgInit + } if s.IsLinkname() || s.Name == "main.main" { flag2 |= goobj.SymFlagLinkname } + if s.IsLinknameStd() { + flag2 |= goobj.SymFlagLinknameStd + } if s.ABIWrapper() { flag2 |= goobj.SymFlagABIWrapper } diff --git a/src/cmd/compile/internal/ssa/llvmdebug.go b/src/cmd/compile/internal/ssa/llvmdebug.go index de76a44f6178d9..7e0c6810e19dcf 100644 --- a/src/cmd/compile/internal/ssa/llvmdebug.go +++ b/src/cmd/compile/internal/ssa/llvmdebug.go @@ -257,6 +257,22 @@ func (lfc *LLVMFuncContext) setDebugLocation(xpos src.XPos) { } locationScope := llvmDIScopeForPos(scope, pos, 0) - lfc.b.SetCurrentDebugLocationMetadata(GlobalCtxt.CreateDebugLocation( - pos.RelLine(), pos.RelCol(), locationScope, inlinedAt)) + location := GlobalCtxt.CreateDebugLocation( + pos.RelLine(), pos.RelCol(), locationScope, inlinedAt) + lfc.b.SetCurrentDebugLocationMetadata(location) + + // Generic LLVM optimization may combine instructions from different Go + // inline frames and keep only one of their DILocations. Record one complete + // frontend location for every inline node independently of the instruction + // stream. The final machine pass uses this only when an inline edge has + // otherwise disappeared, so optimization remains unconstrained while Go's + // pcinline tree still has a real final-layout PC for every source edge. + if len(chain) != 0 && !lfc.RequiredInlinePos[pos.Base().InliningIndex()] { + lfc.RequiredInlinePos[pos.Base().InliningIndex()] = true + CurrentModule.AddNamedMetadataOperand(goObjDebugInlineRequiredMD, + GlobalCtxt.MDNode([]llvm.Metadata{ + lfc.LF.ConstantAsMetadata(), + location, + })) + } } diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index ff6c67c2990f46..268186b6ad1caa 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -25,6 +25,7 @@ type LLVMFuncContext struct { ClosureCodeLoads map[ID]bool DeferResults map[llvmLocalKey]bool DeferResultKeys map[ID]llvmLocalKey + RequiredInlinePos map[int]bool OpenDeferBits llvmLocalKey HasOpenDeferBits bool OpenDeferSlots map[llvmLocalKey]int @@ -70,6 +71,8 @@ const goResultsTupleAttr = "go_results_tuple" const goGCStrategy = "goallc" const goGCLeafFunctionAttr = "gc-leaf-function" const goStackGrowthStatepointAttr = "go-stack-growth-statepoint" +const goNoSplitAttr = "go-nosplit" +const goSystemStackAttr = "go-systemstack" const goAsyncUnsafeAttr = "go-async-unsafe" const goWriteBarrierIntrinsic = "llvm.go.gc.write.barrier" const goDeferEdgeIntrinsic = "llvm.go.defer.edge" @@ -79,6 +82,7 @@ const goOpenDeferBitsMD = "goallc.open_defer_bits" const goOpenDeferSlotsMD = "goallc.open_defer_slots" const goObjMarkerRelocMD = "goobj.marker_reloc" const goObjSymbolIndexMD = "goobj.symbol.index" +const goObjDebugInlineRequiredMD = "goobj.debug.inline.required" const llvmFramePointerAttr = "frame-pointer" const llvmFramePointerNonLeaf = "non-leaf" const llvmTargetCPUAttr = "target-cpu" @@ -144,11 +148,37 @@ func llvmTypeContainsABIPad(typ llvm.Type) bool { return false } +// getLLVMABIStorageType removes Go's nominal aggregate identity from a +// function ABI carrier. Compiler-generated runtime calls can describe the +// same physical ABI value using a substituted builtin type (for example []T), +// while the runtime definition uses a named implementation type (for example +// runtime.slice). The native backends join those at the symbol's physical ABI; +// using literal LLVM aggregates does the same without weakening the semantic +// types used inside either function body. +func getLLVMABIStorageType(typ *types.Type) llvm.Type { + switch typ.Kind() { + case types.TARRAY: + return llvm.ArrayType(getLLVMABIStorageType(typ.Elem()), int(typ.NumElem())) + case types.TSTRUCT: + fields := make([]llvm.Type, typ.NumFields(), typ.NumFields()+1) + for i := 0; i < typ.NumFields(); i++ { + fields[i] = getLLVMABIStorageType(typ.FieldType(i)) + } + if llvmStructHasTailPad(typ) { + fields = append(fields, getLLVMABIPadType()) + } + return llvm.StructType(fields, false) + default: + return getLLVMType(typ) + } +} + // getLLVMABIType makes a non-empty carrier only at a top-level zero-sized ABI -// boundary. The original zero-sized type remains in the wrapper, so DataLayout -// supplies its Go alignment without storing that alignment in an attribute. +// boundary. The original zero-sized layout remains in the wrapper, so +// DataLayout supplies its Go alignment without storing that alignment in an +// attribute. func getLLVMABIType(typ *types.Type) llvm.Type { - storage := getLLVMType(typ) + storage := getLLVMABIStorageType(typ) if typ.Size() == 0 { return llvm.StructType([]llvm.Type{storage, getLLVMABIPadType()}, false) } @@ -580,7 +610,12 @@ func (lfc *LLVMFuncContext) currentG(v *Value) llvm.Value { } i64 := GlobalCtxt.Int64Type() - registerName := GlobalCtxt.MetadataAsValue(GlobalCtxt.MDString(register)) + // llvm.read_register requires a metadata node whose first operand is the + // register-name string. A bare MDString passes IR verification but crashes + // SelectionDAG's intrinsic lowering when it casts the operand to MDNode. + registerName := GlobalCtxt.MetadataAsValue(GlobalCtxt.MDNode([]llvm.Metadata{ + GlobalCtxt.MDString(register), + })) sig := llvm.FunctionType(i64, []llvm.Type{registerName.Type()}, false) fn := getOrInsertLLVMIntrinsic("llvm.read_register.i64", sig) raw := lfc.b.CreateCall(sig, fn, []llvm.Value{registerName}, v.String()+".register") @@ -943,9 +978,8 @@ func (lfc *LLVMFuncContext) cgoUnsafeArgAddress(name *ir.Name, llvmName string) return lfc.b.CreateGEP(GlobalCtxt.Int8Type(), lfc.ABI0FrameBase, []llvm.Value{index}, llvmName+".frame") } -func markLLVMGCLeaf(fn, call llvm.Value) { +func markLLVMGCLeafCall(call llvm.Value) { attr := GlobalCtxt.CreateStringAttribute(goGCLeafFunctionAttr, "") - fn.AddFunctionAttr(attr) call.AddCallSiteAttribute(llvmAttributeFunctionIndex, attr) } @@ -1069,7 +1103,7 @@ func (lfc *LLVMFuncContext) llvmRuntimeMemmove(dst, src, length llvm.Value) llvm attachGoObjABISymbolRef(fn, "runtime.memmove", obj.ABIInternal) call := lfc.b.CreateCall(sig.Type, fn, []llvm.Value{dst, src, length}, "") call.SetInstructionCallConv(goABIInternalCallConv) - markLLVMGCLeaf(fn, call) + markLLVMGCLeafCall(call) return call } @@ -1129,7 +1163,7 @@ func (lfc *LLVMFuncContext) llvmMemEq(v *Value) llvm.Value { attachGoObjABISymbolRef(fn, "runtime.memequal", obj.ABIInternal) call := lfc.b.CreateCall(sig.Type, fn, []llvm.Value{left, right, size}, v.String()) call.SetInstructionCallConv(goABIInternalCallConv) - markLLVMGCLeaf(fn, call) + markLLVMGCLeafCall(call) return call } @@ -1731,11 +1765,32 @@ func (lfc *LLVMFuncContext) llvmIData(v *Value) llvm.Value { return result } -// reshapeLLVMValue converts between the distinct nominal LLVM aggregate types -// used for a generic shape and one of its concrete instantiations. Go's type -// system already records these as identical for shape-aware operations; keep -// that decision as the authority and rebuild only the affected first-class -// struct or array value. Scalar leaves and memory keep their normal lowering. +func llvmValueTypesCanReshape(from, to *types.Type) bool { + if from == nil || to == nil { + return false + } + if types.Identical(from, to) && !types.IdenticalStrict(from, to) && (from.HasShape() || to.HasShape()) { + return true + } + if from.Kind() != to.Kind() { + return false + } + switch from.Kind() { + case types.TSTRUCT, types.TARRAY: + // The frontend lowers explicit conversions between defined aggregate + // types with identical underlying types (ignoring struct tags) to Copy. + // LLVM still gives each defined struct its own nominal identity. + return types.IdenticalIgnoreTags(from.Underlying(), to.Underlying()) + } + return false +} + +// reshapeLLVMValue converts between distinct nominal LLVM aggregate types +// that Go has already established are representation-preserving: either a +// generic shape and its concrete instantiation, or an explicit conversion +// between aggregates with identical underlying types. Rebuild only the +// affected first-class struct or array value; scalar leaves and memory keep +// their normal lowering. func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, to *types.Type, name string) llvm.Value { if value.IsNil() { v.Fatalf("cannot reshape an empty LLVM value") @@ -1756,14 +1811,14 @@ func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, t return lfc.b.CreateBitCast(value, want, name) } } - if from == nil || to == nil || !types.Identical(from, to) || types.IdenticalStrict(from, to) || (!from.HasShape() && !to.HasShape()) { + if !llvmValueTypesCanReshape(from, to) { v.Fatalf("cannot reshape LLVM value from Go type %v to %v", from, to) } switch from.Kind() { case types.TSTRUCT: if to.Kind() != types.TSTRUCT || value.Type().TypeKind() != llvm.StructTypeKind || want.TypeKind() != llvm.StructTypeKind { - v.Fatalf("shape-identical structs have incompatible LLVM aggregate kinds") + v.Fatalf("representation-identical structs have incompatible LLVM aggregate kinds") } fromElements := value.Type().StructElementTypes() toElements := want.StructElementTypes() @@ -1776,7 +1831,7 @@ func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, t toElementCount++ } if from.NumFields() != to.NumFields() || len(fromElements) != fromElementCount || len(toElements) != toElementCount { - v.Fatalf("shape-identical structs have incompatible field counts") + v.Fatalf("representation-identical structs have incompatible field counts") } result := llvm.Undef(want) for i := 0; i < from.NumFields(); i++ { @@ -1792,7 +1847,7 @@ func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, t case types.TARRAY: if to.Kind() != types.TARRAY || from.NumElem() != to.NumElem() || value.Type().TypeKind() != llvm.ArrayTypeKind || want.TypeKind() != llvm.ArrayTypeKind { - v.Fatalf("shape-identical arrays have incompatible LLVM aggregate layouts") + v.Fatalf("representation-identical arrays have incompatible LLVM aggregate layouts") } result := llvm.Undef(want) for i := int64(0); i < from.NumElem(); i++ { @@ -1804,7 +1859,56 @@ func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, t return result default: - v.Fatalf("shape-identical Go types %v and %v require unsupported LLVM reshaping", from, to) + v.Fatalf("representation-identical Go types %v and %v require unsupported LLVM reshaping", from, to) + return llvm.Value{} + } +} + +// reshapeLLVMABICarrier rebuilds a value between semantically distinct LLVM +// aggregate types that have the same physical Go ABI structure. This is used +// only at function boundaries: named aggregate identity remains intact in the +// function body, while the signature uses literal aggregates so independently +// constructed runtime helper declarations and definitions agree. +func (lfc *LLVMFuncContext) reshapeLLVMABICarrier(v *Value, value llvm.Value, target llvm.Type, name string) llvm.Value { + if value.Type() == target { + return value + } + + source := value.Type() + switch target.TypeKind() { + case llvm.StructTypeKind: + if source.TypeKind() != llvm.StructTypeKind { + v.Fatalf("cannot reshape non-struct LLVM ABI carrier to struct") + } + sourceFields := source.StructElementTypes() + targetFields := target.StructElementTypes() + if len(sourceFields) != len(targetFields) { + v.Fatalf("cannot reshape LLVM ABI structs with different field counts") + } + result := llvm.Undef(target) + for i := range sourceFields { + fieldName := fmt.Sprintf("%s.abi.field%d", name, i) + field := lfc.b.CreateExtractValue(value, i, fieldName+".extract") + field = lfc.reshapeLLVMABICarrier(v, field, targetFields[i], fieldName) + result = lfc.b.CreateInsertValue(result, field, i, fieldName+".insert") + } + return result + + case llvm.ArrayTypeKind: + if source.TypeKind() != llvm.ArrayTypeKind || source.ArrayLength() != target.ArrayLength() { + v.Fatalf("cannot reshape incompatible LLVM ABI arrays") + } + result := llvm.Undef(target) + for i := 0; i < source.ArrayLength(); i++ { + elementName := fmt.Sprintf("%s.abi.element%d", name, i) + element := lfc.b.CreateExtractValue(value, i, elementName+".extract") + element = lfc.reshapeLLVMABICarrier(v, element, target.ElementType(), elementName) + result = lfc.b.CreateInsertValue(result, element, i, elementName+".insert") + } + return result + + default: + v.Fatalf("cannot reshape LLVM ABI carrier from %v to %v", source, target) return llvm.Value{} } } @@ -1817,10 +1921,7 @@ func (lfc *LLVMFuncContext) llvmValueToABI(v *Value, value llvm.Value, from, log return llvm.Undef(abiType) } value = lfc.reshapeLLVMValue(v, value, from, logical, name) - if value.Type() != abiType { - v.Fatalf("Go ABI value has incompatible LLVM carrier") - } - return value + return lfc.reshapeLLVMABICarrier(v, value, abiType, name) } func (lfc *LLVMFuncContext) llvmValueFromABI(v *Value, value llvm.Value, logical, to *types.Type, name string) llvm.Value { @@ -1830,12 +1931,15 @@ func (lfc *LLVMFuncContext) llvmValueFromABI(v *Value, value llvm.Value, logical } return llvm.Undef(getLLVMType(to)) } + value = lfc.reshapeLLVMABICarrier(v, value, getLLVMType(logical), name) return lfc.reshapeLLVMValue(v, value, logical, to, name) } // llvmStaticCallSignature restores semantic pointer types for compiler-built // runtime calls whose AuxCall uses uintptr only to compute physical ABI -// assignments. AuxCall remains the physical ABI authority; the LLVM operands +// assignments. When compiling runtime itself, an ordinary source call to the +// same helper already has its semantic pointer type and needs no rewrite. +// AuxCall remains the physical ABI authority in both cases; the LLVM operands // and runtime helper parameters are pointers. func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvmFuncSignature { if aux == nil || aux.Fn == nil { @@ -1863,8 +1967,9 @@ func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvm v.Fatalf("%s uses unsupported ABI %v", aux.Fn.Name, aux.ABI().Which()) } if aux.NArgs() != wantArgs || aux.NResults() != 0 { - v.Fatalf("%s has unexpected raw call signature: %d arguments, %d results", aux.Fn.Name, aux.NArgs(), aux.NResults()) + v.Fatalf("%s has unexpected call signature: %d arguments, %d results", aux.Fn.Name, aux.NArgs(), aux.NResults()) } + params := append([]llvm.Type(nil), sig.Type.ParamTypes()...) for i := int64(0); i < pointerArgs; i++ { if int(i) >= len(v.Args)-1 || v.Args[i].Type == nil { v.Fatalf("argument %d to %s is not pointer-shaped", i, aux.Fn.Name) @@ -1880,14 +1985,21 @@ func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvm if !pointerShaped && !writeBarrierTypeAddr { v.Fatalf("argument %d to %s is not pointer-shaped", i, aux.Fn.Name) } - if typ := aux.TypeOfArg(i); typ == nil || !typ.IsUintptr() { - v.Fatalf("argument %d to %s is not raw uintptr", i, aux.Fn.Name) + typ := aux.TypeOfArg(i) + switch { + case typ != nil && typ.IsUintptr(): + params[i] = GlobalCtxt.PointerType(0) + case typ != nil && typ.IsPtrShaped(): + if !pointerShaped { + v.Fatalf("semantic pointer argument %d to %s is not pointer-shaped", i, aux.Fn.Name) + } + if params[i].TypeKind() != llvm.PointerTypeKind { + v.Fatalf("argument %d to %s has non-pointer LLVM type", i, aux.Fn.Name) + } + default: + v.Fatalf("argument %d to %s is neither raw uintptr nor a semantic pointer", i, aux.Fn.Name) } } - params := append([]llvm.Type(nil), sig.Type.ParamTypes()...) - for i := int64(0); i < pointerArgs; i++ { - params[i] = GlobalCtxt.PointerType(0) - } sig.Type = llvm.FunctionType(sig.ReturnType, params, false) return sig } @@ -1931,7 +2043,7 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { configureLLVMCall(call, sig) lfc.materializeAddressedResults(v, call, aux) if llvmGCLeaf { - markLLVMGCLeaf(fn, call) + markLLVMGCLeafCall(call) } return call } @@ -2040,6 +2152,46 @@ func llvmFunctionUsesClosureContext(f *Func) bool { return hasContext } +// llvmRuntimeConstructedClosure reports whether call uses the runtime's +// trusted hand-built funcval shape. The runtime receives some callback entry +// points as unsafe.Pointer and stores that code word into a local funcval. +// Scalar replacement can forward the pointer-to-uintptr Convert directly to +// the closure call instead of reloading the funcval's first word. +// +// Follow the call's memory chain and require the exact forwarded code value to +// have been stored at offset zero of the context. This keeps arbitrary integer +// indirect calls fail-closed and preserves the code/context identity that the +// ordinary Load form establishes structurally. +func llvmRuntimeConstructedClosure(call, code, context *Value) bool { + if code.Op != OpConvert || !code.Type.IsUintptr() || len(code.Args) != 2 || + !code.Args[0].Type.IsUnsafePtr() || !code.Args[1].Type.IsMemory() || code.Uses != 2 || + context == nil || !context.Type.IsPtr() || len(call.Args) == 0 { + return false + } + for mem, steps := call.Args[len(call.Args)-1], 0; mem != nil && steps < 32; steps++ { + switch mem.Op { + case OpStore: + if len(mem.Args) != 3 { + return false + } + addr := mem.Args[0] + if mem.Args[1] == code && (addr == context || + (addr.Op == OpOffPtr && auxIntToInt64(addr.AuxInt) == 0 && len(addr.Args) == 1 && addr.Args[0] == context)) { + return true + } + mem = mem.Args[2] + case OpVarDef, OpVarLive: + if len(mem.Args) != 1 { + return false + } + mem = mem.Args[0] + default: + return false + } + } + return false +} + func llvmLocalName(v *Value) (*ir.Name, llvmLocalKey) { sym := auxToSym(v.Aux) name, ok := sym.(*ir.Name) @@ -2822,10 +2974,17 @@ func (lfc *LLVMFuncContext) CompileBlock(BB *Block, values []*Value) { lfc.b.CreateUnreachable() case BlockJumpTable: index := lfc.GenLV(BB.Controls[0]) - table := lfc.b.CreateSwitch(index, lfc.BBs[BB.Succs[0].Block().ID], len(BB.Succs)) + // BlockJumpTable's control is already proven to be in range, and every + // Go SSA edge is represented by one indexed successor. A real successor + // as LLVM's default would add an extra CFG edge and make PHIs on a + // repeated target have one too few incoming values. + defaultBlock := GlobalCtxt.AddBasicBlock(lfc.LF, BB.String()+".jump.default") + table := lfc.b.CreateSwitch(index, defaultBlock, len(BB.Succs)) for i, succ := range BB.Succs { table.AddCase(llvm.ConstInt(index.Type(), uint64(i), false), lfc.BBs[succ.Block().ID]) } + lfc.b.SetInsertPointAtEnd(defaultBlock) + lfc.b.CreateUnreachable() default: BB.Func.fe.Fatalf(BB.Pos, "unsupported SSA block kind in LLVM lowering: %s", BB.Kind) } @@ -2845,14 +3004,25 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { attachGoObjABISymbolRef(deferReturn, "runtime.deferreturn", obj.ABIInternal) lfc.b.SetInsertPointAtEnd(lfc.OpenDeferRecovery) + frontendFunc := lfc.F.Frontend().Func() + if frontendFunc == nil || !frontendFunc.Endlineno.IsKnown() { + lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer recovery has no function-end source position") + } + // Match the native shared deferreturn convention: its synthetic call is + // attributed to the function end, after every source-level defer. Besides + // giving PCLN a stable line, a call in an LLVM debug-info function must + // carry a !dbg location even when it lives in a disconnected recovery block. + lfc.setDebugLocation(frontendFunc.Endlineno) call := lfc.b.CreateCall(deferReturnSig.Type, deferReturn, nil, "") call.SetInstructionCallConv(goABIInternalCallConv) + lfc.b.ClearCurrentDebugLocation() outParams := lfc.F.OwnAux.ABIInfo().OutParams() if len(outParams) != lfc.ResultCount { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result count %d does not match LLVM signature result count %d", len(outParams), lfc.ResultCount) } results := make([]llvm.Value, len(outParams)) + reshapeContext := &Value{Block: lfc.F.Entry, Pos: lfc.F.Entry.Pos} for i, result := range outParams { var abiType llvm.Type if lfc.ResultCount == 1 { @@ -2874,6 +3044,7 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { value := lfc.b.CreateLoad(getLLVMType(result.Type), slot.Value, fmt.Sprintf("open.defer.result%d", i)) value.SetAlignment(int(result.Type.Alignment())) value.SetVolatile(true) + value = lfc.llvmValueToABI(reshapeContext, value, result.Type, lfc.F.OwnAux.TypeOfResult(int64(i)), abiType, fmt.Sprintf("open.defer.result%d.abi", i)) if value.Type() != abiType { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result %d has incompatible LLVM ABI type", i) } @@ -2966,6 +3137,11 @@ func llvmFuncCalls(f *Func, target string) bool { return false } +func llvmPreserveRecoverFrame(f *Func) bool { + return f.OwnAux != nil && f.OwnAux.Fn != nil && f.OwnAux.Fn.Name == "runtime.gorecover" || + llvmFuncCalls(f, "runtime.gorecover") +} + func LLVMCompile(f *Func) { if f.OwnAux == nil || f.OwnAux.Fn == nil || f.OwnAux.ABIInfo() == nil { f.fe.Fatalf(f.Entry.Pos, "missing function ABI information in LLVM lowering for %s", f.Name) @@ -2976,20 +3152,21 @@ func LLVMCompile(f *Func) { } cc := llvmCallConv(f.OwnAux.ABI().Which()) FCtxt := &LLVMFuncContext{ - BBs: map[ID]llvm.BasicBlock{}, - Vs: map[ID]llvm.Value{}, - Locals: map[llvmLocalKey]llvmStackSlot{}, - AddressedResults: map[ID][]llvmAddressedResult{}, - ResultSlots: map[ID]llvm.Value{}, - ItabMethods: map[ID]bool{}, - ClosureCodeLoads: map[ID]bool{}, - DeferResults: map[llvmLocalKey]bool{}, - DeferResultKeys: map[ID]llvmLocalKey{}, - OpenDeferSlots: map[llvmLocalKey]int{}, - F: f, - b: GlobalCtxt.NewBuilder(), - ReturnType: sig.ReturnType, - ResultCount: sig.ResultCount, + BBs: map[ID]llvm.BasicBlock{}, + Vs: map[ID]llvm.Value{}, + Locals: map[llvmLocalKey]llvmStackSlot{}, + AddressedResults: map[ID][]llvmAddressedResult{}, + ResultSlots: map[ID]llvm.Value{}, + ItabMethods: map[ID]bool{}, + ClosureCodeLoads: map[ID]bool{}, + DeferResults: map[llvmLocalKey]bool{}, + DeferResultKeys: map[ID]llvmLocalKey{}, + RequiredInlinePos: map[int]bool{}, + OpenDeferSlots: map[llvmLocalKey]int{}, + F: f, + b: GlobalCtxt.NewBuilder(), + ReturnType: sig.ReturnType, + ResultCount: sig.ResultCount, } defer FCtxt.b.Dispose() @@ -3042,8 +3219,20 @@ func LLVMCompile(f *Func) { } } cgoUnsafeArgs := frontendFunc != nil && frontendFunc.Pragma&ir.CgoUnsafeArgs != 0 - frontendNoInline := frontendFunc != nil && (frontendFunc.Pragma&ir.Noinline != 0 || frontendFunc.HasDefer() || cgoUnsafeArgs) - if frontendNoInline || llvmFuncCalls(f, "runtime.gorecover") { + // Native Go performs all inlining before it computes and checks the nosplit + // call graph. Do not let LLVM perform a second, invisible round of inlining + // for a nosplit callee: doing so can inflate a caller's frame after Go's + // budget decisions and make an otherwise valid runtime nosplit chain fail at + // link time. + frontendNoInline := f.NoSplit || frontendFunc != nil && (frontendFunc.Pragma&ir.Noinline != 0 || frontendFunc.HasDefer() || cgoUnsafeArgs) + // Go 1.27's gorecover implementation unwinds physical frames and skips its + // own frame before counting the deferred caller. LLVM is capable of inlining + // this much larger function even though the Go inliner does not; doing so + // removes the frame that the runtime algorithm deliberately skips and can + // make an unrelated active panic recoverable. Preserve both gorecover itself + // and every direct recover caller as physical frame boundaries. + preserveRecoverFrame := llvmPreserveRecoverFrame(f) + if frontendNoInline || preserveRecoverFrame { FCtxt.LF.AddFunctionAttr(llvmNoInlineAttribute()) } if f.OpenDeferBits != nil { @@ -3070,10 +3259,24 @@ func LLVMCompile(f *Func) { } } FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goAsyncUnsafeAttr, "")) - // TODO(goallc): Once LLVM lowering propagates the compiler's precise - // morestack policy, attach this only to functions whose prologue can grow - // the Go stack. + // The stack-growth attribute supplies the target's entry-argument map and, + // when a split prologue is permitted, asks it to represent the late + // morestack call as a root-free statepoint. FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goStackGrowthStatepointAttr, "")) + // A //go:nosplit function must not acquire that late morestack edge. Besides + // violating the runtime's nosplit call graph, it would expose a safepoint the + // frontend deliberately prohibited. Give target frame lowering the source + // policy directly instead of asking it to infer a pragma from GoObj metadata. + if f.NoSplit { + FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goNoSplitAttr, "")) + } + // Native Go gives //go:systemstack functions a distinct stack-growth + // prologue: it checks g.stackguard1 and calls runtime.morestackc. Carry the + // source pragma through AttrCFunc so target frame lowering cannot silently + // use the ordinary goroutine stack-growth protocol for runtime code. + if f.OwnAux.Fn.CFunc() { + FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goSystemStackAttr, "")) + } FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(llvmFramePointerAttr, llvmFramePointerNonLeaf)) if sig.HasClosureContext { FCtxt.ClosureContext = FCtxt.LF.Param(sig.ClosureContextIndex) @@ -3121,6 +3324,10 @@ func LLVMCompile(f *Func) { if !code.Type.IsUintptr() { v.Fatalf("direct closure call code address has type %v", code.Type) } + case OpConvert: + if !base.Flag.CompilingRuntime || !llvmRuntimeConstructedClosure(v, code, v.Args[1]) { + v.Fatalf("closure call has unsupported converted code pointer") + } default: v.Fatalf("closure call code pointer has unsupported form %s", code.Op) } @@ -3247,6 +3454,27 @@ func LLVMCompile(f *Func) { } var parameterHomes []*Value var parameterLifetimeSlots []llvmStackSlot + type cgoUnsafeParameterHome struct { + index int + name *ir.Name + typ *types.Type + slot llvmStackSlot + } + var cgoUnsafeParameterHomes []cgoUnsafeParameterHome + if cgoUnsafeArgs { + for i, param := range inParams { + if param.Name == nil || param.Type.Size() == 0 { + continue + } + slot, _ := preallocateLocal(param.Name, param.Name.Sym().Name+".cgo") + cgoUnsafeParameterHomes = append(cgoUnsafeParameterHomes, cgoUnsafeParameterHome{ + index: i, + name: param.Name, + typ: param.Type, + slot: slot, + }) + } + } for _, BB := range f.Blocks { for _, v := range BB.Values { if v.Op != OpLocalAddr || v.Uses == 0 { @@ -3359,16 +3587,38 @@ func LLVMCompile(f *Func) { // incoming register piece separately and addresses stack-assigned parameters // in their incoming slots. The full aggregate store makes any existing piece // loads and stores redundant and lets normal LLVM memory optimization remove - // them. A future optimization may bind wholly stack-assigned parameters - // directly to their incoming fixed stack slots. + // them. Store the physical ABI carrier directly through the opaque pointer: + // reconstructing its semantic named aggregate first obscures the formal + // argument store from SelectionDAG and prevents a wholly stack-assigned + // parameter home from being folded back to its incoming fixed stack slot. FCtxt.b.SetInsertPointAtEnd(FCtxt.BBs[f.Entry.ID]) + if len(cgoUnsafeParameterHomes) != 0 { + var owner *Value + for _, block := range f.Blocks { + if len(block.Values) != 0 { + owner = block.Values[0] + break + } + } + if owner == nil { + f.fe.Fatalf(f.Entry.Pos, "cgo unsafe argument function has no SSA value for diagnostics") + } + for _, home := range cgoUnsafeParameterHomes { + param := FCtxt.LF.Param(home.index) + param = FCtxt.llvmValueFromABI(owner, param, home.typ, home.slot.Type, home.name.Sym().Name+".cgo.home") + if param.Type() != getLLVMType(home.slot.Type) { + f.fe.Fatalf(home.name.Pos(), "cgo unsafe argument home changes LLVM representation") + } + init := FCtxt.b.CreateStore(param, home.slot.Value) + init.SetAlignment(int(home.slot.Type.Alignment())) + } + } for _, v := range parameterHomes { name, key := llvmLocalName(v) slot := FCtxt.Locals[key] param, paramType := FCtxt.paramForArgNameAndType(v, name) - param = FCtxt.llvmValueFromABI(v, param, paramType, slot.Type, v.String()+".home") - if param.Type() != getLLVMType(slot.Type) { - v.Fatalf("parameter home changes LLVM representation") + if paramType.Size() != slot.Type.Size() || param.Type() != getLLVMABIType(slot.Type) { + v.Fatalf("parameter home has incompatible physical ABI carrier") } init := FCtxt.b.CreateStore(param, slot.Value) init.SetAlignment(int(slot.Type.Alignment())) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index a25aeab2f93848..7be0cd3fd889c2 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -10,12 +10,168 @@ import ( "strings" "testing" + "cmd/compile/internal/ir" + "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/src" "github.com/goallc/go-llvm" ) +type llvmTestTypeName struct { + sym *types.Sym +} + +func (n *llvmTestTypeName) Sym() *types.Sym { return n.sym } +func (*llvmTestTypeName) Pos() src.XPos { return src.NoXPos } +func (*llvmTestTypeName) Type() *types.Type { return nil } + +func TestLLVMABICarrierErasesNamedAggregateIdentity(t *testing.T) { + pkg := types.NewPkg("runtime", "runtime") + namedSlice := types.NewNamed(&llvmTestTypeName{sym: pkg.Lookup("slice")}) + namedSlice.SetUnderlying(types.NewStruct([]*types.Field{ + types.NewField(src.NoXPos, pkg.Lookup("array"), types.Types[types.TUNSAFEPTR]), + types.NewField(src.NoXPos, pkg.Lookup("len"), types.Types[types.TINT]), + types.NewField(src.NoXPos, pkg.Lookup("cap"), types.Types[types.TINT]), + })) + types.CalcSize(namedSlice) + builtinSlice := types.NewSlice(types.Types[types.TUINT8]) + types.CalcSize(builtinSlice) + + if getLLVMType(namedSlice) == getLLVMType(builtinSlice) { + t.Fatal("semantic LLVM types unexpectedly lost named aggregate identity") + } + if got, want := getLLVMABIType(namedSlice), getLLVMABIType(builtinSlice); got != want { + t.Fatalf("physical ABI carriers differ: named=%v builtin=%v", got, want) + } +} + +func TestLLVMUntypedABI0FunctionAddressCreatesFunctionDeclaration(t *testing.T) { + oldModule := CurrentModule + oldLowerer := currentLLVMDataLowerer + oldTarget := typecheck.Target + module := GlobalCtxt.NewModule("abi0_function_address") + CurrentModule = module + currentLLVMDataLowerer = nil + typecheck.Target = new(ir.Package) + t.Cleanup(func() { + typecheck.Target = oldTarget + currentLLVMDataLowerer = oldLowerer + CurrentModule = oldModule + module.Dispose() + }) + + typ := llvm.FunctionType(GlobalCtxt.VoidType(), nil, false) + internal := llvm.AddFunction(module, "runtime.asyncPreempt", typ) + internal.SetFunctionCallConv(goABIInternalCallConv) + + pkg := types.NewPkg("runtime", "runtime") + fn := ir.NewFunc(src.NoXPos, src.NoXPos, pkg.Lookup("asyncPreempt"), nil) + fn.ABI = obj.ABI0 + typecheck.Target.Funcs = append(typecheck.Target.Funcs, fn) + sym := fn.LinksymABI(fn.ABI) + if sym.Type != objabi.Sxxx { + t.Fatalf("test requires an unresolved bodyless LSym, got %v", sym.Type) + } + got := llvmGoDataRef(sym) + if got.IsAFunction().IsNil() || got.Name() != "runtime.asyncPreempt.goallc.abi0" { + t.Fatalf("ABI0 function address resolved to %q, want ABI0 function declaration", got.Name()) + } +} + +func TestLLVMNamedAggregateConversionCanReshape(t *testing.T) { + pkg := types.NewPkg("runtime", "runtime") + newCacheType := func(name string, scavType *types.Type) *types.Type { + typ := types.NewNamed(&llvmTestTypeName{sym: pkg.Lookup(name)}) + typ.SetUnderlying(types.NewStruct([]*types.Field{ + types.NewField(src.NoXPos, pkg.Lookup("base"), types.Types[types.TUINTPTR]), + types.NewField(src.NoXPos, pkg.Lookup("cache"), types.Types[types.TUINT64]), + types.NewField(src.NoXPos, pkg.Lookup("scav"), scavType), + })) + return typ + } + pageCache := newCacheType("pageCache", types.Types[types.TUINT64]) + exportedPageCache := newCacheType("PageCache", types.Types[types.TUINT64]) + wrongPageCache := newCacheType("WrongPageCache", types.Types[types.TUINT32]) + + if !llvmValueTypesCanReshape(pageCache, exportedPageCache) { + t.Fatal("defined structs with identical underlying types cannot reshape") + } + if llvmValueTypesCanReshape(pageCache, wrongPageCache) { + t.Fatal("defined structs with different underlying types can reshape") + } +} + +func TestLLVMRuntimeConstructedClosure(t *testing.T) { + mem := &Value{ID: 1, Op: OpInitMem, Type: types.TypeMem} + rawCode := &Value{ID: 2, Op: OpArg, Type: types.Types[types.TUNSAFEPTR]} + code := &Value{ID: 3, Op: OpConvert, Type: types.Types[types.TUINTPTR], Args: []*Value{rawCode, mem}, Uses: 2} + context := &Value{ID: 4, Op: OpLocalAddr, Type: types.NewPtr(types.Types[types.TUINTPTR])} + codeAddress := &Value{ID: 5, Op: OpOffPtr, Type: types.NewPtr(types.Types[types.TUINTPTR]), Args: []*Value{context}} + codeStore := &Value{ID: 6, Op: OpStore, Type: types.TypeMem, Args: []*Value{codeAddress, code, mem}} + otherStore := &Value{ID: 7, Op: OpStore, Type: types.TypeMem, Args: []*Value{context, context, codeStore}} + argument := &Value{ID: 8, Op: OpArg, Type: types.Types[types.TUNSAFEPTR]} + call := &Value{ID: 9, Op: OpClosureLECall, Type: types.TypeMem, Args: []*Value{code, context, argument, otherStore}} + + if !llvmRuntimeConstructedClosure(call, code, context) { + t.Fatal("runtime-constructed funcval was not recognized") + } + + wrongCode := &Value{ID: 10, Op: OpConvert, Type: types.Types[types.TUINTPTR], Args: []*Value{rawCode, mem}, Uses: 2} + call.Args[0] = wrongCode + if llvmRuntimeConstructedClosure(call, wrongCode, context) { + t.Fatal("code value not stored in the funcval context was accepted") + } +} + +func TestLLVMJumpTableDefaultIsUnreachable(t *testing.T) { + module := GlobalCtxt.NewModule("jump_table_default") + builder := GlobalCtxt.NewBuilder() + t.Cleanup(module.Dispose) + t.Cleanup(builder.Dispose) + + i64 := GlobalCtxt.Int64Type() + function := llvm.AddFunction(module, "jump_table_default", llvm.FunctionType(i64, []llvm.Type{i64}, false)) + jumpLLVM := llvm.AddBasicBlock(function, "jump") + mergeLLVM := llvm.AddBasicBlock(function, "merge") + otherLLVM := llvm.AddBasicBlock(function, "other") + + jump := &Block{ID: 1, Kind: BlockJumpTable} + merge := &Block{ID: 2} + other := &Block{ID: 3} + control := &Value{ID: 1, Type: types.Types[types.TINT]} + jump.Controls[0] = control + jump.Succs = []Edge{{b: merge}, {b: merge}, {b: other}} + context := &LLVMFuncContext{ + BBs: map[ID]llvm.BasicBlock{ + jump.ID: jumpLLVM, + merge.ID: mergeLLVM, + other.ID: otherLLVM, + }, + Vs: map[ID]llvm.Value{control.ID: function.Param(0)}, + LF: function, + b: builder, + } + context.CompileBlock(jump, nil) + + builder.SetInsertPointAtEnd(mergeLLVM) + phi := builder.CreatePHI(i64, "carried") + seven := llvm.ConstInt(i64, 7, false) + phi.AddIncoming([]llvm.Value{seven, seven}, []llvm.BasicBlock{jumpLLVM, jumpLLVM}) + builder.CreateRet(phi) + builder.SetInsertPointAtEnd(otherLLVM) + builder.CreateRet(llvm.ConstInt(i64, 9, false)) + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("jump table added a non-SSA default edge: %v\n%s", err, module.String()) + } + if ir := module.String(); !strings.Contains(ir, "b1.jump.default") || !strings.Contains(ir, "unreachable") { + t.Fatalf("jump table has no unreachable default block\n%s", ir) + } +} + func TestLLVMCurrentGRegister(t *testing.T) { for _, test := range []struct { name string @@ -184,3 +340,31 @@ func TestLLVMTargetCPU(t *testing.T) { }) } } + +func TestLLVMPreserveRecoverFrameUsesLinkSymbolName(t *testing.T) { + recoverFn := &Func{ + Name: "gorecover", + OwnAux: &AuxCall{Fn: &obj.LSym{Name: "runtime.gorecover"}}, + } + if !llvmPreserveRecoverFrame(recoverFn) { + t.Fatal("gorecover definition was not recognized from its qualified link symbol") + } + + unrelated := &Func{ + Name: "gorecover", + OwnAux: &AuxCall{Fn: &obj.LSym{Name: "other.gorecover"}}, + } + if llvmPreserveRecoverFrame(unrelated) { + t.Fatal("unqualified SSA function name incorrectly identified another package's gorecover") + } + + caller := &Func{ + OwnAux: &AuxCall{Fn: &obj.LSym{Name: "runtime.preprintpanics.func1"}}, + Blocks: []*Block{{ + Values: []*Value{{Aux: &AuxCall{Fn: &obj.LSym{Name: "runtime.gorecover"}}}}, + }}, + } + if !llvmPreserveRecoverFrame(caller) { + t.Fatal("direct gorecover caller was not preserved as a physical frame") + } +} diff --git a/src/cmd/compile/internal/ssagen/nowb.go b/src/cmd/compile/internal/ssagen/nowb.go index 8e776695e3cef5..d8a82c067cd057 100644 --- a/src/cmd/compile/internal/ssagen/nowb.go +++ b/src/cmd/compile/internal/ssagen/nowb.go @@ -10,6 +10,7 @@ import ( "cmd/compile/internal/base" "cmd/compile/internal/ir" + "cmd/compile/internal/ssa" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/obj" @@ -117,6 +118,27 @@ func (c *nowritebarrierrecChecker) recordCall(fn *ir.Func, to *obj.LSym, pos src *fn.NWBRCalls = append(*fn.NWBRCalls, ir.SymAndPos{Sym: to, Pos: pos}) } +// recordLLVMNoWriteBarrierCalls records the direct calls that remain in the +// final SSA graph for the LLVM IR-only backend. The native backend normally +// records these from State.PrepareCall while emitting machine instructions, +// but the LLVM pipeline deliberately returns before genssa because llc emits +// the linker object instead. Keep the runtime's //go:nowritebarrierrec check +// attached to the same post-optimization SSA call graph in both pipelines. +func recordLLVMNoWriteBarrierCalls(fn *ir.Func, f *ssa.Func) { + if nowritebarrierrecCheck == nil { + return + } + for _, b := range f.Blocks { + for _, v := range b.Values { + call, ok := v.Aux.(*ssa.AuxCall) + if !ok || call.Fn == nil { + continue + } + nowritebarrierrecCheck.recordCall(fn, call.Fn, v.Pos) + } + } +} + func (c *nowritebarrierrecChecker) check() { // We walk the call graph as late as possible so we can // capture all calls created by lowering, but this means we diff --git a/src/cmd/compile/internal/ssagen/pgen.go b/src/cmd/compile/internal/ssagen/pgen.go index b7dbda4a350170..25d91191ae52d6 100644 --- a/src/cmd/compile/internal/ssagen/pgen.go +++ b/src/cmd/compile/internal/ssagen/pgen.go @@ -307,6 +307,7 @@ func Compile(fn *ir.Func, worker int, profile *pgoir.Profile) { // enter genssa: ssa.Compile has already emitted LLVM IR, while genssa // consumes native register-allocation state and emits the _go_.o // member that llc is replacing. + recordLLVMNoWriteBarrierCalls(fn, f) return } // Note: check arg size to fix issue 25507. diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 0bfbf05d58484a..3ffec4ec24ba5f 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -21,14 +21,29 @@ import ( type llvmABIDocument struct { Members []struct { GoObject *struct { - Symbols []llvmABISymbol `json:"symbols"` + Symbols []llvmABISymbol `json:"symbols"` + References []llvmABISymbol `json:"references"` } `json:"go_object"` } `json:"members"` } -type llvmABISymbol struct { +type llvmABIReference struct { + PkgKind string `json:"pkg_kind"` + SymIndex uint32 `json:"sym_index"` Name string `json:"name"` - ABI uint16 `json:"abi"` +} + +type llvmABISymbol struct { + Class string `json:"class"` + ClassIndex uint32 `json:"class_index"` + Name string `json:"name"` + ABI uint16 `json:"abi"` + Flags uint64 `json:"flags"` + FlagNames []string `json:"flag_names"` + Relocations []struct { + Type string `json:"type"` + Target llvmABIReference `json:"target"` + } `json:"relocations"` Function *struct { Info *struct { Args uint32 `json:"args"` @@ -190,9 +205,17 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") + // The runtime wrapper applies its configured LLVM optimization pipeline + // before llc. In particular, InstCombine removes the field-by-field bridge + // between physical ABI carriers and semantic named aggregates, allowing + // stack-assigned pointer arguments to remain in their canonical fixed homes. + optimizedLLVMIR := llvmArchive + ".opt.ll" + runLLVMABICommand(t, nil, opt, "-passes=default", "-S", llvmIR, + "-o", optimizedLLVMIR) + machineIR := runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", - "-o", "-", llvmIR) + "-o", "-", optimizedLLVMIR) for _, pattern := range []string{ `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.[02].*?LDRXui\s+%fixed-stack\.[02]`, @@ -205,7 +228,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } goallcAssembly := runLLVMABICommand(t, nil, llc, - "-load-pass-plugin="+plugin, "-filetype=asm", llvmIR, "-o", "-") + "-load-pass-plugin="+plugin, "-filetype=asm", optimizedLLVMIR, "-o", "-") for _, name := range []string{ "main.mixedABI", "main.liveScalarStackArgument", "main.livePointerSequenceStackArguments", @@ -220,7 +243,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { checkLLVMABIAssembly(t, nativeAssembly, goallcAssembly) runLLVMABICommand(t, nil, llc, - "-load-pass-plugin="+plugin, "-filetype=obj", llvmIR, "-o", goallcObject) + "-load-pass-plugin="+plugin, "-filetype=obj", optimizedLLVMIR, "-o", goallcObject) native := readLLVMABIObject(t, nativeObject) goallc := readLLVMABIObject(t, goallcObject) @@ -230,7 +253,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0}, nil}, goallcArgsMaps: [][]int{{0}, nil, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 1, 0, 2}, + goallcStackMaps: []int32{-1, 0, 1, 2}, }, { name: "mixedABI", args: 152, pointerBits: []int{2, 4, 18}, @@ -239,8 +262,8 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { // homes in LocalsPointerMaps through locals-only alloca records. goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, - goallcQueryMaps: [][]int{{2}, {2}, {2}, {2}, {2, 4, 18}}, + goallcStackMaps: []int32{-1, 0, 1}, + goallcQueryMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}, {2}}, }, { name: "liveScalarStackArgument", args: 136, pointerBits: []int{0}, @@ -289,35 +312,35 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0}, nil}, goallcArgsMaps: [][]int{{0}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, + goallcStackMaps: []int32{-1, 0, 1}, }, { name: "overflowResults", args: 48, pointerBits: []int{2, 3, 4, 5}, nativeArgsMaps: [][]int{{2, 3, 4, 5}, nil}, goallcArgsMaps: [][]int{{2, 3, 4, 5}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, + goallcStackMaps: []int32{-1, 0, 1}, }, { name: "initializedStackResult", args: 16, pointerBits: []int{1}, nativeArgsMaps: [][]int{{1}, nil}, goallcArgsMaps: [][]int{{1}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, + goallcStackMaps: []int32{-1, 0, 1}, }, { name: "stackAggregateResult", args: 40, pointerBits: []int{4}, nativeArgsMaps: [][]int{{4}, nil}, goallcArgsMaps: [][]int{{4}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 1, 0}, + goallcStackMaps: []int32{-1, 0, 1}, }, { name: "bothOverflow", args: 168, pointerBits: []int{2, 6, 20}, nativeArgsMaps: [][]int{{2, 6, 20}, nil}, goallcArgsMaps: [][]int{{2, 6, 20}, {2}, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 1, 0, 2}, + goallcStackMaps: []int32{-1, 0, 1, 2}, }, { name: "pointerAggregateBothOverflow", args: 152, pointerBits: []int{0, 2}, @@ -338,7 +361,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{4}, nil}, goallcArgsMaps: [][]int{{4}, nil, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 1, 2, 0}, + goallcStackMaps: []int32{-1, 0, 1, 2}, }, } for _, tc := range cases { @@ -616,10 +639,13 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p } runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") + optimizedGoallcIR := goallcArchive + ".opt.ll" + runLLVMABICommand(t, nil, opt, "-passes=default", "-S", goallcIR, + "-o", optimizedGoallcIR) machineIR := runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", - "-o", "-", goallcIR) + "-o", "-", optimizedGoallcIR) for _, pattern := range []string{ `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, @@ -629,7 +655,7 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p } } runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, - "-filetype=obj", goallcIR, "-o", goallcObject) + "-filetype=obj", optimizedGoallcIR, "-o", goallcObject) native := readLLVMABIObject(t, nativeObject) goallc := readLLVMABIObject(t, goallcObject) @@ -654,16 +680,16 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p nativeLocals: 8, goallcLocals: 24, nativeArgs: [][]int{{1}, nil}, goallcArgs: [][]int{{1}, nil}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {1}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, 0}, - nativeQueries: []int32{0, -1}, goallcQueries: []int32{1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, 1}, + nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 1}, }, { name: "partiallyInitializedAggregateResult", args: 40, entryBits: []int{3, 4}, nativeLocals: 8, goallcLocals: 24, nativeArgs: [][]int{{3, 4}, nil}, goallcArgs: [][]int{{3, 4}, nil}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {0, 1}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, 0}, - nativeQueries: []int32{0, 0, -1}, goallcQueries: []int32{1, 1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, 1}, + nativeQueries: []int32{0, 0, -1}, goallcQueries: []int32{0, 1, 1}, }, { name: "liveScalarStackArgument", args: 136, entryBits: []int{0}, @@ -746,7 +772,7 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string t.Helper() for _, callee := range callees { call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + - regexp.QuoteMeta(callee) + `.*#([0-9]+)$`).FindSubmatch(ir) + regexp.QuoteMeta(callee) + `.*#([0-9]+)(?:,|$)`).FindSubmatch(ir) if len(call) != 2 { t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) } @@ -936,6 +962,42 @@ func findLLVMABISymbol(t *testing.T, document llvmABIDocument, name string) llvm return llvmABISymbol{} } +func llvmABIRelocationTargetName(document llvmABIDocument, target llvmABIReference) string { + if target.Name != "" || target.PkgKind != "none" { + return target.Name + } + for _, member := range document.Members { + if member.GoObject == nil { + continue + } + var nonpackageDefinitions uint32 + for _, symbol := range member.GoObject.Symbols { + if symbol.Class == "nonpackage" && symbol.ClassIndex >= nonpackageDefinitions { + nonpackageDefinitions = symbol.ClassIndex + 1 + } + } + if target.SymIndex < nonpackageDefinitions { + continue + } + classIndex := target.SymIndex - nonpackageDefinitions + for _, reference := range member.GoObject.References { + if reference.Class == "nonpackage_reference" && reference.ClassIndex == classIndex { + return reference.Name + } + } + } + return "" +} + +func llvmABIHasRelocationTo(document llvmABIDocument, symbol llvmABISymbol, name string) bool { + for _, relocation := range symbol.Relocations { + if llvmABIRelocationTargetName(document, relocation.Target) == name { + return true + } + } + return false +} + func checkLLVMABISymbol(t *testing.T, backend string, symbol llvmABISymbol, tc llvmABICase) { t.Helper() if symbol.ABI != 1 { diff --git a/src/cmd/internal/testdir/llvm_alloca_test.go b/src/cmd/internal/testdir/llvm_alloca_test.go index 41c4da5be11ccb..de76274715320d 100644 --- a/src/cmd/internal/testdir/llvm_alloca_test.go +++ b/src/cmd/internal/testdir/llvm_alloca_test.go @@ -20,6 +20,8 @@ type llvmAllocaArchitectureChecks struct { restoredStorePattern string goallcLocals uint32 goallcPointerBits []int + goallcPCData []int32 + goallcQueries []int32 } var llvmAllocaChecks = map[string]llvmAllocaArchitectureChecks{ @@ -28,12 +30,16 @@ var llvmAllocaChecks = map[string]llvmAllocaArchitectureChecks{ restoredStorePattern: `(?m)^\s*(?:str|stp)\b`, goallcLocals: 88, goallcPointerBits: []int{5, 7, 8, 9}, + goallcPCData: []int32{-1, 0, 1}, + goallcQueries: []int32{0, 1, 1, 1, 1}, }, "linux/amd64": { betweenCallsPattern: `(?s)\bcallq\s+p\.mutateLocal\n(.*?)\bcallq\s+p\.safepoint`, restoredStorePattern: `(?m)^\s*mov[a-z]*\s+[^,\n]+,\s*-[0-9]+\(%rbp\)`, goallcLocals: 88, goallcPointerBits: []int{5, 7, 8, 9}, + goallcPCData: []int32{-1, 1, 0}, + goallcQueries: []int32{1, 1, 1, 1, 0}, }, } @@ -86,10 +92,10 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { parameterInputFunction := llvmAllocaIRFunction(t, inputIR, "p.parameterAcrossSafepoints") for _, pattern := range []string{ - `define goabiinternal void @p\.parameterAcrossSafepoints\(%p\.pointerLocal %value\)`, + `define goabiinternal void @p\.parameterAcrossSafepoints\(\{ ptr, i64, ptr, \[2 x ptr\] \} %value\)`, `alloca %p\.pointerLocal, align 8`, `call void @llvm\.lifetime\.start\.p0\(ptr %v[0-9]+\)`, - `store %p\.pointerLocal %value, ptr %v[0-9]+, align 8`, + `store \{ ptr, i64, ptr, \[2 x ptr\] \} %value, ptr %v[0-9]+, align 8`, } { if !regexp.MustCompile(pattern).Match(parameterInputFunction) { t.Fatalf("input parameter-home IR does not match %q\n%s", @@ -283,8 +289,8 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { checks.goallcLocals, [][]int{nil, nil}, [][]int{nil, checks.goallcPointerBits}, - []int32{-1, 1, 0}, - []int32{1, 1, 1, 1, 0}) + checks.goallcPCData, + checks.goallcQueries) goallcStackObjects := llvmABIStackObjects(t, symbol) if len(goallcStackObjects) != 0 { diff --git a/src/cmd/internal/testdir/llvm_test.go b/src/cmd/internal/testdir/llvm_test.go index 353951f1004078..5285031c29a0fd 100644 --- a/src/cmd/internal/testdir/llvm_test.go +++ b/src/cmd/internal/testdir/llvm_test.go @@ -15,7 +15,9 @@ import ( "os/exec" "path" "path/filepath" + "regexp" "runtime" + "slices" "sort" "strings" "sync" @@ -189,6 +191,8 @@ func runLLVMInfrastructureTests(t *testing.T, common testCommon) { runLLVMCallerStateTest(t, common.gorootTestDir) }) t.Run("getg-abi0-fail-closed", runLLVMGetGABI0FailClosedTest) + t.Run("nosplit", runLLVMNoSplitTest) + t.Run("nowritebarrierrec", runLLVMNoWriteBarrierRecTest) t.Run("writebarrier-helpers", runLLVMWriteBarrierHelperTest) t.Run("compile-only-regressions", func(t *testing.T) { for _, name := range []string{"cmp.go", "typeparam/issue47684c.go"} { @@ -200,6 +204,112 @@ func runLLVMInfrastructureTests(t *testing.T, common testCommon) { t.Run("writebarrier-ir", runLLVMWriteBarrierIRTests) } +func runLLVMNoSplitTest(t *testing.T) { + t.Helper() + dir := t.TempDir() + source := filepath.Join(dir, "nosplit.go") + program := `package p + +func use(*[32]uintptr) + +//go:nosplit +//go:noinline +func NoSplit(pointer *int) *int { + var words [32]uintptr + use(&words) + return pointer +} + +//go:noinline +func Split(pointer *int) *int { + var words [32]uintptr + use(&words) + return pointer +} +` + if err := os.WriteFile(source, []byte(program), 0o666); err != nil { + t.Fatal(err) + } + archive := filepath.Join(dir, "nosplit.a") + runLLVMABICommand(t, nil, goTool, "tool", "compile", + "-p=p", "-enablellvm", "-llvmironly", "-o", archive, source) + ir, err := os.ReadFile(archive + ".ll") + if err != nil { + t.Fatal(err) + } + attributeLine := func(name string) []byte { + definition := regexp.MustCompile(`(?m)^define goabiinternal .*@` + regexp.QuoteMeta(name) + `\([^\n]*\) #([0-9]+)`).FindSubmatch(ir) + if definition == nil { + t.Fatalf("LLVM IR has no attributed definition for %s\n%s", name, ir) + } + pattern := regexp.MustCompile(`(?m)^attributes #` + string(definition[1]) + ` = \{.*$`) + line := pattern.Find(ir) + if line == nil { + t.Fatalf("LLVM IR has no attribute group for %s\n%s", name, ir) + } + return line + } + noSplitAttrs := attributeLine("p.NoSplit") + if !bytes.Contains(noSplitAttrs, []byte(`"go-nosplit"`)) || + !bytes.Contains(noSplitAttrs, []byte(`"go-stack-growth-statepoint"`)) || + !bytes.Contains(noSplitAttrs, []byte(`noinline`)) { + t.Fatalf("LLVM nosplit attributes do not select the nosplit prologue policy: %s", noSplitAttrs) + } + splitAttrs := attributeLine("p.Split") + if !bytes.Contains(splitAttrs, []byte(`"go-stack-growth-statepoint"`)) || + bytes.Contains(splitAttrs, []byte(`"go-nosplit"`)) { + t.Fatalf("LLVM split attributes do not select the stack-growth prologue policy: %s", splitAttrs) + } + + llc := llvmToolPath(t, "llc", "GOALLC_LLC") + plugin := llvmABIPassPlugin(t, llc) + object := filepath.Join(dir, "nosplit.o") + runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, + "-verify-machineinstrs", "-filetype=obj", "-o", object, archive+".ll") + document := readLLVMABIObject(t, object) + noSplit := findLLVMABISymbol(t, document, "p.NoSplit") + if !slices.Contains(noSplit.FlagNames, "nosplit") { + t.Fatalf("p.NoSplit GoObj flags %v do not contain nosplit", noSplit.FlagNames) + } + if llvmABIHasRelocationTo(document, noSplit, "runtime.morestack_noctxt") { + t.Fatal("p.NoSplit unexpectedly calls runtime.morestack_noctxt") + } + split := findLLVMABISymbol(t, document, "p.Split") + if !llvmABIHasRelocationTo(document, split, "runtime.morestack_noctxt") { + t.Fatal("p.Split has no runtime.morestack_noctxt relocation") + } +} + +func runLLVMNoWriteBarrierRecTest(t *testing.T) { + t.Helper() + archive := filepath.Join(t.TempDir(), "nowritebarrier.a") + source := filepath.Join(testenv.GOROOT(t), "test", "nowritebarrier.go") + cmd := exec.Command(goTool, "tool", "compile", + "-p=runtime", + "-+", + "-C", + "-e", + "-d=ssa/check/on", + "-enablellvm", + "-llvmironly", + "-o", archive, + source, + ) + cmd.Env = append(os.Environ(), "GOENV=off", "GOFLAGS=") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("LLVM accepted prohibited runtime write barriers\n%s", out) + } + for _, want := range []string{ + "write barrier prohibited by caller; b2", + "write barrier prohibited by caller; d3", + } { + if !bytes.Contains(out, []byte(want)) { + t.Fatalf("LLVM //go:nowritebarrierrec check missed %q: %v\n%s", want, err, out) + } + } +} + func runLLVMGetGABI0FailClosedTest(t *testing.T) { t.Helper() dir := t.TempDir() diff --git a/src/cmd/llvmplugin/GoALLCInlineAnchors.cpp b/src/cmd/llvmplugin/GoALLCInlineAnchors.cpp index 8f854545f28ede..d74c779ea3e265 100644 --- a/src/cmd/llvmplugin/GoALLCInlineAnchors.cpp +++ b/src/cmd/llvmplugin/GoALLCInlineAnchors.cpp @@ -3,14 +3,17 @@ // license that can be found in the LICENSE file. #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/ErrorHandling.h" @@ -71,6 +74,42 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass { Loc->getAtomRank()); } + SmallVector + requiredInlineLocations(const MachineFunction &MF) const { + SmallVector Required; + const Module *M = MF.getFunction().getParent(); + const NamedMDNode *Locations = + M ? M->getNamedMetadata("goobj.debug.inline.required") : nullptr; + if (!Locations) + return Required; + + for (const MDNode *Entry : Locations->operands()) { + if (Entry->getNumOperands() != 2) + report_fatal_error("expected !goobj.debug.inline.required entries to " + "have two operands"); + const auto *CAM = + dyn_cast_or_null(Entry->getOperand(0)); + const auto *GV = CAM ? dyn_cast(CAM->getValue()) : nullptr; + const auto *Loc = dyn_cast_or_null(Entry->getOperand(1)); + if (!GV || !Loc || !Loc->getInlinedAt()) + report_fatal_error("invalid !goobj.debug.inline.required entry"); + if (GV == &MF.getFunction()) + Required.push_back(Loc); + } + + llvm::stable_sort( + Required, [](const DILocation *LHS, const DILocation *RHS) { + auto Depth = [](const DILocation *Loc) { + unsigned Result = 0; + for (; Loc && Loc->getInlinedAt(); Loc = Loc->getInlinedAt()) + ++Result; + return Result; + }; + return Depth(LHS) > Depth(RHS); + }); + return Required; + } + public: static char ID; @@ -88,6 +127,51 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass { const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); CanonicalSites.clear(); + // LLVM may merge instructions from distinct frontend inline frames (for + // example, SLP-vectorizing adjacent stores) and retain only one debug + // location. Materialize a source NOP for each required inline edge that no + // longer occurs in the optimized MachineFunction. Deeper chains are + // considered first because one such marker also preserves all its parents. + DenseSet SurvivingCallsites; + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &MI : MBB) + if (!MI.isMetaInstruction()) + for (const DILocation *Loc = MI.getDebugLoc().get(); + Loc && Loc->getInlinedAt(); Loc = Loc->getInlinedAt()) + SurvivingCallsites.insert(Loc->getInlinedAt()); + + bool Changed = false; + if (!MF.empty()) { + MachineBasicBlock *InsertBlock = nullptr; + MachineBasicBlock::iterator InsertAt; + for (MachineBasicBlock &MBB : MF) { + auto It = MBB.begin(); + while (It != MBB.end() && (It->isMetaInstruction() || + It->getFlag(MachineInstr::FrameSetup))) + ++It; + if (It != MBB.end()) { + InsertBlock = &MBB; + InsertAt = It; + break; + } + } + for (const DILocation *Loc : requiredInlineLocations(MF)) { + const DILocation *Innermost = Loc->getInlinedAt(); + if (SurvivingCallsites.contains(Innermost)) + continue; + if (!InsertBlock) + report_fatal_error("GoALLC required inline location has no " + "post-prologue insertion point"); + TII.insertNoop(*InsertBlock, InsertAt); + MachineInstr &Marker = *std::prev(InsertAt); + Marker.setDebugLoc(DebugLoc(Loc)); + for (const DILocation *Site = Loc; Site && Site->getInlinedAt(); + Site = Site->getInlinedAt()) + SurvivingCallsites.insert(Site->getInlinedAt()); + Changed = true; + } + } + // Normalize complete inline chains before inspecting them. This keeps the // anchor pass and the later GoObj debug handler on the same edge identity. for (MachineBasicBlock &MBB : MF) @@ -97,7 +181,6 @@ class GoALLCInlineAnchorPass final : public MachineFunctionPass { DebugLoc(canonicalizeLocation(MI.getDebugLoc().get()))); DenseSet AnchoredCallsites; - bool Changed = false; for (MachineBasicBlock &MBB : MF) { for (auto It = MBB.begin(), End = MBB.end(); It != End; ++It) { diff --git a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp b/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp index d8e88ffc55df66..07ba7e50d2d397 100644 --- a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp +++ b/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp @@ -77,25 +77,32 @@ bool GoALLCStackMapPrinter::emitStackMaps(StackMaps &SM, AsmPrinter &AP) { // operands, GC base/derived pairs, and GC allocas into Locations. The // first three entries and the deopt operands are not GC roots. const StackMaps::CallsiteInfo &CSI = *Callsite++; - if (CSI.Locations.size() < 3) - report_fatal_error("malformed GoALLC statepoint location list"); - (void)getNonnegativeConstant(CSI.Locations[0], "calling convention"); - (void)getNonnegativeConstant(CSI.Locations[1], "flags"); - uint64_t NumDeopts = - getNonnegativeConstant(CSI.Locations[2], "deopt count"); - if (NumDeopts > CSI.Locations.size() - 3) - report_fatal_error("malformed GoALLC statepoint deopt operands"); - if (NumDeopts > std::numeric_limits::max()) - report_fatal_error("GoALLC statepoint has too many deopt operands"); - - MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, CSI.ID, + bool IsNoSplitEntry = CSI.ID == goabi::NoSplitEntryStackMapID; + uint64_t NumDeopts = 0; + ArrayRef Locations = CSI.Locations; + if (!IsNoSplitEntry) { + if (Locations.size() < 3) + report_fatal_error("malformed GoALLC statepoint location list"); + (void)getNonnegativeConstant(Locations[0], "calling convention"); + (void)getNonnegativeConstant(Locations[1], "flags"); + NumDeopts = getNonnegativeConstant(Locations[2], "deopt count"); + if (NumDeopts > Locations.size() - 3) + report_fatal_error("malformed GoALLC statepoint deopt operands"); + if (NumDeopts > std::numeric_limits::max()) + report_fatal_error("GoALLC statepoint has too many deopt operands"); + Locations = Locations.drop_front(3); + } + + MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, + IsNoSplitEntry + ? goabi::StackGrowthStatepointID + : CSI.ID, CSI.IsIndirectCall, Info.StackSize, PointerSize, static_cast(NumDeopts), {}}; - Entry.Locations.reserve(CSI.Locations.size() - 3); - for (const StackMaps::Location &Location : - ArrayRef(CSI.Locations).drop_front(3)) { + Entry.Locations.reserve(Locations.size()); + for (const StackMaps::Location &Location : Locations) { auto Type = convertLocationType(Location.Type); int64_t Offset = Location.Offset; if (Location.Type == StackMaps::Location::Constant || diff --git a/src/cmd/llvmplugin/testdata/debug-inline.ll b/src/cmd/llvmplugin/testdata/debug-inline.ll index 707f9f62ea5240..0b0b7e566395e6 100644 --- a/src/cmd/llvmplugin/testdata/debug-inline.ll +++ b/src/cmd/llvmplugin/testdata/debug-inline.ll @@ -115,9 +115,28 @@ entry: ret i64 %x, !dbg !55 } +; The optimized instruction stream has no location for erasedInner. Frontend +; required-location metadata must make the final machine pass materialize the +; missing nested inline edge without constraining IR optimization. +define goabiinternal void @main.erased() !dbg !19 { +entry: + ret void, !dbg !65 +} + +define goabiinternal void @main.erasedMid() !dbg !20 { +entry: + ret void, !dbg !66 +} + +define goabiinternal void @main.erasedInner() !dbg !21 { +entry: + ret void, !dbg !67 +} + !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!5, !6} -!goobj.debug.funcs = !{!40, !41, !42, !43, !44, !45, !46, !47, !48} +!goobj.debug.funcs = !{!40, !41, !42, !43, !44, !45, !46, !47, !48, !49, !56, !57} +!goobj.debug.inline.required = !{!58} !0 = distinct !DICompileUnit(language: DW_LANG_Go, file: !1, producer: "goallc-test", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: true, nameTableKind: None) !1 = !DIFile(filename: "outer.go", directory: "/tmp/goobj-inline") @@ -136,6 +155,9 @@ entry: !16 = distinct !DISubprogram(name: "main.shared", linkageName: "main.shared", scope: !1, file: !1, line: 70, type: !3, scopeLine: 70, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !17 = distinct !DISubprogram(name: "main.sharedLeft", linkageName: "main.sharedLeft", scope: !1, file: !1, line: 80, type: !3, scopeLine: 80, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !18 = distinct !DISubprogram(name: "main.sharedRight", linkageName: "main.sharedRight", scope: !1, file: !1, line: 90, type: !3, scopeLine: 90, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!19 = distinct !DISubprogram(name: "main.erased", linkageName: "main.erased", scope: !1, file: !1, line: 100, type: !3, scopeLine: 100, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!20 = distinct !DISubprogram(name: "main.erasedMid", linkageName: "main.erasedMid", scope: !1, file: !1, line: 110, type: !3, scopeLine: 110, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!21 = distinct !DISubprogram(name: "main.erasedInner", linkageName: "main.erasedInner", scope: !1, file: !1, line: 120, type: !3, scopeLine: 120, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !30 = !DILocation(line: 30, column: 3, scope: !12, inlinedAt: !31) !31 = distinct !DILocation(line: 20, column: 3, scope: !11, inlinedAt: !32) @@ -151,6 +173,12 @@ entry: !53 = !DILocation(line: 72, column: 2, scope: !16) !54 = !DILocation(line: 81, column: 2, scope: !17) !55 = !DILocation(line: 91, column: 2, scope: !18) +!60 = !DILocation(line: 121, column: 2, scope: !21, inlinedAt: !61) +!61 = distinct !DILocation(line: 111, column: 2, scope: !20, inlinedAt: !62) +!62 = distinct !DILocation(line: 101, column: 2, scope: !19) +!65 = !DILocation(line: 102, column: 2, scope: !19) +!66 = !DILocation(line: 112, column: 2, scope: !20) +!67 = !DILocation(line: 122, column: 2, scope: !21) !40 = !{!10, ptr @main.outer} !41 = !{!11, ptr @main.mid} @@ -161,3 +189,7 @@ entry: !46 = !{!16, ptr @main.shared} !47 = !{!17, ptr @main.sharedLeft} !48 = !{!18, ptr @main.sharedRight} +!49 = !{!19, ptr @main.erased} +!56 = !{!20, ptr @main.erasedMid} +!57 = !{!21, ptr @main.erasedInner} +!58 = !{ptr @main.erased, !60} diff --git a/src/cmd/llvmtoolexec/main.go b/src/cmd/llvmtoolexec/main.go index 83adb1e5b2f23c..77266a1f48c38f 100644 --- a/src/cmd/llvmtoolexec/main.go +++ b/src/cmd/llvmtoolexec/main.go @@ -21,18 +21,49 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strconv" "strings" ) +type stringSetFlag map[string]struct{} + +func (f *stringSetFlag) Set(value string) error { + if value == "" { + return errors.New("package path must not be empty") + } + if *f == nil { + *f = make(map[string]struct{}) + } + (*f)[value] = struct{}{} + return nil +} + +func (f *stringSetFlag) String() string { + if f == nil { + return "" + } + values := make([]string, 0, len(*f)) + for value := range *f { + values = append(values, value) + } + sort.Strings(values) + return strings.Join(values, ",") +} + var ( llcPath = flag.String("llc", os.Getenv("GOALLC_LLC"), "path to llc") optPath = flag.String("opt", os.Getenv("GOALLC_OPT"), "path to opt") optPasses = flag.String("opt-passes", "", "optional LLVM optimization pipeline to run before llc") passPluginPath = flag.String("pass-plugin", os.Getenv("GOALLC_PASS_PLUGIN"), "path to the GoALLC LLVM pass plugin (default next to llc)") keepIR = flag.Bool("keep-ir", false, "keep the compiler-generated .ll sidecar") + nativePackages stringSetFlag ) +func init() { + flag.Var(&nativePackages, "native-package", "compile this exact -p package with the native Go backend even when inherited gcflags select LLVM (repeatable)") +} + func main() { flag.Parse() if flag.NArg() < 1 { @@ -55,6 +86,10 @@ func main() { run(tool, args...) return } + if useNativeCompiler(args, nativePackages) { + run(tool, withoutLLVMCompileFlags(args)...) + return + } if !isCompileAction(args) { run(tool, args...) return @@ -290,6 +325,8 @@ func printToolIdentity(tool string, args []string, llc, configuredOpt, optPasses identityInput := append([]byte(nil), out...) identityInput = append(identityInput, "\x00opt-passes="...) identityInput = append(identityInput, optPasses...) + identityInput = append(identityInput, "\x00native-packages="...) + identityInput = append(identityInput, nativePackages.String()...) identity, err := backendIdentity(identityInput, append([]string{wrapper}, backendFiles...)...) if err != nil { fatalf("computing backend identity: %v", err) @@ -426,6 +463,27 @@ func toolFlag(args []string, name string) (string, bool) { return "", false } +func useNativeCompiler(args []string, packages stringSetFlag) bool { + pkg, ok := toolFlag(args, "-p") + if !ok { + return false + } + _, ok = packages[pkg] + return ok +} + +func withoutLLVMCompileFlags(args []string) []string { + native := make([]string, 0, len(args)) + for _, arg := range args { + if arg == "-enablellvm" || strings.HasPrefix(arg, "-enablellvm=") || + arg == "-llvmironly" || strings.HasPrefix(arg, "-llvmironly=") { + continue + } + native = append(native, arg) + } + return native +} + func run(path string, args ...string) { cmd := exec.Command(path, args...) cmd.Stdin = os.Stdin diff --git a/src/cmd/llvmtoolexec/main_test.go b/src/cmd/llvmtoolexec/main_test.go index 42468c50a7619b..db873e9e13c71d 100644 --- a/src/cmd/llvmtoolexec/main_test.go +++ b/src/cmd/llvmtoolexec/main_test.go @@ -384,6 +384,27 @@ func TestCompileInvocationClassification(t *testing.T) { } } +func TestNativePackageOverride(t *testing.T) { + packages := stringSetFlag{"runtime_test": {}} + args := []string{ + "-p", "runtime_test", "-enablellvm", "-llvmironly=true", + "-o", "out.a", "callers_test.go", + } + if !useNativeCompiler(args, packages) { + t.Fatal("exact native package was not recognized") + } + native := withoutLLVMCompileFlags(args) + if hasLLVMCompileFlags(native) { + t.Fatalf("LLVM selection survived native override: %q", native) + } + if got, ok := toolFlag(native, "-p"); !ok || got != "runtime_test" { + t.Fatalf("native override changed package flag to %q, %v", got, ok) + } + if useNativeCompiler([]string{"-p=runtime", "-enablellvm", "-llvmironly"}, packages) { + t.Fatal("native package override matched a different package") + } +} + func TestBoolToolFlag(t *testing.T) { tests := []struct { name string diff --git a/src/cmd/vendor/github.com/goallc/go-llvm/ir.go b/src/cmd/vendor/github.com/goallc/go-llvm/ir.go index c24fe620fb7b89..34ee4fc69cd320 100644 --- a/src/cmd/vendor/github.com/goallc/go-llvm/ir.go +++ b/src/cmd/vendor/github.com/goallc/go-llvm/ir.go @@ -742,6 +742,9 @@ func (v Value) SetMetadata(kind int, node Metadata) { func (v Value) SetGlobalMetadata(kind int, node Metadata) { C.LLVMGlobalSetMetadata(v.C, C.unsigned(kind), node.C) } +func (v Value) EraseGlobalMetadata(kind int) { + C.LLVMGlobalEraseMetadata(v.C, C.unsigned(kind)) +} // Obtain the string value of the instruction. Same as would be printed with // Value.Dump() (with two spaces at the start but no newline at the end). diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index c0f1d979061948..7899734573a515 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -607,9 +607,23 @@ func G0StackOverflow() { }) } +var stackOverflowTestState uint32 + func stackOverflow(x *byte) { var buf [256]byte + // Keep this recursion from becoming a tail-recursive loop. The test needs + // real stack growth, but the LLVM backend is otherwise free to optimize tail + // calls more aggressively than the native Go backend. An atomic load makes + // the return path reachable to the optimizer, and the post-call atomic + // operation keeps state from the current frame live across the call. + buf[0] = byte(atomic.Load(&stackOverflowTestState)) + if buf[0] != 0 { + return + } stackOverflow(&buf[0]) + if x != nil { + atomic.Xadd(&stackOverflowTestState, int32(*x)) + } } func RunGetgThreadSwitchTest() { diff --git a/test/codegen/_cgo_llvm_unsafe_args.go b/test/codegen/_cgo_llvm_unsafe_args.go index b3889b0bdf39b1..4cec88cf2d522e 100644 --- a/test/codegen/_cgo_llvm_unsafe_args.go +++ b/test/codegen/_cgo_llvm_unsafe_args.go @@ -10,30 +10,34 @@ package codegen func llvmCgoUnsafeSink(*uintptr) // LLVM-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-SAME: i64 %p) #[[NOINLINE:[0-9]+]] gc "goallc" +// LLVM-SAME: i64 %p, i64 %q) #[[NOINLINE:[0-9]+]] gc "goallc" // LLVM-NOT: alloca // LLVM: [[FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-NOT: llvm.addressofreturnaddress // LLVM-NOT: llvm.sponentry -// LLVM: [[RESULT:%.*]] = getelementptr i8, ptr [[FRAME]], i64 8 +// LLVM: [[Q:%.*]] = getelementptr i8, ptr [[FRAME]], i64 8 +// LLVM: [[RESULT:%.*]] = getelementptr i8, ptr [[FRAME]], i64 16 // LLVM: store i64 %p, ptr [[FRAME]] +// LLVM: store i64 %q, ptr [[Q]] // LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[FRAME]]) // LLVM: {{%.*}} = load i64, ptr [[RESULT]] // LLVM: attributes #[[NOINLINE]] = { {{.*}}noinline // LLVM-OPT-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-OPT-SAME: i64 %p) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" +// LLVM-OPT-SAME: i64 %p, i64 %q) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" // LLVM-OPT-NOT: alloca // LLVM-OPT: [[OPT_FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-OPT-NOT: llvm.addressofreturnaddress // LLVM-OPT-NOT: llvm.sponentry -// LLVM-OPT: [[OPT_RESULT:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 8 +// LLVM-OPT: [[OPT_Q:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 8 +// LLVM-OPT: [[OPT_RESULT:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 16 // LLVM-OPT: store i64 %p, ptr [[OPT_FRAME]] +// LLVM-OPT: store i64 %q, ptr [[OPT_Q]] // LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[OPT_FRAME]]) // LLVM-OPT: {{%.*}} = load i64, ptr [[OPT_RESULT]] // LLVM-OPT: attributes #[[OPT_NOINLINE]] = { {{.*}}noinline // //go:cgo_unsafe_args -func llvmCgoUnsafeFrame(p uintptr) (r uintptr) { +func llvmCgoUnsafeFrame(p, q uintptr) (r uintptr) { llvmCgoUnsafeSink(&p) return } diff --git a/test/codegen/llvm_opendefer.go b/test/codegen/llvm_opendefer.go index 67ac2b38dd9264..16f8eba8d921a8 100644 --- a/test/codegen/llvm_opendefer.go +++ b/test/codegen/llvm_opendefer.go @@ -6,6 +6,10 @@ package codegen +type llvmOpenDeferNamedResult struct { + value int +} + // LLVM-LABEL: define goabiinternal i64 @codegen.llvmOpenDeferTwo(i64 %value) // LLVM: [[SLOTS:%.*]] = alloca [2 x ptr], align 8, !goallc.open_defer_slots ![[SLOTS_MD:[0-9]+]] // LLVM: [[SLOT0:%.*]] = getelementptr i8, ptr [[SLOTS]], i64 0 @@ -16,8 +20,7 @@ package codegen // LLVM: store volatile ptr {{.*}}, ptr [[SLOT0]] // LLVM: store volatile ptr {{.*}}, ptr [[SLOT1]] // LLVM: [[RECOVERY]]: -// LLVM-NEXT: call goabiinternal void @runtime.deferreturn() -// LLVM: ![[SLOTS_MD]] = !{i32 2} +// LLVM-NEXT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmOpenDeferTwo(i64 %value) // LLVM-OPT: [[SLOTS_OPT:%.*]] = alloca [2 x ptr], align 8, !goallc.open_defer_slots ![[SLOTS_OPT_MD:[0-9]+]] // LLVM-OPT: [[SLOT1_OPT:%.*]] = getelementptr {{.*}}i8, ptr [[SLOTS_OPT]], i64 8 @@ -27,7 +30,22 @@ package codegen // LLVM-OPT: store volatile ptr {{.*}}, ptr [[SLOTS_OPT]] // LLVM-OPT: store volatile ptr {{.*}}, ptr [[SLOT1_OPT]] // LLVM-OPT: [[RECOVERY_OPT]]: -// LLVM-OPT: call goabiinternal void @runtime.deferreturn() +// LLVM-OPT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} + +// LLVM-LABEL: define goabiinternal { i64 } @codegen.llvmOpenDeferNamed( +// LLVM: open.defer.recovery: +// LLVM-NEXT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} +// LLVM: load volatile %codegen.llvmOpenDeferNamedResult +// LLVM: insertvalue { i64 } +// LLVM: ret { i64 } +// LLVM: ![[SLOTS_MD]] = !{i32 2} +// LLVM-OPT-LABEL: define goabiinternal { i64 } @codegen.llvmOpenDeferNamed( +// LLVM-OPT: common.ret: +// LLVM-OPT: load volatile %codegen.llvmOpenDeferNamedResult +// LLVM-OPT: insertvalue { i64 } +// LLVM-OPT: ret { i64 } +// LLVM-OPT: open.defer.recovery: +// LLVM-OPT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} // LLVM-OPT: ![[SLOTS_OPT_MD]] = !{i32 2} func llvmOpenDeferTwo(value int) (result int) { @@ -39,3 +57,10 @@ func llvmOpenDeferTwo(value int) (result int) { }() return 3 } + +func llvmOpenDeferNamed(value int) (result llvmOpenDeferNamedResult) { + defer func() { + result.value += value + }() + return llvmOpenDeferNamedResult{value: 3} +} From 67a9a9ec89650ca700211153a1c30585a3635833 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 00:38:32 +0800 Subject: [PATCH 2/8] cmd/compile: separate entry args maps from stack growth --- doc/goallc-llvm-goobj.md | 37 ++--- src/cmd/compile/internal/ssa/ssa2llvm.go | 5 - src/cmd/internal/testdir/llvm_abi_test.go | 4 +- src/cmd/internal/testdir/llvm_test.go | 15 +- .../testdata/llvm_args_pointer_maps.mir | 4 +- src/cmd/llvmplugin/CMakeLists.txt | 4 +- .../llvmplugin/GoALLCGCMetadataPrinter.cpp | 20 +++ src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp | 130 ------------------ src/cmd/llvmplugin/README.md | 48 ++++--- src/cmd/llvmplugin/testdata/aarch64-frame.ll | 2 +- .../testdata/aggregate-call-result-goobj.ll | 2 +- src/cmd/llvmplugin/testdata/aggregate-cfg.ll | 22 +-- .../testdata/aggregate-conditional-stage-b.ll | 2 +- .../testdata/aggregate-fixed-vector.ll | 2 +- .../testdata/aggregate-loop-stage-b.ll | 2 +- .../aggregate-scalable-vector-unsupported.ll | 2 +- .../testdata/aggregate-scalarization.ll | 32 ++--- src/cmd/llvmplugin/testdata/alloca-derived.ll | 2 +- .../alloca-pointer-dynamic-unsupported.ll | 2 +- .../alloca-pointer-lifetime-ambiguous.ll | 2 +- .../alloca-pointer-lifetime-unsupported.ll | 16 +-- .../alloca-pointer-nonentry-unsupported.ll | 2 +- .../alloca-pointer-realigned-unsupported.ll | 2 +- .../testdata/alloca-pointer-roots.ll | 40 +++--- .../alloca-pointer-select-unsupported.ll | 2 +- .../alloca-pointer-vector-unsupported.ll | 2 +- .../alloca-pointer-volatile-unsupported.ll | 2 +- .../alloca-ptrmap-malformed-bad-kind.ll | 3 +- .../alloca-ptrmap-malformed-bad-length.ll | 3 +- .../alloca-ptrmap-malformed-duplicate.ll | 3 +- .../alloca-ptrmap-malformed-non-direct.ll | 3 +- .../alloca-ptrmap-malformed-overlap.ll | 3 +- .../alloca-ptrmap-malformed-padding.ll | 3 +- .../alloca-ptrmap-malformed-truncated.ll | 3 +- .../testdata/alloca-zero-pointer-array.ll | 2 +- .../testdata/branch-safepoints-relocation.ll | 2 +- .../testdata/conditional-relocation.ll | 4 +- src/cmd/llvmplugin/testdata/debug-inline.ll | 10 +- src/cmd/llvmplugin/testdata/defer-edge.ll | 6 +- .../derived-pointer-rematerialization.ll | 10 +- .../testdata/function-marker-inline.ll | 4 +- .../llvmplugin/testdata/gc-leaf-markers.ll | 2 +- .../llvmplugin/testdata/indirect-callee.ll | 2 +- .../testdata/invalid-gc-leaf-definition.ll | 2 +- .../testdata/irreducible-relocation.ll | 2 +- src/cmd/llvmplugin/testdata/live-aggregate.ll | 2 +- .../llvmplugin/testdata/loop-relocation.ll | 2 +- src/cmd/llvmplugin/testdata/multiple-calls.ll | 2 +- .../llvmplugin/testdata/nest-param-attr.ll | 2 +- .../testdata/open-defer-incomplete.ll | 4 +- src/cmd/llvmplugin/testdata/open-defer.ll | 4 +- .../testdata/pointer-address-observation.ll | 2 +- .../sequential-conditional-relocation.ll | 2 +- src/cmd/llvmplugin/testdata/statepoint.ll | 15 +- .../testdata/supported-param-attrs.ll | 2 +- .../llvmplugin/testdata/unsupported-invoke.ll | 2 +- .../testdata/unsupported-param-attr.ll | 2 +- test/codegen/statepoint.go | 3 +- 58 files changed, 199 insertions(+), 317 deletions(-) create mode 100644 src/cmd/llvmplugin/GoALLCGCMetadataPrinter.cpp delete mode 100644 src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp diff --git a/doc/goallc-llvm-goobj.md b/doc/goallc-llvm-goobj.md index 260d28956f7a98..96f727264b9336 100644 --- a/doc/goallc-llvm-goobj.md +++ b/doc/goallc-llvm-goobj.md @@ -171,10 +171,10 @@ use 的 local 不生成 LLVM 栈槽,避免无意义地增大 frame。 GoObj stack growth、SP 恢复和当前 stack-map 路径只支持编译期固定 frame。 因此 amd64 和 arm64 GoObj target 对 variable-sized `alloca` 都确定性报错, -不能把动态栈分配交给 LLVM 通用 lowering。当前 GoObj writer 的 args/locals -pointer maps 仍为空;本实现只保证现有固定栈槽的 dominance、frame placement -和 stack-growth 约束,不代表已经支持完整 precise-GC stack maps、goroutine、 -defer 或 panic unwind。 +不能把动态栈分配交给 LLVM 通用 lowering。当前 GoObj writer 已为所有 GoObj Go +函数生成类型推导的入口 ArgsPointerMaps,并为已支持的普通 statepoint 和固定 +`alloca` 生成 LocalsPointerMaps;这仍不代表已经支持完整 precise-GC stack maps、 +goroutine、defer 或 panic unwind。 对应回归包括: @@ -273,9 +273,10 @@ pass plugin 默认从 `llc` 所属 LLVM payload 的 wrapper 覆盖的 compile action 重新编译。 插件的功能性源码和测试位于 Go 仓库 -`src/cmd/llvmplugin`,不放入 LLVM 源码树。LLVM 只提供通用的 -`-load-pass-plugin` 和 pre-codegen callback;GoALLC statepoint rewrite 及其 -pass 顺序都应继续在这个 Go-owned 工程中实现。当前 +`src/cmd/llvmplugin`,不放入 LLVM 源码树。LLVM 提供通用的 +`-load-pass-plugin`、pre-codegen callback,以及 GoObj 对 Machine StackMaps +的对象格式适配;GoALLC statepoint rewrite 及其 pass 顺序都应继续在这个 +Go-owned 工程中实现。当前 `runPreCodeGenPipeline` 调用 Go-owned statepoint pass:它对 Go ABI 函数执行 CFG 逆向数据流活跃性分析,为普通调用分配稳定 callsite ID,生成 `gc.statepoint` / `gc.relocate`,并识别 `gc-leaf-function`。受管指针分类当前 @@ -285,21 +286,21 @@ aggregate、`invoke`、`musttail` 和非 leaf inline asm fail closed。未来 compile 进程内集成 LLVM 时直接复用该 core 入口,不经过 plugin adapter。 机器位置不通过修改 LLVM 通用 `StackMaps.cpp` 截获。SSA→LLVM IR 前端为 Go ABI -函数声明 `gc "goallc"` 和 `go-stack-growth-statepoint`,插件只负责注册对应的 -GC strategy 与 `GCMetadataPrinter::emitStackMaps`,并消费这些前端标记。在 +函数声明 `gc "goallc"`;GoObj Go 函数默认采用原生 Go 的可扩栈策略,只有 +`go-nosplit`、`go-systemstack` 这样的例外策略需要额外属性。插件负责注册对应的 +GC strategy、执行 statepoint rewrite 并消费这些前端标记。在 LLVM GoObj AsmPrinter 模块收尾阶段 读取标准 `FnInfos/CSInfos`,跳过 statepoint 的 CC、flags 和 deopt 前缀后,把 原始 GC locations 写入 MCContext。GoObj writer 在最终 layout 后完成 SP 校验、`Direct`/`Indirect` 解释、LocalsPointerMaps 和 PCDATA_StackMapIndex -编码。GoALLC 要求 StackMaps 记录 CALL 起点;map 从 CALL 开始。前端添加的 -`go-stack-growth-statepoint` 属性使 LLVM 在 PEI 阶段把 -`runtime.morestack` 调用生成为物理 MIR `STATEPOINT`。其 deopt 和 GC alloca -区为空,GC pointer 区则记录类型推导出的入口参数 home。它从 morestack CALL -起点选择入口 ArgsPointerMaps 和空 locals bitmap,因此普通调用与栈增长调用 -走同一 Machine StackMaps 链路,且不依赖 return PC 反推调用范围。已有 -Machine `STATEPOINT` 但缺少该前端属性时, -LLVM target lowering 会 fail closed;不再使用 slow-path reset label 兼容普通 -morestack CALL。GoObj 先写索引 0 的 +编码。GoALLC 要求 StackMaps 记录 CALL 起点;map 从 CALL 开始。LLVM target +formal lowering 为每个 GoObj Go 函数推导入口参数 home,并用一个独立、零字节的 +`EntryArgsStackMapID` 表达函数级 ArgsPointerMaps;该记录不是调用点,不生成 +PCDATA。非 nosplit 函数在 PEI 阶段额外把 `runtime.morestack` 调用生成为物理、 +root-free 的 `StackGrowthStatepointID`,从 CALL 起点选择入口 ArgsPointerMaps 和 +空 locals bitmap。nosplit 函数不得包含这个 statepoint。这样普通调用与栈增长 +调用仍走同一 Machine StackMaps 链路,且不依赖 return PC 反推调用范围,同时 +函数级入口图不再伪装成 morestack 调用。GoObj 先写索引 0 的 `PCDATA_UnsafePoint`(当前恒为 safe 的 `-1`),再写索引 1 的 `PCDATA_StackMapIndex`;不能只写后一张表,否则 linker 会把它误认成索引 0。 `Direct SP+offset` 是栈地址本身,不表示该 slot 存有 pointer,因此不会设置 diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 268186b6ad1caa..4dbcd14a75ba57 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -70,7 +70,6 @@ const goABI0SymbolSuffix = "" const goResultsTupleAttr = "go_results_tuple" const goGCStrategy = "goallc" const goGCLeafFunctionAttr = "gc-leaf-function" -const goStackGrowthStatepointAttr = "go-stack-growth-statepoint" const goNoSplitAttr = "go-nosplit" const goSystemStackAttr = "go-systemstack" const goAsyncUnsafeAttr = "go-async-unsafe" @@ -3259,10 +3258,6 @@ func LLVMCompile(f *Func) { } } FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goAsyncUnsafeAttr, "")) - // The stack-growth attribute supplies the target's entry-argument map and, - // when a split prologue is permitted, asks it to represent the late - // morestack call as a root-free statepoint. - FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goStackGrowthStatepointAttr, "")) // A //go:nosplit function must not acquire that late morestack edge. Besides // violating the runtime's nosplit call graph, it would expose a safepoint the // frontend deliberately prohibited. Give target frame lowering the source diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 3ffec4ec24ba5f..05cda1afc21f10 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -146,12 +146,14 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { []byte("define goabiinternal"), []byte(`"go_results_tuple"`), []byte(`gc "goallc"`), - []byte(`"go-stack-growth-statepoint"`), } { if !bytes.Contains(ir, needle) { t.Fatalf("GoALLC IR does not contain %q", needle) } } + if bytes.Contains(ir, []byte(`"go-stack-growth-statepoint"`)) { + t.Fatal("GoALLC IR still contains the obsolete stack-growth attribute") + } opt := llvmToolPath(t, "opt", "GOALLC_OPT") runLLVMABICommand(t, ir, opt, "-passes=verify", "-disable-output") diff --git a/src/cmd/internal/testdir/llvm_test.go b/src/cmd/internal/testdir/llvm_test.go index 5285031c29a0fd..22166029a34ebf 100644 --- a/src/cmd/internal/testdir/llvm_test.go +++ b/src/cmd/internal/testdir/llvm_test.go @@ -251,15 +251,16 @@ func Split(pointer *int) *int { } noSplitAttrs := attributeLine("p.NoSplit") if !bytes.Contains(noSplitAttrs, []byte(`"go-nosplit"`)) || - !bytes.Contains(noSplitAttrs, []byte(`"go-stack-growth-statepoint"`)) || !bytes.Contains(noSplitAttrs, []byte(`noinline`)) { t.Fatalf("LLVM nosplit attributes do not select the nosplit prologue policy: %s", noSplitAttrs) } splitAttrs := attributeLine("p.Split") - if !bytes.Contains(splitAttrs, []byte(`"go-stack-growth-statepoint"`)) || - bytes.Contains(splitAttrs, []byte(`"go-nosplit"`)) { + if bytes.Contains(splitAttrs, []byte(`"go-nosplit"`)) { t.Fatalf("LLVM split attributes do not select the stack-growth prologue policy: %s", splitAttrs) } + if bytes.Contains(ir, []byte(`"go-stack-growth-statepoint"`)) { + t.Fatal("LLVM IR still contains the obsolete stack-growth attribute") + } llc := llvmToolPath(t, "llc", "GOALLC_LLC") plugin := llvmABIPassPlugin(t, llc) @@ -274,10 +275,18 @@ func Split(pointer *int) *int { if llvmABIHasRelocationTo(document, noSplit, "runtime.morestack_noctxt") { t.Fatal("p.NoSplit unexpectedly calls runtime.morestack_noctxt") } + noSplitArgs := llvmABIArgsPointerBitmaps(t, noSplit) + if len(noSplitArgs) == 0 || !slices.Equal(noSplitArgs[0], []int{0}) { + t.Fatalf("p.NoSplit entry ArgsPointerMaps = %v, want pointer bit 0", noSplitArgs) + } split := findLLVMABISymbol(t, document, "p.Split") if !llvmABIHasRelocationTo(document, split, "runtime.morestack_noctxt") { t.Fatal("p.Split has no runtime.morestack_noctxt relocation") } + splitArgs := llvmABIArgsPointerBitmaps(t, split) + if len(splitArgs) == 0 || !slices.Equal(splitArgs[0], []int{0}) { + t.Fatalf("p.Split entry ArgsPointerMaps = %v, want pointer bit 0", splitArgs) + } } func runLLVMNoWriteBarrierRecTest(t *testing.T) { diff --git a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir index eeecaa518796b7..fac4976a3ed72f 100644 --- a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir +++ b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir @@ -18,7 +18,6 @@ attributes #0 = { "frame-pointer"="non-leaf" - "go-stack-growth-statepoint" "go_results_tuple" } ... @@ -43,8 +42,9 @@ body: | bb.0.entry: liveins: $lr + STACKMAP 5147419139155979380, 0, 1, 8, $sp, 16 $x3 = COPY $lr - STATEPOINT 5147424658422983495, 0, 0, &runtime.morestack_noctxt, 2, 22, 2, 0, 2, 0, 2, 1, 1, 8, $sp, 16, 2, 0, 2, 1, 0, 0, csr_aarch64_go, implicit-def $sp, implicit-def dead early-clobber $lr, implicit $x3 + STATEPOINT 5147424658422983495, 0, 0, &runtime.morestack_noctxt, 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_aarch64_go, implicit-def $sp, implicit-def dead early-clobber $lr, implicit $x3 STATEPOINT 1, 0, 0, &p.safepoint, 2, 22, 2, 0, 2, 0, 2, 1, 1, 8, $sp, 8, 2, 0, 2, 1, 0, 0, csr_aarch64_go, implicit-def $sp, implicit-def dead early-clobber $lr RET_ReallyLR ... diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index c91bb6715e6b24..b7a465b9ef0f17 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -21,9 +21,9 @@ add_definitions(${LLVM_DEFINITIONS}) include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) add_llvm_pass_plugin(GoALLCStatepoints + GoALLCGCMetadataPrinter.cpp GoALLCInlineAnchors.cpp GoALLCPreCodeGen.cpp - GoALLCStackMapPrinter.cpp GoALLCStatepoints.cpp GoALLCStatepointPlugin.cpp ) @@ -252,7 +252,7 @@ if(BUILD_TESTING) ) set_tests_properties(GoALLCStatepoints.FrontendMarkersPreserved PROPERTIES PASS_REGULAR_EXPRESSION - "go-stack-growth-statepoint" + "gc \"goallc\"" ) foreach(GOALLC_DEBUG_TARGET IN ITEMS X86 AArch64) diff --git a/src/cmd/llvmplugin/GoALLCGCMetadataPrinter.cpp b/src/cmd/llvmplugin/GoALLCGCMetadataPrinter.cpp new file mode 100644 index 00000000000000..1ffe6b65d19adb --- /dev/null +++ b/src/cmd/llvmplugin/GoALLCGCMetadataPrinter.cpp @@ -0,0 +1,20 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "llvm/CodeGen/GCMetadataPrinter.h" + +using namespace llvm; + +namespace { + +// AsmPrinter requires every GC strategy to have a registered metadata +// printer. GoObj itself consumes Machine StackMaps in LLVM's AsmPrinter path, +// so the Go-owned plugin only needs this marker registration. +class GoALLCGCMetadataPrinter final : public GCMetadataPrinter {}; + +} // namespace + +static GCMetadataPrinterRegistry::Add + GoALLCGCMetadataPrinterRegistration("goallc", + "GoALLC GC metadata marker"); diff --git a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp b/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp deleted file mode 100644 index 07ba7e50d2d397..00000000000000 --- a/src/cmd/llvmplugin/GoALLCStackMapPrinter.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2026 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/GCMetadataPrinter.h" -#include "llvm/CodeGen/GoCallingConv.h" -#include "llvm/CodeGen/StackMaps.h" -#include "llvm/MC/MCContext.h" -#include "llvm/Support/ErrorHandling.h" - -#include -#include - -using namespace llvm; - -namespace { - -class GoALLCStackMapPrinter final : public GCMetadataPrinter { -public: - bool emitStackMaps(StackMaps &SM, AsmPrinter &AP) override; -}; - -MCContext::GoObjStackMapLocation::LocationType -convertLocationType(StackMaps::Location::LocationType Type) { - using GoLocation = MCContext::GoObjStackMapLocation; - switch (Type) { - case StackMaps::Location::Unprocessed: - return GoLocation::Unprocessed; - case StackMaps::Location::Register: - return GoLocation::Register; - case StackMaps::Location::Direct: - return GoLocation::Direct; - case StackMaps::Location::Indirect: - return GoLocation::Indirect; - case StackMaps::Location::Constant: - return GoLocation::Constant; - case StackMaps::Location::ConstantIndex: - return GoLocation::ConstantIndex; - } - llvm_unreachable("unknown StackMaps location type"); -} - -uint64_t getNonnegativeConstant(const StackMaps::Location &Location, - StringRef Description) { - if (Location.Type != StackMaps::Location::Constant || Location.Offset < 0) - report_fatal_error("malformed GoALLC stackmap " + Description); - return static_cast(Location.Offset); -} - -} // namespace - -static GCMetadataPrinterRegistry::Add - GoALLCStackMapPrinterRegistration("goallc", - "GoALLC GoObj Machine StackMaps bridge"); - -bool GoALLCStackMapPrinter::emitStackMaps(StackMaps &SM, AsmPrinter &AP) { - // Other object formats can continue to use LLVM's standard stackmap - // serialization while the GoObj bridge is being brought up. - if (!AP.OutContext.isGoObj()) - return false; - - auto &Callsites = SM.getCSInfos(); - auto Callsite = Callsites.begin(); - uint32_t PointerSize = AP.getPointerSize(); - if (!PointerSize) - report_fatal_error("GoALLC statepoint target has no pointer size"); - - for (const auto &[Function, Info] : SM.getFnInfos()) { - for (uint64_t I = 0; I != Info.RecordCount; ++I) { - if (Callsite == Callsites.end()) - report_fatal_error( - "GoALLC statepoint function record count exceeds callsites"); - - // LLVM's statepoint parser flattens CC, flags, deopt count, deopt - // operands, GC base/derived pairs, and GC allocas into Locations. The - // first three entries and the deopt operands are not GC roots. - const StackMaps::CallsiteInfo &CSI = *Callsite++; - bool IsNoSplitEntry = CSI.ID == goabi::NoSplitEntryStackMapID; - uint64_t NumDeopts = 0; - ArrayRef Locations = CSI.Locations; - if (!IsNoSplitEntry) { - if (Locations.size() < 3) - report_fatal_error("malformed GoALLC statepoint location list"); - (void)getNonnegativeConstant(Locations[0], "calling convention"); - (void)getNonnegativeConstant(Locations[1], "flags"); - NumDeopts = getNonnegativeConstant(Locations[2], "deopt count"); - if (NumDeopts > Locations.size() - 3) - report_fatal_error("malformed GoALLC statepoint deopt operands"); - if (NumDeopts > std::numeric_limits::max()) - report_fatal_error("GoALLC statepoint has too many deopt operands"); - Locations = Locations.drop_front(3); - } - - MCContext::GoObjStackMapEntry Entry{CSI.CSOffsetExpr, - IsNoSplitEntry - ? goabi::StackGrowthStatepointID - : CSI.ID, - CSI.IsIndirectCall, Info.StackSize, - PointerSize, - static_cast(NumDeopts), - {}}; - Entry.Locations.reserve(Locations.size()); - for (const StackMaps::Location &Location : Locations) { - auto Type = convertLocationType(Location.Type); - int64_t Offset = Location.Offset; - if (Location.Type == StackMaps::Location::Constant || - Location.Type == StackMaps::Location::ConstantIndex) { - std::optional Constant = SM.getConstantValue(Location); - if (!Constant) - report_fatal_error( - "GoALLC statepoint contains an invalid constant-pool index"); - Type = MCContext::GoObjStackMapLocation::Constant; - Offset = *Constant; - } - Entry.Locations.push_back( - {Type, Location.Size, Location.Reg, Offset}); - } - AP.OutContext.addGoObjSymbolStackMapEntry(Function, std::move(Entry)); - } - } - - if (Callsite != Callsites.end()) - report_fatal_error( - "GoALLC statepoint callsites exceed function record counts"); - - SM.reset(); - return true; -} diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index 1b7b0b954f0095..7140b480805241 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -12,9 +12,11 @@ future in-process `cmd/compile` integration can call the same pipeline without going through `llc`. The SSA-to-LLVM lowering owns the function-level contract: Go ABI definitions -carry `gc "goallc"` and the `go-stack-growth-statepoint` attribute. Loading this -plugin registers the named GC strategy and its metadata printer; the rewrite -pass consumes the frontend markers rather than adding them. +carry `gc "goallc"`, and only source-level exceptions to the native Go stack +policy need target attributes such as `go-nosplit` or `go-systemstack`. Loading +this plugin registers the named GC strategy and the no-op metadata-printer +marker required by AsmPrinter; the rewrite pass consumes the frontend markers +rather than adding them. GoObj stack-map adaptation lives in LLVM core. Entry argument pointer maps do not use an IR marker or intrinsic. After LLVM optimization and inlining, target formal-argument lowering recursively derives @@ -156,7 +158,7 @@ constants in Machine StackMaps. For `gc "goallc"`, a static alloca used only as a deopt layout carrier is not implicitly promoted to a GC root; explicit `gc-live` is the activity signal. The resulting `gc.relocate(alloca)` is `NoRelocate` and rematerializes the same frame index, so no root spill is -created. The Go-owned StackMaps bridge retains the deopt prefix and resolves +created. LLVM's GoObj StackMaps bridge retains the deopt prefix and resolves both inline constants and `ConstantIndex` values. The GoObj writer strictly parses the suffix and maps every layout with a matching direct `gc-live` alloca plus bitmap bit to that callsite's `LocalsPointerMaps`. An unmatched layout is @@ -231,28 +233,24 @@ non-trivial implementation block is copied verbatim, it must live in a separately attributed source file retaining LLVM's Apache-2.0-with-exception notice; BSD-only Go source files should not silently absorb copied code. -`GoALLCStackMapPrinter.cpp` is the Go-owned boundary between LLVM Machine -StackMaps and GoObj. It uses the standard -`AsmPrinter -> GCMetadataPrinter::emitStackMaps` hook, retains deopt and GC -locations in `MCContext`, and resolves StackMaps constant-pool indexes without -adding a parallel serializer. LLVM's generic `StackMaps.cpp` only exposes the -constant resolver; it has no GoALLC grammar or GoObj policy. LLVM records GoObj -statepoint callsites at the CALL +The project-owned LLVM GoObj AsmPrinter path is the boundary between Machine +StackMaps and GoObj. It retains deopt and GC locations in `MCContext` and +resolves StackMaps constant-pool indexes without adding a parallel serializer; +the Go-owned plugin remains responsible for the frontend statepoint rewrite, +not object-format adaptation. LLVM records GoObj statepoint callsites at the CALL start, matching Go's `PCDATA_StackMapIndex` convention without a command-line -mode. The frontend's stack-growth attribute -asks LLVM to express the late-generated `runtime.morestack` call as a physical -MIR `STATEPOINT` with empty deopt and GC-alloca sections. Before frame -allocation, AArch64 formal lowering maps every type-derived input pointer word -onto the existing fixed home reserved for that ABI input. Frame lowering encodes -those homes in the morestack statepoint's GC pointer section as indirect -`SP+offset` locations with base-equals-derived pairs; it never asks the -statepoint machinery to allocate another spill. The GoObj writer recognizes -the stack-growth ID, interprets those offsets in the entry-SP geometry, and -selects pair 0: non-empty `EntryArgs` when present and empty locals. Ordinary -and stack-growth calls use the same Machine StackMaps pipeline without relying -on a return-PC convention. GoObj functions that already contain a Machine -`STATEPOINT` but lack the frontend stack-growth attribute fail closed; there is -no slow-path reset-label fallback for a raw morestack call. +mode. GoObj Go functions use native Go's split-stack policy by default: unless +`go-nosplit` is present, target frame lowering expresses the late-generated +`runtime.morestack` call as a physical, root-free MIR `STATEPOINT`. Before frame +allocation, formal lowering maps every type-derived input pointer word onto the +existing fixed home reserved for that ABI input. Frame lowering records those +homes in a separate zero-byte `EntryArgsStackMapID` record for every GoObj Go +function. This is function metadata rather than a callsite, so it emits no +`PCDATA`. The GoObj writer uses it as pair 0: non-empty `EntryArgs` when present +and empty locals. A split function must additionally contain exactly one real +`StackGrowthStatepointID`, which selects pair 0 at the morestack call; a nosplit +function must contain none. Ordinary and stack-growth calls use the same +Machine StackMaps pipeline without relying on a return-PC convention. GoObj emits the currently constant safe `PCDATA_UnsafePoint` table first and the statepoint-derived `PCDATA_StackMapIndex` table second, as required by their Go ABI indexes 0 and 1. diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 7e95610d3ff4b0..c9df7dc4a35cea 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -135,4 +135,4 @@ entry: ret i64 %regarg } -attributes #0 = { "frame-pointer"="non-leaf" "go-stack-growth-statepoint" } +attributes #0 = { "frame-pointer"="non-leaf" } diff --git a/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll b/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll index 1eb42cffc7a0fe..5a7f8e240ce54e 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll @@ -70,7 +70,7 @@ declare goabiinternal void @leaf_consume_pair(%pair) #0 define goabiinternal ptr @aggregate_call_result_goobj( ptr %seed, i1 %take_call) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %value = call goabiinternal %pair @make_pair(ptr %seed, i64 17) br i1 %take_call, label %call, label %skip diff --git a/src/cmd/llvmplugin/testdata/aggregate-cfg.ll b/src/cmd/llvmplugin/testdata/aggregate-cfg.ll index 30b98fd1942fd4..4e2b94bb6e59da 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-cfg.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-cfg.ll @@ -62,7 +62,7 @@ declare goabiinternal void @leaf_consume_pair(%pair) #0 define goabiinternal ptr @aggregate_diamond_call_skip( i1 %take_call, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_call, label %call, label %skip @@ -81,7 +81,7 @@ merge: define goabiinternal ptr @aggregate_branch_safepoints( i1 %take_left, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_left, label %left, label %right @@ -101,7 +101,7 @@ merge: define goabiinternal ptr @aggregate_sequential_conditional( i1 %take_first, i1 %take_second, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_first, label %first_call, label %first_skip @@ -130,7 +130,7 @@ second_merge: define goabiinternal ptr @aggregate_natural_loop( i32 %count, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br label %header @@ -149,7 +149,7 @@ exit: define goabiinternal ptr @aggregate_irreducible( i1 %enter_b, i1 %leave_a, i1 %leave_b, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %enter_b, label %b, label %a @@ -168,7 +168,7 @@ exit: define goabiinternal ptr @aggregate_phi_edge_use( i1 %take_call, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_call, label %call, label %skip @@ -189,7 +189,7 @@ merge: define goabiinternal ptr @aggregate_phi_duplicate_edge( i32 %which, %pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: call goabiinternal void @safepoint() switch i32 %which, label %merge [ @@ -206,7 +206,7 @@ merge: define goabiinternal ptr @aggregate_call_result_conditional( ptr %seed, i1 %take_call) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %value = call goabiinternal %pair @make_pair(ptr %seed, i64 7) br i1 %take_call, label %call, label %skip @@ -226,7 +226,7 @@ merge: define goabiinternal ptr @aggregate_call_result_loop( ptr %seed, i32 %count) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %value = call goabiinternal %pair @make_pair(ptr %seed, i64 11) br label %header @@ -246,7 +246,7 @@ exit: define goabiinternal ptr @aggregate_call_result_irreducible( ptr %seed, i1 %enter_b, i1 %leave_a, i1 %leave_b) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %value = call goabiinternal %pair @make_pair(ptr %seed, i64 13) br i1 %enter_b, label %b, label %a @@ -265,7 +265,7 @@ exit: } define goabiinternal ptr @aggregate_multiple_safepoints(%pair %value) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: call goabiinternal void @safepoint() call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/aggregate-conditional-stage-b.ll b/src/cmd/llvmplugin/testdata/aggregate-conditional-stage-b.ll index 6c059d34e3f95a..ce42b9425bfcf4 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-conditional-stage-b.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-conditional-stage-b.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal ptr @aggregate_conditional_relocation(i1 %take_call, %pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @aggregate_conditional_relocation(i1 %take_call, %pair %value) gc "goallc" { entry: br i1 %take_call, label %call, label %skip diff --git a/src/cmd/llvmplugin/testdata/aggregate-fixed-vector.ll b/src/cmd/llvmplugin/testdata/aggregate-fixed-vector.ll index 56a9222dfc6542..4ea420f13110cb 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-fixed-vector.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-fixed-vector.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal ptr @fixed_vector(ptr %source) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @fixed_vector(ptr %source) gc "goallc" { entry: %value = load <2 x ptr>, ptr %source, align 8 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/aggregate-loop-stage-b.ll b/src/cmd/llvmplugin/testdata/aggregate-loop-stage-b.ll index 53216f66bd2d13..2cbd66a25159b7 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-loop-stage-b.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-loop-stage-b.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal ptr @aggregate_loop_relocation(i1 %take_call, i1 %again, %pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @aggregate_loop_relocation(i1 %take_call, i1 %again, %pair %value) gc "goallc" { entry: br label %header diff --git a/src/cmd/llvmplugin/testdata/aggregate-scalable-vector-unsupported.ll b/src/cmd/llvmplugin/testdata/aggregate-scalable-vector-unsupported.ll index 44f7936209cbb1..2d36d129a1d385 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-scalable-vector-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-scalable-vector-unsupported.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal ptr @scalable_vector(ptr %source) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @scalable_vector(ptr %source) gc "goallc" { entry: %value = load , ptr %source, align 8 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/aggregate-scalarization.ll b/src/cmd/llvmplugin/testdata/aggregate-scalarization.ll index 48df3f3718e39a..01da4f405e8374 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-scalarization.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-scalarization.ll @@ -59,14 +59,14 @@ declare goabiinternal void @leaf_consume_pair(%pair) #0 declare goabiinternal void @leaf_consume_nested(%nested) #0 declare goabiinternal void @leaf_consume_vector_pair(%vector_pair) #0 -define goabiinternal ptr @pair_across_call(%pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @pair_across_call(%pair %value) gc "goallc" { entry: call goabiinternal void @safepoint() %pointer = extractvalue %pair %value, 0 ret ptr %pointer } -define goabiinternal ptr @triple_across_call(%triple %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @triple_across_call(%triple %value) gc "goallc" { entry: call goabiinternal void @safepoint() %first = extractvalue %triple %value, 1 @@ -76,14 +76,14 @@ entry: ret ptr %result } -define goabiinternal void @nested_across_call(%nested %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @nested_across_call(%nested %value) gc "goallc" { entry: call goabiinternal void @safepoint() call goabiinternal void @leaf_consume_nested(%nested %value) ret void } -define goabiinternal ptr @fixed_vector_across_call(ptr %source) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @fixed_vector_across_call(ptr %source) gc "goallc" { entry: %value = load <2 x ptr>, ptr %source, align 8 call goabiinternal void @safepoint() @@ -91,7 +91,7 @@ entry: ret ptr %result } -define goabiinternal ptr @nested_fixed_vector_across_call(ptr %source, i64 %number) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @nested_fixed_vector_across_call(ptr %source, i64 %number) gc "goallc" { entry: %vector.value = load <2 x ptr>, ptr %source, align 8 %with_vector = insertvalue %vector_pair poison, <2 x ptr> %vector.value, 0 @@ -103,7 +103,7 @@ entry: ret ptr %result } -define goabiinternal ptr @insertvalue_across_call(ptr %pointer, i64 %number) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @insertvalue_across_call(ptr %pointer, i64 %number) gc "goallc" { entry: %with_pointer = insertvalue %pair zeroinitializer, ptr %pointer, 0 %value = insertvalue %pair %with_pointer, i64 %number, 1 @@ -117,7 +117,7 @@ entry: ; pointer roots for poison leaves. Those leaves can be overwritten after the ; safepoint without ever being observed, as happens while reflect.Value is ; assembled for a later call. -define goabiinternal ptr @partial_insertvalue_across_call(ptr %pointer, i64 %number) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @partial_insertvalue_across_call(ptr %pointer, i64 %number) gc "goallc" { entry: %partial = insertvalue %reflect_value poison, ptr %pointer, 0 call goabiinternal void @safepoint() @@ -127,7 +127,7 @@ entry: ret ptr %leaf } -define goabiinternal ptr @phi_across_call(i1 %choose, %pair %left_value, %pair %right_value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @phi_across_call(i1 %choose, %pair %left_value, %pair %right_value) gc "goallc" { entry: br i1 %choose, label %left, label %right @@ -144,7 +144,7 @@ merge: ret ptr %result } -define goabiinternal ptr @select_across_call(i1 %choose, %pair %left_value, %pair %right_value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @select_across_call(i1 %choose, %pair %left_value, %pair %right_value) gc "goallc" { entry: %value = select i1 %choose, %pair %left_value, %pair %right_value call goabiinternal void @safepoint() @@ -152,7 +152,7 @@ entry: ret ptr %result } -define goabiinternal ptr @phi_edge_use(%pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @phi_edge_use(%pair %value) gc "goallc" { entry: call goabiinternal void @safepoint() br label %merge @@ -163,7 +163,7 @@ merge: ret ptr %result } -define goabiinternal ptr @multiple_calls(%pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @multiple_calls(%pair %value) gc "goallc" { entry: call goabiinternal void @safepoint() call goabiinternal void @safepoint() @@ -171,7 +171,7 @@ entry: ret ptr %result } -define goabiinternal ptr @aggregate_call_result(ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @aggregate_call_result(ptr %pointer) gc "goallc" { entry: %value = call goabiinternal %pair @make_pair(ptr %pointer, i64 7) call goabiinternal void @safepoint() @@ -179,13 +179,13 @@ entry: ret ptr %result } -define goabiinternal void @aggregate_current_call_argument(%pair %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @aggregate_current_call_argument(%pair %value) gc "goallc" { entry: call goabiinternal void @consume_pair(%pair %value) ret void } -define goabiinternal void @aggregate_load_store(ptr %source, ptr %destination) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @aggregate_load_store(ptr %source, ptr %destination) gc "goallc" { entry: %value = load %pair, ptr %source, align 8 call goabiinternal void @safepoint() @@ -193,7 +193,7 @@ entry: ret void } -define goabiinternal ptr @alloca_derived_leaf(i64 %number) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @alloca_derived_leaf(i64 %number) gc "goallc" { entry: %slot = alloca i64, align 8 store i64 %number, ptr %slot, align 8 @@ -203,7 +203,7 @@ entry: ret ptr %result } -define goabiinternal void @frozen_aggregate() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @frozen_aggregate() gc "goallc" { entry: %value = freeze %pair poison call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/alloca-derived.ll b/src/cmd/llvmplugin/testdata/alloca-derived.ll index 24073270dc22a6..d6bcd985d96d9d 100644 --- a/src/cmd/llvmplugin/testdata/alloca-derived.ll +++ b/src/cmd/llvmplugin/testdata/alloca-derived.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() -define goabiinternal i64 @selected_stack_address_live_across_call(i1 %choose_a) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal i64 @selected_stack_address_live_across_call(i1 %choose_a) gc "goallc" { entry: %a = alloca i64, align 8 %b = alloca i64, align 8 diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-dynamic-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-dynamic-unsupported.ll index b59d10421aa11e..c8df1c47743dc0 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-dynamic-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-dynamic-unsupported.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal void @dynamic_pointer_alloca(i64 %count) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @dynamic_pointer_alloca(i64 %count) gc "goallc" { entry: %slot = alloca ptr, i64 %count, align 8 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-ambiguous.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-ambiguous.ll index 4eeaed12b2b48f..030c6d5e399730 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-ambiguous.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-ambiguous.ll @@ -4,7 +4,7 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) declare void @llvm.fake.use(...) declare goabiinternal void @safepoint() -define goabiinternal void @path_local_pointer_alloca_lifetime(i1 %start) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @path_local_pointer_alloca_lifetime(i1 %start) gc "goallc" { entry: %slot = alloca ptr, align 8 br i1 %start, label %live, label %join diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-unsupported.ll index 511f8e20fbf6b0..7e9691262844b2 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-lifetime-unsupported.ll @@ -89,7 +89,7 @@ declare goabiinternal void @safepoint() declare goabiinternal void @observe(ptr) declare goabiinternal void @observe_slice({ ptr, i64, i64 }) -define goabiinternal void @locals_pointer_alloca_with_lifetime() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @locals_pointer_alloca_with_lifetime() gc "goallc" { entry: %slot = alloca ptr, align 8 call goabiinternal void @safepoint() @@ -101,7 +101,7 @@ entry: ret void } -define goabiinternal void @stack_object_alloca_with_lifetime() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @stack_object_alloca_with_lifetime() gc "goallc" { entry: %slot = alloca ptr, align 8 call goabiinternal void @safepoint() @@ -112,7 +112,7 @@ entry: ret void } -define goabiinternal void @loop_reinitialized_pointer_alloca(i1 %again) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @loop_reinitialized_pointer_alloca(i1 %again) gc "goallc" { entry: %slot = alloca ptr, align 8 br label %loop @@ -129,7 +129,7 @@ exit: ret void } -define goabiinternal void @preinitialized_pointer_alloca() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @preinitialized_pointer_alloca() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 call void @llvm.lifetime.start.p0(i64 16, ptr %slot) @@ -140,7 +140,7 @@ entry: } define goabiinternal void @store_initialized_pointer_alloca( - ptr %first, ptr %second) "go-stack-growth-statepoint" gc "goallc" { + ptr %first, ptr %second) gc "goallc" { entry: %slot = alloca %pointer_gap, align 8 call void @llvm.lifetime.start.p0(i64 24, ptr %slot) @@ -154,7 +154,7 @@ entry: } define goabiinternal void @partially_stored_pointer_alloca( - ptr %first, ptr %second) "go-stack-growth-statepoint" gc "goallc" { + ptr %first, ptr %second) gc "goallc" { entry: %slot = alloca %pointer_gap, align 8 call void @llvm.lifetime.start.p0(i64 24, ptr %slot) @@ -168,7 +168,7 @@ entry: } define goabiinternal void @phi_edge_pointer_alloca( - i1 %use_stack, ptr %other) "go-stack-growth-statepoint" gc "goallc" { + i1 %use_stack, ptr %other) gc "goallc" { entry: %slot = alloca ptr, align 8 br i1 %use_stack, label %initialize, label %external @@ -188,7 +188,7 @@ merge: ret void } -define goabiinternal void @hoisted_aggregate_pointer_alloca() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @hoisted_aggregate_pointer_alloca() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 %slice = insertvalue { ptr, i64, i64 } poison, ptr %slot, 0 diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll index d9e8e78af24960..470f8bbe569779 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-nonentry-unsupported.ll @@ -5,7 +5,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() define goabiinternal void @nonentry_pointer_alloca( - i1 %allocate) "go-stack-growth-statepoint" gc "goallc" { + i1 %allocate) gc "goallc" { entry: call goabiinternal void @safepoint() br i1 %allocate, label %allocate.block, label %exit diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-realigned-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-realigned-unsupported.ll index 38b61897e42496..d1ada39373384e 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-realigned-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-realigned-unsupported.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal void @realigned_pointer_alloca() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @realigned_pointer_alloca() gc "goallc" { entry: %slot = alloca ptr, align 32 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index 4ee65b76db5794..e89c46dbebca71 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -132,7 +132,7 @@ declare goabiinternal i64 @readonly_pointer_slot(ptr readonly) memory(read) declare goabiinternal i64 @readnone_callee() memory(none) declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) -define goabiinternal ptr @pointer_slot(ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @pointer_slot(ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -143,7 +143,7 @@ entry: } define goabiinternal ptr @nested_whole_aggregate( - ptr %first, ptr %second, ptr %third) "go-stack-growth-statepoint" gc "goallc" { + ptr %first, ptr %second, ptr %third) gc "goallc" { entry: ; The optimized use graph contains only direct memory operations, so this ; must use fixed homes and remain eligible for SROA. @@ -159,7 +159,7 @@ entry: } define goabiinternal ptr @alloca_call_skip( - i1 %take_call, ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + i1 %take_call, ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -175,7 +175,7 @@ merge: } define goabiinternal ptr @alloca_multiple_calls( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -186,7 +186,7 @@ entry: } define goabiinternal ptr @alloca_partial_initialization() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: ; Each field is initialized from a different safepointing call. The plugin ; must zero the whole object before the first call so the complete bitmap is @@ -204,7 +204,7 @@ entry: } define goabiinternal ptr @alloca_loop( - i1 %again, ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + i1 %again, ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -220,7 +220,7 @@ exit: } define goabiinternal ptr @alloca_gep_address_across_call( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca %pointer_field, align 8 %field = getelementptr inbounds %pointer_field, ptr %slot, i32 0, i32 1 @@ -231,7 +231,7 @@ entry: } define goabiinternal void @alloca_direct_address_across_calls() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr null, ptr %slot, align 8 @@ -242,7 +242,7 @@ entry: } define goabiinternal void @argument_home_address_across_calls(ptr %pointer) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: ; This canonical parameter alloca becomes the argument's fixed ABI home. ; Its last callsite has no direct gc-live base, so the function also needs @@ -257,7 +257,7 @@ entry: } define goabiinternal void @argument_aggregate_home_address_across_calls( - %nested %value) "go-stack-growth-statepoint" gc "goallc" { + %nested %value) gc "goallc" { entry: ; The split aggregate parameter still has one complete fixed home and one ; argp-relative StackObject covering all ABI pieces and padding. @@ -271,7 +271,7 @@ entry: } define goabiinternal void @alloca_gep_value_across_calls() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %slot = alloca %pointer_field, align 8 %field = getelementptr inbounds %pointer_field, ptr %slot, i32 0, i32 1 @@ -283,7 +283,7 @@ entry: } define goabiinternal void @alloca_pointer_free_address_across_calls() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %slot = alloca i64, align 8 store i64 0, ptr %slot, align 8 @@ -294,7 +294,7 @@ entry: } define goabiinternal ptr @alloca_address_passed_to_callee( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: ; The structural call use makes the address observable. %slot = alloca ptr, align 8 @@ -305,7 +305,7 @@ entry: } define goabiinternal void @alloca_marker_free_at_safepoint( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr null, ptr %slot, align 8 @@ -315,7 +315,7 @@ entry: } define goabiinternal ptr @alloca_high_bitmap_word( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca %high_bitmap, align 8 %field = getelementptr inbounds %high_bitmap, ptr %slot, i32 0, i32 1 @@ -326,7 +326,7 @@ entry: } define goabiinternal ptr @alloca_multiple_records( - ptr %first, ptr %second) "go-stack-growth-statepoint" gc "goallc" { + ptr %first, ptr %second) gc "goallc" { entry: %left = alloca ptr, align 8 %right = alloca ptr, align 8 @@ -338,7 +338,7 @@ entry: } define goabiinternal ptr @alloca_select_same_base( - i1 %choose, ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + i1 %choose, ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 %same = getelementptr inbounds i8, ptr %slot, i64 0 @@ -350,7 +350,7 @@ entry: } define goabiinternal ptr @alloca_nocapture_writable( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -360,7 +360,7 @@ entry: } define goabiinternal ptr @alloca_escaped_before_unknown_write( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 @@ -371,7 +371,7 @@ entry: } define goabiinternal i64 @alloca_readonly_and_readnone( - ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { + ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store ptr %pointer, ptr %slot, align 8 diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll index c66e005c806494..cdff2808f1629d 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-select-unsupported.ll @@ -3,7 +3,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() define goabiinternal void @select_different_pointer_allocas( - i1 %choose) "go-stack-growth-statepoint" gc "goallc" { + i1 %choose) gc "goallc" { entry: %left = alloca ptr, align 8 %right = alloca ptr, align 8 diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-vector-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-vector-unsupported.ll index ccc415570ef502..efcae4508f524c 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-vector-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-vector-unsupported.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal void @pointer_vector_alloca() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @pointer_vector_alloca() gc "goallc" { entry: %slot = alloca <2 x ptr>, align 8 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-volatile-unsupported.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-volatile-unsupported.ll index 65bb822e37b1de..a271948d165312 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-volatile-unsupported.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-volatile-unsupported.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal void @volatile_pointer_alloca(ptr %pointer) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @volatile_pointer_alloca(ptr %pointer) gc "goallc" { entry: %slot = alloca ptr, align 8 store volatile ptr %pointer, ptr %slot, align 8 diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-kind.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-kind.ll index 103ec3196b83fb..cf85824fb9d4e4 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-kind.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-kind.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, i64 1347703373, i64 11, ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 15), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-length.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-length.ll index b040f5b94c29d7..68487e456e8a54 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-length.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-bad-length.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 14, i64 1, i64 1095520067, i64 11, ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 15), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-duplicate.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-duplicate.ll index a62444f1b208b2..9935a0788dbf7b 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-duplicate.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-duplicate.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 26, i64 2, i64 1095520067, i64 11, ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095520067, i64 11, ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 26), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-non-direct.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-non-direct.ll index 40f13094dc8bfb..9721ff56160d79 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-non-direct.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-non-direct.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, i64 1095520067, i64 11, i64 0, i64 0, i64 8, i64 8, i64 8, i64 1, i64 64, i64 1, i64 1, i64 1095519299, i64 15), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-overlap.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-overlap.ll index 2ee72172006279..544d75dfddd824 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-overlap.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-overlap.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 26, i64 2, i64 1095520067, i64 11, ptr %slot, i64 0, i64 8, i64 8, i64 8, i64 1, i64 64, i64 1, i64 1, i64 1095520067, i64 11, ptr %slot, i64 0, i64 16, i64 8, i64 8, i64 2, i64 64, i64 1, i64 3, i64 1095519299, i64 26), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-padding.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-padding.ll index 33c98ae023adc5..7a49b46a587adf 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-padding.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-padding.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, i64 1095520067, i64 11, ptr %slot, i64 0, i64 8, i64 8, i64 8, i64 1, i64 64, i64 1, i64 3, i64 1095519299, i64 15), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-truncated.ll b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-truncated.ll index c8adc37247705f..1a4cbde8e27059 100644 --- a/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-truncated.ll +++ b/src/cmd/llvmplugin/testdata/alloca-ptrmap-malformed-truncated.ll @@ -3,11 +3,10 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...) -define goabiinternal void @test() #0 gc "goallc" { +define goabiinternal void @test() gc "goallc" { entry: %slot = alloca [2 x ptr], align 8 store [2 x ptr] zeroinitializer, ptr %slot, align 8 %statepoint = call goabiinternal token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 1, i32 0, ptr elementtype(void ()) @callee, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i64 1195461697, i64 15, i64 1, i64 1095520067), "gc-live"(ptr %slot) ] ret void } -attributes #0 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll b/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll index bf0ae1fbeb7131..42720029f9fc9b 100644 --- a/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll +++ b/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll @@ -3,7 +3,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() define goabiinternal void @zero_length_pointer_array() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %slot = alloca [0 x ptr], align 8 call goabiinternal void @safepoint() diff --git a/src/cmd/llvmplugin/testdata/branch-safepoints-relocation.ll b/src/cmd/llvmplugin/testdata/branch-safepoints-relocation.ll index 862f84459f5ed2..05419ab311f681 100644 --- a/src/cmd/llvmplugin/testdata/branch-safepoints-relocation.ll +++ b/src/cmd/llvmplugin/testdata/branch-safepoints-relocation.ll @@ -3,7 +3,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @left_callee() declare goabiinternal void @right_callee() -define goabiinternal i64 @branch_safepoints(ptr %p, i1 %take_left) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal i64 @branch_safepoints(ptr %p, i1 %take_left) gc "goallc" { entry: br i1 %take_left, label %left, label %right diff --git a/src/cmd/llvmplugin/testdata/conditional-relocation.ll b/src/cmd/llvmplugin/testdata/conditional-relocation.ll index d74187db6bccc9..a5e54a126e5689 100644 --- a/src/cmd/llvmplugin/testdata/conditional-relocation.ll +++ b/src/cmd/llvmplugin/testdata/conditional-relocation.ll @@ -30,7 +30,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() -define goabiinternal i64 @conditional_safepoint(ptr %p, i1 %take_call) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal i64 @conditional_safepoint(ptr %p, i1 %take_call) gc "goallc" { entry: br i1 %take_call, label %call, label %skip @@ -50,7 +50,7 @@ merge: define goabiinternal i64 @conditional_phi_edge_use( ptr %p, ptr %q, i1 %take_call) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_call, label %call, label %skip diff --git a/src/cmd/llvmplugin/testdata/debug-inline.ll b/src/cmd/llvmplugin/testdata/debug-inline.ll index 0b0b7e566395e6..11d8d7004b27aa 100644 --- a/src/cmd/llvmplugin/testdata/debug-inline.ll +++ b/src/cmd/llvmplugin/testdata/debug-inline.ll @@ -7,12 +7,12 @@ target triple = "x86_64-unknown-linux-goobj" ; DEBUG: "line": 10, ; DEBUG: "name": "main.mid", ; DEBUG-X86: "parent_pc": 6 -; DEBUG-AARCH64: "parent_pc": 0 +; DEBUG-AARCH64: "parent_pc": 32 ; DEBUG: "parent": 0, ; DEBUG: "line": 20, ; DEBUG: "name": "main.inner", ; DEBUG-X86: "parent_pc": 7 -; DEBUG-AARCH64: "parent_pc": 4 +; DEBUG-AARCH64: "parent_pc": 36 ; DEBUG-X86: "pc_quantum": 1, ; DEBUG-AARCH64: "pc_quantum": 4, ; DEBUG: "kind": "pcfile", @@ -47,7 +47,7 @@ target triple = "x86_64-unknown-linux-goobj" ; DEBUG: "line": 0, ; DEBUG: "name": "main.zeroCallee", ; DEBUG-X86: "parent_pc": 6 -; DEBUG-AARCH64: "parent_pc": 0 +; DEBUG-AARCH64: "parent_pc": 32 ; DEBUG-LABEL: "name": "main.shared", ; DEBUG: "start_line": 70, @@ -56,12 +56,12 @@ target triple = "x86_64-unknown-linux-goobj" ; DEBUG: "line": 71, ; DEBUG: "name": "main.sharedLeft", ; DEBUG-X86: "parent_pc": 6 -; DEBUG-AARCH64: "parent_pc": 0 +; DEBUG-AARCH64: "parent_pc": 32 ; DEBUG: "parent": -1, ; DEBUG: "line": 71, ; DEBUG: "name": "main.sharedRight", ; DEBUG-X86: "parent_pc": 14 -; DEBUG-AARCH64: "parent_pc": 12 +; DEBUG-AARCH64: "parent_pc": 44 @main.sink = global i64 0 diff --git a/src/cmd/llvmplugin/testdata/defer-edge.ll b/src/cmd/llvmplugin/testdata/defer-edge.ll index 43a620ce678feb..1ce15e46817819 100644 --- a/src/cmd/llvmplugin/testdata/defer-edge.ll +++ b/src/cmd/llvmplugin/testdata/defer-edge.ll @@ -26,7 +26,7 @@ declare goabiinternal void @runtime.panicmem() declare void @llvm.go.defer.edge() declare void @llvm.lifetime.start.p0(ptr captures(none)) -define goabiinternal void @defer_edge() #0 gc "goallc" { +define goabiinternal void @defer_edge() gc "goallc" { entry: call goabiinternal void @runtime.deferproc() callbr void @llvm.go.defer.edge() to label %normal [label %recover] @@ -39,7 +39,7 @@ recover: ret void } -define goabiinternal ptr @defer_result(ptr %pointer) #0 gc "goallc" { +define goabiinternal ptr @defer_result(ptr %pointer) gc "goallc" { entry: %result = alloca ptr, align 8, !goallc.defer_result !1 call void @llvm.lifetime.start.p0(ptr %result) @@ -63,7 +63,5 @@ entry: ret void } -attributes #0 = { "go-stack-growth-statepoint" } - !0 = !{i8 23, i8 0} !1 = !{} diff --git a/src/cmd/llvmplugin/testdata/derived-pointer-rematerialization.ll b/src/cmd/llvmplugin/testdata/derived-pointer-rematerialization.ll index 31ce288dcef9f2..0ce9c23b6663d0 100644 --- a/src/cmd/llvmplugin/testdata/derived-pointer-rematerialization.ll +++ b/src/cmd/llvmplugin/testdata/derived-pointer-rematerialization.ll @@ -29,7 +29,7 @@ declare goabiinternal void @callee() ; above the source nil guard. The non-heap value null+96 must not become a Go ; GC root at the intervening safepoint. define goabiinternal i1 @hoisted_null_offset(ptr %base) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %derived = getelementptr i8, ptr %base, i64 96 call goabiinternal void @callee() @@ -42,7 +42,7 @@ entry: ; Rebuild the full address-expression chain from the relocated base, not from ; an independently relocated interior pointer. define goabiinternal i8 @derived_chain(ptr %base) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %field = getelementptr i8, ptr %base, i64 16 %element = getelementptr i8, ptr %field, i64 8 @@ -55,7 +55,7 @@ entry: ; a safepoint: the merge must select the rebuilt address on that path and the ; original address on the other path. define goabiinternal i8 @conditional_derived(ptr %base, i1 %take_call) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %derived = getelementptr i8, ptr %base, i64 32 br i1 %take_call, label %call, label %skip @@ -76,7 +76,7 @@ merge: ; derived pointer. Relocate the vector base as one value, then rebuild the ; vector GEP instead of exposing its interior-pointer lanes as Go GC roots. define goabiinternal <2 x ptr> @derived_vector(<2 x ptr> %base) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %derived = getelementptr i8, <2 x ptr> %base, <2 x i64> @@ -88,7 +88,7 @@ entry: ; base. Keep that scalar base live and rebuild the vector result after it is ; relocated. define goabiinternal <2 x ptr> @derived_vector_from_scalar(ptr %base) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %derived = getelementptr i8, ptr %base, <2 x i64> diff --git a/src/cmd/llvmplugin/testdata/function-marker-inline.ll b/src/cmd/llvmplugin/testdata/function-marker-inline.ll index 2b2ab0831dbee4..be07f764e500f7 100644 --- a/src/cmd/llvmplugin/testdata/function-marker-inline.ll +++ b/src/cmd/llvmplugin/testdata/function-marker-inline.ll @@ -14,14 +14,14 @@ target triple = "x86_64-unknown-linux-goobj" declare void @llvm.sideeffect() define internal goabiinternal void @callee() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: call void @llvm.sideeffect(), !goobj.marker_reloc !0 ret void } define goabiinternal void @caller() - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: call goabiinternal void @callee() ret void diff --git a/src/cmd/llvmplugin/testdata/gc-leaf-markers.ll b/src/cmd/llvmplugin/testdata/gc-leaf-markers.ll index b0db0b1d0111b7..64d9c7c1e5e288 100644 --- a/src/cmd/llvmplugin/testdata/gc-leaf-markers.ll +++ b/src/cmd/llvmplugin/testdata/gc-leaf-markers.ll @@ -3,7 +3,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee_leaf() "gc-leaf-function" declare goabiinternal void @callsite_leaf() -define goabiinternal void @leaf_calls() "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @leaf_calls() gc "goallc" { entry: call goabiinternal void @callee_leaf() call goabiinternal void @callsite_leaf() "gc-leaf-function" diff --git a/src/cmd/llvmplugin/testdata/indirect-callee.ll b/src/cmd/llvmplugin/testdata/indirect-callee.ll index 5f58bd365ad177..652172f1e88554 100644 --- a/src/cmd/llvmplugin/testdata/indirect-callee.ll +++ b/src/cmd/llvmplugin/testdata/indirect-callee.ll @@ -54,4 +54,4 @@ entry: ret void } -attributes #0 = { "frame-pointer"="non-leaf" "go-stack-growth-statepoint" } +attributes #0 = { "frame-pointer"="non-leaf" } diff --git a/src/cmd/llvmplugin/testdata/invalid-gc-leaf-definition.ll b/src/cmd/llvmplugin/testdata/invalid-gc-leaf-definition.ll index e5807484c08e41..542bf9bfbca0f0 100644 --- a/src/cmd/llvmplugin/testdata/invalid-gc-leaf-definition.ll +++ b/src/cmd/llvmplugin/testdata/invalid-gc-leaf-definition.ll @@ -8,4 +8,4 @@ entry: ret void } -attributes #0 = { "gc-leaf-function" "go-stack-growth-statepoint" } +attributes #0 = { "gc-leaf-function" } diff --git a/src/cmd/llvmplugin/testdata/irreducible-relocation.ll b/src/cmd/llvmplugin/testdata/irreducible-relocation.ll index f8e74a22e5d6d8..ca7e378a7edc6a 100644 --- a/src/cmd/llvmplugin/testdata/irreducible-relocation.ll +++ b/src/cmd/llvmplugin/testdata/irreducible-relocation.ll @@ -4,7 +4,7 @@ declare goabiinternal void @callee() define goabiinternal i64 @irreducible_relocation( ptr %p, i1 %enter_b, i1 %leave_a, i1 %leave_b) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %enter_b, label %b, label %a diff --git a/src/cmd/llvmplugin/testdata/live-aggregate.ll b/src/cmd/llvmplugin/testdata/live-aggregate.ll index 96f4e9d4517a80..de743e0d94decf 100644 --- a/src/cmd/llvmplugin/testdata/live-aggregate.ll +++ b/src/cmd/llvmplugin/testdata/live-aggregate.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @callee() -define goabiinternal ptr @live_pointer_aggregate({ ptr, i64 } %value) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal ptr @live_pointer_aggregate({ ptr, i64 } %value) gc "goallc" { entry: call goabiinternal void @callee() %pointer = extractvalue { ptr, i64 } %value, 0 diff --git a/src/cmd/llvmplugin/testdata/loop-relocation.ll b/src/cmd/llvmplugin/testdata/loop-relocation.ll index 5b0293f8f37791..bdc6b1b0b0105c 100644 --- a/src/cmd/llvmplugin/testdata/loop-relocation.ll +++ b/src/cmd/llvmplugin/testdata/loop-relocation.ll @@ -4,7 +4,7 @@ declare goabiinternal void @callee() define goabiinternal i64 @loop_relocation( ptr %p, i1 %take_call, i1 %again) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br label %header diff --git a/src/cmd/llvmplugin/testdata/multiple-calls.ll b/src/cmd/llvmplugin/testdata/multiple-calls.ll index b45b643c8d1412..1c7cca7580dd86 100644 --- a/src/cmd/llvmplugin/testdata/multiple-calls.ll +++ b/src/cmd/llvmplugin/testdata/multiple-calls.ll @@ -40,7 +40,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @first_callee() declare goabiinternal void @second_callee() -define goabiinternal i64 @different_pointer_sets_across_calls(ptr %p, ptr %q) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal i64 @different_pointer_sets_across_calls(ptr %p, ptr %q) gc "goallc" { entry: call goabiinternal void @first_callee() %first = load i64, ptr %p, align 8 diff --git a/src/cmd/llvmplugin/testdata/nest-param-attr.ll b/src/cmd/llvmplugin/testdata/nest-param-attr.ll index 52bc3bbe2f9130..5fbce840c5a7e2 100644 --- a/src/cmd/llvmplugin/testdata/nest-param-attr.ll +++ b/src/cmd/llvmplugin/testdata/nest-param-attr.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @closure_callee(ptr nest) -define goabiinternal void @nest_param_attr(ptr %context) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @nest_param_attr(ptr %context) gc "goallc" { entry: call goabiinternal void @closure_callee(ptr nest %context) ret void diff --git a/src/cmd/llvmplugin/testdata/open-defer-incomplete.ll b/src/cmd/llvmplugin/testdata/open-defer-incomplete.ll index 31b9109d085c40..08c8a5d26cc1a3 100644 --- a/src/cmd/llvmplugin/testdata/open-defer-incomplete.ll +++ b/src/cmd/llvmplugin/testdata/open-defer-incomplete.ll @@ -4,7 +4,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal void @open_defer_missing_slots() #0 gc "goallc" { +define goabiinternal void @open_defer_missing_slots() gc "goallc" { entry: %bits = alloca i8, align 1, !goallc.open_defer_bits !0 store volatile i8 0, ptr %bits, align 1 @@ -12,6 +12,4 @@ entry: ret void } -attributes #0 = { "go-stack-growth-statepoint" } - !0 = !{} diff --git a/src/cmd/llvmplugin/testdata/open-defer.ll b/src/cmd/llvmplugin/testdata/open-defer.ll index e66d1650026fb6..422d75c9cd2760 100644 --- a/src/cmd/llvmplugin/testdata/open-defer.ll +++ b/src/cmd/llvmplugin/testdata/open-defer.ll @@ -30,7 +30,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @safepoint() -define goabiinternal ptr @open_defer(ptr %value) #0 gc "goallc" { +define goabiinternal ptr @open_defer(ptr %value) gc "goallc" { entry: %bits = alloca i8, align 1, !goallc.open_defer_bits !0 %slots = alloca [2 x ptr], align 8, !goallc.open_defer_slots !1 @@ -47,7 +47,5 @@ entry: ret ptr %result } -attributes #0 = { "go-stack-growth-statepoint" } - !0 = !{} !1 = !{i32 2} diff --git a/src/cmd/llvmplugin/testdata/pointer-address-observation.ll b/src/cmd/llvmplugin/testdata/pointer-address-observation.ll index e41f84427b50b9..b098de3333a6d4 100644 --- a/src/cmd/llvmplugin/testdata/pointer-address-observation.ll +++ b/src/cmd/llvmplugin/testdata/pointer-address-observation.ll @@ -14,7 +14,7 @@ declare i64 @llvm.go.pointer.address.i64.p0(ptr) declare goabiinternal void @callee() define goabiinternal i1 @observe(ptr %pointer) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: %before = call i64 @llvm.go.pointer.address.i64.p0(ptr %pointer) call goabiinternal void @callee() diff --git a/src/cmd/llvmplugin/testdata/sequential-conditional-relocation.ll b/src/cmd/llvmplugin/testdata/sequential-conditional-relocation.ll index 59d8c591cc1c10..5b3e59fb11964a 100644 --- a/src/cmd/llvmplugin/testdata/sequential-conditional-relocation.ll +++ b/src/cmd/llvmplugin/testdata/sequential-conditional-relocation.ll @@ -5,7 +5,7 @@ declare goabiinternal void @second_callee() define goabiinternal i64 @sequential_conditional_safepoints( ptr %p, i1 %take_first, i1 %take_second) - "go-stack-growth-statepoint" gc "goallc" { + gc "goallc" { entry: br i1 %take_first, label %first_call, label %first_skip diff --git a/src/cmd/llvmplugin/testdata/statepoint.ll b/src/cmd/llvmplugin/testdata/statepoint.ll index 9178dd6c4f21a7..0485fd0f9c4f68 100644 --- a/src/cmd/llvmplugin/testdata/statepoint.ll +++ b/src/cmd/llvmplugin/testdata/statepoint.ll @@ -4,14 +4,14 @@ declare goabiinternal void @callee() declare goabiinternal void @leaf_callee() #0 declare goabiinternal ptr @make_pointer() -define goabiinternal i64 @pointer_live_across_call(ptr %p) #1 gc "goallc" { +define goabiinternal i64 @pointer_live_across_call(ptr %p) gc "goallc" { entry: call goabiinternal void @callee() %value = load i64, ptr %p, align 8 ret i64 %value } -define goabiinternal i64 @stack_address_live_across_call() #1 gc "goallc" { +define goabiinternal i64 @stack_address_live_across_call() gc "goallc" { entry: %slot = alloca i64, align 8 store i64 42, ptr %slot, align 8 @@ -20,13 +20,13 @@ entry: ret i64 %value } -define goabiinternal void @explicit_leaf_call() #1 gc "goallc" { +define goabiinternal void @explicit_leaf_call() gc "goallc" { entry: call goabiinternal void @leaf_callee() ret void } -define goabiinternal i64 @pointer_live_across_two_calls(ptr %p) #1 gc "goallc" { +define goabiinternal i64 @pointer_live_across_two_calls(ptr %p) gc "goallc" { entry: call goabiinternal void @callee() call goabiinternal void @callee() @@ -34,7 +34,7 @@ entry: ret i64 %value } -define goabiinternal i64 @pointer_live_into_cfg(ptr %p, i1 %take_left) #1 gc "goallc" { +define goabiinternal i64 @pointer_live_into_cfg(ptr %p, i1 %take_left) gc "goallc" { entry: call goabiinternal void @callee() br i1 %take_left, label %left, label %right @@ -52,7 +52,7 @@ done: ret i64 %value } -define goabiinternal ptr @call_result_live_across_call() #1 gc "goallc" { +define goabiinternal ptr @call_result_live_across_call() gc "goallc" { entry: %pointer = call goabiinternal ptr @make_pointer() call goabiinternal void @callee() @@ -62,7 +62,7 @@ entry: ; Function block layout is intentionally different from CFG dominance. The ; safepoint in %use is visited before the pointer-producing call in %define, ; but its liveness set contains that call result. -define goabiinternal ptr @out_of_layout_call_result() #1 gc "goallc" { +define goabiinternal ptr @out_of_layout_call_result() gc "goallc" { entry: br label %define @@ -76,4 +76,3 @@ define: } attributes #0 = { "gc-leaf-function" } -attributes #1 = { "go-stack-growth-statepoint" } diff --git a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll index a1fb5f2f3fcbd1..cdd741ddc3e7fb 100644 --- a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll +++ b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll @@ -8,4 +8,4 @@ entry: ret void } -attributes #0 = { "frame-pointer"="non-leaf" "go-stack-growth-statepoint" } +attributes #0 = { "frame-pointer"="non-leaf" } diff --git a/src/cmd/llvmplugin/testdata/unsupported-invoke.ll b/src/cmd/llvmplugin/testdata/unsupported-invoke.ll index 8c65eba7e923df..6e84d2eb71cadc 100644 --- a/src/cmd/llvmplugin/testdata/unsupported-invoke.ll +++ b/src/cmd/llvmplugin/testdata/unsupported-invoke.ll @@ -4,7 +4,7 @@ declare goabiinternal void @callee() declare i32 @__gxx_personality_v0(...) define goabiinternal void @unsupported_invoke() - "go-stack-growth-statepoint" gc "goallc" + gc "goallc" personality ptr @__gxx_personality_v0 { entry: invoke goabiinternal void @callee() diff --git a/src/cmd/llvmplugin/testdata/unsupported-param-attr.ll b/src/cmd/llvmplugin/testdata/unsupported-param-attr.ll index c1911175b6bb68..1898ce3c354129 100644 --- a/src/cmd/llvmplugin/testdata/unsupported-param-attr.ll +++ b/src/cmd/llvmplugin/testdata/unsupported-param-attr.ll @@ -2,7 +2,7 @@ target triple = "x86_64-unknown-linux-goobj" declare goabiinternal void @unsupported_callee(ptr noalias) -define goabiinternal void @unsupported_param_attr(ptr %context) "go-stack-growth-statepoint" gc "goallc" { +define goabiinternal void @unsupported_param_attr(ptr %context) gc "goallc" { entry: call goabiinternal void @unsupported_callee(ptr noalias %context) ret void diff --git a/test/codegen/statepoint.go b/test/codegen/statepoint.go index 3cbae7970769f5..026c0e98151013 100644 --- a/test/codegen/statepoint.go +++ b/test/codegen/statepoint.go @@ -19,7 +19,8 @@ import "unsafe" // LLVM-NOT: llvm.experimental.stackmap // LLVM-LABEL: define goabiinternal i64 @codegen.goABIStatepointAttributes( // LLVM-SAME: i64 %x) #[[ATTRS:[0-9]+]] gc "goallc" {{.*}}{ -// LLVM: attributes #[[ATTRS]] = { {{.*}}"frame-pointer"="non-leaf"{{.*}}"go-stack-growth-statepoint"{{.*}} } +// LLVM: attributes #[[ATTRS]] = { {{.*}}"frame-pointer"="non-leaf"{{.*}} } +// LLVM-NOT: go-stack-growth-statepoint func goABIStatepointAttributes(x int) int { return x + 1 } From 8cf2909ec01cefad5f85aa34fce69a7110d17f4f Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 10:21:20 +0800 Subject: [PATCH 3/8] cmd/compile: preserve named ABI types at call boundaries --- src/cmd/compile/internal/ssa/ssa2llvm.go | 113 +++++++++++------- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 49 +++++++- 2 files changed, 119 insertions(+), 43 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 4dbcd14a75ba57..fe2e678ceb6784 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -5,6 +5,7 @@ package ssa import ( "cmd/compile/internal/base" "cmd/compile/internal/ir" + "cmd/compile/internal/typecheck" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/src" @@ -147,37 +148,12 @@ func llvmTypeContainsABIPad(typ llvm.Type) bool { return false } -// getLLVMABIStorageType removes Go's nominal aggregate identity from a -// function ABI carrier. Compiler-generated runtime calls can describe the -// same physical ABI value using a substituted builtin type (for example []T), -// while the runtime definition uses a named implementation type (for example -// runtime.slice). The native backends join those at the symbol's physical ABI; -// using literal LLVM aggregates does the same without weakening the semantic -// types used inside either function body. -func getLLVMABIStorageType(typ *types.Type) llvm.Type { - switch typ.Kind() { - case types.TARRAY: - return llvm.ArrayType(getLLVMABIStorageType(typ.Elem()), int(typ.NumElem())) - case types.TSTRUCT: - fields := make([]llvm.Type, typ.NumFields(), typ.NumFields()+1) - for i := 0; i < typ.NumFields(); i++ { - fields[i] = getLLVMABIStorageType(typ.FieldType(i)) - } - if llvmStructHasTailPad(typ) { - fields = append(fields, getLLVMABIPadType()) - } - return llvm.StructType(fields, false) - default: - return getLLVMType(typ) - } -} - // getLLVMABIType makes a non-empty carrier only at a top-level zero-sized ABI // boundary. The original zero-sized layout remains in the wrapper, so // DataLayout supplies its Go alignment without storing that alignment in an // attribute. func getLLVMABIType(typ *types.Type) llvm.Type { - storage := getLLVMABIStorageType(typ) + storage := getLLVMType(typ) if typ.Size() == 0 { return llvm.StructType([]llvm.Type{storage, getLLVMABIPadType()}, false) } @@ -1864,10 +1840,9 @@ func (lfc *LLVMFuncContext) reshapeLLVMValue(v *Value, value llvm.Value, from, t } // reshapeLLVMABICarrier rebuilds a value between semantically distinct LLVM -// aggregate types that have the same physical Go ABI structure. This is used -// only at function boundaries: named aggregate identity remains intact in the -// function body, while the signature uses literal aggregates so independently -// constructed runtime helper declarations and definitions agree. +// aggregate types that have the same physical Go ABI structure. It is used +// only by callers at function boundaries; definitions and declarations retain +// the named aggregate types in their own Go signatures. func (lfc *LLVMFuncContext) reshapeLLVMABICarrier(v *Value, value llvm.Value, target llvm.Type, name string) llvm.Value { if value.Type() == target { return value @@ -2003,6 +1978,37 @@ func llvmStaticCallSignature(v *Value, aux *AuxCall, sig llvmFuncSignature) llvm return sig } +// llvmStaticCalleeAux returns the source signature of a function defined in +// the package currently being compiled. Compiler-built runtime calls use the +// deliberately approximate declarations in the go.runtime pseudo-package; +// their AuxCall can therefore describe a physically equivalent but nominally +// different signature from the real runtime definition. Keep the definition's +// named types in LLVM and make the caller bridge that boundary instead. +func llvmStaticCalleeAux(aux *AuxCall) *AuxCall { + if aux == nil || aux.Fn == nil || aux.ABIInfo() == nil { + return aux + } + if llvmStaticCalleeAuxCache == nil { + llvmStaticCalleeAuxCache = make(map[*obj.LSym]*AuxCall) + } + if callee := llvmStaticCalleeAuxCache[aux.Fn]; callee != nil { + return callee + } + abi := aux.ABI().Which() + for _, fn := range typecheck.Target.Funcs { + if fn == nil || fn.Nname == nil || fn.Type() == nil || fn.ABI != abi { + continue + } + if fn.LinksymABI(abi) != aux.Fn { + continue + } + callee := StaticAuxCall(aux.Fn, aux.ABI().ABIAnalyze(fn.Type(), false)) + llvmStaticCalleeAuxCache[aux.Fn] = callee + return callee + } + return aux +} + func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { aux := auxToCall(v.Aux) if aux == nil || aux.Fn == nil { @@ -2012,9 +2018,26 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { v.Fatalf("static call to %s has %d LLVM arguments, want %d", aux.Fn.Name, got, want) } - sig := llvmStaticCallSignature(v, aux, llvmSignature(aux)) + callerSig := llvmStaticCallSignature(v, aux, llvmSignature(aux)) + calleeAux := llvmStaticCalleeAux(aux) + calleeSig := llvmSignature(calleeAux) + if calleeAux == aux { + calleeSig = llvmStaticCallSignature(v, calleeAux, calleeSig) + } + if got, want := calleeAux.NArgs(), aux.NArgs(); got != want { + v.Fatalf("static call to %s has %d callee arguments, want %d", aux.Fn.Name, got, want) + } + if got, want := calleeAux.ABIInfo().InRegistersUsed(), aux.ABIInfo().InRegistersUsed(); got != want { + v.Fatalf("static call to %s has %d callee input registers, want %d", aux.Fn.Name, got, want) + } + if got, want := calleeAux.ABIInfo().OutRegistersUsed(), aux.ABIInfo().OutRegistersUsed(); got != want { + v.Fatalf("static call to %s has %d callee output registers, want %d", aux.Fn.Name, got, want) + } + if got, want := calleeAux.ArgWidth(), aux.ArgWidth(); got != want { + v.Fatalf("static call to %s has callee argument width %d, want %d", aux.Fn.Name, got, want) + } cc := llvmCallConv(aux.ABI().Which()) - fn := getOrInsertLLVMFunction(aux.Fn.Name, sig, cc) + fn := getOrInsertLLVMFunction(aux.Fn.Name, calleeSig, cc) attachGoObjSymbolRef(fn, aux.Fn) // AMD64 rewrites some Move and Eq operations to static runtime calls before // LLVM emission. Keep the same leaf contract as the dedicated LLVM lowering @@ -2025,26 +2048,34 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { args := make([]llvm.Value, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { arg := lfc.GenLV(v.Args[i]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + if arg.Type() != callerSig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), callerSig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d.caller", v, i)) } - if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { + arg = lfc.reshapeLLVMABICarrier(v, arg, calleeSig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d.callee", v, i)) + if got, want := arg.Type(), calleeSig.Type.ParamTypes()[i]; got != want { v.Fatalf("argument %d to %s has incompatible LLVM type", i, aux.Fn.Name) } args = append(args, arg) } name := v.String() - if sig.ResultCount == 0 { + if calleeSig.ResultCount == 0 { name = "" } - call := lfc.b.CreateCall(sig.Type, fn, args, name) + call := lfc.b.CreateCall(calleeSig.Type, fn, args, name) call.SetInstructionCallConv(cc) - configureLLVMCall(call, sig) - lfc.materializeAddressedResults(v, call, aux) + configureLLVMCall(call, calleeSig) + if (callerSig.ResultCount == 0) != (calleeSig.ResultCount == 0) { + v.Fatalf("static call to %s has incompatible caller and callee result presence", aux.Fn.Name) + } + result := call + if callerSig.ResultCount != 0 { + result = lfc.reshapeLLVMABICarrier(v, call, callerSig.ReturnType, v.String()+".caller.result") + } + lfc.materializeAddressedResults(v, result, aux) if llvmGCLeaf { markLLVMGCLeafCall(call) } - return call + return result } func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext bool) llvm.Value { @@ -3663,6 +3694,7 @@ var goObjImportsWritten bool var currentLLVMDataLowerer *llvmDataLowerer var goObjCompilerUsed []llvm.Value var goObjCompilerUsedNames map[string]bool +var llvmStaticCalleeAuxCache map[*obj.LSym]*AuxCall var GlobalCtxt = llvm.GlobalContext() @@ -3803,6 +3835,7 @@ func InitModule(pkg *types.Pkg) { currentLLVMDataLowerer = newLLVMDataLowerer(make(map[*obj.LSym]bool)) goObjCompilerUsed = nil goObjCompilerUsedNames = make(map[string]bool) + llvmStaticCalleeAuxCache = make(map[*obj.LSym]*AuxCall) initLLVMDebugInfo(pkg) } diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index 7be0cd3fd889c2..7bad74a44f22d7 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "cmd/compile/internal/abi" "cmd/compile/internal/ir" "cmd/compile/internal/typecheck" "cmd/compile/internal/types" @@ -28,7 +29,7 @@ func (n *llvmTestTypeName) Sym() *types.Sym { return n.sym } func (*llvmTestTypeName) Pos() src.XPos { return src.NoXPos } func (*llvmTestTypeName) Type() *types.Type { return nil } -func TestLLVMABICarrierErasesNamedAggregateIdentity(t *testing.T) { +func TestLLVMABICarrierPreservesNamedAggregateIdentity(t *testing.T) { pkg := types.NewPkg("runtime", "runtime") namedSlice := types.NewNamed(&llvmTestTypeName{sym: pkg.Lookup("slice")}) namedSlice.SetUnderlying(types.NewStruct([]*types.Field{ @@ -43,8 +44,50 @@ func TestLLVMABICarrierErasesNamedAggregateIdentity(t *testing.T) { if getLLVMType(namedSlice) == getLLVMType(builtinSlice) { t.Fatal("semantic LLVM types unexpectedly lost named aggregate identity") } - if got, want := getLLVMABIType(namedSlice), getLLVMABIType(builtinSlice); got != want { - t.Fatalf("physical ABI carriers differ: named=%v builtin=%v", got, want) + if got, want := getLLVMABIType(namedSlice), getLLVMType(namedSlice); got != want { + t.Fatalf("named ABI carrier = %v, want semantic type %v", got, want) + } + if got, other := getLLVMABIType(namedSlice), getLLVMABIType(builtinSlice); got == other { + t.Fatalf("named ABI carrier unexpectedly collapsed to builtin carrier %v", got) + } +} + +func TestLLVMStaticCalleeAuxUsesPackageDefinition(t *testing.T) { + oldTarget := typecheck.Target + oldCache := llvmStaticCalleeAuxCache + typecheck.Target = new(ir.Package) + llvmStaticCalleeAuxCache = make(map[*obj.LSym]*AuxCall) + t.Cleanup(func() { + typecheck.Target = oldTarget + llvmStaticCalleeAuxCache = oldCache + }) + + pkg := types.NewPkg("runtime", "runtime") + namedSlice := types.NewNamed(&llvmTestTypeName{sym: pkg.Lookup("slice")}) + namedSlice.SetUnderlying(types.NewStruct([]*types.Field{ + types.NewField(src.NoXPos, pkg.Lookup("array"), types.Types[types.TUNSAFEPTR]), + types.NewField(src.NoXPos, pkg.Lookup("len"), types.Types[types.TINT]), + types.NewField(src.NoXPos, pkg.Lookup("cap"), types.Types[types.TINT]), + })) + types.CalcSize(namedSlice) + builtinSlice := types.NewSlice(types.Types[types.TUINT8]) + types.CalcSize(builtinSlice) + + fn := ir.NewFunc(src.NoXPos, src.NoXPos, pkg.Lookup("growslice"), types.NewSignature(nil, nil, []*types.Field{ + types.NewField(src.NoXPos, nil, namedSlice), + })) + typecheck.Target.Funcs = []*ir.Func{fn} + config := abi.NewABIConfig(9, 15, 0, uint8(obj.ABIInternal)) + caller := StaticAuxCall(fn.LinksymABI(obj.ABIInternal), config.ABIAnalyzeTypes(nil, []*types.Type{builtinSlice})) + callee := llvmStaticCalleeAux(caller) + if callee == caller { + t.Fatal("static call retained the caller's approximate runtime signature") + } + if got := callee.TypeOfResult(0); got != namedSlice { + t.Fatalf("callee result type = %v, want named runtime result %v", got, namedSlice) + } + if cached := llvmStaticCalleeAux(caller); cached != callee { + t.Fatal("static callee signature was not cached") } } From 53b5cb259ac026dfc31878ea34394d8e62f6c190 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 10:21:24 +0800 Subject: [PATCH 4/8] test: qualify LLVM-built runtime dependency closure --- src/cmd/internal/testdir/llvm_abi_test.go | 11 +- src/cmd/internal/testdir/llvm_stdlib_test.go | 113 +++++++++++++++++-- src/cmd/internal/testdir/llvm_test.go | 7 ++ test/codegen/llvm_memops.go | 4 +- test/llvm_stdlib_packages.json | 4 + 5 files changed, 121 insertions(+), 18 deletions(-) diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 05cda1afc21f10..97e039447eb70d 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -463,24 +463,27 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri "-o", "-", goallcIR) machinePatterns := map[string][]string{ "p.initializedPointerResult": { - `STATEPOINT 5147424658422983495,[^\n]*\$rsp, 72,`, `STATEPOINT -[0-9]+,[^\n]*\$rsp, 0,`, }, "p.partiallyInitializedAggregateResult": { - `STATEPOINT 5147424658422983495,[^\n]*\$rsp, 80,[^\n]*\$rsp, 88,`, `STATEPOINT -[0-9]+,[^\n]*\$rsp, 0,[^\n]*\$rsp, 8,`, }, "p.liveScalarStackArgument": { - `STATEPOINT 5147424658422983495,[^\n]*\$rsp, 64,`, `STATEPOINT -[0-9]+,[^\n]*\$rsp, 72,`, }, "p.liveAggregateStackArgument": { - `STATEPOINT 5147424658422983495,[^\n]*\$rsp, 48,[^\n]*\$rsp, 64,`, `STATEPOINT -[0-9]+,[^\n]*\$rsp, 56,[^\n]*\$rsp, 72,`, }, } for name, patterns := range machinePatterns { body := llvmABIMachineFunction(t, machineIR, name) + stackGrowth := regexp.MustCompile(`(?m)^.*STATEPOINT 5147424658422983495,[^\n]*&runtime\.morestack_noctxt[^\n]*$`).Find(body) + if stackGrowth == nil { + t.Fatalf("%s PEI MIR has no stack-growth statepoint\n%s", name, body) + } + if bytes.Contains(stackGrowth, []byte(`1, 8, $rsp`)) { + t.Fatalf("%s stack-growth statepoint unexpectedly contains GC roots: %s", name, stackGrowth) + } for _, pattern := range patterns { if !regexp.MustCompile(pattern).Match(body) { t.Fatalf("%s PEI MIR does not match %q\n%s", name, pattern, body) diff --git a/src/cmd/internal/testdir/llvm_stdlib_test.go b/src/cmd/internal/testdir/llvm_stdlib_test.go index e4a1e94670d4e2..e0bc74078bd084 100644 --- a/src/cmd/internal/testdir/llvm_stdlib_test.go +++ b/src/cmd/internal/testdir/llvm_stdlib_test.go @@ -31,6 +31,7 @@ type llvmStdlibTestSet struct { Whitelist map[string]string `json:"whitelist"` Blacklist map[string]string `json:"blacklist"` PlatformBlacklist map[string]map[string]string `json:"platform_blacklist,omitempty"` + DependencyClosure []string `json:"dependency_closure,omitempty"` } type llvmStdlibPolicy struct { @@ -100,8 +101,9 @@ func classifyLLVMStdlibPackage(set llvmStdlibTestSet, name string) llvmStdlibCla func effectiveLLVMStdlibTestSet(set llvmStdlibTestSet, platform string) llvmStdlibTestSet { effective := llvmStdlibTestSet{ - Whitelist: make(map[string]string, len(set.Whitelist)), - Blacklist: make(map[string]string, len(set.Blacklist)), + Whitelist: make(map[string]string, len(set.Whitelist)), + Blacklist: make(map[string]string, len(set.Blacklist)), + DependencyClosure: make([]string, 0, len(set.DependencyClosure)), } for name, reason := range set.Whitelist { effective.Whitelist[name] = reason @@ -113,9 +115,23 @@ func effectiveLLVMStdlibTestSet(set llvmStdlibTestSet, platform string) llvmStdl delete(effective.Whitelist, name) effective.Blacklist[name] = reason } + for _, name := range set.DependencyClosure { + if _, ok := effective.Whitelist[name]; ok { + effective.DependencyClosure = append(effective.DependencyClosure, name) + } + } return effective } +func llvmStdlibUsesDependencyClosure(set llvmStdlibTestSet, name string) bool { + for _, entry := range set.DependencyClosure { + if entry == name { + return true + } + } + return false +} + func validateLLVMStdlibPolicy(t *testing.T, packages map[string]bool, set llvmStdlibTestSet) { t.Helper() failed := false @@ -179,6 +195,22 @@ func validateLLVMStdlibPolicy(t *testing.T, packages map[string]bool, set llvmSt } } } + closureEntries := make(map[string]bool, len(set.DependencyClosure)) + for _, name := range set.DependencyClosure { + if name == "*" || strings.ContainsAny(name, "*?[\\") { + t.Errorf("LLVM standard library dependency-closure entry %q is not an exact package", name) + failed = true + } + if closureEntries[name] { + t.Errorf("LLVM standard library dependency-closure entry %q is duplicated", name) + failed = true + } + closureEntries[name] = true + if _, ok := set.Whitelist[name]; !ok { + t.Errorf("LLVM standard library dependency-closure entry %q is not in the whitelist", name) + failed = true + } + } for name := range packages { if classifyLLVMStdlibPackage(set, name) == llvmStdlibUnclassified { t.Errorf("standard library package %q is not classified as white or black", name) @@ -216,8 +248,9 @@ func TestClassifyLLVMStdlibPackage(t *testing.T) { func TestEffectiveLLVMStdlibTestSet(t *testing.T) { set := llvmStdlibTestSet{ - Whitelist: map[string]string{"bytes": "qualified", "cmp": "qualified"}, - Blacklist: map[string]string{"*": "not yet qualified"}, + Whitelist: map[string]string{"bytes": "qualified", "cmp": "qualified"}, + Blacklist: map[string]string{"*": "not yet qualified"}, + DependencyClosure: []string{"bytes", "cmp"}, PlatformBlacklist: map[string]map[string]string{ "linux/amd64": {"bytes": "known failure"}, }, @@ -232,6 +265,38 @@ func TestEffectiveLLVMStdlibTestSet(t *testing.T) { if got := classifyLLVMStdlibPackage(set, "bytes"); got != llvmStdlibWhite { t.Errorf("platform selection modified the common set: classifyLLVMStdlibPackage(bytes) = %v, want %v", got, llvmStdlibWhite) } + if llvmStdlibUsesDependencyClosure(effective, "bytes") { + t.Error("platform-blacklisted bytes retained dependency-closure qualification") + } + if !llvmStdlibUsesDependencyClosure(effective, "cmp") { + t.Error("effective policy lost cmp dependency-closure qualification") + } +} + +func llvmStdlibDependencyPackages(t *testing.T, packages map[string]bool, name string) []string { + t.Helper() + cmd := testenv.Command(t, llvmStdlibGoTool(t), "list", "-deps", "-f={{.ImportPath}}", name) + cmd.Env = append(os.Environ(), "GOENV=off", "GOFLAGS=", "GOROOT="+testenv.GOROOT(t)) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("list dependencies for standard library package %q: %v\n%s", name, err, out) + } + seen := make(map[string]bool) + var dependencies []string + for _, dependency := range strings.Fields(string(out)) { + if !packages[dependency] { + t.Fatalf("dependency-closure package %q has non-standard dependency %q", name, dependency) + } + if !seen[dependency] { + seen[dependency] = true + dependencies = append(dependencies, dependency) + } + } + if !seen[name] { + t.Fatalf("dependency closure for %q does not contain the package itself", name) + } + sort.Strings(dependencies) + return dependencies } func TestLLVMStdlib(t *testing.T) { @@ -252,13 +317,20 @@ func TestLLVMStdlib(t *testing.T) { set := effectiveLLVMStdlibTestSet(policySet, platform) configureLLVMTestToolchain(t) toolexec := llvmToolexec(t, "default") + runtimeToolexec := llvmToolexecWithNativePackages(t, "default", "runtime_test", "runtime.test") whitelist := make([]string, 0, len(set.Whitelist)) for name := range set.Whitelist { whitelist = append(whitelist, name) } sort.Strings(whitelist) - t.Logf("LLVM standard library entry-package policy: %d white, %d black (%d packages)", len(whitelist), len(packages)-len(whitelist), len(packages)) + t.Logf("LLVM standard library policy: %d white, %d black (%d packages), %d dependency closures", len(whitelist), len(packages)-len(whitelist), len(packages), len(set.DependencyClosure)) + + dependencyPackages := make(map[string][]string, len(set.DependencyClosure)) + for _, name := range set.DependencyClosure { + dependencyPackages[name] = llvmStdlibDependencyPackages(t, packages, name) + t.Logf("LLVM stdlib dependency closure: package=%q packages=%d", name, len(dependencyPackages[name])) + } knownBlacklist := make([]string, 0, len(set.Blacklist)-1) for name := range set.Blacklist { @@ -279,16 +351,33 @@ func TestLLVMStdlib(t *testing.T) { cache := t.TempDir() for _, name := range whitelist { t.Run(name, func(t *testing.T) { + compilePackages := []string{name} + packageToolexec := toolexec + testTimeout := "2m" + processTimeout := 5 * time.Minute + if closure := dependencyPackages[name]; len(closure) != 0 { + compilePackages = closure + testTimeout = "5m" + processTimeout = 8 * time.Minute + if name == "runtime" { + // runtime_test and the generated runtime.test main are test + // scaffolding rather than part of the qualified runtime closure. + packageToolexec = runtimeToolexec + } + } for run := 1; run <= llvmStdlibWhitelistRuns; run++ { - ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 5*time.Minute) - cmd := testenv.CommandContext(t, ctx, llvmStdlibGoTool(t), + ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), processTimeout) + args := []string{ "test", "-count=1", - "-timeout=2m", - "-toolexec="+toolexec, - fmt.Sprintf("-gcflags=%s=-enablellvm -llvmironly", name), - name, - ) + "-timeout=" + testTimeout, + "-toolexec=" + packageToolexec, + } + for _, compilePackage := range compilePackages { + args = append(args, fmt.Sprintf("-gcflags=%s=-enablellvm -llvmironly", compilePackage)) + } + args = append(args, name) + cmd := testenv.CommandContext(t, ctx, llvmStdlibGoTool(t), args...) cmd.Env = append(os.Environ(), "GOENV=off", "GOFLAGS=", diff --git a/src/cmd/internal/testdir/llvm_test.go b/src/cmd/internal/testdir/llvm_test.go index 22166029a34ebf..06ba2e3a09160a 100644 --- a/src/cmd/internal/testdir/llvm_test.go +++ b/src/cmd/internal/testdir/llvm_test.go @@ -917,6 +917,10 @@ func runLLVMWriteBarrierIRTests(t *testing.T) { } func llvmToolexec(t *testing.T, optPasses string) string { + return llvmToolexecWithNativePackages(t, optPasses) +} + +func llvmToolexecWithNativePackages(t *testing.T, optPasses string, nativePackages ...string) string { t.Helper() wrapper := llvmToolexecPath(t) @@ -931,6 +935,9 @@ func llvmToolexec(t *testing.T, optPasses string) string { opt := llvmToolPath(t, "opt", "GOALLC_OPT") args = append(args, "-opt="+opt, "-opt-passes="+optPasses) } + for _, name := range nativePackages { + args = append(args, "-native-package="+name) + } value, err := quoted.Join(args) if err != nil { t.Fatal(err) diff --git a/test/codegen/llvm_memops.go b/test/codegen/llvm_memops.go index 801d1221b9425d..97666d49f8a947 100644 --- a/test/codegen/llvm_memops.go +++ b/test/codegen/llvm_memops.go @@ -90,14 +90,14 @@ func llvmMoveAligned(dst, src *[3]uint64) { // LLVM-DAG: define goabiinternal void @codegen.llvmMoveLarge( // LLVM-DAG: call goabiinternal void @runtime.memmove(ptr %dst, ptr {{%.*}}, i64 128) #{{[0-9]+}} -// LLVM-DAG: declare !goobj.builtin !{{[0-9]+}} goabiinternal void @runtime.memmove(ptr, ptr, i64) #{{[0-9]+}} +// LLVM-DAG: declare !goobj.builtin !{{[0-9]+}} goabiinternal void @runtime.memmove(ptr, ptr, i64) func llvmMoveLarge(dst *[128]byte, src [128]byte) { *dst = src } // LLVM-DAG: define goabiinternal i8 @codegen.llvmMemEq( // LLVM-DAG: call goabiinternal i8 @runtime.memequal(ptr {{%.*}}, ptr {{%.*}}, i64 {{%.*}}) #{{[0-9]+}} -// LLVM-DAG: declare !goobj.builtin !{{[0-9]+}} goabiinternal i8 @runtime.memequal(ptr, ptr, i64) #{{[0-9]+}} +// LLVM-DAG: declare !goobj.builtin !{{[0-9]+}} goabiinternal i8 @runtime.memequal(ptr, ptr, i64) func llvmMemEq(a, b string) bool { return a == b } diff --git a/test/llvm_stdlib_packages.json b/test/llvm_stdlib_packages.json index da500406b38be2..5654bfb84cfcb4 100644 --- a/test/llvm_stdlib_packages.json +++ b/test/llvm_stdlib_packages.json @@ -58,6 +58,7 @@ "path/filepath": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "regexp": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "regexp/syntax": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", + "runtime": "qualified through LLVM O2 compilation of its exact dependency closure, GoObj/archive, link, and runtime package tests with test scaffolding compiled natively", "sort": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "strconv": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "strings": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", @@ -68,6 +69,9 @@ "unicode/utf16": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "unicode/utf8": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests" }, + "dependency_closure": [ + "runtime" + ], "blacklist": { "*": "entry package has not yet been qualified through LLVM O2 compile, GoObj/archive, link, and package tests", "compress/flate": "llc/GoObj: X86 and AArch64 SelectionDAG instruction-selection assertions" From c5cdf52161682a5202ab05f10d0dcc68c441accd Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 10:47:25 +0800 Subject: [PATCH 5/8] test: skip DWARF for LLVM runtime qualification --- src/cmd/internal/testdir/llvm_stdlib_test.go | 7 +++++++ test/llvm_stdlib_packages.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cmd/internal/testdir/llvm_stdlib_test.go b/src/cmd/internal/testdir/llvm_stdlib_test.go index e0bc74078bd084..2bcb751256c77f 100644 --- a/src/cmd/internal/testdir/llvm_stdlib_test.go +++ b/src/cmd/internal/testdir/llvm_stdlib_test.go @@ -373,6 +373,13 @@ func TestLLVMStdlib(t *testing.T) { "-timeout=" + testTimeout, "-toolexec=" + packageToolexec, } + if name == "runtime" { + // LLVM GoObj does not yet emit the complete per-function + // DWARF carrier set expected by the Go linker. Runtime + // qualification currently covers code generation, GoObj, + // linking, and execution, but not debug information. + args = append(args, "-ldflags=-w") + } for _, compilePackage := range compilePackages { args = append(args, fmt.Sprintf("-gcflags=%s=-enablellvm -llvmironly", compilePackage)) } diff --git a/test/llvm_stdlib_packages.json b/test/llvm_stdlib_packages.json index 5654bfb84cfcb4..c6e67306daca50 100644 --- a/test/llvm_stdlib_packages.json +++ b/test/llvm_stdlib_packages.json @@ -58,7 +58,7 @@ "path/filepath": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "regexp": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "regexp/syntax": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", - "runtime": "qualified through LLVM O2 compilation of its exact dependency closure, GoObj/archive, link, and runtime package tests with test scaffolding compiled natively", + "runtime": "qualified through LLVM O2 compilation of its exact dependency closure, GoObj/archive, stripped link (-w), and runtime package tests with test scaffolding compiled natively; DWARF is not yet qualified", "sort": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "strconv": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", "strings": "qualified through entry-package LLVM O2 compile, GoObj/archive, link, and package tests", From df6059708b37e703d82133a8b99d403f21eb0cc7 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 12:40:27 +0800 Subject: [PATCH 6/8] cmd/compile: bridge promoted receivers at LLVM calls --- src/cmd/compile/internal/ssa/ssa2llvm.go | 37 ++++++++++++++++++- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 34 +++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index fe2e678ceb6784..c9487a9dd1b6b1 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -1849,10 +1849,45 @@ func (lfc *LLVMFuncContext) reshapeLLVMABICarrier(v *Value, value llvm.Value, ta } source := value.Type() + // Go ABI analysis may describe a promoted method receiver using its single + // physical register carrier while the generated wrapper definition retains + // the named aggregate receiver type. Peel and rebuild singleton aggregates + // at that caller boundary. This keeps the callee signature semantic without + // introducing an anonymous aggregate signature shared by both sides. + if source.TypeKind() != target.TypeKind() { + switch source.TypeKind() { + case llvm.StructTypeKind: + fields := source.StructElementTypes() + if len(fields) == 1 { + field := lfc.b.CreateExtractValue(value, 0, name+".abi.unwrap") + return lfc.reshapeLLVMABICarrier(v, field, target, name) + } + case llvm.ArrayTypeKind: + if source.ArrayLength() == 1 { + element := lfc.b.CreateExtractValue(value, 0, name+".abi.unwrap") + return lfc.reshapeLLVMABICarrier(v, element, target, name) + } + } + + switch target.TypeKind() { + case llvm.StructTypeKind: + fields := target.StructElementTypes() + if len(fields) == 1 { + field := lfc.reshapeLLVMABICarrier(v, value, fields[0], name) + return lfc.b.CreateInsertValue(llvm.Undef(target), field, 0, name+".abi.wrap") + } + case llvm.ArrayTypeKind: + if target.ArrayLength() == 1 { + element := lfc.reshapeLLVMABICarrier(v, value, target.ElementType(), name) + return lfc.b.CreateInsertValue(llvm.Undef(target), element, 0, name+".abi.wrap") + } + } + } + switch target.TypeKind() { case llvm.StructTypeKind: if source.TypeKind() != llvm.StructTypeKind { - v.Fatalf("cannot reshape non-struct LLVM ABI carrier to struct") + v.Fatalf("cannot reshape LLVM ABI carrier kind %d to struct at %s", source.TypeKind(), name) } sourceFields := source.StructElementTypes() targetFields := target.StructElementTypes() diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index 7bad74a44f22d7..0162401f19478f 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -52,6 +52,40 @@ func TestLLVMABICarrierPreservesNamedAggregateIdentity(t *testing.T) { } } +func TestLLVMABICarrierBridgesPromotedReceiverAtCaller(t *testing.T) { + module := GlobalCtxt.NewModule("promoted_receiver_carrier") + builder := GlobalCtxt.NewBuilder() + t.Cleanup(module.Dispose) + t.Cleanup(builder.Dispose) + + pointer := GlobalCtxt.PointerType(0) + receiver := GlobalCtxt.StructCreateNamed("goallc.test.promoted.receiver") + receiver.StructSetBody([]llvm.Type{pointer}, false) + + wrap := llvm.AddFunction(module, "wrap", llvm.FunctionType(receiver, []llvm.Type{pointer}, false)) + builder.SetInsertPointAtEnd(llvm.AddBasicBlock(wrap, "entry")) + context := &LLVMFuncContext{b: builder} + value := &Value{ID: 1, Type: types.Types[types.TUNSAFEPTR]} + builder.CreateRet(context.reshapeLLVMABICarrier(value, wrap.Param(0), receiver, "receiver")) + + unwrap := llvm.AddFunction(module, "unwrap", llvm.FunctionType(pointer, []llvm.Type{receiver}, false)) + builder.SetInsertPointAtEnd(llvm.AddBasicBlock(unwrap, "entry")) + builder.CreateRet(context.reshapeLLVMABICarrier(value, unwrap.Param(0), pointer, "receiver")) + + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("LLVM verifier rejected promoted receiver carrier bridge: %v\n%s", err, module.String()) + } + ir := module.String() + for _, want := range []string{ + "insertvalue %goallc.test.promoted.receiver undef, ptr %0, 0", + "extractvalue %goallc.test.promoted.receiver %0, 0", + } { + if !strings.Contains(ir, want) { + t.Errorf("promoted receiver bridge does not contain %q\n%s", want, ir) + } + } +} + func TestLLVMStaticCalleeAuxUsesPackageDefinition(t *testing.T) { oldTarget := typecheck.Target oldCache := llvmStaticCalleeAuxCache From 2c533ad05ebf355b53403f70a78687cb625393b2 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 12:57:44 +0800 Subject: [PATCH 7/8] test: expect named LLVM ABI aggregates --- src/cmd/internal/testdir/llvm_alloca_test.go | 4 ++-- test/codegen/llvm_opendefer.go | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cmd/internal/testdir/llvm_alloca_test.go b/src/cmd/internal/testdir/llvm_alloca_test.go index de76274715320d..95cf2fd84c1001 100644 --- a/src/cmd/internal/testdir/llvm_alloca_test.go +++ b/src/cmd/internal/testdir/llvm_alloca_test.go @@ -92,10 +92,10 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { parameterInputFunction := llvmAllocaIRFunction(t, inputIR, "p.parameterAcrossSafepoints") for _, pattern := range []string{ - `define goabiinternal void @p\.parameterAcrossSafepoints\(\{ ptr, i64, ptr, \[2 x ptr\] \} %value\)`, + `define goabiinternal void @p\.parameterAcrossSafepoints\(%p\.pointerLocal %value\)`, `alloca %p\.pointerLocal, align 8`, `call void @llvm\.lifetime\.start\.p0\(ptr %v[0-9]+\)`, - `store \{ ptr, i64, ptr, \[2 x ptr\] \} %value, ptr %v[0-9]+, align 8`, + `store %p\.pointerLocal %value, ptr %v[0-9]+, align 8`, } { if !regexp.MustCompile(pattern).Match(parameterInputFunction) { t.Fatalf("input parameter-home IR does not match %q\n%s", diff --git a/test/codegen/llvm_opendefer.go b/test/codegen/llvm_opendefer.go index 16f8eba8d921a8..64c9cbbe498323 100644 --- a/test/codegen/llvm_opendefer.go +++ b/test/codegen/llvm_opendefer.go @@ -32,18 +32,16 @@ type llvmOpenDeferNamedResult struct { // LLVM-OPT: [[RECOVERY_OPT]]: // LLVM-OPT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} -// LLVM-LABEL: define goabiinternal { i64 } @codegen.llvmOpenDeferNamed( +// LLVM-LABEL: define goabiinternal %codegen.llvmOpenDeferNamedResult @codegen.llvmOpenDeferNamed( // LLVM: open.defer.recovery: // LLVM-NEXT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} // LLVM: load volatile %codegen.llvmOpenDeferNamedResult -// LLVM: insertvalue { i64 } -// LLVM: ret { i64 } +// LLVM: ret %codegen.llvmOpenDeferNamedResult // LLVM: ![[SLOTS_MD]] = !{i32 2} -// LLVM-OPT-LABEL: define goabiinternal { i64 } @codegen.llvmOpenDeferNamed( +// LLVM-OPT-LABEL: define goabiinternal %codegen.llvmOpenDeferNamedResult @codegen.llvmOpenDeferNamed( // LLVM-OPT: common.ret: // LLVM-OPT: load volatile %codegen.llvmOpenDeferNamedResult -// LLVM-OPT: insertvalue { i64 } -// LLVM-OPT: ret { i64 } +// LLVM-OPT: ret %codegen.llvmOpenDeferNamedResult // LLVM-OPT: open.defer.recovery: // LLVM-OPT: call goabiinternal void @runtime.deferreturn(), !dbg !{{[0-9]+}} // LLVM-OPT: ![[SLOTS_OPT_MD]] = !{i32 2} From d96d1156b41d234e2550eca11e9c85909d5de6a7 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Thu, 13 Aug 2026 20:58:55 +0800 Subject: [PATCH 8/8] cmd/compile: derive entry stack maps from CFG state Treat the function-level entry map as map zero and expect GoObj PCDATA to return to it whenever CFG-derived PCSP reaches the entry stack depth. Verify that morestack is a raw ABI0 call rather than a statepoint, and update GoObj/plugin fixtures for the new initial map and slow-path range. --- src/cmd/internal/testdir/llvm_abi_test.go | 52 +++++++++---------- src/cmd/internal/testdir/llvm_alloca_test.go | 4 +- .../testdata/llvm_args_pointer_maps.mir | 4 +- src/cmd/llvmplugin/README.md | 24 +++++---- src/cmd/llvmplugin/testdata/aarch64-frame.ll | 3 +- .../testdata/aggregate-call-result-goobj.ll | 7 ++- .../testdata/conditional-relocation.ll | 7 ++- src/cmd/llvmplugin/testdata/multiple-calls.ll | 7 ++- test/llvm_tests.json | 2 +- 9 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 97e039447eb70d..4e3e7054bf2d47 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -255,7 +255,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0}, nil}, goallcArgsMaps: [][]int{{0}, nil, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 0, 1, 2}, + goallcStackMaps: []int32{0, 1, 0, 2}, }, { name: "mixedABI", args: 152, pointerBits: []int{2, 4, 18}, @@ -264,7 +264,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { // homes in LocalsPointerMaps through locals-only alloca records. goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, 1}, + goallcStackMaps: []int32{0, 1, 0, 1}, goallcQueryMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}, {2}}, }, { @@ -272,7 +272,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0}, nil}, goallcArgsMaps: [][]int{{0}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0}, + goallcStackMaps: []int32{0}, checkFullMaps: true, nativeLocals: 8, goallcLocals: 24, @@ -286,7 +286,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0, 2}, nil}, goallcArgsMaps: [][]int{{0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0}, + goallcStackMaps: []int32{0}, checkFullMaps: true, nativeLocals: 8, goallcLocals: 24, @@ -300,7 +300,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0, 2}, nil}, goallcArgsMaps: [][]int{{0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0}, + goallcStackMaps: []int32{0}, checkFullMaps: true, nativeLocals: 8, goallcLocals: 24, @@ -314,42 +314,42 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{0}, nil}, goallcArgsMaps: [][]int{{0}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, 1}, + goallcStackMaps: []int32{0, 1, 0}, }, { name: "overflowResults", args: 48, pointerBits: []int{2, 3, 4, 5}, nativeArgsMaps: [][]int{{2, 3, 4, 5}, nil}, goallcArgsMaps: [][]int{{2, 3, 4, 5}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, 1}, + goallcStackMaps: []int32{0, 1, 0}, }, { name: "initializedStackResult", args: 16, pointerBits: []int{1}, nativeArgsMaps: [][]int{{1}, nil}, goallcArgsMaps: [][]int{{1}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, 1}, + goallcStackMaps: []int32{0, 1, 0}, }, { name: "stackAggregateResult", args: 40, pointerBits: []int{4}, nativeArgsMaps: [][]int{{4}, nil}, goallcArgsMaps: [][]int{{4}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, 1}, + goallcStackMaps: []int32{0, 1, 0}, }, { name: "bothOverflow", args: 168, pointerBits: []int{2, 6, 20}, nativeArgsMaps: [][]int{{2, 6, 20}, nil}, goallcArgsMaps: [][]int{{2, 6, 20}, {2}, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 0, 1, 2}, + goallcStackMaps: []int32{0, 1, 0, 2}, }, { name: "pointerAggregateBothOverflow", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, goallcArgsMaps: [][]int{{0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0}, + goallcStackMaps: []int32{0}, checkFullMaps: true, nativeLocals: 8, goallcLocals: 136, @@ -363,7 +363,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{4}, nil}, goallcArgsMaps: [][]int{{4}, nil, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, - goallcStackMaps: []int32{-1, 0, 1, 2}, + goallcStackMaps: []int32{0, 1, 0, 2}, }, } for _, tc := range cases { @@ -477,12 +477,12 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri } for name, patterns := range machinePatterns { body := llvmABIMachineFunction(t, machineIR, name) - stackGrowth := regexp.MustCompile(`(?m)^.*STATEPOINT 5147424658422983495,[^\n]*&runtime\.morestack_noctxt[^\n]*$`).Find(body) - if stackGrowth == nil { - t.Fatalf("%s PEI MIR has no stack-growth statepoint\n%s", name, body) + morestackCall := regexp.MustCompile(`(?m)^.*CALL64pcrel32 &"?runtime\.morestack_noctxt"?[^\n]*$`).Find(body) + if morestackCall == nil { + t.Fatalf("%s PEI MIR has no raw ABI0 morestack call\n%s", name, body) } - if bytes.Contains(stackGrowth, []byte(`1, 8, $rsp`)) { - t.Fatalf("%s stack-growth statepoint unexpectedly contains GC roots: %s", name, stackGrowth) + if regexp.MustCompile(`(?m)^.*STATEPOINT[^\n]*runtime\.morestack_noctxt[^\n]*$`).Match(body) { + t.Fatalf("%s PEI MIR still represents morestack as a statepoint\n%s", name, body) } for _, pattern := range patterns { if !regexp.MustCompile(pattern).Match(body) { @@ -529,7 +529,7 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri name: "initializedPointerResult", args: 72, entryBits: []int{8}, goallcLocals: 16, goallcArgs: [][]int{{8}, nil}, goallcMaps: [][]int{nil, {1}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0, 1, 0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{1, 0}, nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 16, 8, 0}, }, @@ -537,21 +537,21 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri name: "partiallyInitializedAggregateResult", args: 88, entryBits: []int{9, 10}, goallcLocals: 24, goallcArgs: [][]int{{9, 10}, nil}, goallcMaps: [][]int{nil, {1, 2}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0, 1, 0}, nativeQueries: []int32{0, 0, -1}, goallcQueries: []int32{1, 1, 0}, nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 24, 8, 0}, }, { name: "liveScalarStackArgument", args: 136, entryBits: []int{7}, goallcLocals: 8, goallcArgs: [][]int{{7}}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 0}, nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 0}, }, { name: "liveAggregateStackArgument", args: 136, entryBits: []int{5, 7}, goallcLocals: 8, goallcArgs: [][]int{{5, 7}}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 0}, nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 0}, }, @@ -685,7 +685,7 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p nativeLocals: 8, goallcLocals: 24, nativeArgs: [][]int{{1}, nil}, goallcArgs: [][]int{{1}, nil}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {1}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, 1}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0, 1, 0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 1}, }, { @@ -693,7 +693,7 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p nativeLocals: 8, goallcLocals: 24, nativeArgs: [][]int{{3, 4}, nil}, goallcArgs: [][]int{{3, 4}, nil}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {0, 1}}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, 1}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0, 1, 0}, nativeQueries: []int32{0, 0, -1}, goallcQueries: []int32{0, 1, 1}, }, { @@ -701,7 +701,7 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p nativeLocals: 8, goallcLocals: 8, nativeArgs: [][]int{{0}, nil}, goallcArgs: [][]int{{0}}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 0}, }, { @@ -709,7 +709,7 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p nativeLocals: 8, goallcLocals: 8, nativeArgs: [][]int{{0, 2}, nil}, goallcArgs: [][]int{{0, 2}}, nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{0}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, 0}, }, } @@ -754,7 +754,7 @@ func runLLVMABIArgsPointerMapMachineTest(t *testing.T, goroot, llc, plugin strin [][]int{nil, nil}; !reflect.DeepEqual(got, want) { t.Fatalf("machine LocalsPointerMaps=%v, want %v", got, want) } - if got, want := llvmABIStackMapRanges(symbol), []int32{-1, 0, 1}; !reflect.DeepEqual(got, want) { + if got, want := llvmABIStackMapRanges(symbol), []int32{0, 1}; !reflect.DeepEqual(got, want) { t.Fatalf("machine PCDATA_StackMapIndex=%v, want %v", got, want) } var queryIndexes []int32 diff --git a/src/cmd/internal/testdir/llvm_alloca_test.go b/src/cmd/internal/testdir/llvm_alloca_test.go index 95cf2fd84c1001..96251c3f027aeb 100644 --- a/src/cmd/internal/testdir/llvm_alloca_test.go +++ b/src/cmd/internal/testdir/llvm_alloca_test.go @@ -30,7 +30,7 @@ var llvmAllocaChecks = map[string]llvmAllocaArchitectureChecks{ restoredStorePattern: `(?m)^\s*(?:str|stp)\b`, goallcLocals: 88, goallcPointerBits: []int{5, 7, 8, 9}, - goallcPCData: []int32{-1, 0, 1}, + goallcPCData: []int32{0, 1, 0}, goallcQueries: []int32{0, 1, 1, 1, 1}, }, "linux/amd64": { @@ -38,7 +38,7 @@ var llvmAllocaChecks = map[string]llvmAllocaArchitectureChecks{ restoredStorePattern: `(?m)^\s*mov[a-z]*\s+[^,\n]+,\s*-[0-9]+\(%rbp\)`, goallcLocals: 88, goallcPointerBits: []int{5, 7, 8, 9}, - goallcPCData: []int32{-1, 1, 0}, + goallcPCData: []int32{0, 1, 0}, goallcQueries: []int32{1, 1, 1, 1, 0}, }, } diff --git a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir index fac4976a3ed72f..6d71b5d8562480 100644 --- a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir +++ b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir @@ -7,7 +7,7 @@ ptr } - declare goabi0 void @runtime.morestack_noctxt() + declare goabi0 void @"runtime.morestack_noctxt"() declare goabiinternal void @p.safepoint() define goabiinternal %results @p.argResult(ptr %input) @@ -44,7 +44,7 @@ body: | STACKMAP 5147419139155979380, 0, 1, 8, $sp, 16 $x3 = COPY $lr - STATEPOINT 5147424658422983495, 0, 0, &runtime.morestack_noctxt, 2, 22, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_aarch64_go, implicit-def $sp, implicit-def dead early-clobber $lr, implicit $x3 + BL &"runtime.morestack_noctxt", implicit-def $lr, implicit $sp, implicit $x3 STATEPOINT 1, 0, 0, &p.safepoint, 2, 22, 2, 0, 2, 0, 2, 1, 1, 8, $sp, 8, 2, 0, 2, 1, 0, 0, csr_aarch64_go, implicit-def $sp, implicit-def dead early-clobber $lr RET_ReallyLR ... diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index 7140b480805241..d1ceffdbce16f4 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -241,16 +241,20 @@ not object-format adaptation. LLVM records GoObj statepoint callsites at the CAL start, matching Go's `PCDATA_StackMapIndex` convention without a command-line mode. GoObj Go functions use native Go's split-stack policy by default: unless `go-nosplit` is present, target frame lowering expresses the late-generated -`runtime.morestack` call as a physical, root-free MIR `STATEPOINT`. Before frame -allocation, formal lowering maps every type-derived input pointer word onto the -existing fixed home reserved for that ABI input. Frame lowering records those -homes in a separate zero-byte `EntryArgsStackMapID` record for every GoObj Go -function. This is function metadata rather than a callsite, so it emits no -`PCDATA`. The GoObj writer uses it as pair 0: non-empty `EntryArgs` when present -and empty locals. A split function must additionally contain exactly one real -`StackGrowthStatepointID`, which selects pair 0 at the morestack call; a nosplit -function must contain none. Ordinary and stack-growth calls use the same -Machine StackMaps pipeline without relying on a return-PC convention. +`runtime.morestack` call as an ordinary ABI0 MIR call. Before frame allocation, +formal lowering maps each live type-derived input pointer word onto the existing +fixed home reserved for that ABI input. It still reserves and saves complete ABI +homes for an unused formal so a morestack retry preserves the register +assignment, but does not scan a word that LLVM callers may replace with poison. +Frame lowering records the live homes in a separate zero-byte +`EntryArgsStackMapID` record for every GoObj Go function. This is function +metadata rather than a callsite. The GoObj writer uses it as pair 0: non-empty +`EntryArgs` when present and empty locals, and initializes +`PCDATA_StackMapIndex` to 0. AsmPrinter's PCSP stream has already resolved the +Machine CFG; every transition back to the entry stack depth selects map 0. This +covers the pre-frame morestack path without identifying `runtime.morestack` by +name or manufacturing a statepoint. An ordinary statepoint selects its actual +live map and overrides a same-PC entry-depth transition. GoObj emits the currently constant safe `PCDATA_UnsafePoint` table first and the statepoint-derived `PCDATA_StackMapIndex` table second, as required by their Go ABI indexes 0 and 1. diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index c9df7dc4a35cea..5baf17a7c8d522 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -23,7 +23,8 @@ target triple = "aarch64-apple-darwin-goobj" ; OBJVIEW-NEXT: 1 ; FRAME-TEXT-LABEL: TEXT aarch64_pointer_and_code_live(SB) -; FRAME-TEXT: R_CALLARM64:runtime.morestack_noctxt{{.*}}PCDATA_StackMapIndex=0{{.*}}ArgsPointerMaps=01{{.*}}LocalsPointerMaps=00 +; FRAME-TEXT: PCDATA_StackMapIndex=0{{.*}}ArgsPointerMaps=01{{.*}}LocalsPointerMaps=00 +; FRAME-TEXT: R_CALLARM64:runtime.morestack_noctxt ; FRAME-TEXT-NEXT: {{.*}}stack-growth safepoint{{.*}}map[0]{{.*}}ArgsPointerMaps=01{{.*}}LocalsPointerMaps=00 ; FRAME-TEXT: R_CALLIND{{.*}}PCDATA_StackMapIndex=1{{.*}}ArgsPointerMaps=00{{.*}}LocalsPointerMaps=01 ; FRAME-TEXT-NEXT: {{.*}}ordinary safepoint{{.*}}map[1]{{.*}}ArgsPointerMaps=00{{.*}}LocalsPointerMaps=01 diff --git a/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll b/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll index 5a7f8e240ce54e..b17d4499d924f3 100644 --- a/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll +++ b/src/cmd/llvmplugin/testdata/aggregate-call-result-goobj.ll @@ -5,14 +5,14 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW: "kind": "stack_map_index" ; OBJVIEW: "start": 0 ; OBJVIEW-NEXT: "end": [[#MAKE_START:]] -; OBJVIEW-NEXT: "value": -1 +; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "start": [[#MAKE_START]] ; OBJVIEW-NEXT: "end": [[#SAFEPOINT_START:]] ; OBJVIEW-NEXT: "value": 1 ; OBJVIEW: "start": [[#SAFEPOINT_START]] -; OBJVIEW-NEXT: "end": [[#MORESTACK_START:]] +; OBJVIEW-NEXT: "end": [[#ENTRY_DEPTH_START:]] ; OBJVIEW-NEXT: "value": 2 -; OBJVIEW: "start": [[#MORESTACK_START]] +; OBJVIEW: "start": [[#ENTRY_DEPTH_START]] ; OBJVIEW-NEXT: "end": [[#SIZE]] ; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "kind": "args_pointer_maps" @@ -48,7 +48,6 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW: "stack_map_index": 2 ; OBJVIEW-NEXT: "relocation_type": "R_CALL" ; OBJVIEW: "sym_index": 2 -; OBJVIEW: "call_offset": [[#MORESTACK_START+1]] ; OBJVIEW: "stack_map_index": 0 ; OBJVIEW-NEXT: "relocation_type": "R_CALL" ; OBJVIEW: "sym_index": 3 diff --git a/src/cmd/llvmplugin/testdata/conditional-relocation.ll b/src/cmd/llvmplugin/testdata/conditional-relocation.ll index a5e54a126e5689..1c9c2e9300ee78 100644 --- a/src/cmd/llvmplugin/testdata/conditional-relocation.ll +++ b/src/cmd/llvmplugin/testdata/conditional-relocation.ll @@ -9,11 +9,11 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW: "kind": "stack_map_index" ; OBJVIEW: "start": 0 ; OBJVIEW-NEXT: "end": [[#SAFEPOINT_START:]] -; OBJVIEW-NEXT: "value": -1 +; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "start": [[#SAFEPOINT_START]] -; OBJVIEW-NEXT: "end": [[#MORESTACK_START:]] +; OBJVIEW-NEXT: "end": [[#ENTRY_DEPTH_START:]] ; OBJVIEW-NEXT: "value": 1 -; OBJVIEW: "start": [[#MORESTACK_START]] +; OBJVIEW: "start": [[#ENTRY_DEPTH_START]] ; OBJVIEW-NEXT: "end": [[#SIZE]] ; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "kind": "locals_pointer_maps" @@ -25,7 +25,6 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW-NEXT: 0 ; OBJVIEW: "call_offset": [[#SAFEPOINT_START+1]] ; OBJVIEW: "stack_map_index": 1 -; OBJVIEW: "call_offset": [[#MORESTACK_START+1]] ; OBJVIEW: "stack_map_index": 0 declare goabiinternal void @callee() diff --git a/src/cmd/llvmplugin/testdata/multiple-calls.ll b/src/cmd/llvmplugin/testdata/multiple-calls.ll index 1c7cca7580dd86..28396c57a509d6 100644 --- a/src/cmd/llvmplugin/testdata/multiple-calls.ll +++ b/src/cmd/llvmplugin/testdata/multiple-calls.ll @@ -9,14 +9,14 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW: "kind": "stack_map_index" ; OBJVIEW: "start": 0 ; OBJVIEW-NEXT: "end": [[#FIRST_START:]] -; OBJVIEW-NEXT: "value": -1 +; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "start": [[#FIRST_START]] ; OBJVIEW-NEXT: "end": [[#SECOND_START:]] ; OBJVIEW-NEXT: "value": 1 ; OBJVIEW: "start": [[#SECOND_START]] -; OBJVIEW-NEXT: "end": [[#MORESTACK_START:]] +; OBJVIEW-NEXT: "end": [[#ENTRY_DEPTH_START:]] ; OBJVIEW-NEXT: "value": 2 -; OBJVIEW: "start": [[#MORESTACK_START]] +; OBJVIEW: "start": [[#ENTRY_DEPTH_START]] ; OBJVIEW-NEXT: "end": [[#SIZE]] ; OBJVIEW-NEXT: "value": 0 ; OBJVIEW: "kind": "locals_pointer_maps" @@ -34,7 +34,6 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW: "stack_map_index": 1 ; OBJVIEW: "call_offset": [[#SECOND_START+1]] ; OBJVIEW: "stack_map_index": 2 -; OBJVIEW: "call_offset": [[#MORESTACK_START+1]] ; OBJVIEW: "stack_map_index": 0 declare goabiinternal void @first_callee() diff --git a/test/llvm_tests.json b/test/llvm_tests.json index f57350aa5db3cb..d3d0113b7b2bd7 100644 --- a/test/llvm_tests.json +++ b/test/llvm_tests.json @@ -71,7 +71,7 @@ "codegen/shortcircuit.go": "empty-interface extraction and concrete type comparison", "codegen/smallintiface.go": "small scalar boxing through runtime.staticuint64s", "codegen/spills.go": "integer and floating-point values preserved across calls", - "codegen/statepoint.go": "frontend-owned Go GC strategy and stack-growth statepoint function markers", + "codegen/statepoint.go": "frontend-owned Go GC strategy and nosplit-only stack-growth policy", "codegen/structs.go": "typed struct initialization and zeroing", "codegen/type_descriptor.go": "compiler-owned runtime type descriptor data and GoObj metadata", "codegen/type_descriptor_kinds.go": "all runtime type descriptor layouts and variable descriptor data",