From 9b12317c857428e6f5a11890aca8a636405c113c Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 14:18:22 +0800 Subject: [PATCH 1/8] cmd/compile: model stack ABI values as LLVM memory carriers --- src/cmd/compile/internal/ssa/ssa2llvm.go | 540 +++++++++++++++--- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 29 + src/cmd/internal/testdir/llvm_abi_test.go | 91 +-- src/cmd/llvmplugin/CMakeLists.txt | 30 + src/cmd/llvmplugin/GoALLCStatepoints.cpp | 156 ++++- src/cmd/llvmplugin/README.md | 52 +- src/cmd/llvmplugin/testdata/aarch64-frame.ll | 14 +- .../testdata/alloca-pointer-roots.ll | 13 +- .../testdata/supported-param-attrs.ll | 26 + 9 files changed, 799 insertions(+), 152 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 1939d5c9ce8774..cc6bc89bba9316 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -10,6 +10,7 @@ import ( "cmd/internal/src" "fmt" "internal/buildcfg" + "strconv" "strings" "github.com/goallc/go-llvm" @@ -21,6 +22,7 @@ type LLVMFuncContext struct { Locals map[llvmLocalKey]llvmStackSlot AddressedResults map[ID][]llvmAddressedResult ResultSlots map[ID]llvm.Value + CallResultSlots map[llvmCallResultKey]llvmStackSlot ItabMethods map[ID]bool ClosureCodeLoads map[ID]bool DeferResults map[llvmLocalKey]bool @@ -39,6 +41,9 @@ type LLVMFuncContext struct { b llvm.Builder ReturnType llvm.Type ResultCount int + ReturnCount int + Params []llvmParamSignature + Results []llvmResultSignature } // SSA may clone an ir.Name while retaining the same logical source @@ -61,6 +66,11 @@ type llvmAddressedResult struct { Owner *Value } +type llvmCallResultKey struct { + Call ID + Index int64 +} + // LLVM's GoABIInternal calling convention has numeric ID 22. Keep the // prototype lowering on the Go register ABI so llc emits GoObj symbols that // the standard Go linker can call directly. @@ -68,6 +78,7 @@ const goABIInternalCallConv llvm.CallConv = 22 const goABI0CallConv llvm.CallConv = 23 const goABI0SymbolSuffix = "" const goResultsTupleAttr = "go_results_tuple" +const goMemoryResultsAttr = "go_memory_results" const goGCStrategy = "goallc" const goGCLeafFunctionAttr = "gc-leaf-function" const goNoSplitAttr = "go-nosplit" @@ -96,10 +107,28 @@ type llvmFuncSignature struct { Type llvm.Type ReturnType llvm.Type ResultCount int + ReturnCount int + Params []llvmParamSignature + Results []llvmResultSignature + MemoryResultIndices []int HasClosureContext bool ClosureContextIndex int } +type llvmParamSignature struct { + ValueType llvm.Type + Alignment int + ByVal bool +} + +type llvmResultSignature struct { + ValueType llvm.Type + Alignment int + InMemory bool + ReturnIndex int + ParamIndex int +} + func llvmCallConv(which obj.ABI) llvm.CallConv { switch which { case obj.ABI0: @@ -165,15 +194,50 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { } params := make([]llvm.Type, 0, aux.NArgs()) + paramSignatures := make([]llvmParamSignature, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { - param := getLLVMABIType(aux.TypeOfArg(i)) - params = append(params, param) + goType := aux.TypeOfArg(i) + valueType := getLLVMABIType(goType) + paramType := valueType + param := llvmParamSignature{ValueType: valueType} + assignment := aux.ABIInfo().InParam(int(i)) + if len(assignment.Registers) == 0 && goType.Size() != 0 { + if goType.Alignment() <= 0 { + base.Fatalf("invalid alignment %d for stack argument %d of type %v", goType.Alignment(), i, goType) + } + param.ByVal = true + param.Alignment = int(goType.Alignment()) + paramType = GlobalCtxt.PointerType(0) + } + params = append(params, paramType) + paramSignatures = append(paramSignatures, param) } results := make([]llvm.Type, 0, aux.NResults()) + resultSignatures := make([]llvmResultSignature, 0, aux.NResults()) + memoryResultIndices := make([]int, 0, aux.NResults()) for i := int64(0); i < aux.NResults(); i++ { - result := getLLVMABIType(aux.TypeOfResult(i)) - results = append(results, result) + goType := aux.TypeOfResult(i) + result := llvmResultSignature{ + ValueType: getLLVMABIType(goType), + ReturnIndex: -1, + ParamIndex: -1, + } + assignment := aux.ABIInfo().OutParam(int(i)) + if len(assignment.Registers) == 0 && goType.Size() != 0 { + if goType.Alignment() <= 0 { + base.Fatalf("invalid alignment %d for stack result %d of type %v", goType.Alignment(), i, goType) + } + result.InMemory = true + result.Alignment = int(goType.Alignment()) + result.ParamIndex = len(params) + params = append(params, GlobalCtxt.PointerType(0)) + memoryResultIndices = append(memoryResultIndices, int(i)) + } else { + result.ReturnIndex = len(results) + results = append(results, result.ValueType) + } + resultSignatures = append(resultSignatures, result) } var ret llvm.Type @@ -188,7 +252,11 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { return llvmFuncSignature{ Type: llvm.FunctionType(ret, params, false), ReturnType: ret, - ResultCount: len(results), + ResultCount: len(resultSignatures), + ReturnCount: len(results), + Params: paramSignatures, + Results: resultSignatures, + MemoryResultIndices: memoryResultIndices, ClosureContextIndex: -1, } } @@ -210,6 +278,30 @@ func llvmNestAttribute() llvm.Attribute { return GlobalCtxt.CreateEnumAttribute(kind, 0) } +func llvmByValAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("byval") + if kind == 0 { + base.Fatalf("LLVM does not provide the byval parameter attribute") + } + return GlobalCtxt.CreateTypeAttribute(kind, t) +} + +func llvmByRefAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("byref") + if kind == 0 { + base.Fatalf("LLVM does not provide the byref parameter attribute") + } + return GlobalCtxt.CreateTypeAttribute(kind, t) +} + +func llvmMemoryResultsAttribute(indices []int) llvm.Attribute { + values := make([]string, len(indices)) + for i, index := range indices { + values[i] = strconv.Itoa(index) + } + return GlobalCtxt.CreateStringAttribute(goMemoryResultsAttr, strings.Join(values, ",")) +} + func llvmNullPointerIsValidAttribute() llvm.Attribute { kind := llvm.AttributeKindID("null_pointer_is_valid") if kind == 0 { @@ -228,9 +320,26 @@ func llvmNoInlineAttribute() llvm.Attribute { func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallConv) { fn.SetFunctionCallConv(cc) - if sig.ResultCount > 1 { + if sig.ReturnCount > 1 { fn.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + if len(sig.MemoryResultIndices) != 0 { + fn.AddFunctionAttr(llvmMemoryResultsAttribute(sig.MemoryResultIndices)) + } + for i, param := range sig.Params { + if !param.ByVal { + continue + } + fn.AddAttributeAtIndex(i+1, llvmByValAttribute(param.ValueType)) + fn.Param(i).SetParamAlignment(param.Alignment) + } + for _, result := range sig.Results { + if !result.InMemory { + continue + } + fn.AddAttributeAtIndex(result.ParamIndex+1, llvmByRefAttribute(result.ValueType)) + fn.Param(result.ParamIndex).SetParamAlignment(result.Alignment) + } if sig.HasClosureContext { // LLVM parameter attribute indexes are one-based. The closure context // is deliberately excluded from the Go ABI argument layout by the @@ -240,9 +349,26 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon } func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { - if sig.ResultCount > 1 { + if sig.ReturnCount > 1 { call.AddCallSiteAttribute(llvmAttributeFunctionIndex, GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + if len(sig.MemoryResultIndices) != 0 { + call.AddCallSiteAttribute(llvmAttributeFunctionIndex, llvmMemoryResultsAttribute(sig.MemoryResultIndices)) + } + for i, param := range sig.Params { + if !param.ByVal { + continue + } + call.AddCallSiteAttribute(i+1, llvmByValAttribute(param.ValueType)) + call.SetInstrParamAlignment(i+1, param.Alignment) + } + for _, result := range sig.Results { + if !result.InMemory { + continue + } + call.AddCallSiteAttribute(result.ParamIndex+1, llvmByRefAttribute(result.ValueType)) + call.SetInstrParamAlignment(result.ParamIndex+1, result.Alignment) + } } func llvmFunctionStorageName(name string, cc llvm.CallConv) string { @@ -319,6 +445,19 @@ func (lfc *LLVMFuncContext) llvmLifetimeStart(slot llvmStackSlot) { lfc.b.CreateCall(sig, fn, []llvm.Value{slot.Value}, "") } +func (lfc *LLVMFuncContext) llvmLifetimeEnd(slot llvmStackSlot) { + if slot.Value.IsAAllocaInst().IsNil() { + return + } + sig := llvm.FunctionType( + GlobalCtxt.VoidType(), + []llvm.Type{GlobalCtxt.PointerType(0)}, + false, + ) + fn := getOrInsertLLVMIntrinsic("llvm.lifetime.end.p0", sig) + lfc.b.CreateCall(sig, fn, []llvm.Value{slot.Value}, "") +} + func (lfc *LLVMFuncContext) llvmKeepAlive(value llvm.Value) { // An operand bundle is a real SSA use that follows the call through // inlining. llvm.donothing survives long enough for the statepoint pass to @@ -1080,6 +1219,33 @@ func (lfc *LLVMFuncContext) llvmRuntimeMemmove(dst, src, length llvm.Value) llvm return call } +func (lfc *LLVMFuncContext) llvmCopyFixedMemory(dst, src llvm.Value, size int64, align int) llvm.Value { + lengthType := getLLVMType(types.Types[types.TUINTPTR]) + length := llvm.ConstInt(lengthType, uint64(size), false) + if size > llvmInlineMemmoveLimit { + return lfc.llvmRuntimeMemmove(dst, src, length) + } + sig := llvm.FunctionType( + GlobalCtxt.VoidType(), + []llvm.Type{dst.Type(), src.Type(), length.Type(), GlobalCtxt.Int1Type()}, + false, + ) + name := "llvm.memmove.p0.p0.i64" + if length.Type().IntTypeWidth() == 32 { + name = "llvm.memmove.p0.p0.i32" + } + fn := getOrInsertLLVMIntrinsic(name, sig) + call := lfc.b.CreateCall(sig, fn, []llvm.Value{ + dst, + src, + length, + llvm.ConstInt(GlobalCtxt.Int1Type(), 0, false), + }, "") + call.SetInstrParamAlignment(1, align) + call.SetInstrParamAlignment(2, align) + return call +} + func (lfc *LLVMFuncContext) llvmMove(v *Value) llvm.Value { size, align := llvmMemoryOpInfo(v) dst := lfc.llvmMemoryPointer(v, 0) @@ -1130,6 +1296,7 @@ func (lfc *LLVMFuncContext) llvmMemEq(v *Value) llvm.Value { Type: llvm.FunctionType(boolType, []llvm.Type{left.Type(), right.Type(), uintptrType}, false), ReturnType: boolType, ResultCount: 1, + ReturnCount: 1, ClosureContextIndex: -1, } fn := getOrInsertLLVMABISymbolRef("runtime.memequal", obj.ABIInternal, sig, goABIInternalCallConv) @@ -1552,13 +1719,108 @@ func (lfc *LLVMFuncContext) paramForArgNameAndType(name *ir.Name) (llvm.Value, * key := llvmLocalKeyForName(name) for i, param := range lfc.F.OwnAux.ABIInfo().InParams() { if param.Name != nil && llvmLocalKeyForName(param.Name) == key { - return lfc.LF.Param(i), lfc.F.OwnAux.TypeOfArg(int64(i)) + value := lfc.LF.Param(i) + if lfc.Params[i].ByVal { + value = lfc.b.CreateLoad(lfc.Params[i].ValueType, value, name.Sym().Name+".byval") + value.SetAlignment(lfc.Params[i].Alignment) + } + return value, lfc.F.OwnAux.TypeOfArg(int64(i)) } } lfc.F.fe.Fatalf(name.Pos(), "could not find LLVM parameter for %v", name) return llvm.Value{}, nil } +func (lfc *LLVMFuncContext) llvmByValCallArgument(v, argValue *Value, index int, logical *types.Type, param llvmParamSignature) (llvm.Value, *llvmStackSlot) { + if !param.ByVal || logical.Size() == 0 { + v.Fatalf("argument %d is not a non-empty byval parameter", index) + } + + // Go SSA keeps non-SSA-able aggregate call arguments in memory and exposes + // the value through a Load or Dereference. The LLVM byval carrier wants the + // address of those bytes, so preserve that address instead of constructing a + // huge first-class aggregate only to store it back to a temporary. + if types.Identical(argValue.Type, logical) && + (argValue.Op == OpLoad || argValue.Op == OpDereference) && len(argValue.Args) != 0 { + address := lfc.GenLV(argValue.Args[0]) + if address.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("byval argument %d has non-pointer source address", index) + } + return address, nil + } + + value := lfc.GenLV(argValue) + value = lfc.llvmValueToABI(v, value, argValue.Type, logical, param.ValueType, fmt.Sprintf("%s.arg%d", v, index)) + if value.Type() != param.ValueType { + v.Fatalf("byval argument %d has incompatible LLVM value type", index) + } + + // A pure SSA value has no source address even though Go assigned the logical + // argument wholly to the stack. Materialize any such value in a fixed entry + // alloca; this is only how the unified byval carrier obtains an address, not + // a second ABI classification path. + entryBuilder := GlobalCtxt.NewBuilder() + defer entryBuilder.Dispose() + entry := lfc.LF.EntryBasicBlock() + if first := entry.FirstInstruction(); first.IsNil() { + entryBuilder.SetInsertPointAtEnd(entry) + } else { + entryBuilder.SetInsertPointBefore(first) + } + address := entryBuilder.CreateAlloca(param.ValueType, fmt.Sprintf("%s.arg%d.byval", v, index)) + address.SetAlignment(param.Alignment) + slot := &llvmStackSlot{Value: address, Type: logical} + lfc.llvmLifetimeStart(*slot) + store := lfc.b.CreateStore(value, address) + store.SetAlignment(param.Alignment) + return address, slot +} + +func (lfc *LLVMFuncContext) llvmMemoryResultCallArguments(v *Value, sig llvmFuncSignature, aux *AuxCall) []llvm.Value { + args := make([]llvm.Value, 0, len(sig.MemoryResultIndices)) + for index, result := range sig.Results { + if !result.InMemory { + continue + } + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: v.ID, Index: int64(index)}] + if !ok || !types.Identical(slot.Type, aux.TypeOfResult(int64(index))) { + v.Fatalf("memory result %d has no compatible caller-owned home", index) + } + // A call reinitializes its caller-owned result home. Keep future result + // homes out of earlier statepoints while allowing SelectN and + // SelectNAddr uses after this call to determine the actual live range. + lfc.llvmLifetimeStart(slot) + args = append(args, slot.Value) + } + return args +} + +func (lfc *LLVMFuncContext) storeMemoryResult(v, value *Value, index int) { + result := lfc.Results[index] + if !result.InMemory || result.ParamIndex < 0 { + v.Fatalf("result %d is not assigned to memory", index) + } + logical := lfc.F.OwnAux.TypeOfResult(int64(index)) + dst := lfc.LF.Param(result.ParamIndex) + if types.Identical(value.Type, logical) && + (value.Op == OpLoad || value.Op == OpDereference) && len(value.Args) != 0 { + src := lfc.GenLV(value.Args[0]) + if src.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("memory result %d has a non-pointer source address", index) + } + lfc.llvmCopyFixedMemory(dst, src, logical.Size(), result.Alignment) + return + } + + lVal := lfc.GenLV(value) + lVal = lfc.llvmValueToABI(v, lVal, value.Type, logical, result.ValueType, fmt.Sprintf("%s.result%d", v, index)) + if lVal.Type() != result.ValueType { + v.Fatalf("memory result %d has incompatible LLVM value type", index) + } + store := lfc.b.CreateStore(lVal, dst) + store.SetAlignment(result.Alignment) +} + func (lfc *LLVMFuncContext) registerArgument(v *Value) llvm.Value { aux, ok := v.Aux.(*AuxNameOffset) if !ok || aux.Name == nil { @@ -1913,21 +2175,38 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { // statepoints. llvmGCLeaf := aux.Fn == ir.Syms.WBZero || aux.Fn == ir.Syms.WBMove || aux.Fn == ir.Syms.Memmove || aux.Fn == ir.Syms.Memequal - args := make([]llvm.Value, 0, aux.NArgs()) + args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) + var byValTemps []llvmStackSlot for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[i]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + var arg llvm.Value + if sig.Params[i].ByVal { + var temp *llvmStackSlot + arg, temp = lfc.llvmByValCallArgument(v, v.Args[i], int(i), aux.TypeOfArg(i), sig.Params[i]) + if temp != nil { + byValTemps = append(byValTemps, *temp) + } + } else { + arg = lfc.GenLV(v.Args[i]) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } + } + if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { + v.Fatalf("argument %d to %s has incompatible LLVM type", i, aux.Fn.Name) } args = append(args, arg) } + args = append(args, lfc.llvmMemoryResultCallArguments(v, sig, aux)...) name := v.String() - if sig.ResultCount == 0 { + if sig.ReturnCount == 0 { name = "" } call := lfc.b.CreateCall(sig.Type, fn, args, name) call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) + for _, slot := range byValTemps { + lfc.llvmLifetimeEnd(slot) + } lfc.materializeAddressedResults(v, call, aux) if llvmGCLeaf { markLLVMGCLeafCall(call) @@ -1962,17 +2241,29 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext if code.Type().TypeKind() != llvm.PointerTypeKind { v.Fatalf("indirect callee has non-pointer LLVM type") } - args := make([]llvm.Value, 0, aux.NArgs()) + args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) + var byValTemps []llvmStackSlot for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[argStart+int(i)]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[argStart+int(i)].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + argValue := v.Args[argStart+int(i)] + var arg llvm.Value + if sig.Params[i].ByVal { + var temp *llvmStackSlot + arg, temp = lfc.llvmByValCallArgument(v, argValue, int(i), aux.TypeOfArg(i), sig.Params[i]) + if temp != nil { + byValTemps = append(byValTemps, *temp) + } + } else { + arg = lfc.GenLV(argValue) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, argValue.Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } } if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { v.Fatalf("argument %d to indirect call has incompatible LLVM type", i) } args = append(args, arg) } + args = append(args, lfc.llvmMemoryResultCallArguments(v, sig, aux)...) if closureContext { context := lfc.GenLV(v.Args[1]) if context.Type().TypeKind() != llvm.PointerTypeKind { @@ -1981,12 +2272,15 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext args = append(args, context) } name := v.String() - if sig.ResultCount == 0 { + if sig.ReturnCount == 0 { name = "" } call := lfc.b.CreateCall(sig.Type, code, args, name) call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) + for _, slot := range byValTemps { + lfc.llvmLifetimeEnd(slot) + } lfc.materializeAddressedResults(v, call, aux) if closureContext { call.AddCallSiteAttribute(sig.ClosureContextIndex+1, llvmNestAttribute()) @@ -2002,13 +2296,18 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext // alignment while allowing ordinary LLVM promotion to remove unnecessary // homes. func (lfc *LLVMFuncContext) materializeAddressedResults(v *Value, call llvm.Value, aux *AuxCall) { + sig := llvmSignature(aux) for _, result := range lfc.AddressedResults[v.ID] { if result.Slot.Type.HasPointers() { lfc.llvmLifetimeStart(result.Slot) } + resultSig := sig.Results[result.Index] + if resultSig.InMemory || resultSig.ReturnIndex < 0 { + result.Owner.Fatalf("addressed register result was assigned to memory") + } value := call - if aux.NResults() > 1 { - value = lfc.b.CreateExtractValue(call, int(result.Index), result.Owner.String()+".value") + if sig.ReturnCount > 1 { + value = lfc.b.CreateExtractValue(call, resultSig.ReturnIndex, result.Owner.String()+".value") } value = lfc.llvmValueFromABI(result.Owner, value, aux.TypeOfResult(result.Index), result.Slot.Type, result.Owner.String()+".reshape") if got, want := value.Type(), getLLVMType(result.Slot.Type); got != want { @@ -2553,10 +2852,21 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { case sel >= int(aux.NResults()): // Selecting the trailing SSA memory dependency only forces the // call to be emitted; it has no LLVM value. - case aux.NResults() == 1: - lVal = call default: - lVal = lfc.b.CreateExtractValue(call, sel, v.String()) + sig := llvmSignature(aux) + result := sig.Results[sel] + if result.InMemory { + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: src.ID, Index: int64(sel)}] + if !ok { + v.Fatalf("memory result %d has no caller-owned home", sel) + } + lVal = lfc.b.CreateLoad(result.ValueType, slot.Value, v.String()) + lVal.SetAlignment(result.Alignment) + } else if sig.ReturnCount == 1 { + lVal = call + } else { + lVal = lfc.b.CreateExtractValue(call, result.ReturnIndex, v.String()) + } } if sel < int(aux.NResults()) { lVal = lfc.llvmValueFromABI(v, lVal, aux.TypeOfResult(int64(sel)), v.Type, v.String()+".reshape") @@ -2590,16 +2900,25 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { v.Fatalf("addressed call result has no LLVM memory home") } case OpMakeResult: - switch lfc.ResultCount { + direct := make([]llvm.Value, lfc.ReturnCount) + for i, result := range lfc.Results { + if result.InMemory { + lfc.storeMemoryResult(v, v.Args[i], i) + continue + } + direct[result.ReturnIndex] = lfc.llvmValueToABI( + v, lfc.GenLV(v.Args[i]), v.Args[i].Type, + lfc.F.OwnAux.TypeOfResult(int64(i)), result.ValueType, + fmt.Sprintf("%s.result%d", v, i), + ) + } + switch lfc.ReturnCount { case 0: case 1: - lVal = lfc.GenLV(v.Args[0]) - lVal = lfc.llvmValueToABI(v, lVal, v.Args[0].Type, lfc.F.OwnAux.TypeOfResult(0), lfc.ReturnType, v.String()+".result0") + lVal = direct[0] default: lVal = llvm.Undef(lfc.ReturnType) - for i := 0; i < lfc.ResultCount; i++ { - resultType := lfc.ReturnType.StructElementTypes()[i] - result := lfc.llvmValueToABI(v, lfc.GenLV(v.Args[i]), v.Args[i].Type, lfc.F.OwnAux.TypeOfResult(int64(i)), resultType, fmt.Sprintf("%s.result%d", v, i)) + for i, result := range direct { lVal = lfc.b.CreateInsertValue(lVal, result, i, "") } lVal.SetName(v.String()) @@ -2793,7 +3112,7 @@ func (lfc *LLVMFuncContext) CompileBlock(BB *Block, values []*Value) { defer lfc.b.ClearCurrentDebugLocation() switch BB.Kind { case BlockRet: - if lfc.ResultCount == 0 { + if lfc.ReturnCount == 0 { lfc.b.CreateRetVoid() } else { lfc.b.CreateRet(lfc.GenLV(BB.Controls[0])) @@ -2866,17 +3185,13 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { if len(outParams) != lfc.ResultCount { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result count %d does not match LLVM signature result count %d", len(outParams), lfc.ResultCount) } - results := make([]llvm.Value, len(outParams)) + results := make([]llvm.Value, lfc.ReturnCount) reshapeContext := &Value{Block: lfc.F.Entry, Pos: lfc.F.Entry.Pos} for i, result := range outParams { - var abiType llvm.Type - if lfc.ResultCount == 1 { - abiType = lfc.ReturnType - } else { - abiType = lfc.ReturnType.StructElementTypes()[i] - } + resultSig := lfc.Results[i] + abiType := resultSig.ValueType if result.Type.Size() == 0 { - results[i] = llvm.Undef(abiType) + results[resultSig.ReturnIndex] = llvm.Undef(abiType) continue } if result.Name == nil { @@ -2893,10 +3208,15 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { if value.Type() != abiType { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result %d has incompatible LLVM ABI type", i) } - results[i] = value + if resultSig.InMemory { + store := lfc.b.CreateStore(value, lfc.LF.Param(resultSig.ParamIndex)) + store.SetAlignment(resultSig.Alignment) + } else { + results[resultSig.ReturnIndex] = value + } } - switch len(results) { + switch lfc.ReturnCount { case 0: lfc.b.CreateRetVoid() case 1: @@ -2935,22 +3255,51 @@ func (lfc *LLVMFuncContext) emitTailCallReturn(b *Block) { } result := lfc.GenLV(call) - switch lfc.ResultCount { + calleeSig := llvmSignature(aux) + direct := make([]llvm.Value, lfc.ReturnCount) + for i, callerResult := range lfc.Results { + calleeResult := calleeSig.Results[i] + if callerResult.InMemory && calleeResult.InMemory { + src, ok := lfc.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(i)}] + if !ok { + call.Fatalf("memory result %d has no caller-owned home", i) + } + lfc.llvmCopyFixedMemory(lfc.LF.Param(callerResult.ParamIndex), src.Value, + lfc.F.OwnAux.TypeOfResult(int64(i)).Size(), callerResult.Alignment) + continue + } + + var field llvm.Value + if calleeResult.InMemory { + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(i)}] + if !ok { + call.Fatalf("memory result %d has no caller-owned home", i) + } + field = lfc.b.CreateLoad(calleeResult.ValueType, slot.Value, fmt.Sprintf("%s.return%d.load", call, i)) + field.SetAlignment(calleeResult.Alignment) + } else if calleeSig.ReturnCount == 1 { + field = result + } else { + field = lfc.b.CreateExtractValue(result, calleeResult.ReturnIndex, fmt.Sprintf("%s.return%d.extract", call, i)) + } + field = lfc.llvmValueFromABI(call, field, aux.TypeOfResult(int64(i)), aux.TypeOfResult(int64(i)), fmt.Sprintf("%s.return%d.fromabi", call, i)) + field = lfc.llvmValueToABI(call, field, aux.TypeOfResult(int64(i)), lfc.F.OwnAux.TypeOfResult(int64(i)), callerResult.ValueType, fmt.Sprintf("%s.return%d", call, i)) + if callerResult.InMemory { + store := lfc.b.CreateStore(field, lfc.LF.Param(callerResult.ParamIndex)) + store.SetAlignment(callerResult.Alignment) + } else { + direct[callerResult.ReturnIndex] = field + } + } + + switch lfc.ReturnCount { case 0: lfc.b.CreateRetVoid() case 1: - result = lfc.llvmValueFromABI(call, result, aux.TypeOfResult(0), aux.TypeOfResult(0), call.String()+".return.fromabi") - result = lfc.llvmValueToABI(call, result, aux.TypeOfResult(0), lfc.F.OwnAux.TypeOfResult(0), lfc.ReturnType, call.String()+".return") - if result.Type() != lfc.ReturnType { - call.Fatalf("tail-call result has incompatible LLVM return type") - } - lfc.b.CreateRet(result) + lfc.b.CreateRet(direct[0]) default: ret := llvm.Undef(lfc.ReturnType) - for i := 0; i < lfc.ResultCount; i++ { - field := lfc.b.CreateExtractValue(result, i, fmt.Sprintf("%s.return%d.extract", call, i)) - field = lfc.llvmValueFromABI(call, field, aux.TypeOfResult(int64(i)), aux.TypeOfResult(int64(i)), fmt.Sprintf("%s.return%d.fromabi", call, i)) - field = lfc.llvmValueToABI(call, field, aux.TypeOfResult(int64(i)), lfc.F.OwnAux.TypeOfResult(int64(i)), lfc.ReturnType.StructElementTypes()[i], fmt.Sprintf("%s.return%d", call, i)) + for i, field := range direct { ret = lfc.b.CreateInsertValue(ret, field, i, fmt.Sprintf("%s.return%d.insert", call, i)) } lfc.b.CreateRet(ret) @@ -2989,6 +3338,7 @@ func LLVMCompile(f *Func) { Locals: map[llvmLocalKey]llvmStackSlot{}, AddressedResults: map[ID][]llvmAddressedResult{}, ResultSlots: map[ID]llvm.Value{}, + CallResultSlots: map[llvmCallResultKey]llvmStackSlot{}, ItabMethods: map[ID]bool{}, ClosureCodeLoads: map[ID]bool{}, DeferResults: map[llvmLocalKey]bool{}, @@ -2999,6 +3349,9 @@ func LLVMCompile(f *Func) { b: GlobalCtxt.NewBuilder(), ReturnType: sig.ReturnType, ResultCount: sig.ResultCount, + ReturnCount: sig.ReturnCount, + Params: sig.Params, + Results: sig.Results, } defer FCtxt.b.Dispose() @@ -3087,6 +3440,11 @@ func LLVMCompile(f *Func) { FCtxt.LF.Param(i).SetName(param.Name.Sym().Name) } } + for i, result := range sig.Results { + if result.InMemory { + FCtxt.LF.Param(result.ParamIndex).SetName(fmt.Sprintf(".result%d", i)) + } + } FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goAsyncUnsafeAttr, "")) // A //go:nosplit function must not acquire that late morestack edge. Besides // violating the runtime's nosplit call graph, it would expose a safepoint the @@ -3186,6 +3544,31 @@ func LLVMCompile(f *Func) { FCtxt.Locals[key] = llvmStackSlot{Value: value, Type: name.Type()} } } + // A non-empty parameter assigned wholly to the Go stack is represented as + // a typed LLVM byval pointer. That pointer is the parameter's ABI-defined + // incoming home, so bind addressable Go SSA uses to it directly. Register + // parameters retain the existing compiler-owned alloca/store path. + for i, param := range sig.Params { + if !param.ByVal { + continue + } + assignment := inParams[i] + if len(assignment.Registers) != 0 { + f.fe.Fatalf(f.Entry.Pos, "LLVM byval parameter %d was assigned Go registers", i) + } + if assignment.Name == nil { + continue + } + goType := f.OwnAux.TypeOfArg(int64(i)) + if goType.Size() == 0 || !types.Identical(goType, assignment.Name.Type()) { + f.fe.Fatalf(assignment.Name.Pos(), "invalid Go type for LLVM byval parameter %v", assignment.Name) + } + key := llvmLocalKeyForName(assignment.Name) + if _, exists := FCtxt.Locals[key]; exists { + f.fe.Fatalf(assignment.Name.Pos(), "duplicate LLVM byval parameter home %v", assignment.Name) + } + FCtxt.Locals[key] = llvmStackSlot{Value: FCtxt.LF.Param(i), Type: assignment.Name.Type()} + } isDeferResultLocal := func(name *ir.Name) bool { return frontendFunc != nil && frontendFunc.HasDefer() && (name.Class == ir.PPARAMOUT || name.IsOutputParamHeapAddr()) @@ -3298,16 +3681,38 @@ func LLVMCompile(f *Func) { } } } - // SelectNAddr denotes an address into a call's outgoing result area. LLVM's - // Go calling convention reconstructs stack-assigned results as first-class - // return values, so reserve equivalent fixed entry-block homes before any - // call or phi emission. Multiple selectors of one call result share a slot, - // matching the aliasing of the native ABI result area. - type addressedResultKey struct { - Call ID - Index int64 + // Results that Go assigned to memory use caller-owned typed byref carriers. + // Reserve their stable destinations before call emission; ordinary SelectN + // loads from the same object that SelectNAddr exposes by address. + for _, BB := range f.Blocks { + for _, call := range BB.Values { + switch call.Op { + case OpStaticCall, OpStaticLECall, OpTailLECall, + OpClosureCall, OpClosureLECall, + OpInterCall, OpInterLECall, OpTailLECallInter: + default: + continue + } + aux := auxToCall(call.Aux) + callSig := llvmSignature(aux) + for index, result := range callSig.Results { + if !result.InMemory { + continue + } + resultType := aux.TypeOfResult(int64(index)) + slot := llvmStackSlot{ + Value: FCtxt.b.CreateAlloca(getLLVMType(resultType), fmt.Sprintf("%s.result%d.home", call, index)), + Type: resultType, + } + slot.Value.SetAlignment(int(resultType.Alignment())) + FCtxt.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(index)}] = slot + } + } } - addressedResultSlots := make(map[addressedResultKey]llvmStackSlot) + + // SelectNAddr for a register result still needs an addressable local copy. + // Multiple selectors of one call result share the same slot. + addressedResultSlots := make(map[llvmCallResultKey]llvmStackSlot) for _, BB := range f.Blocks { for _, v := range BB.Values { if v.Op != OpSelectNAddr || v.Uses == 0 { @@ -3326,7 +3731,11 @@ func LLVMCompile(f *Func) { if resultType.Alignment() <= 0 || !types.Identical(resultType, aux.TypeOfResult(index)) { v.Fatalf("SelectNAddr result type %v does not match call result %v", resultType, aux.TypeOfResult(index)) } - key := addressedResultKey{Call: call.ID, Index: index} + key := llvmCallResultKey{Call: call.ID, Index: index} + if memorySlot, ok := FCtxt.CallResultSlots[key]; ok { + FCtxt.ResultSlots[v.ID] = memorySlot.Value + continue + } slot, ok := addressedResultSlots[key] if !ok { slot = llvmStackSlot{ @@ -3382,11 +3791,10 @@ func LLVMCompile(f *Func) { } } } - // Go's ABI assigns each parameter either wholly to registers or wholly to - // the stack. Give only parameters that already have an addressable Go SSA - // LocalAddr a complete LLVM memory home. Ordinary register parameters remain - // direct LLVM SSA values, while the backend remains responsible for the - // physical Go ABI assignment. + // Give addressable register parameters a complete LLVM memory home. Wholly + // stack-assigned parameters were bound directly to their typed byval fixed + // homes above, while the backend remains responsible for the physical Go ABI + // assignment. // // This intentionally differs from the native lowering, which stores each // incoming register piece separately and addresses stack-assigned parameters diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index 33c20fffe7a554..3b8770575d9555 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -182,6 +182,35 @@ func TestLLVMGoObjCompilerUsedOnlyKeepsExternalDataRoots(t *testing.T) { } } +func TestLLVMStaticCallMemoryResultsCompatibility(t *testing.T) { + config := abi.NewABIConfig(1, 0, 0, uint8(obj.ABIInternal)) + intType := types.Types[types.TINT] + ptrType := types.Types[types.TUNSAFEPTR] + structType := types.NewStruct([]*types.Field{ + types.NewField(src.NoXPos, nil, ptrType), + types.NewField(src.NoXPos, nil, intType), + }) + types.CalcSize(structType) + makeAux := func(results ...*types.Type) *AuxCall { + return StaticAuxCall(new(obj.LSym), config.ABIAnalyzeTypes(nil, results)) + } + + // A caller's aggregate result and a callee's expanded fields may use the + // same register count, but once either side spills, goret needs identical + // logical result boundaries. + aggregate := makeAux(structType) + expanded := makeAux(ptrType, intType) + if llvmStaticCallMemoryResultsCompatible(aggregate, expanded, llvmSignature(aggregate), llvmSignature(expanded)) { + t.Fatal("differently grouped memory results reported compatible") + } + + left := makeAux(ptrType, intType) + right := makeAux(ptrType, intType) + if !llvmStaticCallMemoryResultsCompatible(left, right, llvmSignature(left), llvmSignature(right)) { + t.Fatal("identical physical memory-result homes reported incompatible") + } +} + func TestLLVMUntypedABI0FunctionAddressCreatesFunctionDeclaration(t *testing.T) { oldModule := CurrentModule oldLowerer := currentLLVMDataLowerer diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index beb40a4d1f1fd6..41f4293ddb9361 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -145,6 +145,9 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } for _, needle := range [][]byte{ []byte("define goabiinternal"), + []byte(" byval("), + []byte(" byref("), + []byte(`"go_memory_results"`), []byte(`"go_results_tuple"`), []byte(`gc "goallc"`), } { @@ -192,10 +195,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } } for _, pattern := range []string{ - `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?"gc-live"\(ptr %second, ptr %first\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, - `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr, ptr \} @main\.pointerAggregateBothOverflow.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?"gc-live"\(ptr %second[[:alnum:]$._-]*, ptr %first[[:alnum:]$._-]*, ptr %first, ptr %second\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, + `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 \} @main\.pointerAggregateBothOverflow.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %\.result17, ptr %[[:alnum:]$._-]+, ptr %\.result16, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten GoALLC ABI IR does not match %q", pattern) @@ -205,6 +208,14 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "liveScalarStackArgument", "livePointerSequenceStackArguments", "livePointerAggregateStackArgument", "overflowResults", "stackAggregateResult", "bothOverflow", "pointerAggregateBothOverflow") + checkLLVMABIStatepointMemoryResultAttrs(t, rewrittenIR, map[string]string{ + "overflowResults": "16,17", + "initializedStackResult": "16", + "stackAggregateResult": "15", + "bothOverflow": "16,17", + "pointerAggregateBothOverflow": "16,17", + "stackResultsAfterGrowth": "16", + }) runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") @@ -220,10 +231,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedLLVMIR) for _, pattern := range []string{ - `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.[02].*?LDRXui\s+%fixed-stack\.[02]`, - `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, - `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?id:\s+4.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.3[^\n]*%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.3.*?STRXui[^\n]*%fixed-stack\.0.*?STRXui[^\n]*%fixed-stack\.1`, + `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?id:\s+0.*?offset:\s+8.*?size:\s+8.*?isImmutable:\s+false.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?id:\s+0.*?offset:\s+24.*?id:\s+1.*?offset:\s+16.*?id:\s+2.*?offset:\s+8.*?LDRXui\s+%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.1.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?id:\s+0.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?id:\s+0.*?offset:\s+40.*?id:\s+1.*?offset:\s+32.*?id:\s+2.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.2.*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?STRXui[^\n]*%fixed-stack\.1.*?STRXui[^\n]*%fixed-stack\.0`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("GoALLC ABI MIR does not match %q", pattern) @@ -263,7 +274,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{2, 4, 18}, nil}, // The explicit panicmem paths keep the still-needed parameter // homes in LocalsPointerMaps through locals-only alloca records. - goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, + goallcArgsMaps: [][]int{{2, 4, 18}, {2}, nil}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 1, -1, 1}, goallcQueryMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}, {2}}, @@ -271,42 +282,42 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "liveScalarStackArgument", args: 136, pointerBits: []int{0}, nativeArgsMaps: [][]int{{0}, nil}, - goallcArgsMaps: [][]int{{0}}, + goallcArgsMaps: [][]int{{0}, {0}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, { name: "livePointerSequenceStackArguments", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, { name: "livePointerAggregateStackArgument", args: 136, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, @@ -348,14 +359,14 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "pointerAggregateBothOverflow", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 136, + goallcLocals: 152, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {16, 17}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, @@ -450,8 +461,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten amd64 IR does not match %q", pattern) @@ -506,8 +517,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri `(?s)TEXT p\.initializedPointerResult.*?MOVQ\s+AX, 0x48\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+CX, 0x40\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+AX, 0x50\(BP\)`, - `(?s)TEXT p\.liveScalarStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), AX`, - `(?s)TEXT p\.liveAggregateStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), BX.*?MOVQ\s+0x38\(BP\), AX`, + `(?s)TEXT p\.liveScalarStackArgument.*?MOVQ\s+0x48\(BP\), AX.*?MOVQ\s+AX, -0x8\(BP\).*?R_CALL:p\.safepoint.*?MOVQ\s+-0x8\(BP\), AX`, + `(?s)TEXT p\.liveAggregateStackArgument.*?MOVQ\s+0x38\(BP\), AX.*?MOVQ\s+0x48\(BP\), CX.*?MOVQ\s+AX, -0x10\(BP\).*?MOVQ\s+CX, -0x8\(BP\).*?R_CALL:p\.safepoint.*?MOVQ\s+-0x8\(BP\), BX.*?MOVQ\s+-0x10\(BP\), AX`, } { if !regexp.MustCompile(pattern).Match(goallcDisassembly) { t.Fatalf("GoALLC amd64 object disassembly does not match %q", pattern) @@ -641,8 +652,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten source IR does not match %q", pattern) @@ -658,8 +669,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedGoallcIR) for _, pattern := range []string{ - `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, + `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?id:\s+0.*?size:\s+8.*?isImmutable:\s+false.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+0.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("source MIR does not match %q", pattern) @@ -783,7 +794,7 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string t.Helper() for _, callee := range callees { call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + - regexp.QuoteMeta(callee) + `.*#([0-9]+)[^\n]*$`).FindSubmatch(ir) + regexp.QuoteMeta(callee) + `.*#([0-9]+)(?: \[|$)`).FindSubmatch(ir) if len(call) != 2 { t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) } @@ -795,6 +806,22 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string } } +func checkLLVMABIStatepointMemoryResultAttrs(t *testing.T, ir []byte, callees map[string]string) { + t.Helper() + for callee, want := range callees { + call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + + regexp.QuoteMeta(callee) + `.*#([0-9]+)(?: \[|$)`).FindSubmatch(ir) + if len(call) != 2 { + t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) + } + attr := regexp.MustCompile(`(?m)^attributes #` + string(call[1]) + + ` = \{[^\n]*"go_memory_results"="` + regexp.QuoteMeta(want) + `"`).Find(ir) + if attr == nil { + t.Fatalf("statepoint call to main.%s lost go_memory_results=%q", callee, want) + } + } +} + func checkLLVMABIAssembly(t *testing.T, native, goallc []byte) { t.Helper() @@ -837,8 +864,8 @@ func checkLLVMABIAssembly(t *testing.T, native, goallc []byte) { `(?m)^main\.pointerAggregateBothOverflow:`, `(?s)\bbl\s+main\.overflowResults.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #8\]`, `(?s)\bbl\s+main\.overflowResults.*?\b(?:mov|stp)\s+[^\n]*x15`, - `(?s)\bbl\s+main\.stackAggregateResult.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #8\].*?\bmov\s+x[0-9]+, x15`, - `(?s)\bbl\s+main\.bothOverflow.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #32\]`, + `(?s)\bbl\s+main\.stackAggregateResult.*?\b(?:ldp\s+x[0-9]+, x[0-9]+|ldur\s+q[0-9]+), \[sp, #8\].*?\bmov\s+x[0-9]+, x15`, + `(?s)\bbl\s+main\.bothOverflow.*?\bldr\s+x[0-9]+, \[sp, #32\].*?\bldr\s+x[0-9]+, \[sp, #40\]`, `(?s)\bbl\s+main\.bothOverflow.*?\b(?:mov|stp)\s+[^\n]*x15`, } { if !regexp.MustCompile(pattern).Match(goallc) { diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 214205fe1df5c5..b1221704102f5c 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -338,6 +338,36 @@ if(BUILD_TESTING) "ptr noundef nonnull readnone align 8 %argument" ) + add_test( + NAME GoALLCStatepoints.ByValParamAttrRewrite + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" + ) + set_tests_properties(GoALLCStatepoints.ByValParamAttrRewrite PROPERTIES + PASS_REGULAR_EXPRESSION + "ptr byval.*5 x ptr.* align 8 %stack_arg.address" + ) + + add_test( + NAME GoALLCStatepoints.ByRefResultAttrRewrite + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" + ) + set_tests_properties(GoALLCStatepoints.ByRefResultAttrRewrite PROPERTIES + PASS_REGULAR_EXPRESSION + "ptr byref.*5 x ptr.* align 8 %stack_result.address;go_memory_results.*0" + ) + add_test( NAME GoALLCStatepoints.SupportedParamAttrsCodegen COMMAND diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 132c94886399e6..91b77ac18d5e97 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -40,6 +40,7 @@ namespace { constexpr StringLiteral GoALLCGCName = "goallc"; constexpr StringLiteral GCLeafAttr = "gc-leaf-function"; constexpr StringLiteral GoResultsTupleAttr = "go_results_tuple"; +constexpr StringLiteral GoMemoryResultsAttr = "go_memory_results"; constexpr StringLiteral GoDeferResultMD = "goallc.defer_result"; constexpr StringLiteral GoOpenDeferBitsMD = "goallc.open_defer_bits"; constexpr StringLiteral GoOpenDeferSlotsMD = "goallc.open_defer_slots"; @@ -118,6 +119,14 @@ struct PointerAllocaRecord { bool ActivityUnclear = false; }; +struct PointerByValRecord { + Argument *Base; + uint64_t ByteSize; + uint64_t Alignment; + uint64_t BitCount; + SmallVector BitmapWords; +}; + struct OpenDeferInfo { AllocaInst *Bits = nullptr; AllocaInst *Slots = nullptr; @@ -1316,6 +1325,62 @@ Error collectPointerAllocas( return Error::success(); } +Error collectPointerByVals(Function &F, + SmallVectorImpl &Records) { + if (!isGoCallingConv(F.getCallingConv())) + return Error::success(); + + const DataLayout &DL = F.getDataLayout(); + uint64_t PointerSize = DL.getPointerSize(0); + for (Argument &Arg : F.args()) { + if (!Arg.hasByValAttr()) + continue; + Type *StorageType = Arg.getParamByValType(); + if (!StorageType || !containsPointer(StorageType)) + continue; + + TypeSize AllocationSize = DL.getTypeAllocSize(StorageType); + if (AllocationSize.isScalable()) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints do not support scalable byval parameter " + "layouts"); + Align Alignment = + Arg.getParamAlign().value_or(DL.getABITypeAlign(StorageType)); + uint64_t ByteSize = AllocationSize.getFixedValue(); + if (!PointerSize || !ByteSize || ByteSize % PointerSize != 0 || + Alignment < DL.getABITypeAlign(StorageType)) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require pointer-aligned fixed byval parameter " + "layouts"); + + SmallVector Leaves; + SmallVector Path; + if (Error Err = + enumeratePointerAllocaLeaves(StorageType, DL, Path, 0, Leaves)) + return std::move(Err); + uint64_t BitCount = ByteSize / PointerSize; + SmallVector BitmapWords((BitCount + 63) / 64, 0); + for (const PointerAllocaLeaf &Leaf : Leaves) { + if (Leaf.Offset % PointerSize != 0 || Leaf.Offset >= ByteSize) + return createStringError( + std::errc::not_supported, + "GoALLC statepoint byval pointer slot is not pointer-aligned"); + uint64_t Bit = Leaf.Offset / PointerSize; + uint64_t Mask = uint64_t(1) << (Bit % 64); + if (BitmapWords[Bit / 64] & Mask) + return createStringError( + std::errc::invalid_argument, + "GoALLC statepoint byval pointer slots overlap"); + BitmapWords[Bit / 64] |= Mask; + } + Records.push_back( + {&Arg, ByteSize, Alignment.value(), BitCount, std::move(BitmapWords)}); + } + return Error::success(); +} + Error validateSafepoint(const SafepointRecord &Record) { const CallInst &Call = *Record.Call; if (Call.isInlineAsm()) @@ -1332,14 +1397,30 @@ Error validateSafepoint(const SafepointRecord &Record) { std::errc::not_supported, "GoALLC statepoints only support a single deopt call operand bundle"); for (unsigned I = 0; I != Call.arg_size(); ++I) { + if (Call.paramHasAttr(I, Attribute::ByVal) && + (!isGoCallingConv(Call.getCallingConv()) || + !Call.getArgOperand(I)->getType()->isPointerTy() || + !Call.getParamByValType(I) || !Call.getParamAlign(I))) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require typed, aligned byval only on Go calls"); + if (Call.paramHasAttr(I, Attribute::ByRef) && + (!isGoCallingConv(Call.getCallingConv()) || + !Call.getArgOperand(I)->getType()->isPointerTy() || + !Call.getParamByRefType(I) || !Call.getParamAlign(I))) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require typed, aligned byref only on Go calls"); for (Attribute Attr : Call.getAttributes().getParamAttrs(I)) { // These non-ABI attributes remain valid after LLVM's generic // RewriteStatepointsForGC pass and are natively accepted by both the // statepoint verifier and SelectionDAG call lowering. O2 commonly // infers them on otherwise ordinary runtime calls. Keep ABI-affecting - // attributes fail closed except for nest, whose Go closure ABI lowering - // is covered separately. + // attributes fail closed except for nest and typed byval, whose Go ABI + // lowering is covered separately. if (!Attr.hasAttribute(Attribute::Nest) && + !Attr.hasAttribute(Attribute::ByVal) && + !Attr.hasAttribute(Attribute::ByRef) && !Attr.hasAttribute(Attribute::Captures) && !Attr.hasAttribute(Attribute::ReadNone) && !Attr.hasAttribute(Attribute::ReadOnly) && @@ -1363,8 +1444,9 @@ Error validateSafepoint(const SafepointRecord &Record) { void appendAllocaPtrMapDeoptOperands( IRBuilder<> &Builder, ArrayRef Allocas, + ArrayRef ByVals, SmallVectorImpl &Deopt) { - if (Allocas.empty()) + if (Allocas.empty() && ByVals.empty()) return; // ProtocolLength covers BEGIN through END, but not the trailing duplicate // length. The envelope itself therefore contributes BEGIN, length, @@ -1372,27 +1454,39 @@ void appendAllocaPtrMapDeoptOperands( uint64_t ProtocolLength = 4; for (const PointerAllocaRecord *Alloca : Allocas) ProtocolLength += 10 + Alloca->BitmapWords.size(); + for (const PointerByValRecord *ByVal : ByVals) + ProtocolLength += 10 + ByVal->BitmapWords.size(); auto AppendConstant = [&](uint64_t Value) { Deopt.push_back(ConstantInt::get(Builder.getInt64Ty(), Value)); }; AppendConstant(GoObj::AllocaPtrMapBeginMagic); AppendConstant(ProtocolLength); - AppendConstant(Allocas.size()); - for (const PointerAllocaRecord *Alloca : Allocas) { + AppendConstant(Allocas.size() + ByVals.size()); + auto AppendRecord = [&](Value *Base, uint64_t ByteSize, uint64_t Alignment, + uint64_t BitCount, ArrayRef BitmapWords) { AppendConstant(GoObj::AllocaPtrMapRecordTag); - AppendConstant(10 + Alloca->BitmapWords.size()); - Deopt.push_back(Alloca->Alloca); - AppendConstant(0); // First contract version describes the whole alloca. - AppendConstant(Alloca->ByteSize); - AppendConstant(Alloca->Alignment); - AppendConstant(Alloca->Alloca->getDataLayout().getPointerSize(0)); - AppendConstant(Alloca->BitCount); + AppendConstant(10 + BitmapWords.size()); + Deopt.push_back(Base); + AppendConstant(0); // First contract version describes the whole object. + AppendConstant(ByteSize); + AppendConstant(Alignment); + AppendConstant( + Builder.GetInsertBlock()->getModule()->getDataLayout().getPointerSize( + 0)); + AppendConstant(BitCount); AppendConstant(GoObj::AllocaPtrMapBitmapWordBits); - AppendConstant(Alloca->BitmapWords.size()); - for (uint64_t Word : Alloca->BitmapWords) + AppendConstant(BitmapWords.size()); + for (uint64_t Word : BitmapWords) AppendConstant(Word); + }; + for (const PointerAllocaRecord *Alloca : Allocas) { + AppendRecord(Alloca->Alloca, Alloca->ByteSize, Alloca->Alignment, + Alloca->BitCount, Alloca->BitmapWords); } + for (const PointerByValRecord *ByVal : ByVals) + AppendRecord(ByVal->Base, ByVal->ByteSize, ByVal->Alignment, + ByVal->BitCount, ByVal->BitmapWords); AppendConstant(GoObj::AllocaPtrMapEndMagic); AppendConstant(ProtocolLength); } @@ -1417,6 +1511,7 @@ void appendOpenDeferDeoptOperands(IRBuilder<> &Builder, Error rewriteCall(SafepointRecord &Record, ArrayRef PointerAllocas, + ArrayRef PointerByVals, const std::optional &OpenDefer) { CallInst *Call = Record.Call; @@ -1433,7 +1528,8 @@ Error rewriteCall(SafepointRecord &Record, // Keep the open-defer envelope before the alloca ptrmap envelope. The latter // deliberately remains the final self-describing suffix for compatibility. appendOpenDeferDeoptOperands(Builder, OpenDefer, Deopt); - appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, Deopt); + appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, PointerByVals, + Deopt); Record.Statepoint = Builder.CreateGCStatepointCall( Record.ID, 0, Callee, CallArgs, Deopt.empty() ? std::nullopt @@ -1443,6 +1539,8 @@ Error rewriteCall(SafepointRecord &Record, if (Call->hasFnAttr(GoResultsTupleAttr)) Record.Statepoint->addFnAttr( Attribute::get(Call->getContext(), GoResultsTupleAttr)); + if (Attribute Attr = Call->getFnAttr(GoMemoryResultsAttr); Attr.isValid()) + Record.Statepoint->addFnAttr(Attr); for (unsigned I = 0; I != Call->arg_size(); ++I) { for (Attribute Attr : Call->getAttributes().getParamAttrs(I)) Record.Statepoint->addParamAttr(GCStatepointInst::CallArgsBeginPos + I, @@ -1515,14 +1613,21 @@ Value *rematerializeAddress(Value *Address, Value *Base, Value *RelocatedBase, void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { - // Each ordinary relocated pointer and each rematerialized alloca-derived - // address is a new reaching definition of its original SSA value. + // Each ordinary relocated pointer and each rematerialized fixed-object + // derived address is a new reaching definition of its original SSA value. + // Static allocas and typed byval/byref arguments are themselves fixed frame + // addresses: SelectionDAG rematerializes either from its frame index at each + // use, so replacing the original IR value with a gc.relocate chain would + // turn later statepoints into ordinary pointer spills. MapVector> Definitions; for (const SafepointRecord &Record : Records) { for (CallInst *RelocateCall : Record.Relocates) { auto *Relocate = cast(RelocateCall); Value *Original = Relocate->getDerivedPtr(); - if (!isa(Original)) + auto *Arg = dyn_cast(Original); + if (!isa(Original) && + !(Arg && (Arg->hasByValAttr() || + Arg->hasAttribute(Attribute::ByRef)))) Definitions[Original].push_back(RelocateCall); } @@ -1658,6 +1763,9 @@ Error rewriteFunction(Function &F) { SmallVector PointerAllocas; if (Error Err = collectPointerAllocas(F, OpenDefer, PointerAllocas)) return Err; + SmallVector PointerByVals; + if (Error Err = collectPointerByVals(F, PointerByVals)) + return Err; if (Error Err = scalarizeLivePointerAggregates(F)) return Err; @@ -1690,6 +1798,13 @@ Error rewriteFunction(Function &F) { // the exact address expression after the statepoint. Record.Live.insert(rematerializableDerivedBase(Address)); } + // A typed byval parameter is the address of a caller-initialized fixed Go + // argument object. Keep that base relocatable at every ordinary safepoint; + // the accompanying layout record tells GoObj which words in the object are + // GC pointers, while the direct base location itself is not a pointer root. + for (SafepointRecord &Record : Records) + for (const PointerByValRecord &ByVal : PointerByVals) + Record.Live.insert(ByVal.Base); for (const SafepointRecord &Record : Records) if (Error Err = validateSafepoint(Record)) return Err; @@ -1731,6 +1846,9 @@ Error rewriteFunction(Function &F) { if (Error Err = promoteAllocasToWholeFunctionLifetime(F, WholeLifetimeAllocas)) return Err; + SmallVector ByValRecords; + for (const PointerByValRecord &ByVal : PointerByVals) + ByValRecords.push_back(&ByVal); for (SafepointRecord &Record : llvm::reverse(Records)) { SmallVector AllocaRecords; for (const PointerAllocaRecord &Alloca : PointerAllocas) { @@ -1749,7 +1867,7 @@ Error rewriteFunction(Function &F) { if (IsActive || Alloca.NeedsStackObject) AllocaRecords.push_back(&Alloca); } - if (Error Err = rewriteCall(Record, AllocaRecords, OpenDefer)) + if (Error Err = rewriteCall(Record, AllocaRecords, ByValRecords, OpenDefer)) return Err; } eraseOriginalCalls(Records); diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index 65b52a2e85161c..e0398ce1307573 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -54,7 +54,7 @@ The current SSA value and CFG rewrite support matrix is: | Value or control-flow shape | Status | Current contract | | --- | --- | --- | -| Pointer arguments | AArch64 GoObj qualified; SelectionDAG home reuse also tested on X86 | Values live after a call use caller statepoints; exact stack inputs stay in their fixed ABI homes, while register inputs and transformed values use normal statepoint spills. Call-only arguments are described by the callee's type-derived entry map. | +| Pointer arguments | AArch64 GoObj qualified; typed stack homes also tested on X86 | Go ABI stack assignments use typed `byval` carriers bound directly to their incoming homes. The home is a memory root when active; pointer values loaded from it and live after a call use normal scalar statepoint tracking. Call-only arguments are described by the callee's type-derived entry map. | | Static `alloca` addresses | Supported | The raw alloca is the storage identity and the only extra `gc-live` root. First-class GEP/cast uses are rebuilt immediately at the use; direct memory uses stay on the FrameIndex and never enter relocation SSA. Pointers loaded from memory remain tracked. | | `select`, GEP, and pointer casts | Supported | Fixed-allocation GEP/no-op cast chains stay as direct FrameIndex uses or are rebuilt immediately at first-class uses. Merged or non-stack pointer values remain ordinary scalar roots. | | Pointer-valued call results | Supported | `gc.result` replaces the ordinary result and later safepoints relocate it. | @@ -63,7 +63,7 @@ The current SSA value and CFG rewrite support matrix is: | Loops and irreducible CFG | Supported | Relocation definitions are propagated through backedge and multi-entry PHIs without a shape-specific algorithm. | | Fixed struct/array SSA aggregates | Supported | Pointer and fixed pointer-vector leaves are extracted before liveness and reconstructed from the current relocated SSA leaves. | | Fixed-width pointer-vector SSA values | Supported | The vector remains one `gc-live` operand and one same-typed `gc.relocate`; it is not split into lanes. Pointer vectors in allocas remain unsupported. | -| Aggregate arguments and call results | Supported for IR rewriting | The wrapped call keeps its real aggregate ABI type. Only leaves live after the call enter caller `gc-live`; supported fixed formal layouts also contribute pointer words to AArch64 entry maps. | +| Aggregate arguments and call results | Supported for IR rewriting and AArch64 runtime ABI tests | Go ABI stack inputs use typed `byval`; stack results use caller-owned typed `byref` carriers plus `go_memory_results`, while register results remain compact LLVM returns. Only first-class leaves live after the call enter caller `gc-live`; fixed input/result homes contribute pointer words to GoObj maps according to their final locations. | | Aggregate load results and store operands | Supported | First-class SSA values use aggregate normalization. Pointer leaves in surviving fixed allocas remain memory roots described by the alloca deopt layout protocol. | | Pointer-containing `alloca` storage | GoObj qualified for fixed layouts | Go `VarDef` emits `llvm.lifetime.start`; parameter homes and addressed result homes have explicit starts at their initialization sites. The statepoint pass uses starts as backward liveness kills and real address uses as gens, so the last use supplies the implicit end. Active contents contribute callsite `LocalsPointerMaps`; address-observable objects additionally get function-wide `FUNCDATA_StackObjects` and one entry initialization because stack growth adjusts them even while source-dead. Locals-only storage is not initialized by the plugin, and no lifetime ends are emitted. | | Scalable vectors | Unsupported | The generic LLVM statepoint rewrite assumes a fixed vector width when constructing relocates; fails closed. | @@ -268,18 +268,24 @@ the same direct location is instead the frame base whose bitmap selects memory slots. Its contents are active only when a matching explicit direct alloca also occurs in the statepoint's GC operands. -Ordinary stack inputs use the same statepoint path. SelectionDAG formal -lowering records a value home only when a Go ABI pointer part is a direct, -non-extending load from an immutable fixed object and its IR aggregate offset, -ABI part offset, load size, and object size all agree. If `gc-live` later -contains that argument or an exact `extractvalue` leaf, statepoint lowering -uses the existing fixed frame index as its indirect memory location, makes the -slot mutable for GC relocation, and reloads `gc.relocate` from that same frame -index. No pre-call copy to a local spill is emitted. A merged, derived, -size-mismatched, or otherwise unproven value falls back to LLVM's normal local -statepoint spill. This is a SelectionDAG contract; it does not introduce -`byval`/`sret`, change GoALLC's LLVM IR emission point, or bypass the standard -statepoint operand format. +Ordinary stack inputs use typed LLVM `byval` parameters whenever Go's ABI +assignment contains no registers. The target lowers that carrier to the exact +mutable incoming Go home, and the frontend binds addressable parameter uses to +the carrier instead of copying the value through a compiler-owned alloca. At a +call site an existing source address is reused; a pure SSA value is materialized +in a short-lived entry alloca solely to provide the `byval` address. This is one +Go ABI classification rule, not a separate scalar-versus-aggregate policy. +Statepoint lowering rematerializes the fixed home itself after stack growth; +pointer values loaded from the home remain ordinary scalar roots when live. + +Results whose Go `OutParam` assignment contains no registers use caller-owned +typed `byref` carriers. The `go_memory_results` function/call attribute records +their logical Go result indexes because LLVM `sret` cannot represent Go's +mixed register-and-stack or multiple stack-result layouts. The target maps a +callee carrier to its fixed outgoing result home and copies that physical home +back to the caller-owned carrier after the call. Register-assigned results keep +the compact LLVM return and `go_results_tuple` mapping. Both carrier kinds keep +the standard statepoint operand format. For AArch64 GoObj, target frame lowering uses Go's frame-chain layout instead of the platform ABI frame record: LR is at `SP+0`, this function writes its FP @@ -294,11 +300,12 @@ moving SP so asynchronous traceback cannot observe a half-built frame. The ArgsPointerMaps phase supports scalar LLVM pointer inputs and receivers, plus pointer leaves in supported fixed struct/array formal layouts, in -ABIInternal register homes, ABIInternal stack-input slots, and ABI0 stack-input -slots on AArch64. Pair 0 is always `(EntryArgs, empty locals)`. Ordinary -statepoints use their actual final machine locations: indirect pointer slots in -the current frame become locals bits, while exact fixed input homes and stack -result slots above the final frame become args bits. The writer jointly +ABIInternal register homes, ABIInternal stack-input slots, ABI0 stack-input +slots, and typed `byref` result slots on AArch64. Pair 0 is always +`(EntryArgs, empty locals)`. Ordinary statepoints use their actual final machine +locations: indirect pointer slots in the current frame become locals bits, +while exact fixed input homes and stack result slots above the final frame +become args bits. The writer jointly deduplicates each complete `(Args, locals)` pair, so the two tables always have the same count. It does not eagerly mark declared result slots; a result slot is an args root only when a statepoint records a live pointer in that physical @@ -362,10 +369,9 @@ forces an entry input home and an ordinary stack-result root into different ArgsPointerMaps entries, then checks their exact objview bitmaps and `PCDATA_StackMapIndex` sequence. The identical-source Go fixture `test/abi/llvm_args_pointer_maps.go` separately forces a scalar pointer and a -three-word pointer aggregate onto the incoming stack. It checks native -assembly stack loads, scalar-only rewritten `gc-live`/`gc.relocate`, alloca -memory roots with no synthetic statepoint spill, and exact Args/Locals/PCDATA -objview data. +three-word pointer aggregate onto the incoming stack. It checks typed `byval` +IR, native assembly stack loads, scalar-only rewritten `gc-live`/`gc.relocate`, +fixed-home memory roots, and exact Args/Locals/PCDATA objview data. The executable identical-source fixture `test/abi/llvm_args_results.go` repeats those checks with `runtime.GC` for a scalar stack pointer, two pointer stack arguments separated by a scalar, a pointer-containing stack aggregate, and the diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 148506f6ca6135..1cb3308cd93526 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -98,12 +98,16 @@ entry: ret ptr %result } -define goabi0 ptr @"aarch64_abi0_pointer_result"(ptr %pointer) #0 gc "goallc" { +define goabi0 void @"aarch64_abi0_pointer_result"( + ptr byval(ptr) align 8 %pointer.home, + ptr byref(ptr) align 8 %result.home) #1 gc "goallc" { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 - ret ptr %pointer + store ptr %pointer, ptr %result.home, align 8 + ret void } define goabiinternal ptr @aarch64_stack_pointer_arg( @@ -111,8 +115,9 @@ define goabiinternal ptr @aarch64_stack_pointer_arg( i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) #0 gc "goallc" { + ptr byval(ptr) align 8 %pointer.home) #0 gc "goallc" { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 @@ -130,10 +135,11 @@ entry: ; The i64 register argument's home starts beyond the 8-byte scaled-uimm12 ; limit (32760), forcing the frameless morestack path to materialize SP+32776. define goabiinternal i64 @aarch64_large_arg_home( - [4096 x i64] %stackarg, i64 %regarg) #0 gc "goallc" { + ptr byval([4096 x i64]) align 8 %stackarg, i64 %regarg) #0 gc "goallc" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg } attributes #0 = { "frame-pointer"="non-leaf" } +attributes #1 = { "frame-pointer"="non-leaf" "go_memory_results"="0" } diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index e89c46dbebca71..4e393bfde62940 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -257,16 +257,13 @@ entry: } define goabiinternal void @argument_aggregate_home_address_across_calls( - %nested %value) gc "goallc" { + ptr byval(%nested) align 8 %value) gc "goallc" { entry: - ; The split aggregate parameter still has one complete fixed home and one - ; argp-relative StackObject covering all ABI pieces and padding. - %slot = alloca %nested, align 8 - call void @llvm.lifetime.start.p0(i64 48, ptr %slot) - store %nested %value, ptr %slot, align 8 - call goabiinternal void @observe_stack_address(ptr %slot) + ; A stack-assigned parameter is already one complete fixed incoming home. + ; Its typed byval layout supplies the entry and call-site argument bitmaps. + call goabiinternal void @observe_stack_address(ptr %value) call goabiinternal void @safepoint() - call goabiinternal void @observe_stack_address(ptr %slot) + call goabiinternal void @observe_stack_address(ptr %value) ret void } diff --git a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll index cdd741ddc3e7fb..78959368c94e76 100644 --- a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll +++ b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll @@ -1,6 +1,12 @@ target triple = "aarch64-unknown-linux-goobj" +%stack_arg = type [5 x ptr] + declare goabiinternal void @supported_callee(ptr) +declare goabiinternal void @supported_byval_callee( + ptr byval(%stack_arg) align 8) +declare goabiinternal void @supported_byref_callee( + ptr byref(%stack_arg) align 8) #1 define goabiinternal void @supported_param_attrs(ptr %argument) #0 gc "goallc" { entry: @@ -8,4 +14,24 @@ entry: ret void } +define goabiinternal void @supported_byval_attr(ptr %argument) #0 gc "goallc" { +entry: + %stack_arg = alloca %stack_arg, align 8 + store %stack_arg zeroinitializer, ptr %stack_arg, align 8 + %first = getelementptr inbounds %stack_arg, ptr %stack_arg, i32 0, i32 0 + store ptr %argument, ptr %first, align 8 + call goabiinternal void @supported_byval_callee( + ptr byval(%stack_arg) align 8 %stack_arg) + ret void +} + +define goabiinternal void @supported_byref_attr() #0 gc "goallc" { +entry: + %stack_result = alloca %stack_arg, align 8 + call goabiinternal void @supported_byref_callee( + ptr byref(%stack_arg) align 8 %stack_result) #1 + ret void +} + attributes #0 = { "frame-pointer"="non-leaf" } +attributes #1 = { "go_memory_results"="0" } From 35d7d749932c85d5b0141d70b8dcad5134006040 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 18:24:40 +0800 Subject: [PATCH 2/8] cmd/compile: use goret for stack results --- src/cmd/compile/internal/ssa/ssa2llvm.go | 40 ++++++------------- src/cmd/internal/testdir/llvm_abi_test.go | 21 +++++----- src/cmd/llvmplugin/CMakeLists.txt | 6 +-- src/cmd/llvmplugin/GoALLCStatepoints.cpp | 24 +++++------ src/cmd/llvmplugin/README.md | 6 +-- src/cmd/llvmplugin/testdata/aarch64-frame.ll | 3 +- .../testdata/supported-param-attrs.ll | 11 +++-- 7 files changed, 48 insertions(+), 63 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index cc6bc89bba9316..ba72477d369687 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -11,7 +11,6 @@ import ( "fmt" "internal/buildcfg" "strconv" - "strings" "github.com/goallc/go-llvm" ) @@ -78,7 +77,6 @@ const goABIInternalCallConv llvm.CallConv = 22 const goABI0CallConv llvm.CallConv = 23 const goABI0SymbolSuffix = "" const goResultsTupleAttr = "go_results_tuple" -const goMemoryResultsAttr = "go_memory_results" const goGCStrategy = "goallc" const goGCLeafFunctionAttr = "gc-leaf-function" const goNoSplitAttr = "go-nosplit" @@ -110,7 +108,6 @@ type llvmFuncSignature struct { ReturnCount int Params []llvmParamSignature Results []llvmResultSignature - MemoryResultIndices []int HasClosureContext bool ClosureContextIndex int } @@ -215,7 +212,6 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { results := make([]llvm.Type, 0, aux.NResults()) resultSignatures := make([]llvmResultSignature, 0, aux.NResults()) - memoryResultIndices := make([]int, 0, aux.NResults()) for i := int64(0); i < aux.NResults(); i++ { goType := aux.TypeOfResult(i) result := llvmResultSignature{ @@ -232,7 +228,6 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { result.Alignment = int(goType.Alignment()) result.ParamIndex = len(params) params = append(params, GlobalCtxt.PointerType(0)) - memoryResultIndices = append(memoryResultIndices, int(i)) } else { result.ReturnIndex = len(results) results = append(results, result.ValueType) @@ -256,7 +251,6 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { ReturnCount: len(results), Params: paramSignatures, Results: resultSignatures, - MemoryResultIndices: memoryResultIndices, ClosureContextIndex: -1, } } @@ -286,20 +280,16 @@ func llvmByValAttribute(t llvm.Type) llvm.Attribute { return GlobalCtxt.CreateTypeAttribute(kind, t) } -func llvmByRefAttribute(t llvm.Type) llvm.Attribute { - kind := llvm.AttributeKindID("byref") +func llvmGoRetAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("goret") if kind == 0 { - base.Fatalf("LLVM does not provide the byref parameter attribute") + base.Fatalf("LLVM does not provide the goret parameter attribute") } return GlobalCtxt.CreateTypeAttribute(kind, t) } -func llvmMemoryResultsAttribute(indices []int) llvm.Attribute { - values := make([]string, len(indices)) - for i, index := range indices { - values[i] = strconv.Itoa(index) - } - return GlobalCtxt.CreateStringAttribute(goMemoryResultsAttr, strings.Join(values, ",")) +func llvmGoRetIndexAttribute(index int) llvm.Attribute { + return GlobalCtxt.CreateStringAttribute("goretindex", strconv.Itoa(index)) } func llvmNullPointerIsValidAttribute() llvm.Attribute { @@ -323,9 +313,6 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon if sig.ReturnCount > 1 { fn.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } - if len(sig.MemoryResultIndices) != 0 { - fn.AddFunctionAttr(llvmMemoryResultsAttribute(sig.MemoryResultIndices)) - } for i, param := range sig.Params { if !param.ByVal { continue @@ -333,11 +320,12 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon fn.AddAttributeAtIndex(i+1, llvmByValAttribute(param.ValueType)) fn.Param(i).SetParamAlignment(param.Alignment) } - for _, result := range sig.Results { + for index, result := range sig.Results { if !result.InMemory { continue } - fn.AddAttributeAtIndex(result.ParamIndex+1, llvmByRefAttribute(result.ValueType)) + fn.AddAttributeAtIndex(result.ParamIndex+1, llvmGoRetAttribute(result.ValueType)) + fn.AddAttributeAtIndex(result.ParamIndex+1, llvmGoRetIndexAttribute(index)) fn.Param(result.ParamIndex).SetParamAlignment(result.Alignment) } if sig.HasClosureContext { @@ -352,9 +340,6 @@ func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { if sig.ReturnCount > 1 { call.AddCallSiteAttribute(llvmAttributeFunctionIndex, GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } - if len(sig.MemoryResultIndices) != 0 { - call.AddCallSiteAttribute(llvmAttributeFunctionIndex, llvmMemoryResultsAttribute(sig.MemoryResultIndices)) - } for i, param := range sig.Params { if !param.ByVal { continue @@ -362,11 +347,12 @@ func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { call.AddCallSiteAttribute(i+1, llvmByValAttribute(param.ValueType)) call.SetInstrParamAlignment(i+1, param.Alignment) } - for _, result := range sig.Results { + for index, result := range sig.Results { if !result.InMemory { continue } - call.AddCallSiteAttribute(result.ParamIndex+1, llvmByRefAttribute(result.ValueType)) + call.AddCallSiteAttribute(result.ParamIndex+1, llvmGoRetAttribute(result.ValueType)) + call.AddCallSiteAttribute(result.ParamIndex+1, llvmGoRetIndexAttribute(index)) call.SetInstrParamAlignment(result.ParamIndex+1, result.Alignment) } } @@ -1777,7 +1763,7 @@ func (lfc *LLVMFuncContext) llvmByValCallArgument(v, argValue *Value, index int, } func (lfc *LLVMFuncContext) llvmMemoryResultCallArguments(v *Value, sig llvmFuncSignature, aux *AuxCall) []llvm.Value { - args := make([]llvm.Value, 0, len(sig.MemoryResultIndices)) + args := make([]llvm.Value, 0, sig.ResultCount-sig.ReturnCount) for index, result := range sig.Results { if !result.InMemory { continue @@ -3681,7 +3667,7 @@ func LLVMCompile(f *Func) { } } } - // Results that Go assigned to memory use caller-owned typed byref carriers. + // Results that Go assigned to memory use caller-owned typed goret carriers. // Reserve their stable destinations before call emission; ordinary SelectN // loads from the same object that SelectNAddr exposes by address. for _, BB := range f.Blocks { diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index 41f4293ddb9361..2e9ea2256e16f6 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -146,8 +146,8 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { for _, needle := range [][]byte{ []byte("define goabiinternal"), []byte(" byval("), - []byte(" byref("), - []byte(`"go_memory_results"`), + []byte(" goret("), + []byte(` "goretindex"="`), []byte(`"go_results_tuple"`), []byte(`gc "goallc"`), } { @@ -208,7 +208,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "liveScalarStackArgument", "livePointerSequenceStackArguments", "livePointerAggregateStackArgument", "overflowResults", "stackAggregateResult", "bothOverflow", "pointerAggregateBothOverflow") - checkLLVMABIStatepointMemoryResultAttrs(t, rewrittenIR, map[string]string{ + checkLLVMABIStatepointGoRetAttrs(t, rewrittenIR, map[string]string{ "overflowResults": "16,17", "initializedStackResult": "16", "stackAggregateResult": "15", @@ -806,18 +806,19 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string } } -func checkLLVMABIStatepointMemoryResultAttrs(t *testing.T, ir []byte, callees map[string]string) { +func checkLLVMABIStatepointGoRetAttrs(t *testing.T, ir []byte, callees map[string]string) { t.Helper() for callee, want := range callees { call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + - regexp.QuoteMeta(callee) + `.*#([0-9]+)(?: \[|$)`).FindSubmatch(ir) - if len(call) != 2 { + regexp.QuoteMeta(callee) + `.*$`).Find(ir) + if call == nil { t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) } - attr := regexp.MustCompile(`(?m)^attributes #` + string(call[1]) + - ` = \{[^\n]*"go_memory_results"="` + regexp.QuoteMeta(want) + `"`).Find(ir) - if attr == nil { - t.Fatalf("statepoint call to main.%s lost go_memory_results=%q", callee, want) + for _, index := range strings.Split(want, ",") { + pattern := `goret\([^)]*\)[^\n,)]*"goretindex"="` + regexp.QuoteMeta(index) + `"` + if !regexp.MustCompile(pattern).Match(call) { + t.Fatalf("statepoint call to main.%s lost goret index %s: %s", callee, index, call) + } } } } diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index b1221704102f5c..67c8c797f50ef0 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -354,7 +354,7 @@ if(BUILD_TESTING) ) add_test( - NAME GoALLCStatepoints.ByRefResultAttrRewrite + NAME GoALLCStatepoints.GoRetResultAttrRewrite COMMAND "${GOALLC_LLC_EXECUTABLE}" "-load-pass-plugin=$" @@ -363,9 +363,9 @@ if(BUILD_TESTING) -o - "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" ) - set_tests_properties(GoALLCStatepoints.ByRefResultAttrRewrite PROPERTIES + set_tests_properties(GoALLCStatepoints.GoRetResultAttrRewrite PROPERTIES PASS_REGULAR_EXPRESSION - "ptr byref.*5 x ptr.* align 8 %stack_result.address;go_memory_results.*0" + "ptr goret.*5 x ptr.* align 8.*goretindex.*0.* %stack_result.address" ) add_test( diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 91b77ac18d5e97..634be20748aad3 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -40,7 +40,6 @@ namespace { constexpr StringLiteral GoALLCGCName = "goallc"; constexpr StringLiteral GCLeafAttr = "gc-leaf-function"; constexpr StringLiteral GoResultsTupleAttr = "go_results_tuple"; -constexpr StringLiteral GoMemoryResultsAttr = "go_memory_results"; constexpr StringLiteral GoDeferResultMD = "goallc.defer_result"; constexpr StringLiteral GoOpenDeferBitsMD = "goallc.open_defer_bits"; constexpr StringLiteral GoOpenDeferSlotsMD = "goallc.open_defer_slots"; @@ -1404,23 +1403,27 @@ Error validateSafepoint(const SafepointRecord &Record) { return createStringError( std::errc::not_supported, "GoALLC statepoints require typed, aligned byval only on Go calls"); - if (Call.paramHasAttr(I, Attribute::ByRef) && + if (Call.paramHasAttr(I, Attribute::GoRet) && (!isGoCallingConv(Call.getCallingConv()) || !Call.getArgOperand(I)->getType()->isPointerTy() || - !Call.getParamByRefType(I) || !Call.getParamAlign(I))) + !Call.getParamGoRetType(I) || + !Call.getParamAttr(I, "goretindex").isValid() || + !Call.getParamAlign(I))) return createStringError( std::errc::not_supported, - "GoALLC statepoints require typed, aligned byref only on Go calls"); + "GoALLC statepoints require indexed, typed, aligned goret only on " + "Go calls"); for (Attribute Attr : Call.getAttributes().getParamAttrs(I)) { // These non-ABI attributes remain valid after LLVM's generic // RewriteStatepointsForGC pass and are natively accepted by both the // statepoint verifier and SelectionDAG call lowering. O2 commonly // infers them on otherwise ordinary runtime calls. Keep ABI-affecting - // attributes fail closed except for nest and typed byval, whose Go ABI - // lowering is covered separately. + // attributes fail closed except for nest and the typed Go ABI memory + // carriers, whose lowering is covered separately. if (!Attr.hasAttribute(Attribute::Nest) && !Attr.hasAttribute(Attribute::ByVal) && - !Attr.hasAttribute(Attribute::ByRef) && + !Attr.hasAttribute(Attribute::GoRet) && + !Attr.hasAttribute("goretindex") && !Attr.hasAttribute(Attribute::Captures) && !Attr.hasAttribute(Attribute::ReadNone) && !Attr.hasAttribute(Attribute::ReadOnly) && @@ -1539,8 +1542,6 @@ Error rewriteCall(SafepointRecord &Record, if (Call->hasFnAttr(GoResultsTupleAttr)) Record.Statepoint->addFnAttr( Attribute::get(Call->getContext(), GoResultsTupleAttr)); - if (Attribute Attr = Call->getFnAttr(GoMemoryResultsAttr); Attr.isValid()) - Record.Statepoint->addFnAttr(Attr); for (unsigned I = 0; I != Call->arg_size(); ++I) { for (Attribute Attr : Call->getAttributes().getParamAttrs(I)) Record.Statepoint->addParamAttr(GCStatepointInst::CallArgsBeginPos + I, @@ -1615,7 +1616,7 @@ void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { // Each ordinary relocated pointer and each rematerialized fixed-object // derived address is a new reaching definition of its original SSA value. - // Static allocas and typed byval/byref arguments are themselves fixed frame + // Static allocas and typed byval/goret arguments are themselves fixed frame // addresses: SelectionDAG rematerializes either from its frame index at each // use, so replacing the original IR value with a gc.relocate chain would // turn later statepoints into ordinary pointer spills. @@ -1626,8 +1627,7 @@ void repairRelocationSSA(Function &F, DominatorTree &DT, Value *Original = Relocate->getDerivedPtr(); auto *Arg = dyn_cast(Original); if (!isa(Original) && - !(Arg && (Arg->hasByValAttr() || - Arg->hasAttribute(Attribute::ByRef)))) + !(Arg && (Arg->hasByValAttr() || Arg->hasGoRetAttr()))) Definitions[Original].push_back(RelocateCall); } diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index e0398ce1307573..e0e94908d9f128 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -63,7 +63,7 @@ The current SSA value and CFG rewrite support matrix is: | Loops and irreducible CFG | Supported | Relocation definitions are propagated through backedge and multi-entry PHIs without a shape-specific algorithm. | | Fixed struct/array SSA aggregates | Supported | Pointer and fixed pointer-vector leaves are extracted before liveness and reconstructed from the current relocated SSA leaves. | | Fixed-width pointer-vector SSA values | Supported | The vector remains one `gc-live` operand and one same-typed `gc.relocate`; it is not split into lanes. Pointer vectors in allocas remain unsupported. | -| Aggregate arguments and call results | Supported for IR rewriting and AArch64 runtime ABI tests | Go ABI stack inputs use typed `byval`; stack results use caller-owned typed `byref` carriers plus `go_memory_results`, while register results remain compact LLVM returns. Only first-class leaves live after the call enter caller `gc-live`; fixed input/result homes contribute pointer words to GoObj maps according to their final locations. | +| Aggregate arguments and call results | Supported for IR rewriting and AArch64 runtime ABI tests | Go ABI stack inputs use typed `byval`; stack results use caller-owned typed `goret(T) "goretindex"="N"` carriers, while register results remain compact LLVM returns. Only first-class leaves live after the call enter caller `gc-live`; fixed input/result homes contribute pointer words to GoObj maps according to their final locations. | | Aggregate load results and store operands | Supported | First-class SSA values use aggregate normalization. Pointer leaves in surviving fixed allocas remain memory roots described by the alloca deopt layout protocol. | | Pointer-containing `alloca` storage | GoObj qualified for fixed layouts | Go `VarDef` emits `llvm.lifetime.start`; parameter homes and addressed result homes have explicit starts at their initialization sites. The statepoint pass uses starts as backward liveness kills and real address uses as gens, so the last use supplies the implicit end. Active contents contribute callsite `LocalsPointerMaps`; address-observable objects additionally get function-wide `FUNCDATA_StackObjects` and one entry initialization because stack growth adjusts them even while source-dead. Locals-only storage is not initialized by the plugin, and no lifetime ends are emitted. | | Scalable vectors | Unsupported | The generic LLVM statepoint rewrite assumes a fixed vector width when constructing relocates; fails closed. | @@ -279,7 +279,7 @@ Statepoint lowering rematerializes the fixed home itself after stack growth; pointer values loaded from the home remain ordinary scalar roots when live. Results whose Go `OutParam` assignment contains no registers use caller-owned -typed `byref` carriers. The `go_memory_results` function/call attribute records +typed `goret(T)` carriers. Each carrier's `"goretindex"="N"` parameter attribute records their logical Go result indexes because LLVM `sret` cannot represent Go's mixed register-and-stack or multiple stack-result layouts. The target maps a callee carrier to its fixed outgoing result home and copies that physical home @@ -301,7 +301,7 @@ moving SP so asynchronous traceback cannot observe a half-built frame. The ArgsPointerMaps phase supports scalar LLVM pointer inputs and receivers, plus pointer leaves in supported fixed struct/array formal layouts, in ABIInternal register homes, ABIInternal stack-input slots, ABI0 stack-input -slots, and typed `byref` result slots on AArch64. Pair 0 is always +slots, and typed `goret` result slots on AArch64. Pair 0 is always `(EntryArgs, empty locals)`. Ordinary statepoints use their actual final machine locations: indirect pointer slots in the current frame become locals bits, while exact fixed input homes and stack result slots above the final frame diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 1cb3308cd93526..8b2590e194970d 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -100,7 +100,7 @@ entry: define goabi0 void @"aarch64_abi0_pointer_result"( ptr byval(ptr) align 8 %pointer.home, - ptr byref(ptr) align 8 %result.home) #1 gc "goallc" { + ptr goret(ptr) "goretindex"="0" align 8 %result.home) #0 gc "goallc" { entry: %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 @@ -142,4 +142,3 @@ entry: } attributes #0 = { "frame-pointer"="non-leaf" } -attributes #1 = { "frame-pointer"="non-leaf" "go_memory_results"="0" } diff --git a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll index 78959368c94e76..90e8add305d104 100644 --- a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll +++ b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll @@ -5,8 +5,8 @@ target triple = "aarch64-unknown-linux-goobj" declare goabiinternal void @supported_callee(ptr) declare goabiinternal void @supported_byval_callee( ptr byval(%stack_arg) align 8) -declare goabiinternal void @supported_byref_callee( - ptr byref(%stack_arg) align 8) #1 +declare goabiinternal void @supported_goret_callee( + ptr goret(%stack_arg) "goretindex"="0" align 8) define goabiinternal void @supported_param_attrs(ptr %argument) #0 gc "goallc" { entry: @@ -25,13 +25,12 @@ entry: ret void } -define goabiinternal void @supported_byref_attr() #0 gc "goallc" { +define goabiinternal void @supported_goret_attr() #0 gc "goallc" { entry: %stack_result = alloca %stack_arg, align 8 - call goabiinternal void @supported_byref_callee( - ptr byref(%stack_arg) align 8 %stack_result) #1 + call goabiinternal void @supported_goret_callee( + ptr goret(%stack_arg) "goretindex"="0" align 8 %stack_result) ret void } attributes #0 = { "frame-pointer"="non-leaf" } -attributes #1 = { "go_memory_results"="0" } From 0511cfcbc7d502ea33db793ebaed610958f60623 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 12 Aug 2026 23:06:38 +0800 Subject: [PATCH 3/8] cmd/llvmplugin: migrate byval checks to FileCheck --- .../testdata/alloca-pointer-roots.ll | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index 4e393bfde62940..cf05286fed6061 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -34,6 +34,16 @@ target triple = "x86_64-unknown-linux-goobj" ; IR-COUNT-2: getelementptr inbounds i8, ptr %slot, i64 0 ; IR-NOT: .address.relocated.merge +; IR-LABEL: define goabiinternal void @argument_aggregate_home_address_across_calls( +; IR-SAME: ptr byval(%nested) align 8 %value +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR-NOT: llvm.memset.inline + ; IR-LABEL: define goabiinternal void @alloca_gep_value_across_calls() ; IR: %field.remat{{[0-9]+}} = getelementptr inbounds %pointer_field, ptr %slot ; IR: "gc-live"(ptr %slot) @@ -100,11 +110,10 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW-NEXT: 0, ; OBJVIEW-NEXT: 3, ; OBJVIEW-NEXT: 5 -; OBJVIEW: "kind": "stack_objects" -; OBJVIEW: "offset": 0 -; OBJVIEW: "size": 48 -; OBJVIEW: "ptr_bytes": 48 -; OBJVIEW: "name": "runtime.gcbits.2900000000000000" +; OBJVIEW: "kind": "locals_pointer_maps" +; OBJVIEW: "set_bits": null +; OBJVIEW-NOT: "kind": "stack_objects" +; OBJVIEW: "stack_map_queries": [ ; OBJVIEW-LABEL: "name": "alloca_pointer_free_address_across_calls" ; OBJVIEW-NOT: "kind": "stack_objects" From c623cd9ae99d7eef5c1827db77d4c7771db0d54a Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 22:06:40 +0800 Subject: [PATCH 4/8] cmd/compile: lower stack inputs through preallocated homes --- src/cmd/compile/internal/ssa/ssa2llvm.go | 157 ++++++++++-------- src/cmd/compile/internal/ssa/ssa2llvm_test.go | 82 +++++---- src/cmd/llvmplugin/CMakeLists.txt | 6 +- src/cmd/llvmplugin/GoALLCStatepoints.cpp | 115 ++++++++----- src/cmd/llvmplugin/testdata/aarch64-frame.ll | 6 +- .../testdata/alloca-pointer-roots.ll | 7 +- .../testdata/supported-param-attrs.ll | 20 ++- 7 files changed, 230 insertions(+), 163 deletions(-) diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index ba72477d369687..5549fa90ad6b87 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -11,6 +11,7 @@ import ( "fmt" "internal/buildcfg" "strconv" + "strings" "github.com/goallc/go-llvm" ) @@ -115,7 +116,7 @@ type llvmFuncSignature struct { type llvmParamSignature struct { ValueType llvm.Type Alignment int - ByVal bool + InMemory bool } type llvmResultSignature struct { @@ -202,7 +203,7 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { if goType.Alignment() <= 0 { base.Fatalf("invalid alignment %d for stack argument %d of type %v", goType.Alignment(), i, goType) } - param.ByVal = true + param.InMemory = true param.Alignment = int(goType.Alignment()) paramType = GlobalCtxt.PointerType(0) } @@ -272,10 +273,10 @@ func llvmNestAttribute() llvm.Attribute { return GlobalCtxt.CreateEnumAttribute(kind, 0) } -func llvmByValAttribute(t llvm.Type) llvm.Attribute { - kind := llvm.AttributeKindID("byval") +func llvmPreallocatedAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("preallocated") if kind == 0 { - base.Fatalf("LLVM does not provide the byval parameter attribute") + base.Fatalf("LLVM does not provide the preallocated parameter attribute") } return GlobalCtxt.CreateTypeAttribute(kind, t) } @@ -314,10 +315,10 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon fn.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } for i, param := range sig.Params { - if !param.ByVal { + if !param.InMemory { continue } - fn.AddAttributeAtIndex(i+1, llvmByValAttribute(param.ValueType)) + fn.AddAttributeAtIndex(i+1, llvmPreallocatedAttribute(param.ValueType)) fn.Param(i).SetParamAlignment(param.Alignment) } for index, result := range sig.Results { @@ -341,10 +342,10 @@ func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { call.AddCallSiteAttribute(llvmAttributeFunctionIndex, GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } for i, param := range sig.Params { - if !param.ByVal { + if !param.InMemory { continue } - call.AddCallSiteAttribute(i+1, llvmByValAttribute(param.ValueType)) + call.AddCallSiteAttribute(i+1, llvmPreallocatedAttribute(param.ValueType)) call.SetInstrParamAlignment(i+1, param.Alignment) } for index, result := range sig.Results { @@ -1706,8 +1707,8 @@ func (lfc *LLVMFuncContext) paramForArgNameAndType(name *ir.Name) (llvm.Value, * for i, param := range lfc.F.OwnAux.ABIInfo().InParams() { if param.Name != nil && llvmLocalKeyForName(param.Name) == key { value := lfc.LF.Param(i) - if lfc.Params[i].ByVal { - value = lfc.b.CreateLoad(lfc.Params[i].ValueType, value, name.Sym().Name+".byval") + if lfc.Params[i].InMemory { + value = lfc.b.CreateLoad(lfc.Params[i].ValueType, value, name.Sym().Name+".memory") value.SetAlignment(lfc.Params[i].Alignment) } return value, lfc.F.OwnAux.TypeOfArg(int64(i)) @@ -1717,49 +1718,59 @@ func (lfc *LLVMFuncContext) paramForArgNameAndType(name *ir.Name) (llvm.Value, * return llvm.Value{}, nil } -func (lfc *LLVMFuncContext) llvmByValCallArgument(v, argValue *Value, index int, logical *types.Type, param llvmParamSignature) (llvm.Value, *llvmStackSlot) { - if !param.ByVal || logical.Size() == 0 { - v.Fatalf("argument %d is not a non-empty byval parameter", index) +func llvmMemoryParameterCount(sig llvmFuncSignature) int { + count := 0 + for _, param := range sig.Params { + if param.InMemory { + count++ + } } + return count +} + +func (lfc *LLVMFuncContext) llvmPreallocatedCallSetup(sig llvmFuncSignature) llvm.Value { + count := llvmMemoryParameterCount(sig) + if count == 0 { + return llvm.Value{} + } + fn := getLLVMIntrinsicDeclaration("llvm.call.preallocated.setup") + return lfc.b.CreateCall(fn.GlobalValueType(), fn, []llvm.Value{ + llvm.ConstInt(GlobalCtxt.Int32Type(), uint64(count), false), + }, "preallocated.setup") +} + +func (lfc *LLVMFuncContext) llvmPreallocatedCallArgument(v, argValue *Value, index, carrierIndex int, logical *types.Type, param llvmParamSignature, setup llvm.Value) llvm.Value { + if !param.InMemory || logical.Size() == 0 || setup.IsNil() { + v.Fatalf("argument %d is not a non-empty memory parameter", index) + } + fn := getLLVMIntrinsicDeclaration("llvm.call.preallocated.arg") + address := lfc.b.CreateCall(fn.GlobalValueType(), fn, []llvm.Value{ + setup, + llvm.ConstInt(GlobalCtxt.Int32Type(), uint64(carrierIndex), false), + }, fmt.Sprintf("%s.arg%d.home", v, index)) + address.AddCallSiteAttribute(llvmAttributeFunctionIndex, llvmPreallocatedAttribute(param.ValueType)) // Go SSA keeps non-SSA-able aggregate call arguments in memory and exposes - // the value through a Load or Dereference. The LLVM byval carrier wants the - // address of those bytes, so preserve that address instead of constructing a - // huge first-class aggregate only to store it back to a temporary. + // the value through a Load or Dereference. Copy those bytes directly into + // the ABI-defined outgoing home instead of constructing a first-class value. if types.Identical(argValue.Type, logical) && (argValue.Op == OpLoad || argValue.Op == OpDereference) && len(argValue.Args) != 0 { - address := lfc.GenLV(argValue.Args[0]) - if address.Type().TypeKind() != llvm.PointerTypeKind { - v.Fatalf("byval argument %d has non-pointer source address", index) + source := lfc.GenLV(argValue.Args[0]) + if source.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("memory argument %d has non-pointer source address", index) } - return address, nil + lfc.llvmCopyFixedMemory(address, source, logical.Size(), param.Alignment) + return address } value := lfc.GenLV(argValue) value = lfc.llvmValueToABI(v, value, argValue.Type, logical, param.ValueType, fmt.Sprintf("%s.arg%d", v, index)) if value.Type() != param.ValueType { - v.Fatalf("byval argument %d has incompatible LLVM value type", index) - } - - // A pure SSA value has no source address even though Go assigned the logical - // argument wholly to the stack. Materialize any such value in a fixed entry - // alloca; this is only how the unified byval carrier obtains an address, not - // a second ABI classification path. - entryBuilder := GlobalCtxt.NewBuilder() - defer entryBuilder.Dispose() - entry := lfc.LF.EntryBasicBlock() - if first := entry.FirstInstruction(); first.IsNil() { - entryBuilder.SetInsertPointAtEnd(entry) - } else { - entryBuilder.SetInsertPointBefore(first) + v.Fatalf("memory argument %d has incompatible LLVM value type", index) } - address := entryBuilder.CreateAlloca(param.ValueType, fmt.Sprintf("%s.arg%d.byval", v, index)) - address.SetAlignment(param.Alignment) - slot := &llvmStackSlot{Value: address, Type: logical} - lfc.llvmLifetimeStart(*slot) store := lfc.b.CreateStore(value, address) store.SetAlignment(param.Alignment) - return address, slot + return address } func (lfc *LLVMFuncContext) llvmMemoryResultCallArguments(v *Value, sig llvmFuncSignature, aux *AuxCall) []llvm.Value { @@ -2162,15 +2173,13 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { llvmGCLeaf := aux.Fn == ir.Syms.WBZero || aux.Fn == ir.Syms.WBMove || aux.Fn == ir.Syms.Memmove || aux.Fn == ir.Syms.Memequal args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) - var byValTemps []llvmStackSlot + setup := lfc.llvmPreallocatedCallSetup(sig) + carrierIndex := 0 for i := int64(0); i < aux.NArgs(); i++ { var arg llvm.Value - if sig.Params[i].ByVal { - var temp *llvmStackSlot - arg, temp = lfc.llvmByValCallArgument(v, v.Args[i], int(i), aux.TypeOfArg(i), sig.Params[i]) - if temp != nil { - byValTemps = append(byValTemps, *temp) - } + if sig.Params[i].InMemory { + arg = lfc.llvmPreallocatedCallArgument(v, v.Args[i], int(i), carrierIndex, aux.TypeOfArg(i), sig.Params[i], setup) + carrierIndex++ } else { arg = lfc.GenLV(v.Args[i]) if arg.Type() != sig.Type.ParamTypes()[i] { @@ -2187,12 +2196,16 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { if sig.ReturnCount == 0 { name = "" } - call := lfc.b.CreateCall(sig.Type, fn, args, name) + var call llvm.Value + if setup.IsNil() { + call = lfc.b.CreateCall(sig.Type, fn, args, name) + } else { + bundle := llvm.NewOperandBundle("preallocated", []llvm.Value{setup}) + call = lfc.b.CreateCallWithOperandBundles(sig.Type, fn, args, []llvm.OperandBundle{bundle}, name) + bundle.Dispose() + } call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) - for _, slot := range byValTemps { - lfc.llvmLifetimeEnd(slot) - } lfc.materializeAddressedResults(v, call, aux) if llvmGCLeaf { markLLVMGCLeafCall(call) @@ -2228,16 +2241,14 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext v.Fatalf("indirect callee has non-pointer LLVM type") } args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) - var byValTemps []llvmStackSlot + setup := lfc.llvmPreallocatedCallSetup(sig) + carrierIndex := 0 for i := int64(0); i < aux.NArgs(); i++ { argValue := v.Args[argStart+int(i)] var arg llvm.Value - if sig.Params[i].ByVal { - var temp *llvmStackSlot - arg, temp = lfc.llvmByValCallArgument(v, argValue, int(i), aux.TypeOfArg(i), sig.Params[i]) - if temp != nil { - byValTemps = append(byValTemps, *temp) - } + if sig.Params[i].InMemory { + arg = lfc.llvmPreallocatedCallArgument(v, argValue, int(i), carrierIndex, aux.TypeOfArg(i), sig.Params[i], setup) + carrierIndex++ } else { arg = lfc.GenLV(argValue) if arg.Type() != sig.Type.ParamTypes()[i] { @@ -2261,12 +2272,16 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext if sig.ReturnCount == 0 { name = "" } - call := lfc.b.CreateCall(sig.Type, code, args, name) + var call llvm.Value + if setup.IsNil() { + call = lfc.b.CreateCall(sig.Type, code, args, name) + } else { + bundle := llvm.NewOperandBundle("preallocated", []llvm.Value{setup}) + call = lfc.b.CreateCallWithOperandBundles(sig.Type, code, args, []llvm.OperandBundle{bundle}, name) + bundle.Dispose() + } call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) - for _, slot := range byValTemps { - lfc.llvmLifetimeEnd(slot) - } lfc.materializeAddressedResults(v, call, aux) if closureContext { call.AddCallSiteAttribute(sig.ClosureContextIndex+1, llvmNestAttribute()) @@ -3530,28 +3545,28 @@ func LLVMCompile(f *Func) { FCtxt.Locals[key] = llvmStackSlot{Value: value, Type: name.Type()} } } - // A non-empty parameter assigned wholly to the Go stack is represented as - // a typed LLVM byval pointer. That pointer is the parameter's ABI-defined + // A non-empty parameter assigned wholly to the Go stack is represented as a + // typed LLVM preallocated pointer. That pointer is the parameter's ABI-defined // incoming home, so bind addressable Go SSA uses to it directly. Register // parameters retain the existing compiler-owned alloca/store path. for i, param := range sig.Params { - if !param.ByVal { + if !param.InMemory { continue } assignment := inParams[i] if len(assignment.Registers) != 0 { - f.fe.Fatalf(f.Entry.Pos, "LLVM byval parameter %d was assigned Go registers", i) + f.fe.Fatalf(f.Entry.Pos, "LLVM memory parameter %d was assigned Go registers", i) } if assignment.Name == nil { continue } goType := f.OwnAux.TypeOfArg(int64(i)) if goType.Size() == 0 || !types.Identical(goType, assignment.Name.Type()) { - f.fe.Fatalf(assignment.Name.Pos(), "invalid Go type for LLVM byval parameter %v", assignment.Name) + f.fe.Fatalf(assignment.Name.Pos(), "invalid Go type for LLVM memory parameter %v", assignment.Name) } key := llvmLocalKeyForName(assignment.Name) if _, exists := FCtxt.Locals[key]; exists { - f.fe.Fatalf(assignment.Name.Pos(), "duplicate LLVM byval parameter home %v", assignment.Name) + f.fe.Fatalf(assignment.Name.Pos(), "duplicate LLVM memory parameter home %v", assignment.Name) } FCtxt.Locals[key] = llvmStackSlot{Value: FCtxt.LF.Param(i), Type: assignment.Name.Type()} } @@ -3778,9 +3793,9 @@ func LLVMCompile(f *Func) { } } // Give addressable register parameters a complete LLVM memory home. Wholly - // stack-assigned parameters were bound directly to their typed byval fixed - // homes above, while the backend remains responsible for the physical Go ABI - // assignment. + // stack-assigned parameters were bound directly to their typed preallocated + // fixed homes above, while the backend remains responsible for the physical + // Go ABI assignment. // // This intentionally differs from the native lowering, which stores each // incoming register piece separately and addresses stack-assigned parameters diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index 3b8770575d9555..fff37317bb298b 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "cmd/compile/internal/abi" "cmd/compile/internal/base" "cmd/compile/internal/ir" "cmd/compile/internal/typecheck" @@ -133,6 +134,58 @@ func TestLLVMBuiltinDeclarationKeepsCallSiteSignatures(t *testing.T) { } } +func TestLLVMGoMemoryCarriersFollowABIAllocation(t *testing.T) { + config := abi.NewABIConfig(1, 0, 0, uint8(obj.ABIInternal)) + intType := types.Types[types.TINT] + aux := StaticAuxCall(new(obj.LSym), config.ABIAnalyzeTypes( + []*types.Type{intType, intType}, + []*types.Type{intType, intType}, + )) + sig := llvmSignature(aux) + + memoryArgs := 0 + for i, param := range sig.Params { + want := len(aux.ABIInfo().InParam(i).Registers) == 0 && aux.TypeOfArg(int64(i)).Size() != 0 + if param.InMemory != want { + t.Errorf("argument %d memory carrier = %v, Go ABI allocation wants %v", i, param.InMemory, want) + } + if want { + memoryArgs++ + if got := sig.Type.ParamTypes()[i].TypeKind(); got != llvm.PointerTypeKind { + t.Errorf("argument %d carrier kind = %v, want pointer", i, got) + } + } + } + if memoryArgs == 0 || memoryArgs == len(sig.Params) { + t.Fatalf("test requires mixed register and memory arguments, got %d of %d in memory", memoryArgs, len(sig.Params)) + } + + memoryResults := 0 + for i, result := range sig.Results { + want := len(aux.ABIInfo().OutParam(i).Registers) == 0 && aux.TypeOfResult(int64(i)).Size() != 0 + if result.InMemory != want { + t.Errorf("result %d memory carrier = %v, Go ABI allocation wants %v", i, result.InMemory, want) + } + if want { + memoryResults++ + } + } + if memoryResults == 0 || memoryResults == len(sig.Results) { + t.Fatalf("test requires mixed register and memory results, got %d of %d in memory", memoryResults, len(sig.Results)) + } + + module := GlobalCtxt.NewModule("go_memory_carriers") + t.Cleanup(module.Dispose) + fn := llvm.AddFunction(module, "mixed", sig.Type) + configureLLVMFunction(fn, sig, goABIInternalCallConv) + ir := module.String() + for _, want := range []string{"preallocated(", "goret(", `"goretindex"=`} { + if !strings.Contains(ir, want) { + t.Errorf("LLVM function does not contain %q:\n%s", want, ir) + } + } +} + func TestLLVMGoObjCompilerUsedOnlyKeepsExternalDataRoots(t *testing.T) { oldModule := CurrentModule oldLowerer := currentLLVMDataLowerer @@ -182,35 +235,6 @@ func TestLLVMGoObjCompilerUsedOnlyKeepsExternalDataRoots(t *testing.T) { } } -func TestLLVMStaticCallMemoryResultsCompatibility(t *testing.T) { - config := abi.NewABIConfig(1, 0, 0, uint8(obj.ABIInternal)) - intType := types.Types[types.TINT] - ptrType := types.Types[types.TUNSAFEPTR] - structType := types.NewStruct([]*types.Field{ - types.NewField(src.NoXPos, nil, ptrType), - types.NewField(src.NoXPos, nil, intType), - }) - types.CalcSize(structType) - makeAux := func(results ...*types.Type) *AuxCall { - return StaticAuxCall(new(obj.LSym), config.ABIAnalyzeTypes(nil, results)) - } - - // A caller's aggregate result and a callee's expanded fields may use the - // same register count, but once either side spills, goret needs identical - // logical result boundaries. - aggregate := makeAux(structType) - expanded := makeAux(ptrType, intType) - if llvmStaticCallMemoryResultsCompatible(aggregate, expanded, llvmSignature(aggregate), llvmSignature(expanded)) { - t.Fatal("differently grouped memory results reported compatible") - } - - left := makeAux(ptrType, intType) - right := makeAux(ptrType, intType) - if !llvmStaticCallMemoryResultsCompatible(left, right, llvmSignature(left), llvmSignature(right)) { - t.Fatal("identical physical memory-result homes reported incompatible") - } -} - func TestLLVMUntypedABI0FunctionAddressCreatesFunctionDeclaration(t *testing.T) { oldModule := CurrentModule oldLowerer := currentLLVMDataLowerer diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 67c8c797f50ef0..1816037be2e358 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -339,7 +339,7 @@ if(BUILD_TESTING) ) add_test( - NAME GoALLCStatepoints.ByValParamAttrRewrite + NAME GoALLCStatepoints.PreallocatedParamAttrRewrite COMMAND "${GOALLC_LLC_EXECUTABLE}" "-load-pass-plugin=$" @@ -348,9 +348,9 @@ if(BUILD_TESTING) -o - "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" ) - set_tests_properties(GoALLCStatepoints.ByValParamAttrRewrite PROPERTIES + set_tests_properties(GoALLCStatepoints.PreallocatedParamAttrRewrite PROPERTIES PASS_REGULAR_EXPRESSION - "ptr byval.*5 x ptr.* align 8 %stack_arg.address" + "ptr preallocated.*5 x ptr.* align 8 %home" ) add_test( diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 634be20748aad3..5faaed2fdc8b02 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -118,7 +118,7 @@ struct PointerAllocaRecord { bool ActivityUnclear = false; }; -struct PointerByValRecord { +struct PointerMemoryParamRecord { Argument *Base; uint64_t ByteSize; uint64_t Alignment; @@ -1324,17 +1324,17 @@ Error collectPointerAllocas( return Error::success(); } -Error collectPointerByVals(Function &F, - SmallVectorImpl &Records) { +Error collectPointerMemoryParams( + Function &F, SmallVectorImpl &Records) { if (!isGoCallingConv(F.getCallingConv())) return Error::success(); const DataLayout &DL = F.getDataLayout(); uint64_t PointerSize = DL.getPointerSize(0); for (Argument &Arg : F.args()) { - if (!Arg.hasByValAttr()) + if (!Arg.hasPreallocatedAttr()) continue; - Type *StorageType = Arg.getParamByValType(); + Type *StorageType = Arg.getAttributes().getPreallocatedType(); if (!StorageType || !containsPointer(StorageType)) continue; @@ -1342,7 +1342,7 @@ Error collectPointerByVals(Function &F, if (AllocationSize.isScalable()) return createStringError( std::errc::not_supported, - "GoALLC statepoints do not support scalable byval parameter " + "GoALLC statepoints do not support scalable preallocated parameter " "layouts"); Align Alignment = Arg.getParamAlign().value_or(DL.getABITypeAlign(StorageType)); @@ -1351,7 +1351,8 @@ Error collectPointerByVals(Function &F, Alignment < DL.getABITypeAlign(StorageType)) return createStringError( std::errc::not_supported, - "GoALLC statepoints require pointer-aligned fixed byval parameter " + "GoALLC statepoints require pointer-aligned fixed preallocated " + "parameter " "layouts"); SmallVector Leaves; @@ -1365,13 +1366,14 @@ Error collectPointerByVals(Function &F, if (Leaf.Offset % PointerSize != 0 || Leaf.Offset >= ByteSize) return createStringError( std::errc::not_supported, - "GoALLC statepoint byval pointer slot is not pointer-aligned"); + "GoALLC statepoint preallocated pointer slot is not " + "pointer-aligned"); uint64_t Bit = Leaf.Offset / PointerSize; uint64_t Mask = uint64_t(1) << (Bit % 64); if (BitmapWords[Bit / 64] & Mask) return createStringError( std::errc::invalid_argument, - "GoALLC statepoint byval pointer slots overlap"); + "GoALLC statepoint preallocated pointer slots overlap"); BitmapWords[Bit / 64] |= Mask; } Records.push_back( @@ -1389,20 +1391,29 @@ Error validateSafepoint(const SafepointRecord &Record) { if (Call.isMustTailCall()) return createStringError(std::errc::not_supported, "GoALLC statepoints do not support musttail"); - if (Call.getNumOperandBundles() != 0 && - (Call.getNumOperandBundles() != 1 || - !Call.getOperandBundle(LLVMContext::OB_deopt))) - return createStringError( - std::errc::not_supported, - "GoALLC statepoints only support a single deopt call operand bundle"); + if (Call.countOperandBundlesOfType(LLVMContext::OB_deopt) > 1 || + Call.countOperandBundlesOfType(LLVMContext::OB_preallocated) > 1) + return createStringError(std::errc::not_supported, + "GoALLC statepoints require unique deopt and " + "preallocated call operand bundles"); + for (unsigned I = 0; I != Call.getNumOperandBundles(); ++I) { + uint32_t Tag = Call.getOperandBundleAt(I).getTagID(); + if (Tag != LLVMContext::OB_deopt && Tag != LLVMContext::OB_preallocated) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints do not support call operand bundle '%s'", + Call.getOperandBundleAt(I).getTagName().str().c_str()); + } for (unsigned I = 0; I != Call.arg_size(); ++I) { - if (Call.paramHasAttr(I, Attribute::ByVal) && + if (Call.paramHasAttr(I, Attribute::Preallocated) && (!isGoCallingConv(Call.getCallingConv()) || !Call.getArgOperand(I)->getType()->isPointerTy() || - !Call.getParamByValType(I) || !Call.getParamAlign(I))) + !Call.getParamPreallocatedType(I) || !Call.getParamAlign(I) || + !Call.getOperandBundle(LLVMContext::OB_preallocated))) return createStringError( std::errc::not_supported, - "GoALLC statepoints require typed, aligned byval only on Go calls"); + "GoALLC statepoints require bundled, typed, aligned preallocated " + "parameters only on Go calls"); if (Call.paramHasAttr(I, Attribute::GoRet) && (!isGoCallingConv(Call.getCallingConv()) || !Call.getArgOperand(I)->getType()->isPointerTy() || @@ -1421,7 +1432,7 @@ Error validateSafepoint(const SafepointRecord &Record) { // attributes fail closed except for nest and the typed Go ABI memory // carriers, whose lowering is covered separately. if (!Attr.hasAttribute(Attribute::Nest) && - !Attr.hasAttribute(Attribute::ByVal) && + !Attr.hasAttribute(Attribute::Preallocated) && !Attr.hasAttribute(Attribute::GoRet) && !Attr.hasAttribute("goretindex") && !Attr.hasAttribute(Attribute::Captures) && @@ -1447,9 +1458,9 @@ Error validateSafepoint(const SafepointRecord &Record) { void appendAllocaPtrMapDeoptOperands( IRBuilder<> &Builder, ArrayRef Allocas, - ArrayRef ByVals, + ArrayRef MemoryParams, SmallVectorImpl &Deopt) { - if (Allocas.empty() && ByVals.empty()) + if (Allocas.empty() && MemoryParams.empty()) return; // ProtocolLength covers BEGIN through END, but not the trailing duplicate // length. The envelope itself therefore contributes BEGIN, length, @@ -1457,15 +1468,15 @@ void appendAllocaPtrMapDeoptOperands( uint64_t ProtocolLength = 4; for (const PointerAllocaRecord *Alloca : Allocas) ProtocolLength += 10 + Alloca->BitmapWords.size(); - for (const PointerByValRecord *ByVal : ByVals) - ProtocolLength += 10 + ByVal->BitmapWords.size(); + for (const PointerMemoryParamRecord *Param : MemoryParams) + ProtocolLength += 10 + Param->BitmapWords.size(); auto AppendConstant = [&](uint64_t Value) { Deopt.push_back(ConstantInt::get(Builder.getInt64Ty(), Value)); }; AppendConstant(GoObj::AllocaPtrMapBeginMagic); AppendConstant(ProtocolLength); - AppendConstant(Allocas.size() + ByVals.size()); + AppendConstant(Allocas.size() + MemoryParams.size()); auto AppendRecord = [&](Value *Base, uint64_t ByteSize, uint64_t Alignment, uint64_t BitCount, ArrayRef BitmapWords) { AppendConstant(GoObj::AllocaPtrMapRecordTag); @@ -1487,9 +1498,9 @@ void appendAllocaPtrMapDeoptOperands( AppendRecord(Alloca->Alloca, Alloca->ByteSize, Alloca->Alignment, Alloca->BitCount, Alloca->BitmapWords); } - for (const PointerByValRecord *ByVal : ByVals) - AppendRecord(ByVal->Base, ByVal->ByteSize, ByVal->Alignment, - ByVal->BitCount, ByVal->BitmapWords); + for (const PointerMemoryParamRecord *Param : MemoryParams) + AppendRecord(Param->Base, Param->ByteSize, Param->Alignment, + Param->BitCount, Param->BitmapWords); AppendConstant(GoObj::AllocaPtrMapEndMagic); AppendConstant(ProtocolLength); } @@ -1514,7 +1525,7 @@ void appendOpenDeferDeoptOperands(IRBuilder<> &Builder, Error rewriteCall(SafepointRecord &Record, ArrayRef PointerAllocas, - ArrayRef PointerByVals, + ArrayRef MemoryParams, const std::optional &OpenDefer) { CallInst *Call = Record.Call; @@ -1531,13 +1542,23 @@ Error rewriteCall(SafepointRecord &Record, // Keep the open-defer envelope before the alloca ptrmap envelope. The latter // deliberately remains the final self-describing suffix for compatibility. appendOpenDeferDeoptOperands(Builder, OpenDefer, Deopt); - appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, PointerByVals, - Deopt); + appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, MemoryParams, Deopt); Record.Statepoint = Builder.CreateGCStatepointCall( Record.ID, 0, Callee, CallArgs, Deopt.empty() ? std::nullopt : std::optional>(ArrayRef(Deopt)), GCLive, "statepoint_token"); + if (auto Bundle = Call->getOperandBundle(LLVMContext::OB_preallocated)) { + SmallVector Bundles; + for (unsigned I = 0; I != Record.Statepoint->getNumOperandBundles(); ++I) + Bundles.emplace_back(Record.Statepoint->getOperandBundleAt(I)); + Bundles.emplace_back(*Bundle); + CallInst *BundledStatepoint = CallInst::Create( + Record.Statepoint, Bundles, Record.Statepoint->getIterator()); + Record.Statepoint->replaceAllUsesWith(BundledStatepoint); + Record.Statepoint->eraseFromParent(); + Record.Statepoint = BundledStatepoint; + } Record.Statepoint->setCallingConv(Call->getCallingConv()); if (Call->hasFnAttr(GoResultsTupleAttr)) Record.Statepoint->addFnAttr( @@ -1616,9 +1637,9 @@ void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { // Each ordinary relocated pointer and each rematerialized fixed-object // derived address is a new reaching definition of its original SSA value. - // Static allocas and typed byval/goret arguments are themselves fixed frame - // addresses: SelectionDAG rematerializes either from its frame index at each - // use, so replacing the original IR value with a gc.relocate chain would + // Static allocas and typed preallocated/goret arguments are themselves fixed + // frame addresses: SelectionDAG rematerializes either from its frame index at + // each use, so replacing the original IR value with a gc.relocate chain would // turn later statepoints into ordinary pointer spills. MapVector> Definitions; for (const SafepointRecord &Record : Records) { @@ -1627,7 +1648,7 @@ void repairRelocationSSA(Function &F, DominatorTree &DT, Value *Original = Relocate->getDerivedPtr(); auto *Arg = dyn_cast(Original); if (!isa(Original) && - !(Arg && (Arg->hasByValAttr() || Arg->hasGoRetAttr()))) + !(Arg && (Arg->hasPreallocatedAttr() || Arg->hasGoRetAttr()))) Definitions[Original].push_back(RelocateCall); } @@ -1763,8 +1784,8 @@ Error rewriteFunction(Function &F) { SmallVector PointerAllocas; if (Error Err = collectPointerAllocas(F, OpenDefer, PointerAllocas)) return Err; - SmallVector PointerByVals; - if (Error Err = collectPointerByVals(F, PointerByVals)) + SmallVector PointerMemoryParams; + if (Error Err = collectPointerMemoryParams(F, PointerMemoryParams)) return Err; if (Error Err = scalarizeLivePointerAggregates(F)) return Err; @@ -1798,13 +1819,14 @@ Error rewriteFunction(Function &F) { // the exact address expression after the statepoint. Record.Live.insert(rematerializableDerivedBase(Address)); } - // A typed byval parameter is the address of a caller-initialized fixed Go - // argument object. Keep that base relocatable at every ordinary safepoint; - // the accompanying layout record tells GoObj which words in the object are - // GC pointers, while the direct base location itself is not a pointer root. + // A typed preallocated parameter is the address of a caller-initialized fixed + // Go argument object. Keep that base relocatable at every ordinary + // safepoint; the accompanying layout record tells GoObj which words in the + // object are GC pointers, while the direct base location itself is not a + // pointer root. for (SafepointRecord &Record : Records) - for (const PointerByValRecord &ByVal : PointerByVals) - Record.Live.insert(ByVal.Base); + for (const PointerMemoryParamRecord &Param : PointerMemoryParams) + Record.Live.insert(Param.Base); for (const SafepointRecord &Record : Records) if (Error Err = validateSafepoint(Record)) return Err; @@ -1846,9 +1868,9 @@ Error rewriteFunction(Function &F) { if (Error Err = promoteAllocasToWholeFunctionLifetime(F, WholeLifetimeAllocas)) return Err; - SmallVector ByValRecords; - for (const PointerByValRecord &ByVal : PointerByVals) - ByValRecords.push_back(&ByVal); + SmallVector MemoryParamRecords; + for (const PointerMemoryParamRecord &Param : PointerMemoryParams) + MemoryParamRecords.push_back(&Param); for (SafepointRecord &Record : llvm::reverse(Records)) { SmallVector AllocaRecords; for (const PointerAllocaRecord &Alloca : PointerAllocas) { @@ -1867,7 +1889,8 @@ Error rewriteFunction(Function &F) { if (IsActive || Alloca.NeedsStackObject) AllocaRecords.push_back(&Alloca); } - if (Error Err = rewriteCall(Record, AllocaRecords, ByValRecords, OpenDefer)) + if (Error Err = + rewriteCall(Record, AllocaRecords, MemoryParamRecords, OpenDefer)) return Err; } eraseOriginalCalls(Records); diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 8b2590e194970d..573b7be5d62ade 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -99,7 +99,7 @@ entry: } define goabi0 void @"aarch64_abi0_pointer_result"( - ptr byval(ptr) align 8 %pointer.home, + ptr preallocated(ptr) align 8 %pointer.home, ptr goret(ptr) "goretindex"="0" align 8 %result.home) #0 gc "goallc" { entry: %pointer = load ptr, ptr %pointer.home, align 8 @@ -115,7 +115,7 @@ define goabiinternal ptr @aarch64_stack_pointer_arg( i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr byval(ptr) align 8 %pointer.home) #0 gc "goallc" { + ptr preallocated(ptr) align 8 %pointer.home) #0 gc "goallc" { entry: %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 @@ -135,7 +135,7 @@ entry: ; The i64 register argument's home starts beyond the 8-byte scaled-uimm12 ; limit (32760), forcing the frameless morestack path to materialize SP+32776. define goabiinternal i64 @aarch64_large_arg_home( - ptr byval([4096 x i64]) align 8 %stackarg, i64 %regarg) #0 gc "goallc" { + ptr preallocated([4096 x i64]) align 8 %stackarg, i64 %regarg) #0 gc "goallc" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index cf05286fed6061..3ecc8076174d5d 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -35,7 +35,7 @@ target triple = "x86_64-unknown-linux-goobj" ; IR-NOT: .address.relocated.merge ; IR-LABEL: define goabiinternal void @argument_aggregate_home_address_across_calls( -; IR-SAME: ptr byval(%nested) align 8 %value +; IR-SAME: ptr preallocated(%nested) align 8 %value ; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) ; IR: call coldcc ptr @llvm.experimental.gc.relocate ; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) @@ -266,10 +266,11 @@ entry: } define goabiinternal void @argument_aggregate_home_address_across_calls( - ptr byval(%nested) align 8 %value) gc "goallc" { + ptr preallocated(%nested) align 8 %value) gc "goallc" { entry: ; A stack-assigned parameter is already one complete fixed incoming home. - ; Its typed byval layout supplies the entry and call-site argument bitmaps. + ; Its typed preallocated layout supplies the entry and call-site argument + ; bitmaps. call goabiinternal void @observe_stack_address(ptr %value) call goabiinternal void @safepoint() call goabiinternal void @observe_stack_address(ptr %value) diff --git a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll index 90e8add305d104..fd4383869132ff 100644 --- a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll +++ b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll @@ -3,8 +3,10 @@ target triple = "aarch64-unknown-linux-goobj" %stack_arg = type [5 x ptr] declare goabiinternal void @supported_callee(ptr) -declare goabiinternal void @supported_byval_callee( - ptr byval(%stack_arg) align 8) +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare goabiinternal void @supported_preallocated_callee( + ptr preallocated(%stack_arg) align 8) declare goabiinternal void @supported_goret_callee( ptr goret(%stack_arg) "goretindex"="0" align 8) @@ -14,14 +16,16 @@ entry: ret void } -define goabiinternal void @supported_byval_attr(ptr %argument) #0 gc "goallc" { +define goabiinternal void @supported_preallocated_attr(ptr %argument) #0 gc "goallc" { entry: - %stack_arg = alloca %stack_arg, align 8 - store %stack_arg zeroinitializer, ptr %stack_arg, align 8 - %first = getelementptr inbounds %stack_arg, ptr %stack_arg, i32 0, i32 0 + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%stack_arg) + store %stack_arg zeroinitializer, ptr %home, align 8 + %first = getelementptr inbounds %stack_arg, ptr %home, i32 0, i32 0 store ptr %argument, ptr %first, align 8 - call goabiinternal void @supported_byval_callee( - ptr byval(%stack_arg) align 8 %stack_arg) + call goabiinternal void @supported_preallocated_callee( + ptr preallocated(%stack_arg) align 8 %home) + ["preallocated"(token %setup)] ret void } From 21a31ae9b7c417d284ceb9b6e295f7c8814ecfef Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 22:06:40 +0800 Subject: [PATCH 5/8] test: qualify LLVM-built compression packages --- test/llvm_stdlib_packages.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/llvm_stdlib_packages.json b/test/llvm_stdlib_packages.json index 08d8012391e14a..6c27967880c460 100644 --- a/test/llvm_stdlib_packages.json +++ b/test/llvm_stdlib_packages.json @@ -2,9 +2,13 @@ "packages": { "whitelist": { "archive/tar": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "archive/zip": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "bufio": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "bytes": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "cmp": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/flate": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/gzip": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/zlib": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/heap": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/list": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/ring": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", @@ -67,11 +71,7 @@ "unicode/utf8": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests" }, "blacklist": { - "*": "package and its standard-library dependency closure have not yet been qualified through LLVM O2 compile, GoObj/archive, link, and package tests", - "archive/zip": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/gzip": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/zlib": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/flate": "llc/GoObj: X86 and AArch64 SelectionDAG instruction-selection assertions" + "*": "package and its standard-library dependency closure have not yet been qualified through LLVM O2 compile, GoObj/archive, link, and package tests" }, "platform_blacklist": { "linux/amd64": { From 027cf6fa9c11462bb9a46ee0cba232e8a75852b8 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sat, 15 Aug 2026 23:33:32 +0800 Subject: [PATCH 6/8] test: check typed stack ABI carriers --- test/codegen/_cgo_llvm_unsafe_args.go | 25 ++++++++++------------- test/codegen/issue25378.go | 4 ++-- test/codegen/llvm_argument_memory_home.go | 21 +++++++++++-------- test/codegen/llvm_dereference.go | 6 ++++-- test/codegen/llvm_direct_iface.go | 5 ++++- test/codegen/llvm_linkname.go | 8 ++++++-- test/codegen/llvm_selectnaddr.go | 4 ++-- 7 files changed, 41 insertions(+), 32 deletions(-) diff --git a/test/codegen/_cgo_llvm_unsafe_args.go b/test/codegen/_cgo_llvm_unsafe_args.go index 4cec88cf2d522e..7588ccf8e18f1d 100644 --- a/test/codegen/_cgo_llvm_unsafe_args.go +++ b/test/codegen/_cgo_llvm_unsafe_args.go @@ -9,31 +9,28 @@ package codegen //go:noescape func llvmCgoUnsafeSink(*uintptr) -// LLVM-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-SAME: i64 %p, i64 %q) #[[NOINLINE:[0-9]+]] gc "goallc" +// LLVM-LABEL: define goabi0 void @"codegen.llvmCgoUnsafeFrame"( +// LLVM-SAME: ptr preallocated(i64) align 8 %p, ptr preallocated(i64) align 8 %q, ptr goret(i64) align 8 "goretindex"="0" [[RETURN:%.*]]) #[[NOINLINE:[0-9]+]] gc "goallc" // LLVM-NOT: alloca // LLVM: [[FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-NOT: llvm.addressofreturnaddress // LLVM-NOT: llvm.sponentry -// LLVM: [[Q:%.*]] = getelementptr i8, ptr [[FRAME]], i64 8 // LLVM: [[RESULT:%.*]] = getelementptr i8, ptr [[FRAME]], i64 16 -// LLVM: store i64 %p, ptr [[FRAME]] -// LLVM: store i64 %q, ptr [[Q]] -// LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[FRAME]]) -// LLVM: {{%.*}} = load i64, ptr [[RESULT]] +// LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} %p) +// LLVM: call void @llvm.memmove.p0.p0.i64(ptr align 8 [[RETURN]], ptr align 8 [[RESULT]], i64 8, i1 false) +// LLVM: ret void // LLVM: attributes #[[NOINLINE]] = { {{.*}}noinline -// LLVM-OPT-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-OPT-SAME: i64 %p, i64 %q) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" +// LLVM-OPT-LABEL: define goabi0 void @"codegen.llvmCgoUnsafeFrame"( +// LLVM-OPT-SAME: ptr preallocated(i64) align 8 %p, ptr preallocated(i64) align 8{{.*}} %q, ptr {{.*}}goret(i64) align 8{{.*}} "goretindex"="0" [[OPT_RETURN:%.*]]) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" // LLVM-OPT-NOT: alloca // LLVM-OPT: [[OPT_FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-OPT-NOT: llvm.addressofreturnaddress // LLVM-OPT-NOT: llvm.sponentry -// LLVM-OPT: [[OPT_Q:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 8 // LLVM-OPT: [[OPT_RESULT:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 16 -// LLVM-OPT: store i64 %p, ptr [[OPT_FRAME]] -// LLVM-OPT: store i64 %q, ptr [[OPT_Q]] -// LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[OPT_FRAME]]) -// LLVM-OPT: {{%.*}} = load i64, ptr [[OPT_RESULT]] +// LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} %p) +// LLVM-OPT: [[OPT_VALUE:%.*]] = load i64, ptr [[OPT_RESULT]], align 8 +// LLVM-OPT-NEXT: store i64 [[OPT_VALUE]], ptr [[OPT_RETURN]], align 8 +// LLVM-OPT: ret void // LLVM-OPT: attributes #[[OPT_NOINLINE]] = { {{.*}}noinline // //go:cgo_unsafe_args diff --git a/test/codegen/issue25378.go b/test/codegen/issue25378.go index 9a704fc40f2800..4cc89753959237 100644 --- a/test/codegen/issue25378.go +++ b/test/codegen/issue25378.go @@ -7,11 +7,11 @@ package codegen // LLVM-DAG: @codegen.wsp = global <{ [256 x i8] }> {{.*}}, section ".noptrdata" -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16([2 x i16] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16(ptr preallocated([2 x i16]) // LLVM-DAG: zext i16 {{%.*}} to i64 // LLVM-DAG: icmp ult i64 {{%.*}}, 256 // LLVM-DAG: getelementptr i8, ptr @codegen.wsp, i64 -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte([2 x i8] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte(ptr preallocated([2 x i8]) // LLVM-DAG: zext i8 {{%.*}} to i64 var wsp = [256]bool{ diff --git a/test/codegen/llvm_argument_memory_home.go b/test/codegen/llvm_argument_memory_home.go index 8fa68f1d76e74e..7a5484e95c7577 100644 --- a/test/codegen/llvm_argument_memory_home.go +++ b/test/codegen/llvm_argument_memory_home.go @@ -42,19 +42,22 @@ func llvmRegisterArgumentMemoryHome(x llvmArgumentStrings3) int { return len(x.a) + len(x.b) + len(x.c) } -// Non-trivial arrays are assigned wholly to the ABI stack. Their Go SSA -// LocalAddr uses the same local-home initialization instead of reading an -// uninitialized alloca. +// Non-trivial arrays are assigned wholly to the ABI stack. The typed +// preallocated carrier binds LocalAddr directly to that incoming home, without +// constructing and initializing another local copy. // -// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) -// LLVM: [[STACK_HOME:%.*]] = alloca [2 x { ptr, i64 }], align 8 -// LLVM: store [2 x { ptr, i64 }] %x, ptr [[STACK_HOME]], align 8 +// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr preallocated([2 x { ptr, i64 }]) align 8 %x) +// LLVM-NOT: alloca +// LLVM-NOT: store +// LLVM: load { ptr, i64 }, ptr +// LLVM: load { ptr, i64 }, ptr // LLVM: ret i64 // -// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) +// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr preallocated([2 x { ptr, i64 }]) align 8{{.*}} %x) // LLVM-OPT-NOT: alloca -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 0 -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 1 +// LLVM-OPT-NOT: store +// LLVM-OPT: load i64, ptr +// LLVM-OPT: load i64, ptr // LLVM-OPT: ret i64 // // LLVM-LABEL: define goabiinternal i64 @codegen.llvmDirectRegisterArgument(i64 %x) diff --git a/test/codegen/llvm_dereference.go b/test/codegen/llvm_dereference.go index 3a109311ea48d5..3bf9d4e855cf5c 100644 --- a/test/codegen/llvm_dereference.go +++ b/test/codegen/llvm_dereference.go @@ -15,10 +15,12 @@ type llvmDereferenceLargeResult struct { values [32]int } -// LLVM-LABEL: define goabiinternal %codegen.llvmDereferenceLargeResult @codegen.llvmNamedLargeStackResult( +// LLVM-LABEL: define goabiinternal void @codegen.llvmNamedLargeStackResult( +// LLVM-SAME: i64 %seed, ptr goret(%codegen.llvmDereferenceLargeResult) align 8 "goretindex"="0" [[RESULT:%.*]]) // LLVM: call goabiinternal void @codegen.llvmFillNamedLargeStackResult( // LLVM: load %codegen.llvmDereferenceLargeResult, ptr {{%.*}}, align 8 -// LLVM: ret %codegen.llvmDereferenceLargeResult +// LLVM: call goabiinternal void @"runtime.memmove"(ptr [[RESULT]], ptr {{%.*}}, i64 256) +// LLVM: ret void // // LLVM-LABEL: define goabiinternal %codegen.llvmDereferenceAddressedResult @codegen.llvmNamedStackResult( // LLVM: call goabiinternal void @codegen.llvmFillNamedStackResult( diff --git a/test/codegen/llvm_direct_iface.go b/test/codegen/llvm_direct_iface.go index 1d29e761a6dc8c..0173b9092c40b9 100644 --- a/test/codegen/llvm_direct_iface.go +++ b/test/codegen/llvm_direct_iface.go @@ -34,7 +34,10 @@ func llvmDirectIfaceSink(llvmDirectIfaceNested) {} // LLVM: [[LEAF:%.*]] = insertvalue %codegen.llvmDirectIfaceLeaf undef, ptr [[DATA]], 0 // LLVM: [[ARRAY:%.*]] = insertvalue [1 x %codegen.llvmDirectIfaceLeaf] undef, %codegen.llvmDirectIfaceLeaf [[LEAF]], 0 // LLVM: [[NESTED:%.*]] = insertvalue %codegen.llvmDirectIfaceNested {{.*}}, [1 x %codegen.llvmDirectIfaceLeaf] [[ARRAY]], 2 -// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(%codegen.llvmDirectIfaceNested [[NESTED]]) +// LLVM: [[SETUP:%.*]] = call token @llvm.call.preallocated.setup(i32 1) +// LLVM: [[HOME:%.*]] = call ptr @llvm.call.preallocated.arg(token [[SETUP]], i32 0) +// LLVM: store %codegen.llvmDirectIfaceNested [[NESTED]], ptr [[HOME]], align 8 +// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(ptr preallocated(%codegen.llvmDirectIfaceNested) align 8 [[HOME]]) [ "preallocated"(token [[SETUP]]) ] func llvmDirectIfaceCall(x any) { switch x := x.(type) { case llvmDirectIfaceNested: diff --git a/test/codegen/llvm_linkname.go b/test/codegen/llvm_linkname.go index a497cd96dcc71f..71a25f2dcfcead 100644 --- a/test/codegen/llvm_linkname.go +++ b/test/codegen/llvm_linkname.go @@ -20,14 +20,18 @@ func llvmLinknameLocal() int { // LLVM: call goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM: declare goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-NOT: @"runtime.llvmLinknameLocal" -// LLVM-LABEL: define weak goabi0 i64 @"runtime.llvmLinknameLocal"() +// LLVM-LABEL: define weak goabi0 void @"runtime.llvmLinknameLocal"( +// LLVM-SAME: ptr goret(i64) align 8 "goretindex"="0" [[RESULT:%.*]]) // LLVM: call goabiinternal i64 @runtime.llvmLinknameLocal() +// LLVM: store i64 {{%.*}}, ptr [[RESULT]], align 8 // LLVM-LABEL: define goabiinternal i64 @runtime.llvmLinknameLocal() // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmLinknameCalls() // LLVM-OPT: call goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-OPT: declare goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-OPT-NOT: @"runtime.llvmLinknameLocal" -// LLVM-OPT-LABEL: define weak goabi0 i64 @"runtime.llvmLinknameLocal"() +// LLVM-OPT-LABEL: define weak goabi0 void @"runtime.llvmLinknameLocal"( +// LLVM-OPT-SAME: ptr goret(i64) align 8 "goretindex"="0" [[OPT_RESULT:%.*]]) +// LLVM-OPT: store i64 7, ptr [[OPT_RESULT]], align 8 // LLVM-OPT-LABEL: define goabiinternal {{.*}}@runtime.llvmLinknameLocal() func llvmLinknameCalls() int { return llvmLinknameExternal() + llvmLinknameLocal() diff --git a/test/codegen/llvm_selectnaddr.go b/test/codegen/llvm_selectnaddr.go index caaeafb43a47b2..c78e263e1d230e 100644 --- a/test/codegen/llvm_selectnaddr.go +++ b/test/codegen/llvm_selectnaddr.go @@ -10,8 +10,8 @@ type llvmAddressedCallResult [20]int // LLVM-LABEL: define goabiinternal i64 @codegen.llvmReadAddressedCallResult( // LLVM: [[HOME:%[^ ]+\.home]] = alloca [20 x i64], align 8 -// LLVM: [[RESULT:%.*]] = call goabiinternal [20 x i64] @codegen.llvmMakeAddressedCallResult(i64 %seed) -// LLVM-NEXT: store [20 x i64] [[RESULT]], ptr [[HOME]], align 8 +// LLVM: call goabiinternal void @codegen.llvmMakeAddressedCallResult(i64 %seed, ptr goret([20 x i64]) align 8 "goretindex"="0" [[HOME]]) +// LLVM-NOT: store [20 x i64] // LLVM: load i64, ptr // LLVM: ret i64 // From f53908277e1a6d9c9d2ecfdfd97ad710bd0d04f5 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 16 Aug 2026 00:26:01 +0800 Subject: [PATCH 7/8] cmd/link: externalize independent ARM64 PC-relative relocations --- src/cmd/link/internal/arm64/asm.go | 108 ++++++++++++++++++++++++ src/cmd/link/internal/arm64/asm_test.go | 33 ++++++++ src/cmd/link/internal/ld/data.go | 2 + src/cmd/link/internal/loader/loader.go | 1 + 4 files changed, 144 insertions(+) create mode 100644 src/cmd/link/internal/arm64/asm_test.go diff --git a/src/cmd/link/internal/arm64/asm.go b/src/cmd/link/internal/arm64/asm.go index 9864788e1b6125..5357b881dfa51f 100644 --- a/src/cmd/link/internal/arm64/asm.go +++ b/src/cmd/link/internal/arm64/asm.go @@ -559,6 +559,34 @@ func elfreloc1(ctxt *ld.Link, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, ldstType = elf.R_AARCH64_LDST64_ABS_LO12_NC } out.Write64(uint64(ldstType) | uint64(elfsym)<<32) + case objabi.R_ARM64_PCREL: + if siz != 4 || r.Off < 0 || int64(r.Off)+4 > int64(len(ldr.Data(s))) { + return false + } + inst := ctxt.Arch.ByteOrder.Uint32(ldr.Data(s)[r.Off:]) + kind, ok := classifyPCRelInstruction(inst) + if !ok { + ldr.Errorf(s, "unsupported instruction for %x R_ARM64_PCREL", inst) + return false + } + var relocType elf.R_AARCH64 + switch kind { + case arm64PCRelADRP: + relocType = elf.R_AARCH64_ADR_PREL_PG_HI21 + case arm64PCRelADD: + relocType = elf.R_AARCH64_ADD_ABS_LO12_NC + case arm64PCRelLDST8: + relocType = elf.R_AARCH64_LDST8_ABS_LO12_NC + case arm64PCRelLDST16: + relocType = elf.R_AARCH64_LDST16_ABS_LO12_NC + case arm64PCRelLDST32: + relocType = elf.R_AARCH64_LDST32_ABS_LO12_NC + case arm64PCRelLDST64: + relocType = elf.R_AARCH64_LDST64_ABS_LO12_NC + case arm64PCRelLDST128: + relocType = elf.R_AARCH64_LDST128_ABS_LO12_NC + } + out.Write64(uint64(relocType) | uint64(elfsym)<<32) case objabi.R_ARM64_TLS_LE: out.Write64(uint64(elf.R_AARCH64_TLSLE_MOVW_TPREL_G0) | uint64(elfsym)<<32) @@ -588,6 +616,43 @@ func elfreloc1(ctxt *ld.Link, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, func signext21(x int64) int64 { return x << (64 - 21) >> (64 - 21) } func signext24(x int64) int64 { return x << (64 - 24) >> (64 - 24) } +type arm64PCRelInstruction uint8 + +const ( + arm64PCRelADRP arm64PCRelInstruction = iota + arm64PCRelADD + arm64PCRelLDST8 + arm64PCRelLDST16 + arm64PCRelLDST32 + arm64PCRelLDST64 + arm64PCRelLDST128 +) + +func classifyPCRelInstruction(inst uint32) (arm64PCRelInstruction, bool) { + switch { + case (inst>>24)&0x9f == 0x90: + return arm64PCRelADRP, true + case (inst>>24)&0x9f == 0x91: + return arm64PCRelADD, true + case (inst>>24)&0x3b == 0x39: + shift := inst >> 30 + if shift == 0 && (inst>>20)&0x048 == 0x048 { + return arm64PCRelLDST128, true + } + switch shift { + case 0: + return arm64PCRelLDST8, true + case 1: + return arm64PCRelLDST16, true + case 2: + return arm64PCRelLDST32, true + case 3: + return arm64PCRelLDST64, true + } + } + return 0, false +} + func machoreloc1(arch *sys.Arch, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, r loader.ExtReloc, sectoff int64) bool { var v uint32 @@ -646,6 +711,26 @@ func machoreloc1(arch *sys.Arch, out *ld.OutBuf, ldr *loader.Loader, s loader.Sy } v |= 1 << 24 // pc-relative bit v |= ld.MACHO_ARM64_RELOC_BRANCH26 << 28 + case objabi.R_ARM64_PCREL: + if siz != 4 || r.Off < 0 || int64(r.Off)+4 > int64(len(ldr.Data(s))) { + return false + } + inst := arch.ByteOrder.Uint32(ldr.Data(s)[r.Off:]) + kind, ok := classifyPCRelInstruction(inst) + if !ok { + ldr.Errorf(s, "unsupported instruction for %x R_ARM64_PCREL", inst) + return false + } + if xadd != 0 { + out.Write32(uint32(sectoff)) + out.Write32((ld.MACHO_ARM64_RELOC_ADDEND << 28) | (2 << 25) | uint32(xadd&0xffffff)) + } + if kind == arm64PCRelADRP { + v |= 1 << 24 + v |= ld.MACHO_ARM64_RELOC_PAGE21 << 28 + } else { + v |= ld.MACHO_ARM64_RELOC_PAGEOFF12 << 28 + } case objabi.R_ADDRARM64, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, @@ -795,6 +880,25 @@ func archreloc(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, r loade nExtReloc := 0 switch rt := r.Type(); rt { default: + case objabi.R_ARM64_PCREL: + // PE currently resolves independent instruction relocations in + // the Go linker. ELF and Mach-O external links may move the final + // target section, so preserve each instruction relocation for the + // platform linker instead of baking in the preliminary layout. + if target.IsWindows() { + break + } + rs, off := ld.FoldSubSymbolOffset(ldr, rs) + xadd := r.Add() + off + rst := ldr.SymType(rs) + if rst != sym.SHOSTOBJ && rst != sym.SDYNIMPORT && ldr.SymSect(rs) == nil { + ldr.Errorf(s, "missing section for %s", ldr.SymName(rs)) + } + nExtReloc = 1 + if target.IsDarwin() && xadd != 0 { + nExtReloc = 2 + } + return val, nExtReloc, isOk case objabi.R_ARM64_GOTPCREL, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, @@ -1127,6 +1231,10 @@ func archrelocvariant(*ld.Target, *loader.Loader, loader.Reloc, sym.RelocVariant func extreloc(target *ld.Target, ldr *loader.Loader, r loader.Reloc, s loader.Sym) (loader.ExtReloc, bool) { switch rt := r.Type(); rt { + case objabi.R_ARM64_PCREL: + if !target.IsWindows() { + return ld.ExtrelocViaOuterSym(ldr, r, s), true + } case objabi.R_ARM64_GOTPCREL, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, diff --git a/src/cmd/link/internal/arm64/asm_test.go b/src/cmd/link/internal/arm64/asm_test.go new file mode 100644 index 00000000000000..89c87e59d2d058 --- /dev/null +++ b/src/cmd/link/internal/arm64/asm_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package arm64 + +import "testing" + +func TestClassifyPCRelInstruction(t *testing.T) { + tests := []struct { + name string + inst uint32 + want arm64PCRelInstruction + ok bool + }{ + {"adrp", 0x90000008, arm64PCRelADRP, true}, + {"add", 0x91000008, arm64PCRelADD, true}, + {"ldrb", 0x39400100, arm64PCRelLDST8, true}, + {"ldrh", 0x79400100, arm64PCRelLDST16, true}, + {"ldrw", 0xb9400100, arm64PCRelLDST32, true}, + {"ldrx", 0xf9400100, arm64PCRelLDST64, true}, + {"strq", 0x3d800000, arm64PCRelLDST128, true}, + {"nop", 0xd503201f, 0, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := classifyPCRelInstruction(test.inst) + if got != test.want || ok != test.ok { + t.Fatalf("classifyPCRelInstruction(%#08x) = (%d, %t), want (%d, %t)", test.inst, got, ok, test.want, test.ok) + } + }) + } +} diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 4a27db5a42ef88..00614f698d0615 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -750,6 +750,7 @@ func ExtrelocSimple(ldr *loader.Loader, r loader.Reloc) loader.ExtReloc { rs := r.Sym() rr.Xsym = rs rr.Xadd = r.Add() + rr.Off = r.Off() rr.Type = r.Type() rr.Size = r.Siz() return rr @@ -768,6 +769,7 @@ func ExtrelocViaOuterSym(ldr *loader.Loader, r loader.Reloc, s loader.Sym) loade ldr.Errorf(s, "missing section for %s", ldr.SymName(rs)) } rr.Xsym = rs + rr.Off = r.Off() rr.Type = r.Type() rr.Size = r.Siz() return rr diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 04601b32a00af6..b07cb4ae26b651 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -45,6 +45,7 @@ type Relocs struct { type ExtReloc struct { Xsym Sym Xadd int64 + Off int32 // offset within the source symbol Type objabi.RelocType Size uint8 } From 511d70282af56fcb1ffbbf3c76c3f335fcc9aa48 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Sun, 16 Aug 2026 00:38:45 +0800 Subject: [PATCH 8/8] test: match LLVM abs lowering by architecture --- test/codegen/llvm_abs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/codegen/llvm_abs.go b/test/codegen/llvm_abs.go index 90ce70838c19eb..baa5ece4958a92 100644 --- a/test/codegen/llvm_abs.go +++ b/test/codegen/llvm_abs.go @@ -9,7 +9,8 @@ package codegen import "math" // LLVM-LABEL: define goabiinternal double @codegen.llvmAbs64(double %x) -// LLVM: call double @llvm.fabs.f64(double %x) +// LLVM-ARM64: call double @llvm.fabs.f64(double %x) +// LLVM-AMD64: and i64 {{.*}}, 9223372036854775807 // LLVM-OPT-LABEL: define goabiinternal double @codegen.llvmAbs64(double %x) // LLVM-OPT: call double @llvm.fabs.f64(double %x) func llvmAbs64(x float64) float64 {