diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 1939d5c9ce8774..5549fa90ad6b87 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -10,6 +10,7 @@ import ( "cmd/internal/src" "fmt" "internal/buildcfg" + "strconv" "strings" "github.com/goallc/go-llvm" @@ -21,6 +22,7 @@ type LLVMFuncContext struct { Locals map[llvmLocalKey]llvmStackSlot AddressedResults map[ID][]llvmAddressedResult ResultSlots map[ID]llvm.Value + CallResultSlots map[llvmCallResultKey]llvmStackSlot ItabMethods map[ID]bool ClosureCodeLoads map[ID]bool DeferResults map[llvmLocalKey]bool @@ -39,6 +41,9 @@ type LLVMFuncContext struct { b llvm.Builder ReturnType llvm.Type ResultCount int + ReturnCount int + Params []llvmParamSignature + Results []llvmResultSignature } // SSA may clone an ir.Name while retaining the same logical source @@ -61,6 +66,11 @@ type llvmAddressedResult struct { Owner *Value } +type llvmCallResultKey struct { + Call ID + Index int64 +} + // LLVM's GoABIInternal calling convention has numeric ID 22. Keep the // prototype lowering on the Go register ABI so llc emits GoObj symbols that // the standard Go linker can call directly. @@ -96,10 +106,27 @@ type llvmFuncSignature struct { Type llvm.Type ReturnType llvm.Type ResultCount int + ReturnCount int + Params []llvmParamSignature + Results []llvmResultSignature HasClosureContext bool ClosureContextIndex int } +type llvmParamSignature struct { + ValueType llvm.Type + Alignment int + InMemory bool +} + +type llvmResultSignature struct { + ValueType llvm.Type + Alignment int + InMemory bool + ReturnIndex int + ParamIndex int +} + func llvmCallConv(which obj.ABI) llvm.CallConv { switch which { case obj.ABI0: @@ -165,15 +192,48 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { } params := make([]llvm.Type, 0, aux.NArgs()) + paramSignatures := make([]llvmParamSignature, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { - param := getLLVMABIType(aux.TypeOfArg(i)) - params = append(params, param) + goType := aux.TypeOfArg(i) + valueType := getLLVMABIType(goType) + paramType := valueType + param := llvmParamSignature{ValueType: valueType} + assignment := aux.ABIInfo().InParam(int(i)) + if len(assignment.Registers) == 0 && goType.Size() != 0 { + if goType.Alignment() <= 0 { + base.Fatalf("invalid alignment %d for stack argument %d of type %v", goType.Alignment(), i, goType) + } + param.InMemory = true + param.Alignment = int(goType.Alignment()) + paramType = GlobalCtxt.PointerType(0) + } + params = append(params, paramType) + paramSignatures = append(paramSignatures, param) } results := make([]llvm.Type, 0, aux.NResults()) + resultSignatures := make([]llvmResultSignature, 0, aux.NResults()) for i := int64(0); i < aux.NResults(); i++ { - result := getLLVMABIType(aux.TypeOfResult(i)) - results = append(results, result) + goType := aux.TypeOfResult(i) + result := llvmResultSignature{ + ValueType: getLLVMABIType(goType), + ReturnIndex: -1, + ParamIndex: -1, + } + assignment := aux.ABIInfo().OutParam(int(i)) + if len(assignment.Registers) == 0 && goType.Size() != 0 { + if goType.Alignment() <= 0 { + base.Fatalf("invalid alignment %d for stack result %d of type %v", goType.Alignment(), i, goType) + } + result.InMemory = true + result.Alignment = int(goType.Alignment()) + result.ParamIndex = len(params) + params = append(params, GlobalCtxt.PointerType(0)) + } else { + result.ReturnIndex = len(results) + results = append(results, result.ValueType) + } + resultSignatures = append(resultSignatures, result) } var ret llvm.Type @@ -188,7 +248,10 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { return llvmFuncSignature{ Type: llvm.FunctionType(ret, params, false), ReturnType: ret, - ResultCount: len(results), + ResultCount: len(resultSignatures), + ReturnCount: len(results), + Params: paramSignatures, + Results: resultSignatures, ClosureContextIndex: -1, } } @@ -210,6 +273,26 @@ func llvmNestAttribute() llvm.Attribute { return GlobalCtxt.CreateEnumAttribute(kind, 0) } +func llvmPreallocatedAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("preallocated") + if kind == 0 { + base.Fatalf("LLVM does not provide the preallocated parameter attribute") + } + return GlobalCtxt.CreateTypeAttribute(kind, t) +} + +func llvmGoRetAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("goret") + if kind == 0 { + base.Fatalf("LLVM does not provide the goret parameter attribute") + } + return GlobalCtxt.CreateTypeAttribute(kind, t) +} + +func llvmGoRetIndexAttribute(index int) llvm.Attribute { + return GlobalCtxt.CreateStringAttribute("goretindex", strconv.Itoa(index)) +} + func llvmNullPointerIsValidAttribute() llvm.Attribute { kind := llvm.AttributeKindID("null_pointer_is_valid") if kind == 0 { @@ -228,9 +311,24 @@ func llvmNoInlineAttribute() llvm.Attribute { func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallConv) { fn.SetFunctionCallConv(cc) - if sig.ResultCount > 1 { + if sig.ReturnCount > 1 { fn.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + for i, param := range sig.Params { + if !param.InMemory { + continue + } + fn.AddAttributeAtIndex(i+1, llvmPreallocatedAttribute(param.ValueType)) + fn.Param(i).SetParamAlignment(param.Alignment) + } + for index, result := range sig.Results { + if !result.InMemory { + continue + } + fn.AddAttributeAtIndex(result.ParamIndex+1, llvmGoRetAttribute(result.ValueType)) + fn.AddAttributeAtIndex(result.ParamIndex+1, llvmGoRetIndexAttribute(index)) + fn.Param(result.ParamIndex).SetParamAlignment(result.Alignment) + } if sig.HasClosureContext { // LLVM parameter attribute indexes are one-based. The closure context // is deliberately excluded from the Go ABI argument layout by the @@ -240,9 +338,24 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon } func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { - if sig.ResultCount > 1 { + if sig.ReturnCount > 1 { call.AddCallSiteAttribute(llvmAttributeFunctionIndex, GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + for i, param := range sig.Params { + if !param.InMemory { + continue + } + call.AddCallSiteAttribute(i+1, llvmPreallocatedAttribute(param.ValueType)) + call.SetInstrParamAlignment(i+1, param.Alignment) + } + for index, result := range sig.Results { + if !result.InMemory { + continue + } + call.AddCallSiteAttribute(result.ParamIndex+1, llvmGoRetAttribute(result.ValueType)) + call.AddCallSiteAttribute(result.ParamIndex+1, llvmGoRetIndexAttribute(index)) + call.SetInstrParamAlignment(result.ParamIndex+1, result.Alignment) + } } func llvmFunctionStorageName(name string, cc llvm.CallConv) string { @@ -319,6 +432,19 @@ func (lfc *LLVMFuncContext) llvmLifetimeStart(slot llvmStackSlot) { lfc.b.CreateCall(sig, fn, []llvm.Value{slot.Value}, "") } +func (lfc *LLVMFuncContext) llvmLifetimeEnd(slot llvmStackSlot) { + if slot.Value.IsAAllocaInst().IsNil() { + return + } + sig := llvm.FunctionType( + GlobalCtxt.VoidType(), + []llvm.Type{GlobalCtxt.PointerType(0)}, + false, + ) + fn := getOrInsertLLVMIntrinsic("llvm.lifetime.end.p0", sig) + lfc.b.CreateCall(sig, fn, []llvm.Value{slot.Value}, "") +} + func (lfc *LLVMFuncContext) llvmKeepAlive(value llvm.Value) { // An operand bundle is a real SSA use that follows the call through // inlining. llvm.donothing survives long enough for the statepoint pass to @@ -1080,6 +1206,33 @@ func (lfc *LLVMFuncContext) llvmRuntimeMemmove(dst, src, length llvm.Value) llvm return call } +func (lfc *LLVMFuncContext) llvmCopyFixedMemory(dst, src llvm.Value, size int64, align int) llvm.Value { + lengthType := getLLVMType(types.Types[types.TUINTPTR]) + length := llvm.ConstInt(lengthType, uint64(size), false) + if size > llvmInlineMemmoveLimit { + return lfc.llvmRuntimeMemmove(dst, src, length) + } + sig := llvm.FunctionType( + GlobalCtxt.VoidType(), + []llvm.Type{dst.Type(), src.Type(), length.Type(), GlobalCtxt.Int1Type()}, + false, + ) + name := "llvm.memmove.p0.p0.i64" + if length.Type().IntTypeWidth() == 32 { + name = "llvm.memmove.p0.p0.i32" + } + fn := getOrInsertLLVMIntrinsic(name, sig) + call := lfc.b.CreateCall(sig, fn, []llvm.Value{ + dst, + src, + length, + llvm.ConstInt(GlobalCtxt.Int1Type(), 0, false), + }, "") + call.SetInstrParamAlignment(1, align) + call.SetInstrParamAlignment(2, align) + return call +} + func (lfc *LLVMFuncContext) llvmMove(v *Value) llvm.Value { size, align := llvmMemoryOpInfo(v) dst := lfc.llvmMemoryPointer(v, 0) @@ -1130,6 +1283,7 @@ func (lfc *LLVMFuncContext) llvmMemEq(v *Value) llvm.Value { Type: llvm.FunctionType(boolType, []llvm.Type{left.Type(), right.Type(), uintptrType}, false), ReturnType: boolType, ResultCount: 1, + ReturnCount: 1, ClosureContextIndex: -1, } fn := getOrInsertLLVMABISymbolRef("runtime.memequal", obj.ABIInternal, sig, goABIInternalCallConv) @@ -1552,13 +1706,118 @@ func (lfc *LLVMFuncContext) paramForArgNameAndType(name *ir.Name) (llvm.Value, * key := llvmLocalKeyForName(name) for i, param := range lfc.F.OwnAux.ABIInfo().InParams() { if param.Name != nil && llvmLocalKeyForName(param.Name) == key { - return lfc.LF.Param(i), lfc.F.OwnAux.TypeOfArg(int64(i)) + value := lfc.LF.Param(i) + if lfc.Params[i].InMemory { + value = lfc.b.CreateLoad(lfc.Params[i].ValueType, value, name.Sym().Name+".memory") + value.SetAlignment(lfc.Params[i].Alignment) + } + return value, lfc.F.OwnAux.TypeOfArg(int64(i)) } } lfc.F.fe.Fatalf(name.Pos(), "could not find LLVM parameter for %v", name) return llvm.Value{}, nil } +func llvmMemoryParameterCount(sig llvmFuncSignature) int { + count := 0 + for _, param := range sig.Params { + if param.InMemory { + count++ + } + } + return count +} + +func (lfc *LLVMFuncContext) llvmPreallocatedCallSetup(sig llvmFuncSignature) llvm.Value { + count := llvmMemoryParameterCount(sig) + if count == 0 { + return llvm.Value{} + } + fn := getLLVMIntrinsicDeclaration("llvm.call.preallocated.setup") + return lfc.b.CreateCall(fn.GlobalValueType(), fn, []llvm.Value{ + llvm.ConstInt(GlobalCtxt.Int32Type(), uint64(count), false), + }, "preallocated.setup") +} + +func (lfc *LLVMFuncContext) llvmPreallocatedCallArgument(v, argValue *Value, index, carrierIndex int, logical *types.Type, param llvmParamSignature, setup llvm.Value) llvm.Value { + if !param.InMemory || logical.Size() == 0 || setup.IsNil() { + v.Fatalf("argument %d is not a non-empty memory parameter", index) + } + fn := getLLVMIntrinsicDeclaration("llvm.call.preallocated.arg") + address := lfc.b.CreateCall(fn.GlobalValueType(), fn, []llvm.Value{ + setup, + llvm.ConstInt(GlobalCtxt.Int32Type(), uint64(carrierIndex), false), + }, fmt.Sprintf("%s.arg%d.home", v, index)) + address.AddCallSiteAttribute(llvmAttributeFunctionIndex, llvmPreallocatedAttribute(param.ValueType)) + + // Go SSA keeps non-SSA-able aggregate call arguments in memory and exposes + // the value through a Load or Dereference. Copy those bytes directly into + // the ABI-defined outgoing home instead of constructing a first-class value. + if types.Identical(argValue.Type, logical) && + (argValue.Op == OpLoad || argValue.Op == OpDereference) && len(argValue.Args) != 0 { + source := lfc.GenLV(argValue.Args[0]) + if source.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("memory argument %d has non-pointer source address", index) + } + lfc.llvmCopyFixedMemory(address, source, logical.Size(), param.Alignment) + return address + } + + value := lfc.GenLV(argValue) + value = lfc.llvmValueToABI(v, value, argValue.Type, logical, param.ValueType, fmt.Sprintf("%s.arg%d", v, index)) + if value.Type() != param.ValueType { + v.Fatalf("memory argument %d has incompatible LLVM value type", index) + } + store := lfc.b.CreateStore(value, address) + store.SetAlignment(param.Alignment) + return address +} + +func (lfc *LLVMFuncContext) llvmMemoryResultCallArguments(v *Value, sig llvmFuncSignature, aux *AuxCall) []llvm.Value { + args := make([]llvm.Value, 0, sig.ResultCount-sig.ReturnCount) + for index, result := range sig.Results { + if !result.InMemory { + continue + } + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: v.ID, Index: int64(index)}] + if !ok || !types.Identical(slot.Type, aux.TypeOfResult(int64(index))) { + v.Fatalf("memory result %d has no compatible caller-owned home", index) + } + // A call reinitializes its caller-owned result home. Keep future result + // homes out of earlier statepoints while allowing SelectN and + // SelectNAddr uses after this call to determine the actual live range. + lfc.llvmLifetimeStart(slot) + args = append(args, slot.Value) + } + return args +} + +func (lfc *LLVMFuncContext) storeMemoryResult(v, value *Value, index int) { + result := lfc.Results[index] + if !result.InMemory || result.ParamIndex < 0 { + v.Fatalf("result %d is not assigned to memory", index) + } + logical := lfc.F.OwnAux.TypeOfResult(int64(index)) + dst := lfc.LF.Param(result.ParamIndex) + if types.Identical(value.Type, logical) && + (value.Op == OpLoad || value.Op == OpDereference) && len(value.Args) != 0 { + src := lfc.GenLV(value.Args[0]) + if src.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("memory result %d has a non-pointer source address", index) + } + lfc.llvmCopyFixedMemory(dst, src, logical.Size(), result.Alignment) + return + } + + lVal := lfc.GenLV(value) + lVal = lfc.llvmValueToABI(v, lVal, value.Type, logical, result.ValueType, fmt.Sprintf("%s.result%d", v, index)) + if lVal.Type() != result.ValueType { + v.Fatalf("memory result %d has incompatible LLVM value type", index) + } + store := lfc.b.CreateStore(lVal, dst) + store.SetAlignment(result.Alignment) +} + func (lfc *LLVMFuncContext) registerArgument(v *Value) llvm.Value { aux, ok := v.Aux.(*AuxNameOffset) if !ok || aux.Name == nil { @@ -1913,19 +2172,38 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { // statepoints. llvmGCLeaf := aux.Fn == ir.Syms.WBZero || aux.Fn == ir.Syms.WBMove || aux.Fn == ir.Syms.Memmove || aux.Fn == ir.Syms.Memequal - args := make([]llvm.Value, 0, aux.NArgs()) + args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) + setup := lfc.llvmPreallocatedCallSetup(sig) + carrierIndex := 0 for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[i]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + var arg llvm.Value + if sig.Params[i].InMemory { + arg = lfc.llvmPreallocatedCallArgument(v, v.Args[i], int(i), carrierIndex, aux.TypeOfArg(i), sig.Params[i], setup) + carrierIndex++ + } else { + arg = lfc.GenLV(v.Args[i]) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } + } + if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { + v.Fatalf("argument %d to %s has incompatible LLVM type", i, aux.Fn.Name) } args = append(args, arg) } + args = append(args, lfc.llvmMemoryResultCallArguments(v, sig, aux)...) name := v.String() - if sig.ResultCount == 0 { + if sig.ReturnCount == 0 { name = "" } - call := lfc.b.CreateCall(sig.Type, fn, args, name) + var call llvm.Value + if setup.IsNil() { + call = lfc.b.CreateCall(sig.Type, fn, args, name) + } else { + bundle := llvm.NewOperandBundle("preallocated", []llvm.Value{setup}) + call = lfc.b.CreateCallWithOperandBundles(sig.Type, fn, args, []llvm.OperandBundle{bundle}, name) + bundle.Dispose() + } call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) lfc.materializeAddressedResults(v, call, aux) @@ -1962,17 +2240,27 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext if code.Type().TypeKind() != llvm.PointerTypeKind { v.Fatalf("indirect callee has non-pointer LLVM type") } - args := make([]llvm.Value, 0, aux.NArgs()) + args := make([]llvm.Value, 0, len(sig.Type.ParamTypes())) + setup := lfc.llvmPreallocatedCallSetup(sig) + carrierIndex := 0 for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[argStart+int(i)]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[argStart+int(i)].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + argValue := v.Args[argStart+int(i)] + var arg llvm.Value + if sig.Params[i].InMemory { + arg = lfc.llvmPreallocatedCallArgument(v, argValue, int(i), carrierIndex, aux.TypeOfArg(i), sig.Params[i], setup) + carrierIndex++ + } else { + arg = lfc.GenLV(argValue) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, argValue.Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } } if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { v.Fatalf("argument %d to indirect call has incompatible LLVM type", i) } args = append(args, arg) } + args = append(args, lfc.llvmMemoryResultCallArguments(v, sig, aux)...) if closureContext { context := lfc.GenLV(v.Args[1]) if context.Type().TypeKind() != llvm.PointerTypeKind { @@ -1981,10 +2269,17 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext args = append(args, context) } name := v.String() - if sig.ResultCount == 0 { + if sig.ReturnCount == 0 { name = "" } - call := lfc.b.CreateCall(sig.Type, code, args, name) + var call llvm.Value + if setup.IsNil() { + call = lfc.b.CreateCall(sig.Type, code, args, name) + } else { + bundle := llvm.NewOperandBundle("preallocated", []llvm.Value{setup}) + call = lfc.b.CreateCallWithOperandBundles(sig.Type, code, args, []llvm.OperandBundle{bundle}, name) + bundle.Dispose() + } call.SetInstructionCallConv(cc) configureLLVMCall(call, sig) lfc.materializeAddressedResults(v, call, aux) @@ -2002,13 +2297,18 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext // alignment while allowing ordinary LLVM promotion to remove unnecessary // homes. func (lfc *LLVMFuncContext) materializeAddressedResults(v *Value, call llvm.Value, aux *AuxCall) { + sig := llvmSignature(aux) for _, result := range lfc.AddressedResults[v.ID] { if result.Slot.Type.HasPointers() { lfc.llvmLifetimeStart(result.Slot) } + resultSig := sig.Results[result.Index] + if resultSig.InMemory || resultSig.ReturnIndex < 0 { + result.Owner.Fatalf("addressed register result was assigned to memory") + } value := call - if aux.NResults() > 1 { - value = lfc.b.CreateExtractValue(call, int(result.Index), result.Owner.String()+".value") + if sig.ReturnCount > 1 { + value = lfc.b.CreateExtractValue(call, resultSig.ReturnIndex, result.Owner.String()+".value") } value = lfc.llvmValueFromABI(result.Owner, value, aux.TypeOfResult(result.Index), result.Slot.Type, result.Owner.String()+".reshape") if got, want := value.Type(), getLLVMType(result.Slot.Type); got != want { @@ -2553,10 +2853,21 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { case sel >= int(aux.NResults()): // Selecting the trailing SSA memory dependency only forces the // call to be emitted; it has no LLVM value. - case aux.NResults() == 1: - lVal = call default: - lVal = lfc.b.CreateExtractValue(call, sel, v.String()) + sig := llvmSignature(aux) + result := sig.Results[sel] + if result.InMemory { + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: src.ID, Index: int64(sel)}] + if !ok { + v.Fatalf("memory result %d has no caller-owned home", sel) + } + lVal = lfc.b.CreateLoad(result.ValueType, slot.Value, v.String()) + lVal.SetAlignment(result.Alignment) + } else if sig.ReturnCount == 1 { + lVal = call + } else { + lVal = lfc.b.CreateExtractValue(call, result.ReturnIndex, v.String()) + } } if sel < int(aux.NResults()) { lVal = lfc.llvmValueFromABI(v, lVal, aux.TypeOfResult(int64(sel)), v.Type, v.String()+".reshape") @@ -2590,16 +2901,25 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { v.Fatalf("addressed call result has no LLVM memory home") } case OpMakeResult: - switch lfc.ResultCount { + direct := make([]llvm.Value, lfc.ReturnCount) + for i, result := range lfc.Results { + if result.InMemory { + lfc.storeMemoryResult(v, v.Args[i], i) + continue + } + direct[result.ReturnIndex] = lfc.llvmValueToABI( + v, lfc.GenLV(v.Args[i]), v.Args[i].Type, + lfc.F.OwnAux.TypeOfResult(int64(i)), result.ValueType, + fmt.Sprintf("%s.result%d", v, i), + ) + } + switch lfc.ReturnCount { case 0: case 1: - lVal = lfc.GenLV(v.Args[0]) - lVal = lfc.llvmValueToABI(v, lVal, v.Args[0].Type, lfc.F.OwnAux.TypeOfResult(0), lfc.ReturnType, v.String()+".result0") + lVal = direct[0] default: lVal = llvm.Undef(lfc.ReturnType) - for i := 0; i < lfc.ResultCount; i++ { - resultType := lfc.ReturnType.StructElementTypes()[i] - result := lfc.llvmValueToABI(v, lfc.GenLV(v.Args[i]), v.Args[i].Type, lfc.F.OwnAux.TypeOfResult(int64(i)), resultType, fmt.Sprintf("%s.result%d", v, i)) + for i, result := range direct { lVal = lfc.b.CreateInsertValue(lVal, result, i, "") } lVal.SetName(v.String()) @@ -2793,7 +3113,7 @@ func (lfc *LLVMFuncContext) CompileBlock(BB *Block, values []*Value) { defer lfc.b.ClearCurrentDebugLocation() switch BB.Kind { case BlockRet: - if lfc.ResultCount == 0 { + if lfc.ReturnCount == 0 { lfc.b.CreateRetVoid() } else { lfc.b.CreateRet(lfc.GenLV(BB.Controls[0])) @@ -2866,17 +3186,13 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { if len(outParams) != lfc.ResultCount { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result count %d does not match LLVM signature result count %d", len(outParams), lfc.ResultCount) } - results := make([]llvm.Value, len(outParams)) + results := make([]llvm.Value, lfc.ReturnCount) reshapeContext := &Value{Block: lfc.F.Entry, Pos: lfc.F.Entry.Pos} for i, result := range outParams { - var abiType llvm.Type - if lfc.ResultCount == 1 { - abiType = lfc.ReturnType - } else { - abiType = lfc.ReturnType.StructElementTypes()[i] - } + resultSig := lfc.Results[i] + abiType := resultSig.ValueType if result.Type.Size() == 0 { - results[i] = llvm.Undef(abiType) + results[resultSig.ReturnIndex] = llvm.Undef(abiType) continue } if result.Name == nil { @@ -2893,10 +3209,15 @@ func (lfc *LLVMFuncContext) emitOpenDeferRecovery() { if value.Type() != abiType { lfc.F.fe.Fatalf(lfc.F.Entry.Pos, "open-coded defer result %d has incompatible LLVM ABI type", i) } - results[i] = value + if resultSig.InMemory { + store := lfc.b.CreateStore(value, lfc.LF.Param(resultSig.ParamIndex)) + store.SetAlignment(resultSig.Alignment) + } else { + results[resultSig.ReturnIndex] = value + } } - switch len(results) { + switch lfc.ReturnCount { case 0: lfc.b.CreateRetVoid() case 1: @@ -2935,22 +3256,51 @@ func (lfc *LLVMFuncContext) emitTailCallReturn(b *Block) { } result := lfc.GenLV(call) - switch lfc.ResultCount { + calleeSig := llvmSignature(aux) + direct := make([]llvm.Value, lfc.ReturnCount) + for i, callerResult := range lfc.Results { + calleeResult := calleeSig.Results[i] + if callerResult.InMemory && calleeResult.InMemory { + src, ok := lfc.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(i)}] + if !ok { + call.Fatalf("memory result %d has no caller-owned home", i) + } + lfc.llvmCopyFixedMemory(lfc.LF.Param(callerResult.ParamIndex), src.Value, + lfc.F.OwnAux.TypeOfResult(int64(i)).Size(), callerResult.Alignment) + continue + } + + var field llvm.Value + if calleeResult.InMemory { + slot, ok := lfc.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(i)}] + if !ok { + call.Fatalf("memory result %d has no caller-owned home", i) + } + field = lfc.b.CreateLoad(calleeResult.ValueType, slot.Value, fmt.Sprintf("%s.return%d.load", call, i)) + field.SetAlignment(calleeResult.Alignment) + } else if calleeSig.ReturnCount == 1 { + field = result + } else { + field = lfc.b.CreateExtractValue(result, calleeResult.ReturnIndex, fmt.Sprintf("%s.return%d.extract", call, i)) + } + field = lfc.llvmValueFromABI(call, field, aux.TypeOfResult(int64(i)), aux.TypeOfResult(int64(i)), fmt.Sprintf("%s.return%d.fromabi", call, i)) + field = lfc.llvmValueToABI(call, field, aux.TypeOfResult(int64(i)), lfc.F.OwnAux.TypeOfResult(int64(i)), callerResult.ValueType, fmt.Sprintf("%s.return%d", call, i)) + if callerResult.InMemory { + store := lfc.b.CreateStore(field, lfc.LF.Param(callerResult.ParamIndex)) + store.SetAlignment(callerResult.Alignment) + } else { + direct[callerResult.ReturnIndex] = field + } + } + + switch lfc.ReturnCount { case 0: lfc.b.CreateRetVoid() case 1: - result = lfc.llvmValueFromABI(call, result, aux.TypeOfResult(0), aux.TypeOfResult(0), call.String()+".return.fromabi") - result = lfc.llvmValueToABI(call, result, aux.TypeOfResult(0), lfc.F.OwnAux.TypeOfResult(0), lfc.ReturnType, call.String()+".return") - if result.Type() != lfc.ReturnType { - call.Fatalf("tail-call result has incompatible LLVM return type") - } - lfc.b.CreateRet(result) + lfc.b.CreateRet(direct[0]) default: ret := llvm.Undef(lfc.ReturnType) - for i := 0; i < lfc.ResultCount; i++ { - field := lfc.b.CreateExtractValue(result, i, fmt.Sprintf("%s.return%d.extract", call, i)) - field = lfc.llvmValueFromABI(call, field, aux.TypeOfResult(int64(i)), aux.TypeOfResult(int64(i)), fmt.Sprintf("%s.return%d.fromabi", call, i)) - field = lfc.llvmValueToABI(call, field, aux.TypeOfResult(int64(i)), lfc.F.OwnAux.TypeOfResult(int64(i)), lfc.ReturnType.StructElementTypes()[i], fmt.Sprintf("%s.return%d", call, i)) + for i, field := range direct { ret = lfc.b.CreateInsertValue(ret, field, i, fmt.Sprintf("%s.return%d.insert", call, i)) } lfc.b.CreateRet(ret) @@ -2989,6 +3339,7 @@ func LLVMCompile(f *Func) { Locals: map[llvmLocalKey]llvmStackSlot{}, AddressedResults: map[ID][]llvmAddressedResult{}, ResultSlots: map[ID]llvm.Value{}, + CallResultSlots: map[llvmCallResultKey]llvmStackSlot{}, ItabMethods: map[ID]bool{}, ClosureCodeLoads: map[ID]bool{}, DeferResults: map[llvmLocalKey]bool{}, @@ -2999,6 +3350,9 @@ func LLVMCompile(f *Func) { b: GlobalCtxt.NewBuilder(), ReturnType: sig.ReturnType, ResultCount: sig.ResultCount, + ReturnCount: sig.ReturnCount, + Params: sig.Params, + Results: sig.Results, } defer FCtxt.b.Dispose() @@ -3087,6 +3441,11 @@ func LLVMCompile(f *Func) { FCtxt.LF.Param(i).SetName(param.Name.Sym().Name) } } + for i, result := range sig.Results { + if result.InMemory { + FCtxt.LF.Param(result.ParamIndex).SetName(fmt.Sprintf(".result%d", i)) + } + } FCtxt.LF.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goAsyncUnsafeAttr, "")) // A //go:nosplit function must not acquire that late morestack edge. Besides // violating the runtime's nosplit call graph, it would expose a safepoint the @@ -3186,6 +3545,31 @@ func LLVMCompile(f *Func) { FCtxt.Locals[key] = llvmStackSlot{Value: value, Type: name.Type()} } } + // A non-empty parameter assigned wholly to the Go stack is represented as a + // typed LLVM preallocated pointer. That pointer is the parameter's ABI-defined + // incoming home, so bind addressable Go SSA uses to it directly. Register + // parameters retain the existing compiler-owned alloca/store path. + for i, param := range sig.Params { + if !param.InMemory { + continue + } + assignment := inParams[i] + if len(assignment.Registers) != 0 { + f.fe.Fatalf(f.Entry.Pos, "LLVM memory parameter %d was assigned Go registers", i) + } + if assignment.Name == nil { + continue + } + goType := f.OwnAux.TypeOfArg(int64(i)) + if goType.Size() == 0 || !types.Identical(goType, assignment.Name.Type()) { + f.fe.Fatalf(assignment.Name.Pos(), "invalid Go type for LLVM memory parameter %v", assignment.Name) + } + key := llvmLocalKeyForName(assignment.Name) + if _, exists := FCtxt.Locals[key]; exists { + f.fe.Fatalf(assignment.Name.Pos(), "duplicate LLVM memory parameter home %v", assignment.Name) + } + FCtxt.Locals[key] = llvmStackSlot{Value: FCtxt.LF.Param(i), Type: assignment.Name.Type()} + } isDeferResultLocal := func(name *ir.Name) bool { return frontendFunc != nil && frontendFunc.HasDefer() && (name.Class == ir.PPARAMOUT || name.IsOutputParamHeapAddr()) @@ -3298,16 +3682,38 @@ func LLVMCompile(f *Func) { } } } - // SelectNAddr denotes an address into a call's outgoing result area. LLVM's - // Go calling convention reconstructs stack-assigned results as first-class - // return values, so reserve equivalent fixed entry-block homes before any - // call or phi emission. Multiple selectors of one call result share a slot, - // matching the aliasing of the native ABI result area. - type addressedResultKey struct { - Call ID - Index int64 + // Results that Go assigned to memory use caller-owned typed goret carriers. + // Reserve their stable destinations before call emission; ordinary SelectN + // loads from the same object that SelectNAddr exposes by address. + for _, BB := range f.Blocks { + for _, call := range BB.Values { + switch call.Op { + case OpStaticCall, OpStaticLECall, OpTailLECall, + OpClosureCall, OpClosureLECall, + OpInterCall, OpInterLECall, OpTailLECallInter: + default: + continue + } + aux := auxToCall(call.Aux) + callSig := llvmSignature(aux) + for index, result := range callSig.Results { + if !result.InMemory { + continue + } + resultType := aux.TypeOfResult(int64(index)) + slot := llvmStackSlot{ + Value: FCtxt.b.CreateAlloca(getLLVMType(resultType), fmt.Sprintf("%s.result%d.home", call, index)), + Type: resultType, + } + slot.Value.SetAlignment(int(resultType.Alignment())) + FCtxt.CallResultSlots[llvmCallResultKey{Call: call.ID, Index: int64(index)}] = slot + } + } } - addressedResultSlots := make(map[addressedResultKey]llvmStackSlot) + + // SelectNAddr for a register result still needs an addressable local copy. + // Multiple selectors of one call result share the same slot. + addressedResultSlots := make(map[llvmCallResultKey]llvmStackSlot) for _, BB := range f.Blocks { for _, v := range BB.Values { if v.Op != OpSelectNAddr || v.Uses == 0 { @@ -3326,7 +3732,11 @@ func LLVMCompile(f *Func) { if resultType.Alignment() <= 0 || !types.Identical(resultType, aux.TypeOfResult(index)) { v.Fatalf("SelectNAddr result type %v does not match call result %v", resultType, aux.TypeOfResult(index)) } - key := addressedResultKey{Call: call.ID, Index: index} + key := llvmCallResultKey{Call: call.ID, Index: index} + if memorySlot, ok := FCtxt.CallResultSlots[key]; ok { + FCtxt.ResultSlots[v.ID] = memorySlot.Value + continue + } slot, ok := addressedResultSlots[key] if !ok { slot = llvmStackSlot{ @@ -3382,11 +3792,10 @@ func LLVMCompile(f *Func) { } } } - // Go's ABI assigns each parameter either wholly to registers or wholly to - // the stack. Give only parameters that already have an addressable Go SSA - // LocalAddr a complete LLVM memory home. Ordinary register parameters remain - // direct LLVM SSA values, while the backend remains responsible for the - // physical Go ABI assignment. + // Give addressable register parameters a complete LLVM memory home. Wholly + // stack-assigned parameters were bound directly to their typed preallocated + // fixed homes above, while the backend remains responsible for the physical + // Go ABI assignment. // // This intentionally differs from the native lowering, which stores each // incoming register piece separately and addresses stack-assigned parameters diff --git a/src/cmd/compile/internal/ssa/ssa2llvm_test.go b/src/cmd/compile/internal/ssa/ssa2llvm_test.go index 33c20fffe7a554..fff37317bb298b 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm_test.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "cmd/compile/internal/abi" "cmd/compile/internal/base" "cmd/compile/internal/ir" "cmd/compile/internal/typecheck" @@ -133,6 +134,58 @@ func TestLLVMBuiltinDeclarationKeepsCallSiteSignatures(t *testing.T) { } } +func TestLLVMGoMemoryCarriersFollowABIAllocation(t *testing.T) { + config := abi.NewABIConfig(1, 0, 0, uint8(obj.ABIInternal)) + intType := types.Types[types.TINT] + aux := StaticAuxCall(new(obj.LSym), config.ABIAnalyzeTypes( + []*types.Type{intType, intType}, + []*types.Type{intType, intType}, + )) + sig := llvmSignature(aux) + + memoryArgs := 0 + for i, param := range sig.Params { + want := len(aux.ABIInfo().InParam(i).Registers) == 0 && aux.TypeOfArg(int64(i)).Size() != 0 + if param.InMemory != want { + t.Errorf("argument %d memory carrier = %v, Go ABI allocation wants %v", i, param.InMemory, want) + } + if want { + memoryArgs++ + if got := sig.Type.ParamTypes()[i].TypeKind(); got != llvm.PointerTypeKind { + t.Errorf("argument %d carrier kind = %v, want pointer", i, got) + } + } + } + if memoryArgs == 0 || memoryArgs == len(sig.Params) { + t.Fatalf("test requires mixed register and memory arguments, got %d of %d in memory", memoryArgs, len(sig.Params)) + } + + memoryResults := 0 + for i, result := range sig.Results { + want := len(aux.ABIInfo().OutParam(i).Registers) == 0 && aux.TypeOfResult(int64(i)).Size() != 0 + if result.InMemory != want { + t.Errorf("result %d memory carrier = %v, Go ABI allocation wants %v", i, result.InMemory, want) + } + if want { + memoryResults++ + } + } + if memoryResults == 0 || memoryResults == len(sig.Results) { + t.Fatalf("test requires mixed register and memory results, got %d of %d in memory", memoryResults, len(sig.Results)) + } + + module := GlobalCtxt.NewModule("go_memory_carriers") + t.Cleanup(module.Dispose) + fn := llvm.AddFunction(module, "mixed", sig.Type) + configureLLVMFunction(fn, sig, goABIInternalCallConv) + ir := module.String() + for _, want := range []string{"preallocated(", "goret(", `"goretindex"=`} { + if !strings.Contains(ir, want) { + t.Errorf("LLVM function does not contain %q:\n%s", want, ir) + } + } +} + func TestLLVMGoObjCompilerUsedOnlyKeepsExternalDataRoots(t *testing.T) { oldModule := CurrentModule oldLowerer := currentLLVMDataLowerer diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index beb40a4d1f1fd6..2e9ea2256e16f6 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -145,6 +145,9 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } for _, needle := range [][]byte{ []byte("define goabiinternal"), + []byte(" byval("), + []byte(" goret("), + []byte(` "goretindex"="`), []byte(`"go_results_tuple"`), []byte(`gc "goallc"`), } { @@ -192,10 +195,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } } for _, pattern := range []string{ - `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?"gc-live"\(ptr %second, ptr %first\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, - `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr, ptr \} @main\.pointerAggregateBothOverflow.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?"gc-live"\(ptr %second[[:alnum:]$._-]*, ptr %first[[:alnum:]$._-]*, ptr %first, ptr %second\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, + `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 \} @main\.pointerAggregateBothOverflow.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %\.result17, ptr %[[:alnum:]$._-]+, ptr %\.result16, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten GoALLC ABI IR does not match %q", pattern) @@ -205,6 +208,14 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "liveScalarStackArgument", "livePointerSequenceStackArguments", "livePointerAggregateStackArgument", "overflowResults", "stackAggregateResult", "bothOverflow", "pointerAggregateBothOverflow") + checkLLVMABIStatepointGoRetAttrs(t, rewrittenIR, map[string]string{ + "overflowResults": "16,17", + "initializedStackResult": "16", + "stackAggregateResult": "15", + "bothOverflow": "16,17", + "pointerAggregateBothOverflow": "16,17", + "stackResultsAfterGrowth": "16", + }) runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") @@ -220,10 +231,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedLLVMIR) for _, pattern := range []string{ - `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.[02].*?LDRXui\s+%fixed-stack\.[02]`, - `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, - `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?id:\s+4.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.3[^\n]*%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.3.*?STRXui[^\n]*%fixed-stack\.0.*?STRXui[^\n]*%fixed-stack\.1`, + `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?id:\s+0.*?offset:\s+8.*?size:\s+8.*?isImmutable:\s+false.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?id:\s+0.*?offset:\s+24.*?id:\s+1.*?offset:\s+16.*?id:\s+2.*?offset:\s+8.*?LDRXui\s+%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.1.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?id:\s+0.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?id:\s+0.*?offset:\s+40.*?id:\s+1.*?offset:\s+32.*?id:\s+2.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.2.*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?STRXui[^\n]*%fixed-stack\.1.*?STRXui[^\n]*%fixed-stack\.0`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("GoALLC ABI MIR does not match %q", pattern) @@ -263,7 +274,7 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { nativeArgsMaps: [][]int{{2, 4, 18}, nil}, // The explicit panicmem paths keep the still-needed parameter // homes in LocalsPointerMaps through locals-only alloca records. - goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, + goallcArgsMaps: [][]int{{2, 4, 18}, {2}, nil}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 1, -1, 1}, goallcQueryMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}, {2}}, @@ -271,42 +282,42 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "liveScalarStackArgument", args: 136, pointerBits: []int{0}, nativeArgsMaps: [][]int{{0}, nil}, - goallcArgsMaps: [][]int{{0}}, + goallcArgsMaps: [][]int{{0}, {0}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, { name: "livePointerSequenceStackArguments", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, { name: "livePointerAggregateStackArgument", args: 136, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, @@ -348,14 +359,14 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "pointerAggregateBothOverflow", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, {0, 2}}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 0, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 136, + goallcLocals: 152, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {16, 17}}, nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, }, @@ -450,8 +461,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten amd64 IR does not match %q", pattern) @@ -506,8 +517,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri `(?s)TEXT p\.initializedPointerResult.*?MOVQ\s+AX, 0x48\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+CX, 0x40\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+AX, 0x50\(BP\)`, - `(?s)TEXT p\.liveScalarStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), AX`, - `(?s)TEXT p\.liveAggregateStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), BX.*?MOVQ\s+0x38\(BP\), AX`, + `(?s)TEXT p\.liveScalarStackArgument.*?MOVQ\s+0x48\(BP\), AX.*?MOVQ\s+AX, -0x8\(BP\).*?R_CALL:p\.safepoint.*?MOVQ\s+-0x8\(BP\), AX`, + `(?s)TEXT p\.liveAggregateStackArgument.*?MOVQ\s+0x38\(BP\), AX.*?MOVQ\s+0x48\(BP\), CX.*?MOVQ\s+AX, -0x10\(BP\).*?MOVQ\s+CX, -0x8\(BP\).*?R_CALL:p\.safepoint.*?MOVQ\s+-0x8\(BP\), BX.*?MOVQ\s+-0x10\(BP\), AX`, } { if !regexp.MustCompile(pattern).Match(goallcDisassembly) { t.Fatalf("GoALLC amd64 object disassembly does not match %q", pattern) @@ -641,8 +652,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer[[:alnum:]$._-]*, ptr %pointer\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+, ptr %value\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten source IR does not match %q", pattern) @@ -658,8 +669,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedGoallcIR) for _, pattern := range []string{ - `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, + `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?id:\s+0.*?size:\s+8.*?isImmutable:\s+false.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, + `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+0.*?size:\s+24.*?isAliased:\s+true.*?LDRXui\s+%fixed-stack\.0.*?STATEPOINT[^\n]*%fixed-stack\.0`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("source MIR does not match %q", pattern) @@ -783,7 +794,7 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string t.Helper() for _, callee := range callees { call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + - regexp.QuoteMeta(callee) + `.*#([0-9]+)[^\n]*$`).FindSubmatch(ir) + regexp.QuoteMeta(callee) + `.*#([0-9]+)(?: \[|$)`).FindSubmatch(ir) if len(call) != 2 { t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) } @@ -795,6 +806,23 @@ func checkLLVMABIStatepointTupleAttrs(t *testing.T, ir []byte, callees ...string } } +func checkLLVMABIStatepointGoRetAttrs(t *testing.T, ir []byte, callees map[string]string) { + t.Helper() + for callee, want := range callees { + call := regexp.MustCompile(`(?m)^.*@llvm\.experimental\.gc\.statepoint.*@main\.` + + regexp.QuoteMeta(callee) + `.*$`).Find(ir) + if call == nil { + t.Fatalf("rewritten IR has no attributed statepoint call to main.%s", callee) + } + for _, index := range strings.Split(want, ",") { + pattern := `goret\([^)]*\)[^\n,)]*"goretindex"="` + regexp.QuoteMeta(index) + `"` + if !regexp.MustCompile(pattern).Match(call) { + t.Fatalf("statepoint call to main.%s lost goret index %s: %s", callee, index, call) + } + } + } +} + func checkLLVMABIAssembly(t *testing.T, native, goallc []byte) { t.Helper() @@ -837,8 +865,8 @@ func checkLLVMABIAssembly(t *testing.T, native, goallc []byte) { `(?m)^main\.pointerAggregateBothOverflow:`, `(?s)\bbl\s+main\.overflowResults.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #8\]`, `(?s)\bbl\s+main\.overflowResults.*?\b(?:mov|stp)\s+[^\n]*x15`, - `(?s)\bbl\s+main\.stackAggregateResult.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #8\].*?\bmov\s+x[0-9]+, x15`, - `(?s)\bbl\s+main\.bothOverflow.*?\bldp\s+x[0-9]+, x[0-9]+, \[sp, #32\]`, + `(?s)\bbl\s+main\.stackAggregateResult.*?\b(?:ldp\s+x[0-9]+, x[0-9]+|ldur\s+q[0-9]+), \[sp, #8\].*?\bmov\s+x[0-9]+, x15`, + `(?s)\bbl\s+main\.bothOverflow.*?\bldr\s+x[0-9]+, \[sp, #32\].*?\bldr\s+x[0-9]+, \[sp, #40\]`, `(?s)\bbl\s+main\.bothOverflow.*?\b(?:mov|stp)\s+[^\n]*x15`, } { if !regexp.MustCompile(pattern).Match(goallc) { diff --git a/src/cmd/link/internal/arm64/asm.go b/src/cmd/link/internal/arm64/asm.go index 9864788e1b6125..5357b881dfa51f 100644 --- a/src/cmd/link/internal/arm64/asm.go +++ b/src/cmd/link/internal/arm64/asm.go @@ -559,6 +559,34 @@ func elfreloc1(ctxt *ld.Link, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, ldstType = elf.R_AARCH64_LDST64_ABS_LO12_NC } out.Write64(uint64(ldstType) | uint64(elfsym)<<32) + case objabi.R_ARM64_PCREL: + if siz != 4 || r.Off < 0 || int64(r.Off)+4 > int64(len(ldr.Data(s))) { + return false + } + inst := ctxt.Arch.ByteOrder.Uint32(ldr.Data(s)[r.Off:]) + kind, ok := classifyPCRelInstruction(inst) + if !ok { + ldr.Errorf(s, "unsupported instruction for %x R_ARM64_PCREL", inst) + return false + } + var relocType elf.R_AARCH64 + switch kind { + case arm64PCRelADRP: + relocType = elf.R_AARCH64_ADR_PREL_PG_HI21 + case arm64PCRelADD: + relocType = elf.R_AARCH64_ADD_ABS_LO12_NC + case arm64PCRelLDST8: + relocType = elf.R_AARCH64_LDST8_ABS_LO12_NC + case arm64PCRelLDST16: + relocType = elf.R_AARCH64_LDST16_ABS_LO12_NC + case arm64PCRelLDST32: + relocType = elf.R_AARCH64_LDST32_ABS_LO12_NC + case arm64PCRelLDST64: + relocType = elf.R_AARCH64_LDST64_ABS_LO12_NC + case arm64PCRelLDST128: + relocType = elf.R_AARCH64_LDST128_ABS_LO12_NC + } + out.Write64(uint64(relocType) | uint64(elfsym)<<32) case objabi.R_ARM64_TLS_LE: out.Write64(uint64(elf.R_AARCH64_TLSLE_MOVW_TPREL_G0) | uint64(elfsym)<<32) @@ -588,6 +616,43 @@ func elfreloc1(ctxt *ld.Link, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, func signext21(x int64) int64 { return x << (64 - 21) >> (64 - 21) } func signext24(x int64) int64 { return x << (64 - 24) >> (64 - 24) } +type arm64PCRelInstruction uint8 + +const ( + arm64PCRelADRP arm64PCRelInstruction = iota + arm64PCRelADD + arm64PCRelLDST8 + arm64PCRelLDST16 + arm64PCRelLDST32 + arm64PCRelLDST64 + arm64PCRelLDST128 +) + +func classifyPCRelInstruction(inst uint32) (arm64PCRelInstruction, bool) { + switch { + case (inst>>24)&0x9f == 0x90: + return arm64PCRelADRP, true + case (inst>>24)&0x9f == 0x91: + return arm64PCRelADD, true + case (inst>>24)&0x3b == 0x39: + shift := inst >> 30 + if shift == 0 && (inst>>20)&0x048 == 0x048 { + return arm64PCRelLDST128, true + } + switch shift { + case 0: + return arm64PCRelLDST8, true + case 1: + return arm64PCRelLDST16, true + case 2: + return arm64PCRelLDST32, true + case 3: + return arm64PCRelLDST64, true + } + } + return 0, false +} + func machoreloc1(arch *sys.Arch, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, r loader.ExtReloc, sectoff int64) bool { var v uint32 @@ -646,6 +711,26 @@ func machoreloc1(arch *sys.Arch, out *ld.OutBuf, ldr *loader.Loader, s loader.Sy } v |= 1 << 24 // pc-relative bit v |= ld.MACHO_ARM64_RELOC_BRANCH26 << 28 + case objabi.R_ARM64_PCREL: + if siz != 4 || r.Off < 0 || int64(r.Off)+4 > int64(len(ldr.Data(s))) { + return false + } + inst := arch.ByteOrder.Uint32(ldr.Data(s)[r.Off:]) + kind, ok := classifyPCRelInstruction(inst) + if !ok { + ldr.Errorf(s, "unsupported instruction for %x R_ARM64_PCREL", inst) + return false + } + if xadd != 0 { + out.Write32(uint32(sectoff)) + out.Write32((ld.MACHO_ARM64_RELOC_ADDEND << 28) | (2 << 25) | uint32(xadd&0xffffff)) + } + if kind == arm64PCRelADRP { + v |= 1 << 24 + v |= ld.MACHO_ARM64_RELOC_PAGE21 << 28 + } else { + v |= ld.MACHO_ARM64_RELOC_PAGEOFF12 << 28 + } case objabi.R_ADDRARM64, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, @@ -795,6 +880,25 @@ func archreloc(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, r loade nExtReloc := 0 switch rt := r.Type(); rt { default: + case objabi.R_ARM64_PCREL: + // PE currently resolves independent instruction relocations in + // the Go linker. ELF and Mach-O external links may move the final + // target section, so preserve each instruction relocation for the + // platform linker instead of baking in the preliminary layout. + if target.IsWindows() { + break + } + rs, off := ld.FoldSubSymbolOffset(ldr, rs) + xadd := r.Add() + off + rst := ldr.SymType(rs) + if rst != sym.SHOSTOBJ && rst != sym.SDYNIMPORT && ldr.SymSect(rs) == nil { + ldr.Errorf(s, "missing section for %s", ldr.SymName(rs)) + } + nExtReloc = 1 + if target.IsDarwin() && xadd != 0 { + nExtReloc = 2 + } + return val, nExtReloc, isOk case objabi.R_ARM64_GOTPCREL, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, @@ -1127,6 +1231,10 @@ func archrelocvariant(*ld.Target, *loader.Loader, loader.Reloc, sym.RelocVariant func extreloc(target *ld.Target, ldr *loader.Loader, r loader.Reloc, s loader.Sym) (loader.ExtReloc, bool) { switch rt := r.Type(); rt { + case objabi.R_ARM64_PCREL: + if !target.IsWindows() { + return ld.ExtrelocViaOuterSym(ldr, r, s), true + } case objabi.R_ARM64_GOTPCREL, objabi.R_ARM64_PCREL_LDST8, objabi.R_ARM64_PCREL_LDST16, diff --git a/src/cmd/link/internal/arm64/asm_test.go b/src/cmd/link/internal/arm64/asm_test.go new file mode 100644 index 00000000000000..89c87e59d2d058 --- /dev/null +++ b/src/cmd/link/internal/arm64/asm_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package arm64 + +import "testing" + +func TestClassifyPCRelInstruction(t *testing.T) { + tests := []struct { + name string + inst uint32 + want arm64PCRelInstruction + ok bool + }{ + {"adrp", 0x90000008, arm64PCRelADRP, true}, + {"add", 0x91000008, arm64PCRelADD, true}, + {"ldrb", 0x39400100, arm64PCRelLDST8, true}, + {"ldrh", 0x79400100, arm64PCRelLDST16, true}, + {"ldrw", 0xb9400100, arm64PCRelLDST32, true}, + {"ldrx", 0xf9400100, arm64PCRelLDST64, true}, + {"strq", 0x3d800000, arm64PCRelLDST128, true}, + {"nop", 0xd503201f, 0, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := classifyPCRelInstruction(test.inst) + if got != test.want || ok != test.ok { + t.Fatalf("classifyPCRelInstruction(%#08x) = (%d, %t), want (%d, %t)", test.inst, got, ok, test.want, test.ok) + } + }) + } +} diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 4a27db5a42ef88..00614f698d0615 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -750,6 +750,7 @@ func ExtrelocSimple(ldr *loader.Loader, r loader.Reloc) loader.ExtReloc { rs := r.Sym() rr.Xsym = rs rr.Xadd = r.Add() + rr.Off = r.Off() rr.Type = r.Type() rr.Size = r.Siz() return rr @@ -768,6 +769,7 @@ func ExtrelocViaOuterSym(ldr *loader.Loader, r loader.Reloc, s loader.Sym) loade ldr.Errorf(s, "missing section for %s", ldr.SymName(rs)) } rr.Xsym = rs + rr.Off = r.Off() rr.Type = r.Type() rr.Size = r.Siz() return rr diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 04601b32a00af6..b07cb4ae26b651 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -45,6 +45,7 @@ type Relocs struct { type ExtReloc struct { Xsym Sym Xadd int64 + Off int32 // offset within the source symbol Type objabi.RelocType Size uint8 } diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 214205fe1df5c5..1816037be2e358 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -338,6 +338,36 @@ if(BUILD_TESTING) "ptr noundef nonnull readnone align 8 %argument" ) + add_test( + NAME GoALLCStatepoints.PreallocatedParamAttrRewrite + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" + ) + set_tests_properties(GoALLCStatepoints.PreallocatedParamAttrRewrite PROPERTIES + PASS_REGULAR_EXPRESSION + "ptr preallocated.*5 x ptr.* align 8 %home" + ) + + add_test( + NAME GoALLCStatepoints.GoRetResultAttrRewrite + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" + ) + set_tests_properties(GoALLCStatepoints.GoRetResultAttrRewrite PROPERTIES + PASS_REGULAR_EXPRESSION + "ptr goret.*5 x ptr.* align 8.*goretindex.*0.* %stack_result.address" + ) + add_test( NAME GoALLCStatepoints.SupportedParamAttrsCodegen COMMAND diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 132c94886399e6..5faaed2fdc8b02 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -118,6 +118,14 @@ struct PointerAllocaRecord { bool ActivityUnclear = false; }; +struct PointerMemoryParamRecord { + Argument *Base; + uint64_t ByteSize; + uint64_t Alignment; + uint64_t BitCount; + SmallVector BitmapWords; +}; + struct OpenDeferInfo { AllocaInst *Bits = nullptr; AllocaInst *Slots = nullptr; @@ -1316,6 +1324,64 @@ Error collectPointerAllocas( return Error::success(); } +Error collectPointerMemoryParams( + Function &F, SmallVectorImpl &Records) { + if (!isGoCallingConv(F.getCallingConv())) + return Error::success(); + + const DataLayout &DL = F.getDataLayout(); + uint64_t PointerSize = DL.getPointerSize(0); + for (Argument &Arg : F.args()) { + if (!Arg.hasPreallocatedAttr()) + continue; + Type *StorageType = Arg.getAttributes().getPreallocatedType(); + if (!StorageType || !containsPointer(StorageType)) + continue; + + TypeSize AllocationSize = DL.getTypeAllocSize(StorageType); + if (AllocationSize.isScalable()) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints do not support scalable preallocated parameter " + "layouts"); + Align Alignment = + Arg.getParamAlign().value_or(DL.getABITypeAlign(StorageType)); + uint64_t ByteSize = AllocationSize.getFixedValue(); + if (!PointerSize || !ByteSize || ByteSize % PointerSize != 0 || + Alignment < DL.getABITypeAlign(StorageType)) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require pointer-aligned fixed preallocated " + "parameter " + "layouts"); + + SmallVector Leaves; + SmallVector Path; + if (Error Err = + enumeratePointerAllocaLeaves(StorageType, DL, Path, 0, Leaves)) + return std::move(Err); + uint64_t BitCount = ByteSize / PointerSize; + SmallVector BitmapWords((BitCount + 63) / 64, 0); + for (const PointerAllocaLeaf &Leaf : Leaves) { + if (Leaf.Offset % PointerSize != 0 || Leaf.Offset >= ByteSize) + return createStringError( + std::errc::not_supported, + "GoALLC statepoint preallocated pointer slot is not " + "pointer-aligned"); + uint64_t Bit = Leaf.Offset / PointerSize; + uint64_t Mask = uint64_t(1) << (Bit % 64); + if (BitmapWords[Bit / 64] & Mask) + return createStringError( + std::errc::invalid_argument, + "GoALLC statepoint preallocated pointer slots overlap"); + BitmapWords[Bit / 64] |= Mask; + } + Records.push_back( + {&Arg, ByteSize, Alignment.value(), BitCount, std::move(BitmapWords)}); + } + return Error::success(); +} + Error validateSafepoint(const SafepointRecord &Record) { const CallInst &Call = *Record.Call; if (Call.isInlineAsm()) @@ -1325,21 +1391,50 @@ Error validateSafepoint(const SafepointRecord &Record) { if (Call.isMustTailCall()) return createStringError(std::errc::not_supported, "GoALLC statepoints do not support musttail"); - if (Call.getNumOperandBundles() != 0 && - (Call.getNumOperandBundles() != 1 || - !Call.getOperandBundle(LLVMContext::OB_deopt))) - return createStringError( - std::errc::not_supported, - "GoALLC statepoints only support a single deopt call operand bundle"); + if (Call.countOperandBundlesOfType(LLVMContext::OB_deopt) > 1 || + Call.countOperandBundlesOfType(LLVMContext::OB_preallocated) > 1) + return createStringError(std::errc::not_supported, + "GoALLC statepoints require unique deopt and " + "preallocated call operand bundles"); + for (unsigned I = 0; I != Call.getNumOperandBundles(); ++I) { + uint32_t Tag = Call.getOperandBundleAt(I).getTagID(); + if (Tag != LLVMContext::OB_deopt && Tag != LLVMContext::OB_preallocated) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints do not support call operand bundle '%s'", + Call.getOperandBundleAt(I).getTagName().str().c_str()); + } for (unsigned I = 0; I != Call.arg_size(); ++I) { + if (Call.paramHasAttr(I, Attribute::Preallocated) && + (!isGoCallingConv(Call.getCallingConv()) || + !Call.getArgOperand(I)->getType()->isPointerTy() || + !Call.getParamPreallocatedType(I) || !Call.getParamAlign(I) || + !Call.getOperandBundle(LLVMContext::OB_preallocated))) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require bundled, typed, aligned preallocated " + "parameters only on Go calls"); + if (Call.paramHasAttr(I, Attribute::GoRet) && + (!isGoCallingConv(Call.getCallingConv()) || + !Call.getArgOperand(I)->getType()->isPointerTy() || + !Call.getParamGoRetType(I) || + !Call.getParamAttr(I, "goretindex").isValid() || + !Call.getParamAlign(I))) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require indexed, typed, aligned goret only on " + "Go calls"); for (Attribute Attr : Call.getAttributes().getParamAttrs(I)) { // These non-ABI attributes remain valid after LLVM's generic // RewriteStatepointsForGC pass and are natively accepted by both the // statepoint verifier and SelectionDAG call lowering. O2 commonly // infers them on otherwise ordinary runtime calls. Keep ABI-affecting - // attributes fail closed except for nest, whose Go closure ABI lowering - // is covered separately. + // attributes fail closed except for nest and the typed Go ABI memory + // carriers, whose lowering is covered separately. if (!Attr.hasAttribute(Attribute::Nest) && + !Attr.hasAttribute(Attribute::Preallocated) && + !Attr.hasAttribute(Attribute::GoRet) && + !Attr.hasAttribute("goretindex") && !Attr.hasAttribute(Attribute::Captures) && !Attr.hasAttribute(Attribute::ReadNone) && !Attr.hasAttribute(Attribute::ReadOnly) && @@ -1363,8 +1458,9 @@ Error validateSafepoint(const SafepointRecord &Record) { void appendAllocaPtrMapDeoptOperands( IRBuilder<> &Builder, ArrayRef Allocas, + ArrayRef MemoryParams, SmallVectorImpl &Deopt) { - if (Allocas.empty()) + if (Allocas.empty() && MemoryParams.empty()) return; // ProtocolLength covers BEGIN through END, but not the trailing duplicate // length. The envelope itself therefore contributes BEGIN, length, @@ -1372,27 +1468,39 @@ void appendAllocaPtrMapDeoptOperands( uint64_t ProtocolLength = 4; for (const PointerAllocaRecord *Alloca : Allocas) ProtocolLength += 10 + Alloca->BitmapWords.size(); + for (const PointerMemoryParamRecord *Param : MemoryParams) + ProtocolLength += 10 + Param->BitmapWords.size(); auto AppendConstant = [&](uint64_t Value) { Deopt.push_back(ConstantInt::get(Builder.getInt64Ty(), Value)); }; AppendConstant(GoObj::AllocaPtrMapBeginMagic); AppendConstant(ProtocolLength); - AppendConstant(Allocas.size()); - for (const PointerAllocaRecord *Alloca : Allocas) { + AppendConstant(Allocas.size() + MemoryParams.size()); + auto AppendRecord = [&](Value *Base, uint64_t ByteSize, uint64_t Alignment, + uint64_t BitCount, ArrayRef BitmapWords) { AppendConstant(GoObj::AllocaPtrMapRecordTag); - AppendConstant(10 + Alloca->BitmapWords.size()); - Deopt.push_back(Alloca->Alloca); - AppendConstant(0); // First contract version describes the whole alloca. - AppendConstant(Alloca->ByteSize); - AppendConstant(Alloca->Alignment); - AppendConstant(Alloca->Alloca->getDataLayout().getPointerSize(0)); - AppendConstant(Alloca->BitCount); + AppendConstant(10 + BitmapWords.size()); + Deopt.push_back(Base); + AppendConstant(0); // First contract version describes the whole object. + AppendConstant(ByteSize); + AppendConstant(Alignment); + AppendConstant( + Builder.GetInsertBlock()->getModule()->getDataLayout().getPointerSize( + 0)); + AppendConstant(BitCount); AppendConstant(GoObj::AllocaPtrMapBitmapWordBits); - AppendConstant(Alloca->BitmapWords.size()); - for (uint64_t Word : Alloca->BitmapWords) + AppendConstant(BitmapWords.size()); + for (uint64_t Word : BitmapWords) AppendConstant(Word); + }; + for (const PointerAllocaRecord *Alloca : Allocas) { + AppendRecord(Alloca->Alloca, Alloca->ByteSize, Alloca->Alignment, + Alloca->BitCount, Alloca->BitmapWords); } + for (const PointerMemoryParamRecord *Param : MemoryParams) + AppendRecord(Param->Base, Param->ByteSize, Param->Alignment, + Param->BitCount, Param->BitmapWords); AppendConstant(GoObj::AllocaPtrMapEndMagic); AppendConstant(ProtocolLength); } @@ -1417,6 +1525,7 @@ void appendOpenDeferDeoptOperands(IRBuilder<> &Builder, Error rewriteCall(SafepointRecord &Record, ArrayRef PointerAllocas, + ArrayRef MemoryParams, const std::optional &OpenDefer) { CallInst *Call = Record.Call; @@ -1433,12 +1542,23 @@ Error rewriteCall(SafepointRecord &Record, // Keep the open-defer envelope before the alloca ptrmap envelope. The latter // deliberately remains the final self-describing suffix for compatibility. appendOpenDeferDeoptOperands(Builder, OpenDefer, Deopt); - appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, Deopt); + appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, MemoryParams, Deopt); Record.Statepoint = Builder.CreateGCStatepointCall( Record.ID, 0, Callee, CallArgs, Deopt.empty() ? std::nullopt : std::optional>(ArrayRef(Deopt)), GCLive, "statepoint_token"); + if (auto Bundle = Call->getOperandBundle(LLVMContext::OB_preallocated)) { + SmallVector Bundles; + for (unsigned I = 0; I != Record.Statepoint->getNumOperandBundles(); ++I) + Bundles.emplace_back(Record.Statepoint->getOperandBundleAt(I)); + Bundles.emplace_back(*Bundle); + CallInst *BundledStatepoint = CallInst::Create( + Record.Statepoint, Bundles, Record.Statepoint->getIterator()); + Record.Statepoint->replaceAllUsesWith(BundledStatepoint); + Record.Statepoint->eraseFromParent(); + Record.Statepoint = BundledStatepoint; + } Record.Statepoint->setCallingConv(Call->getCallingConv()); if (Call->hasFnAttr(GoResultsTupleAttr)) Record.Statepoint->addFnAttr( @@ -1515,14 +1635,20 @@ Value *rematerializeAddress(Value *Address, Value *Base, Value *RelocatedBase, void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { - // Each ordinary relocated pointer and each rematerialized alloca-derived - // address is a new reaching definition of its original SSA value. + // Each ordinary relocated pointer and each rematerialized fixed-object + // derived address is a new reaching definition of its original SSA value. + // Static allocas and typed preallocated/goret arguments are themselves fixed + // frame addresses: SelectionDAG rematerializes either from its frame index at + // each use, so replacing the original IR value with a gc.relocate chain would + // turn later statepoints into ordinary pointer spills. MapVector> Definitions; for (const SafepointRecord &Record : Records) { for (CallInst *RelocateCall : Record.Relocates) { auto *Relocate = cast(RelocateCall); Value *Original = Relocate->getDerivedPtr(); - if (!isa(Original)) + auto *Arg = dyn_cast(Original); + if (!isa(Original) && + !(Arg && (Arg->hasPreallocatedAttr() || Arg->hasGoRetAttr()))) Definitions[Original].push_back(RelocateCall); } @@ -1658,6 +1784,9 @@ Error rewriteFunction(Function &F) { SmallVector PointerAllocas; if (Error Err = collectPointerAllocas(F, OpenDefer, PointerAllocas)) return Err; + SmallVector PointerMemoryParams; + if (Error Err = collectPointerMemoryParams(F, PointerMemoryParams)) + return Err; if (Error Err = scalarizeLivePointerAggregates(F)) return Err; @@ -1690,6 +1819,14 @@ Error rewriteFunction(Function &F) { // the exact address expression after the statepoint. Record.Live.insert(rematerializableDerivedBase(Address)); } + // A typed preallocated parameter is the address of a caller-initialized fixed + // Go argument object. Keep that base relocatable at every ordinary + // safepoint; the accompanying layout record tells GoObj which words in the + // object are GC pointers, while the direct base location itself is not a + // pointer root. + for (SafepointRecord &Record : Records) + for (const PointerMemoryParamRecord &Param : PointerMemoryParams) + Record.Live.insert(Param.Base); for (const SafepointRecord &Record : Records) if (Error Err = validateSafepoint(Record)) return Err; @@ -1731,6 +1868,9 @@ Error rewriteFunction(Function &F) { if (Error Err = promoteAllocasToWholeFunctionLifetime(F, WholeLifetimeAllocas)) return Err; + SmallVector MemoryParamRecords; + for (const PointerMemoryParamRecord &Param : PointerMemoryParams) + MemoryParamRecords.push_back(&Param); for (SafepointRecord &Record : llvm::reverse(Records)) { SmallVector AllocaRecords; for (const PointerAllocaRecord &Alloca : PointerAllocas) { @@ -1749,7 +1889,8 @@ Error rewriteFunction(Function &F) { if (IsActive || Alloca.NeedsStackObject) AllocaRecords.push_back(&Alloca); } - if (Error Err = rewriteCall(Record, AllocaRecords, OpenDefer)) + if (Error Err = + rewriteCall(Record, AllocaRecords, MemoryParamRecords, OpenDefer)) return Err; } eraseOriginalCalls(Records); diff --git a/src/cmd/llvmplugin/README.md b/src/cmd/llvmplugin/README.md index 65b52a2e85161c..e0e94908d9f128 100644 --- a/src/cmd/llvmplugin/README.md +++ b/src/cmd/llvmplugin/README.md @@ -54,7 +54,7 @@ The current SSA value and CFG rewrite support matrix is: | Value or control-flow shape | Status | Current contract | | --- | --- | --- | -| Pointer arguments | AArch64 GoObj qualified; SelectionDAG home reuse also tested on X86 | Values live after a call use caller statepoints; exact stack inputs stay in their fixed ABI homes, while register inputs and transformed values use normal statepoint spills. Call-only arguments are described by the callee's type-derived entry map. | +| Pointer arguments | AArch64 GoObj qualified; typed stack homes also tested on X86 | Go ABI stack assignments use typed `byval` carriers bound directly to their incoming homes. The home is a memory root when active; pointer values loaded from it and live after a call use normal scalar statepoint tracking. Call-only arguments are described by the callee's type-derived entry map. | | Static `alloca` addresses | Supported | The raw alloca is the storage identity and the only extra `gc-live` root. First-class GEP/cast uses are rebuilt immediately at the use; direct memory uses stay on the FrameIndex and never enter relocation SSA. Pointers loaded from memory remain tracked. | | `select`, GEP, and pointer casts | Supported | Fixed-allocation GEP/no-op cast chains stay as direct FrameIndex uses or are rebuilt immediately at first-class uses. Merged or non-stack pointer values remain ordinary scalar roots. | | Pointer-valued call results | Supported | `gc.result` replaces the ordinary result and later safepoints relocate it. | @@ -63,7 +63,7 @@ The current SSA value and CFG rewrite support matrix is: | Loops and irreducible CFG | Supported | Relocation definitions are propagated through backedge and multi-entry PHIs without a shape-specific algorithm. | | Fixed struct/array SSA aggregates | Supported | Pointer and fixed pointer-vector leaves are extracted before liveness and reconstructed from the current relocated SSA leaves. | | Fixed-width pointer-vector SSA values | Supported | The vector remains one `gc-live` operand and one same-typed `gc.relocate`; it is not split into lanes. Pointer vectors in allocas remain unsupported. | -| Aggregate arguments and call results | Supported for IR rewriting | The wrapped call keeps its real aggregate ABI type. Only leaves live after the call enter caller `gc-live`; supported fixed formal layouts also contribute pointer words to AArch64 entry maps. | +| Aggregate arguments and call results | Supported for IR rewriting and AArch64 runtime ABI tests | Go ABI stack inputs use typed `byval`; stack results use caller-owned typed `goret(T) "goretindex"="N"` carriers, while register results remain compact LLVM returns. Only first-class leaves live after the call enter caller `gc-live`; fixed input/result homes contribute pointer words to GoObj maps according to their final locations. | | Aggregate load results and store operands | Supported | First-class SSA values use aggregate normalization. Pointer leaves in surviving fixed allocas remain memory roots described by the alloca deopt layout protocol. | | Pointer-containing `alloca` storage | GoObj qualified for fixed layouts | Go `VarDef` emits `llvm.lifetime.start`; parameter homes and addressed result homes have explicit starts at their initialization sites. The statepoint pass uses starts as backward liveness kills and real address uses as gens, so the last use supplies the implicit end. Active contents contribute callsite `LocalsPointerMaps`; address-observable objects additionally get function-wide `FUNCDATA_StackObjects` and one entry initialization because stack growth adjusts them even while source-dead. Locals-only storage is not initialized by the plugin, and no lifetime ends are emitted. | | Scalable vectors | Unsupported | The generic LLVM statepoint rewrite assumes a fixed vector width when constructing relocates; fails closed. | @@ -268,18 +268,24 @@ the same direct location is instead the frame base whose bitmap selects memory slots. Its contents are active only when a matching explicit direct alloca also occurs in the statepoint's GC operands. -Ordinary stack inputs use the same statepoint path. SelectionDAG formal -lowering records a value home only when a Go ABI pointer part is a direct, -non-extending load from an immutable fixed object and its IR aggregate offset, -ABI part offset, load size, and object size all agree. If `gc-live` later -contains that argument or an exact `extractvalue` leaf, statepoint lowering -uses the existing fixed frame index as its indirect memory location, makes the -slot mutable for GC relocation, and reloads `gc.relocate` from that same frame -index. No pre-call copy to a local spill is emitted. A merged, derived, -size-mismatched, or otherwise unproven value falls back to LLVM's normal local -statepoint spill. This is a SelectionDAG contract; it does not introduce -`byval`/`sret`, change GoALLC's LLVM IR emission point, or bypass the standard -statepoint operand format. +Ordinary stack inputs use typed LLVM `byval` parameters whenever Go's ABI +assignment contains no registers. The target lowers that carrier to the exact +mutable incoming Go home, and the frontend binds addressable parameter uses to +the carrier instead of copying the value through a compiler-owned alloca. At a +call site an existing source address is reused; a pure SSA value is materialized +in a short-lived entry alloca solely to provide the `byval` address. This is one +Go ABI classification rule, not a separate scalar-versus-aggregate policy. +Statepoint lowering rematerializes the fixed home itself after stack growth; +pointer values loaded from the home remain ordinary scalar roots when live. + +Results whose Go `OutParam` assignment contains no registers use caller-owned +typed `goret(T)` carriers. Each carrier's `"goretindex"="N"` parameter attribute records +their logical Go result indexes because LLVM `sret` cannot represent Go's +mixed register-and-stack or multiple stack-result layouts. The target maps a +callee carrier to its fixed outgoing result home and copies that physical home +back to the caller-owned carrier after the call. Register-assigned results keep +the compact LLVM return and `go_results_tuple` mapping. Both carrier kinds keep +the standard statepoint operand format. For AArch64 GoObj, target frame lowering uses Go's frame-chain layout instead of the platform ABI frame record: LR is at `SP+0`, this function writes its FP @@ -294,11 +300,12 @@ moving SP so asynchronous traceback cannot observe a half-built frame. The ArgsPointerMaps phase supports scalar LLVM pointer inputs and receivers, plus pointer leaves in supported fixed struct/array formal layouts, in -ABIInternal register homes, ABIInternal stack-input slots, and ABI0 stack-input -slots on AArch64. Pair 0 is always `(EntryArgs, empty locals)`. Ordinary -statepoints use their actual final machine locations: indirect pointer slots in -the current frame become locals bits, while exact fixed input homes and stack -result slots above the final frame become args bits. The writer jointly +ABIInternal register homes, ABIInternal stack-input slots, ABI0 stack-input +slots, and typed `goret` result slots on AArch64. Pair 0 is always +`(EntryArgs, empty locals)`. Ordinary statepoints use their actual final machine +locations: indirect pointer slots in the current frame become locals bits, +while exact fixed input homes and stack result slots above the final frame +become args bits. The writer jointly deduplicates each complete `(Args, locals)` pair, so the two tables always have the same count. It does not eagerly mark declared result slots; a result slot is an args root only when a statepoint records a live pointer in that physical @@ -362,10 +369,9 @@ forces an entry input home and an ordinary stack-result root into different ArgsPointerMaps entries, then checks their exact objview bitmaps and `PCDATA_StackMapIndex` sequence. The identical-source Go fixture `test/abi/llvm_args_pointer_maps.go` separately forces a scalar pointer and a -three-word pointer aggregate onto the incoming stack. It checks native -assembly stack loads, scalar-only rewritten `gc-live`/`gc.relocate`, alloca -memory roots with no synthetic statepoint spill, and exact Args/Locals/PCDATA -objview data. +three-word pointer aggregate onto the incoming stack. It checks typed `byval` +IR, native assembly stack loads, scalar-only rewritten `gc-live`/`gc.relocate`, +fixed-home memory roots, and exact Args/Locals/PCDATA objview data. The executable identical-source fixture `test/abi/llvm_args_results.go` repeats those checks with `runtime.GC` for a scalar stack pointer, two pointer stack arguments separated by a scalar, a pointer-containing stack aggregate, and the diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 148506f6ca6135..573b7be5d62ade 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -98,12 +98,16 @@ entry: ret ptr %result } -define goabi0 ptr @"aarch64_abi0_pointer_result"(ptr %pointer) #0 gc "goallc" { +define goabi0 void @"aarch64_abi0_pointer_result"( + ptr preallocated(ptr) align 8 %pointer.home, + ptr goret(ptr) "goretindex"="0" align 8 %result.home) #0 gc "goallc" { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 - ret ptr %pointer + store ptr %pointer, ptr %result.home, align 8 + ret void } define goabiinternal ptr @aarch64_stack_pointer_arg( @@ -111,8 +115,9 @@ define goabiinternal ptr @aarch64_stack_pointer_arg( i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) #0 gc "goallc" { + ptr preallocated(ptr) align 8 %pointer.home) #0 gc "goallc" { entry: + %pointer = load ptr, ptr %pointer.home, align 8 %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 @@ -130,7 +135,7 @@ entry: ; The i64 register argument's home starts beyond the 8-byte scaled-uimm12 ; limit (32760), forcing the frameless morestack path to materialize SP+32776. define goabiinternal i64 @aarch64_large_arg_home( - [4096 x i64] %stackarg, i64 %regarg) #0 gc "goallc" { + ptr preallocated([4096 x i64]) align 8 %stackarg, i64 %regarg) #0 gc "goallc" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index e89c46dbebca71..3ecc8076174d5d 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -34,6 +34,16 @@ target triple = "x86_64-unknown-linux-goobj" ; IR-COUNT-2: getelementptr inbounds i8, ptr %slot, i64 0 ; IR-NOT: .address.relocated.merge +; IR-LABEL: define goabiinternal void @argument_aggregate_home_address_across_calls( +; IR-SAME: ptr preallocated(%nested) align 8 %value +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR: "deopt"({{.*}}ptr %value{{.*}}i64 48{{.*}}i64 6{{.*}}i64 41{{.*}}), "gc-live"(ptr %value) +; IR: call coldcc ptr @llvm.experimental.gc.relocate +; IR-NOT: llvm.memset.inline + ; IR-LABEL: define goabiinternal void @alloca_gep_value_across_calls() ; IR: %field.remat{{[0-9]+}} = getelementptr inbounds %pointer_field, ptr %slot ; IR: "gc-live"(ptr %slot) @@ -100,11 +110,10 @@ target triple = "x86_64-unknown-linux-goobj" ; OBJVIEW-NEXT: 0, ; OBJVIEW-NEXT: 3, ; OBJVIEW-NEXT: 5 -; OBJVIEW: "kind": "stack_objects" -; OBJVIEW: "offset": 0 -; OBJVIEW: "size": 48 -; OBJVIEW: "ptr_bytes": 48 -; OBJVIEW: "name": "runtime.gcbits.2900000000000000" +; OBJVIEW: "kind": "locals_pointer_maps" +; OBJVIEW: "set_bits": null +; OBJVIEW-NOT: "kind": "stack_objects" +; OBJVIEW: "stack_map_queries": [ ; OBJVIEW-LABEL: "name": "alloca_pointer_free_address_across_calls" ; OBJVIEW-NOT: "kind": "stack_objects" @@ -257,16 +266,14 @@ entry: } define goabiinternal void @argument_aggregate_home_address_across_calls( - %nested %value) gc "goallc" { + ptr preallocated(%nested) align 8 %value) gc "goallc" { entry: - ; The split aggregate parameter still has one complete fixed home and one - ; argp-relative StackObject covering all ABI pieces and padding. - %slot = alloca %nested, align 8 - call void @llvm.lifetime.start.p0(i64 48, ptr %slot) - store %nested %value, ptr %slot, align 8 - call goabiinternal void @observe_stack_address(ptr %slot) + ; A stack-assigned parameter is already one complete fixed incoming home. + ; Its typed preallocated layout supplies the entry and call-site argument + ; bitmaps. + call goabiinternal void @observe_stack_address(ptr %value) call goabiinternal void @safepoint() - call goabiinternal void @observe_stack_address(ptr %slot) + call goabiinternal void @observe_stack_address(ptr %value) ret void } diff --git a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll index cdd741ddc3e7fb..fd4383869132ff 100644 --- a/src/cmd/llvmplugin/testdata/supported-param-attrs.ll +++ b/src/cmd/llvmplugin/testdata/supported-param-attrs.ll @@ -1,6 +1,14 @@ target triple = "aarch64-unknown-linux-goobj" +%stack_arg = type [5 x ptr] + declare goabiinternal void @supported_callee(ptr) +declare token @llvm.call.preallocated.setup(i32) +declare ptr @llvm.call.preallocated.arg(token, i32) +declare goabiinternal void @supported_preallocated_callee( + ptr preallocated(%stack_arg) align 8) +declare goabiinternal void @supported_goret_callee( + ptr goret(%stack_arg) "goretindex"="0" align 8) define goabiinternal void @supported_param_attrs(ptr %argument) #0 gc "goallc" { entry: @@ -8,4 +16,25 @@ entry: ret void } +define goabiinternal void @supported_preallocated_attr(ptr %argument) #0 gc "goallc" { +entry: + %setup = call token @llvm.call.preallocated.setup(i32 1) + %home = call ptr @llvm.call.preallocated.arg(token %setup, i32 0) preallocated(%stack_arg) + store %stack_arg zeroinitializer, ptr %home, align 8 + %first = getelementptr inbounds %stack_arg, ptr %home, i32 0, i32 0 + store ptr %argument, ptr %first, align 8 + call goabiinternal void @supported_preallocated_callee( + ptr preallocated(%stack_arg) align 8 %home) + ["preallocated"(token %setup)] + ret void +} + +define goabiinternal void @supported_goret_attr() #0 gc "goallc" { +entry: + %stack_result = alloca %stack_arg, align 8 + call goabiinternal void @supported_goret_callee( + ptr goret(%stack_arg) "goretindex"="0" align 8 %stack_result) + ret void +} + attributes #0 = { "frame-pointer"="non-leaf" } diff --git a/test/codegen/_cgo_llvm_unsafe_args.go b/test/codegen/_cgo_llvm_unsafe_args.go index 4cec88cf2d522e..7588ccf8e18f1d 100644 --- a/test/codegen/_cgo_llvm_unsafe_args.go +++ b/test/codegen/_cgo_llvm_unsafe_args.go @@ -9,31 +9,28 @@ package codegen //go:noescape func llvmCgoUnsafeSink(*uintptr) -// LLVM-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-SAME: i64 %p, i64 %q) #[[NOINLINE:[0-9]+]] gc "goallc" +// LLVM-LABEL: define goabi0 void @"codegen.llvmCgoUnsafeFrame"( +// LLVM-SAME: ptr preallocated(i64) align 8 %p, ptr preallocated(i64) align 8 %q, ptr goret(i64) align 8 "goretindex"="0" [[RETURN:%.*]]) #[[NOINLINE:[0-9]+]] gc "goallc" // LLVM-NOT: alloca // LLVM: [[FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-NOT: llvm.addressofreturnaddress // LLVM-NOT: llvm.sponentry -// LLVM: [[Q:%.*]] = getelementptr i8, ptr [[FRAME]], i64 8 // LLVM: [[RESULT:%.*]] = getelementptr i8, ptr [[FRAME]], i64 16 -// LLVM: store i64 %p, ptr [[FRAME]] -// LLVM: store i64 %q, ptr [[Q]] -// LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[FRAME]]) -// LLVM: {{%.*}} = load i64, ptr [[RESULT]] +// LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} %p) +// LLVM: call void @llvm.memmove.p0.p0.i64(ptr align 8 [[RETURN]], ptr align 8 [[RESULT]], i64 8, i1 false) +// LLVM: ret void // LLVM: attributes #[[NOINLINE]] = { {{.*}}noinline -// LLVM-OPT-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-OPT-SAME: i64 %p, i64 %q) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" +// LLVM-OPT-LABEL: define goabi0 void @"codegen.llvmCgoUnsafeFrame"( +// LLVM-OPT-SAME: ptr preallocated(i64) align 8 %p, ptr preallocated(i64) align 8{{.*}} %q, ptr {{.*}}goret(i64) align 8{{.*}} "goretindex"="0" [[OPT_RETURN:%.*]]) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" // LLVM-OPT-NOT: alloca // LLVM-OPT: [[OPT_FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-OPT-NOT: llvm.addressofreturnaddress // LLVM-OPT-NOT: llvm.sponentry -// LLVM-OPT: [[OPT_Q:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 8 // LLVM-OPT: [[OPT_RESULT:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 16 -// LLVM-OPT: store i64 %p, ptr [[OPT_FRAME]] -// LLVM-OPT: store i64 %q, ptr [[OPT_Q]] -// LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[OPT_FRAME]]) -// LLVM-OPT: {{%.*}} = load i64, ptr [[OPT_RESULT]] +// LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} %p) +// LLVM-OPT: [[OPT_VALUE:%.*]] = load i64, ptr [[OPT_RESULT]], align 8 +// LLVM-OPT-NEXT: store i64 [[OPT_VALUE]], ptr [[OPT_RETURN]], align 8 +// LLVM-OPT: ret void // LLVM-OPT: attributes #[[OPT_NOINLINE]] = { {{.*}}noinline // //go:cgo_unsafe_args diff --git a/test/codegen/issue25378.go b/test/codegen/issue25378.go index 9a704fc40f2800..4cc89753959237 100644 --- a/test/codegen/issue25378.go +++ b/test/codegen/issue25378.go @@ -7,11 +7,11 @@ package codegen // LLVM-DAG: @codegen.wsp = global <{ [256 x i8] }> {{.*}}, section ".noptrdata" -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16([2 x i16] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16(ptr preallocated([2 x i16]) // LLVM-DAG: zext i16 {{%.*}} to i64 // LLVM-DAG: icmp ult i64 {{%.*}}, 256 // LLVM-DAG: getelementptr i8, ptr @codegen.wsp, i64 -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte([2 x i8] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte(ptr preallocated([2 x i8]) // LLVM-DAG: zext i8 {{%.*}} to i64 var wsp = [256]bool{ diff --git a/test/codegen/llvm_abs.go b/test/codegen/llvm_abs.go index 90ce70838c19eb..baa5ece4958a92 100644 --- a/test/codegen/llvm_abs.go +++ b/test/codegen/llvm_abs.go @@ -9,7 +9,8 @@ package codegen import "math" // LLVM-LABEL: define goabiinternal double @codegen.llvmAbs64(double %x) -// LLVM: call double @llvm.fabs.f64(double %x) +// LLVM-ARM64: call double @llvm.fabs.f64(double %x) +// LLVM-AMD64: and i64 {{.*}}, 9223372036854775807 // LLVM-OPT-LABEL: define goabiinternal double @codegen.llvmAbs64(double %x) // LLVM-OPT: call double @llvm.fabs.f64(double %x) func llvmAbs64(x float64) float64 { diff --git a/test/codegen/llvm_argument_memory_home.go b/test/codegen/llvm_argument_memory_home.go index 8fa68f1d76e74e..7a5484e95c7577 100644 --- a/test/codegen/llvm_argument_memory_home.go +++ b/test/codegen/llvm_argument_memory_home.go @@ -42,19 +42,22 @@ func llvmRegisterArgumentMemoryHome(x llvmArgumentStrings3) int { return len(x.a) + len(x.b) + len(x.c) } -// Non-trivial arrays are assigned wholly to the ABI stack. Their Go SSA -// LocalAddr uses the same local-home initialization instead of reading an -// uninitialized alloca. +// Non-trivial arrays are assigned wholly to the ABI stack. The typed +// preallocated carrier binds LocalAddr directly to that incoming home, without +// constructing and initializing another local copy. // -// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) -// LLVM: [[STACK_HOME:%.*]] = alloca [2 x { ptr, i64 }], align 8 -// LLVM: store [2 x { ptr, i64 }] %x, ptr [[STACK_HOME]], align 8 +// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr preallocated([2 x { ptr, i64 }]) align 8 %x) +// LLVM-NOT: alloca +// LLVM-NOT: store +// LLVM: load { ptr, i64 }, ptr +// LLVM: load { ptr, i64 }, ptr // LLVM: ret i64 // -// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) +// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr preallocated([2 x { ptr, i64 }]) align 8{{.*}} %x) // LLVM-OPT-NOT: alloca -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 0 -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 1 +// LLVM-OPT-NOT: store +// LLVM-OPT: load i64, ptr +// LLVM-OPT: load i64, ptr // LLVM-OPT: ret i64 // // LLVM-LABEL: define goabiinternal i64 @codegen.llvmDirectRegisterArgument(i64 %x) diff --git a/test/codegen/llvm_dereference.go b/test/codegen/llvm_dereference.go index 3a109311ea48d5..3bf9d4e855cf5c 100644 --- a/test/codegen/llvm_dereference.go +++ b/test/codegen/llvm_dereference.go @@ -15,10 +15,12 @@ type llvmDereferenceLargeResult struct { values [32]int } -// LLVM-LABEL: define goabiinternal %codegen.llvmDereferenceLargeResult @codegen.llvmNamedLargeStackResult( +// LLVM-LABEL: define goabiinternal void @codegen.llvmNamedLargeStackResult( +// LLVM-SAME: i64 %seed, ptr goret(%codegen.llvmDereferenceLargeResult) align 8 "goretindex"="0" [[RESULT:%.*]]) // LLVM: call goabiinternal void @codegen.llvmFillNamedLargeStackResult( // LLVM: load %codegen.llvmDereferenceLargeResult, ptr {{%.*}}, align 8 -// LLVM: ret %codegen.llvmDereferenceLargeResult +// LLVM: call goabiinternal void @"runtime.memmove"(ptr [[RESULT]], ptr {{%.*}}, i64 256) +// LLVM: ret void // // LLVM-LABEL: define goabiinternal %codegen.llvmDereferenceAddressedResult @codegen.llvmNamedStackResult( // LLVM: call goabiinternal void @codegen.llvmFillNamedStackResult( diff --git a/test/codegen/llvm_direct_iface.go b/test/codegen/llvm_direct_iface.go index 1d29e761a6dc8c..0173b9092c40b9 100644 --- a/test/codegen/llvm_direct_iface.go +++ b/test/codegen/llvm_direct_iface.go @@ -34,7 +34,10 @@ func llvmDirectIfaceSink(llvmDirectIfaceNested) {} // LLVM: [[LEAF:%.*]] = insertvalue %codegen.llvmDirectIfaceLeaf undef, ptr [[DATA]], 0 // LLVM: [[ARRAY:%.*]] = insertvalue [1 x %codegen.llvmDirectIfaceLeaf] undef, %codegen.llvmDirectIfaceLeaf [[LEAF]], 0 // LLVM: [[NESTED:%.*]] = insertvalue %codegen.llvmDirectIfaceNested {{.*}}, [1 x %codegen.llvmDirectIfaceLeaf] [[ARRAY]], 2 -// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(%codegen.llvmDirectIfaceNested [[NESTED]]) +// LLVM: [[SETUP:%.*]] = call token @llvm.call.preallocated.setup(i32 1) +// LLVM: [[HOME:%.*]] = call ptr @llvm.call.preallocated.arg(token [[SETUP]], i32 0) +// LLVM: store %codegen.llvmDirectIfaceNested [[NESTED]], ptr [[HOME]], align 8 +// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(ptr preallocated(%codegen.llvmDirectIfaceNested) align 8 [[HOME]]) [ "preallocated"(token [[SETUP]]) ] func llvmDirectIfaceCall(x any) { switch x := x.(type) { case llvmDirectIfaceNested: diff --git a/test/codegen/llvm_linkname.go b/test/codegen/llvm_linkname.go index a497cd96dcc71f..71a25f2dcfcead 100644 --- a/test/codegen/llvm_linkname.go +++ b/test/codegen/llvm_linkname.go @@ -20,14 +20,18 @@ func llvmLinknameLocal() int { // LLVM: call goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM: declare goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-NOT: @"runtime.llvmLinknameLocal" -// LLVM-LABEL: define weak goabi0 i64 @"runtime.llvmLinknameLocal"() +// LLVM-LABEL: define weak goabi0 void @"runtime.llvmLinknameLocal"( +// LLVM-SAME: ptr goret(i64) align 8 "goretindex"="0" [[RESULT:%.*]]) // LLVM: call goabiinternal i64 @runtime.llvmLinknameLocal() +// LLVM: store i64 {{%.*}}, ptr [[RESULT]], align 8 // LLVM-LABEL: define goabiinternal i64 @runtime.llvmLinknameLocal() // LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmLinknameCalls() // LLVM-OPT: call goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-OPT: declare goabiinternal i64 @"runtime.llvmLinknameExternal"() // LLVM-OPT-NOT: @"runtime.llvmLinknameLocal" -// LLVM-OPT-LABEL: define weak goabi0 i64 @"runtime.llvmLinknameLocal"() +// LLVM-OPT-LABEL: define weak goabi0 void @"runtime.llvmLinknameLocal"( +// LLVM-OPT-SAME: ptr goret(i64) align 8 "goretindex"="0" [[OPT_RESULT:%.*]]) +// LLVM-OPT: store i64 7, ptr [[OPT_RESULT]], align 8 // LLVM-OPT-LABEL: define goabiinternal {{.*}}@runtime.llvmLinknameLocal() func llvmLinknameCalls() int { return llvmLinknameExternal() + llvmLinknameLocal() diff --git a/test/codegen/llvm_selectnaddr.go b/test/codegen/llvm_selectnaddr.go index caaeafb43a47b2..c78e263e1d230e 100644 --- a/test/codegen/llvm_selectnaddr.go +++ b/test/codegen/llvm_selectnaddr.go @@ -10,8 +10,8 @@ type llvmAddressedCallResult [20]int // LLVM-LABEL: define goabiinternal i64 @codegen.llvmReadAddressedCallResult( // LLVM: [[HOME:%[^ ]+\.home]] = alloca [20 x i64], align 8 -// LLVM: [[RESULT:%.*]] = call goabiinternal [20 x i64] @codegen.llvmMakeAddressedCallResult(i64 %seed) -// LLVM-NEXT: store [20 x i64] [[RESULT]], ptr [[HOME]], align 8 +// LLVM: call goabiinternal void @codegen.llvmMakeAddressedCallResult(i64 %seed, ptr goret([20 x i64]) align 8 "goretindex"="0" [[HOME]]) +// LLVM-NOT: store [20 x i64] // LLVM: load i64, ptr // LLVM: ret i64 // diff --git a/test/llvm_stdlib_packages.json b/test/llvm_stdlib_packages.json index 08d8012391e14a..6c27967880c460 100644 --- a/test/llvm_stdlib_packages.json +++ b/test/llvm_stdlib_packages.json @@ -2,9 +2,13 @@ "packages": { "whitelist": { "archive/tar": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "archive/zip": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "bufio": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "bytes": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "cmp": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/flate": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/gzip": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", + "compress/zlib": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/heap": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/list": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", "container/ring": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests", @@ -67,11 +71,7 @@ "unicode/utf8": "qualified through LLVM O2 compilation of its standard-library dependency closure, GoObj/archive, link, and package tests" }, "blacklist": { - "*": "package and its standard-library dependency closure have not yet been qualified through LLVM O2 compile, GoObj/archive, link, and package tests", - "archive/zip": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/gzip": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/zlib": "dependency closure includes compress/flate, whose LLVM SelectionDAG lowering is not yet qualified", - "compress/flate": "llc/GoObj: X86 and AArch64 SelectionDAG instruction-selection assertions" + "*": "package and its standard-library dependency closure have not yet been qualified through LLVM O2 compile, GoObj/archive, link, and package tests" }, "platform_blacklist": { "linux/amd64": {