diff --git a/.github/workflows/goallc.yml b/.github/workflows/goallc.yml index c99944909f4aef..c3c3a73c003424 100644 --- a/.github/workflows/goallc.yml +++ b/.github/workflows/goallc.yml @@ -25,8 +25,8 @@ concurrency: cancel-in-progress: true env: - PINNED_LLVM_RELEASE: goallc-llvm23.1.0-20260815T075241Z - PINNED_LLVM_REVISION: 9bd4c90aca1584de810fd081bc1fd6a0cce602dd + PINNED_LLVM_RELEASE: goallc-llvm23.1.0-20260818T010709Z + PINNED_LLVM_REVISION: 040059c00d16bfb643a87bf77d162f926a96466f jobs: llvm-payload: diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 1939d5c9ce8774..6af4210b913d47 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -39,6 +39,7 @@ type LLVMFuncContext struct { b llvm.Builder ReturnType llvm.Type ResultCount int + Params []llvmParamSignature } // SSA may clone an ir.Name while retaining the same logical source @@ -96,10 +97,17 @@ type llvmFuncSignature struct { Type llvm.Type ReturnType llvm.Type ResultCount int + Params []llvmParamSignature HasClosureContext bool ClosureContextIndex int } +type llvmParamSignature struct { + ValueType llvm.Type + Alignment int + ByVal bool +} + func llvmCallConv(which obj.ABI) llvm.CallConv { switch which { case obj.ABI0: @@ -165,9 +173,23 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { } params := make([]llvm.Type, 0, aux.NArgs()) + paramSignatures := make([]llvmParamSignature, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { - param := getLLVMABIType(aux.TypeOfArg(i)) - params = append(params, param) + goType := aux.TypeOfArg(i) + valueType := getLLVMABIType(goType) + paramType := valueType + param := llvmParamSignature{ValueType: valueType} + assignment := aux.ABIInfo().InParam(int(i)) + if len(assignment.Registers) == 0 && goType.Size() != 0 { + if goType.Alignment() <= 0 { + base.Fatalf("invalid alignment %d for stack argument %d of type %v", goType.Alignment(), i, goType) + } + param.ByVal = true + param.Alignment = int(goType.Alignment()) + paramType = GlobalCtxt.PointerType(0) + } + params = append(params, paramType) + paramSignatures = append(paramSignatures, param) } results := make([]llvm.Type, 0, aux.NResults()) @@ -189,6 +211,7 @@ func llvmSignature(aux *AuxCall) llvmFuncSignature { Type: llvm.FunctionType(ret, params, false), ReturnType: ret, ResultCount: len(results), + Params: paramSignatures, ClosureContextIndex: -1, } } @@ -210,6 +233,14 @@ func llvmNestAttribute() llvm.Attribute { return GlobalCtxt.CreateEnumAttribute(kind, 0) } +func llvmByValAttribute(t llvm.Type) llvm.Attribute { + kind := llvm.AttributeKindID("byval") + if kind == 0 { + base.Fatalf("LLVM does not provide the byval parameter attribute") + } + return GlobalCtxt.CreateTypeAttribute(kind, t) +} + func llvmNullPointerIsValidAttribute() llvm.Attribute { kind := llvm.AttributeKindID("null_pointer_is_valid") if kind == 0 { @@ -231,6 +262,13 @@ func configureLLVMFunction(fn llvm.Value, sig llvmFuncSignature, cc llvm.CallCon if sig.ResultCount > 1 { fn.AddFunctionAttr(GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + for i, param := range sig.Params { + if !param.ByVal { + continue + } + fn.AddAttributeAtIndex(i+1, llvmByValAttribute(param.ValueType)) + fn.Param(i).SetParamAlignment(param.Alignment) + } if sig.HasClosureContext { // LLVM parameter attribute indexes are one-based. The closure context // is deliberately excluded from the Go ABI argument layout by the @@ -243,6 +281,13 @@ func configureLLVMCall(call llvm.Value, sig llvmFuncSignature) { if sig.ResultCount > 1 { call.AddCallSiteAttribute(llvmAttributeFunctionIndex, GlobalCtxt.CreateStringAttribute(goResultsTupleAttr, "")) } + for i, param := range sig.Params { + if !param.ByVal { + continue + } + call.AddCallSiteAttribute(i+1, llvmByValAttribute(param.ValueType)) + call.SetInstrParamAlignment(i+1, param.Alignment) + } } func llvmFunctionStorageName(name string, cc llvm.CallConv) string { @@ -1552,13 +1597,64 @@ func (lfc *LLVMFuncContext) paramForArgNameAndType(name *ir.Name) (llvm.Value, * key := llvmLocalKeyForName(name) for i, param := range lfc.F.OwnAux.ABIInfo().InParams() { if param.Name != nil && llvmLocalKeyForName(param.Name) == key { - return lfc.LF.Param(i), lfc.F.OwnAux.TypeOfArg(int64(i)) + value := lfc.LF.Param(i) + if lfc.Params[i].ByVal { + value = lfc.b.CreateLoad(lfc.Params[i].ValueType, value, name.Sym().Name+".byval") + value.SetAlignment(lfc.Params[i].Alignment) + } + return value, lfc.F.OwnAux.TypeOfArg(int64(i)) } } lfc.F.fe.Fatalf(name.Pos(), "could not find LLVM parameter for %v", name) return llvm.Value{}, nil } +func (lfc *LLVMFuncContext) llvmByValCallArgument(v, argValue *Value, index int, logical *types.Type, param llvmParamSignature) llvm.Value { + if !param.ByVal || logical.Size() == 0 { + v.Fatalf("argument %d is not a non-empty byval parameter", index) + } + + // Preserve an existing Go memory source when no memory effect separates its + // load from the call. Ordinary LLVM byval lowering will copy these bytes + // into the callee's Go ABI stack home; manufacturing an intermediate + // aggregate value and a second alloca would only obscure that source from + // ordinary memcpy forwarding. A different memory state requires the SSA + // value path below so later argument evaluation cannot change the snapshot. + if types.Identical(argValue.Type, logical) && + (argValue.Op == OpLoad || argValue.Op == OpDereference) && len(argValue.Args) != 0 && + argValue.MemoryArg() == v.MemoryArg() { + address := lfc.GenLV(argValue.Args[0]) + if address.Type().TypeKind() != llvm.PointerTypeKind { + v.Fatalf("byval argument %d has non-pointer source address", index) + } + return address + } + + value := lfc.GenLV(argValue) + value = lfc.llvmValueToABI(v, value, argValue.Type, logical, param.ValueType, fmt.Sprintf("%s.arg%d", v, index)) + if value.Type() != param.ValueType { + v.Fatalf("byval argument %d has incompatible LLVM value type", index) + } + + // LLVM byval takes an address. A pure SSA value therefore needs a canonical + // source object in IR. It is intentionally an ordinary alloca-backed byval + // source: generic DAG combines may fold individual copies, while ordinary + // byval copy semantics remain the correctness path. + entryBuilder := GlobalCtxt.NewBuilder() + defer entryBuilder.Dispose() + entry := lfc.LF.EntryBasicBlock() + if first := entry.FirstInstruction(); first.IsNil() { + entryBuilder.SetInsertPointAtEnd(entry) + } else { + entryBuilder.SetInsertPointBefore(first) + } + address := entryBuilder.CreateAlloca(param.ValueType, fmt.Sprintf("%s.arg%d.byval", v, index)) + address.SetAlignment(param.Alignment) + store := lfc.b.CreateStore(value, address) + store.SetAlignment(param.Alignment) + return address +} + func (lfc *LLVMFuncContext) registerArgument(v *Value) llvm.Value { aux, ok := v.Aux.(*AuxNameOffset) if !ok || aux.Name == nil { @@ -1915,9 +2011,17 @@ func (lfc *LLVMFuncContext) staticCall(v *Value) llvm.Value { aux.Fn == ir.Syms.Memmove || aux.Fn == ir.Syms.Memequal args := make([]llvm.Value, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[i]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + var arg llvm.Value + if sig.Params[i].ByVal { + arg = lfc.llvmByValCallArgument(v, v.Args[i], int(i), aux.TypeOfArg(i), sig.Params[i]) + } else { + arg = lfc.GenLV(v.Args[i]) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, v.Args[i].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } + } + if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { + v.Fatalf("argument %d to %s has incompatible LLVM type", i, aux.Fn.Name) } args = append(args, arg) } @@ -1964,9 +2068,15 @@ func (lfc *LLVMFuncContext) indirectCall(v *Value, argStart int, closureContext } args := make([]llvm.Value, 0, aux.NArgs()) for i := int64(0); i < aux.NArgs(); i++ { - arg := lfc.GenLV(v.Args[argStart+int(i)]) - if arg.Type() != sig.Type.ParamTypes()[i] { - arg = lfc.llvmValueToABI(v, arg, v.Args[argStart+int(i)].Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + argValue := v.Args[argStart+int(i)] + var arg llvm.Value + if sig.Params[i].ByVal { + arg = lfc.llvmByValCallArgument(v, argValue, int(i), aux.TypeOfArg(i), sig.Params[i]) + } else { + arg = lfc.GenLV(argValue) + if arg.Type() != sig.Type.ParamTypes()[i] { + arg = lfc.llvmValueToABI(v, arg, argValue.Type, aux.TypeOfArg(i), sig.Type.ParamTypes()[i], fmt.Sprintf("%s.arg%d", v, i)) + } } if got, want := arg.Type(), sig.Type.ParamTypes()[i]; got != want { v.Fatalf("argument %d to indirect call has incompatible LLVM type", i) @@ -2999,6 +3109,7 @@ func LLVMCompile(f *Func) { b: GlobalCtxt.NewBuilder(), ReturnType: sig.ReturnType, ResultCount: sig.ResultCount, + Params: sig.Params, } defer FCtxt.b.Dispose() @@ -3232,6 +3343,39 @@ func LLVMCompile(f *Func) { FCtxt.Locals[key] = slot return slot, true } + // A typed byval parameter already denotes the callee-private Go ABI stack + // copy. Bind addressable parameter uses directly to that incoming home; + // creating a second alloca and copying the value again would defeat the ABI + // model. + for i, param := range sig.Params { + if !param.ByVal { + continue + } + // CgoUnsafeArgs exposes one parameter address as the base of the + // complete contiguous ABI0 input/result frame. A typed byval argument + // denotes only its own object, so keep addressable uses on the existing + // llvm.go.abi0.frame-derived view below. The physical incoming home is + // still the same slot and therefore needs no initialization copy. + if cgoUnsafeArgs { + continue + } + assignment := inParams[i] + if len(assignment.Registers) != 0 { + f.fe.Fatalf(f.Entry.Pos, "LLVM byval parameter %d was assigned Go registers", i) + } + if assignment.Name == nil { + continue + } + goType := f.OwnAux.TypeOfArg(int64(i)) + if goType.Size() == 0 || !types.Identical(goType, assignment.Name.Type()) { + f.fe.Fatalf(assignment.Name.Pos(), "invalid Go type for LLVM byval parameter %v", assignment.Name) + } + key := llvmLocalKeyForName(assignment.Name) + if _, exists := FCtxt.Locals[key]; exists { + f.fe.Fatalf(assignment.Name.Pos(), "duplicate LLVM byval parameter home %v", assignment.Name) + } + FCtxt.Locals[key] = llvmStackSlot{Value: FCtxt.LF.Param(i), Type: assignment.Name.Type()} + } // Escape analysis names the pointer to a heap-backed result &result. SSA can // keep that pointer only in a register, but panic recovery needs the same // stable, whole-function local-slot semantics as an ordinary named result. @@ -3276,6 +3420,12 @@ func LLVMCompile(f *Func) { continue } if _, created := preallocateLocal(param.Name, param.Name.Sym().Name+".cgo"); created { + // Keep an explicit IR data dependence between every formal input + // and the complete frame address passed to C. Per-argument byval + // semantics cannot describe CgoUnsafeArgs' permitted access beyond + // the first parameter object into adjacent arguments/results. The + // two addresses lower to the same incoming fixed home, so target + // copy folding may remove the physical self-copy. parameterHomes = append(parameterHomes, param.Name) } } diff --git a/src/cmd/internal/testdir/llvm_abi_test.go b/src/cmd/internal/testdir/llvm_abi_test.go index beb40a4d1f1fd6..ba4d75c1cf106b 100644 --- a/src/cmd/internal/testdir/llvm_abi_test.go +++ b/src/cmd/internal/testdir/llvm_abi_test.go @@ -192,10 +192,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { } } for _, pattern := range []string{ - `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?"gc-live"\(ptr %second, ptr %first\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, - `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr, ptr \} @main\.pointerAggregateBothOverflow.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, i64 \} @main\.liveScalarStackArgument.*?ptr byval\(ptr\) align 8 %pointer.*?load ptr, ptr %pointer.*?"gc-live"\(ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerSequenceStackArguments.*?ptr byval\(ptr\) align 8 %first.*?ptr byval\(ptr\) align 8 %second.*?load ptr, ptr %first.*?load ptr, ptr %second.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr, i64 \} @main\.livePointerAggregateStackArgument.*?ptr byval\(%main\.pointerStackAggregate\) align 8 %value.*?load %main\.pointerStackAggregate, ptr %value.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr, ptr \} @main\.pointerAggregateBothOverflow.*?ptr byval\(%main\.pointerStackAggregate\) align 8 %value.*?load %main\.pointerStackAggregate, ptr %value.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten GoALLC ABI IR does not match %q", pattern) @@ -208,10 +208,9 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") - // The runtime wrapper applies its configured LLVM optimization pipeline - // before llc. In particular, InstCombine removes the field-by-field bridge - // between physical ABI carriers and semantic named aggregates, allowing - // stack-assigned pointer arguments to remain in their canonical fixed homes. + // Match the production optimization pipeline before checking the final + // lowering. Loaded pointers deliberately use ordinary statepoint spill slots; + // this test does not require the optional fixed-home reuse optimization. optimizedLLVMIR := llvmArchive + ".opt.ll" runLLVMABICommand(t, nil, opt, "-passes=default", "-S", llvmIR, "-o", optimizedLLVMIR) @@ -220,10 +219,10 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedLLVMIR) for _, pattern := range []string{ - `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.2[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.[02].*?LDRXui\s+%fixed-stack\.[02]`, - `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, - `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?id:\s+4.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.3[^\n]*%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.2.*?LDRXui\s+%fixed-stack\.3.*?STRXui[^\n]*%fixed-stack\.0.*?STRXui[^\n]*%fixed-stack\.1`, + `(?s)name:\s+main\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?size:\s+8.*?isAliased:\s+true.*?stack:\s+.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*1, 8, %stack\.[0-9]+, 0.*?LDRXui\s+%stack\.[0-9]+`, + `(?s)name:\s+main\.livePointerSequenceStackArguments.*?fixedStack:.*?offset:\s+24.*?offset:\s+8.*?stack:\s+.*?size:\s+8.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*1, 8, %stack\.[0-9]+, 0[^\n]*1, 8, %stack\.[0-9]+, 0.*?LDRXui\s+%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+`, + `(?s)name:\s+main\.livePointerAggregateStackArgument.*?fixedStack:.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+.*?size:\s+8.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*1, 8, %stack\.[0-9]+, 0[^\n]*1, 8, %stack\.[0-9]+, 0.*?LDRXui\s+%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+`, + `(?s)name:\s+main\.pointerAggregateBothOverflow.*?fixedStack:.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+.*?size:\s+8.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*1, 8, %stack\.[0-9]+, 0[^\n]*1, 8, %stack\.[0-9]+, 0.*?LDRXui\s+%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("GoALLC ABI MIR does not match %q", pattern) @@ -260,55 +259,53 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { }, { name: "mixedABI", args: 152, pointerBits: []int{2, 4, 18}, - nativeArgsMaps: [][]int{{2, 4, 18}, nil}, - // The explicit panicmem paths keep the still-needed parameter - // homes in LocalsPointerMaps through locals-only alloca records. - goallcArgsMaps: [][]int{{2, 4, 18}, {2}}, + nativeArgsMaps: [][]int{{2, 4, 18}, nil}, + goallcArgsMaps: [][]int{{2, 4, 18}, nil}, nativeStackMaps: []int32{-1, 0, -1}, goallcStackMaps: []int32{-1, 1, -1, 1}, - goallcQueryMaps: [][]int{{2, 4, 18}, {2}, {2}, {2}, {2}}, + goallcQueryMaps: [][]int{{2, 4, 18}, nil, nil, nil, nil}, }, { name: "liveScalarStackArgument", args: 136, pointerBits: []int{0}, nativeArgsMaps: [][]int{{0}, nil}, - goallcArgsMaps: [][]int{{0}}, + goallcArgsMaps: [][]int{{0}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, -1}, + goallcStackMaps: []int32{-1, 1, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {3}}, nativeQueries: []int32{0, -1}, - goallcQueries: []int32{-1, 0}, + goallcQueries: []int32{-1, 1}, }, { name: "livePointerSequenceStackArguments", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, -1}, + goallcStackMaps: []int32{-1, 1, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, - goallcQueries: []int32{-1, 0}, + goallcQueries: []int32{-1, 1}, }, { name: "livePointerAggregateStackArgument", args: 136, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, -1}, + goallcStackMaps: []int32{-1, 1, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 24, + goallcLocals: 40, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {2, 3}}, nativeQueries: []int32{0, -1}, - goallcQueries: []int32{-1, 0}, + goallcQueries: []int32{-1, 1}, }, { name: "growPointer", args: 16, pointerBits: []int{0}, @@ -341,23 +338,23 @@ func runLLVMAArch64ABIDifferentialTest(t *testing.T, gorootTestDir string) { { name: "bothOverflow", args: 168, pointerBits: []int{2, 6, 20}, nativeArgsMaps: [][]int{{2, 6, 20}, nil}, - goallcArgsMaps: [][]int{{2, 6, 20}, {2}, nil}, + goallcArgsMaps: [][]int{{2, 6, 20}, nil, nil}, nativeStackMaps: []int32{-1, 0, 1, -1}, goallcStackMaps: []int32{-1, 1, -1, 2}, }, { name: "pointerAggregateBothOverflow", args: 152, pointerBits: []int{0, 2}, nativeArgsMaps: [][]int{{0, 2}, nil}, - goallcArgsMaps: [][]int{{0, 2}}, + goallcArgsMaps: [][]int{{0, 2}, nil}, nativeStackMaps: []int32{-1, 0, -1}, - goallcStackMaps: []int32{-1, 0, -1}, + goallcStackMaps: []int32{-1, 1, -1}, checkFullMaps: true, nativeLocals: 8, - goallcLocals: 136, + goallcLocals: 152, nativeLocalMaps: [][]int{nil, nil}, - goallcLocalMaps: [][]int{nil}, + goallcLocalMaps: [][]int{nil, {16, 17}}, nativeQueries: []int32{0, -1}, - goallcQueries: []int32{-1, 0}, + goallcQueries: []int32{-1, 1}, }, { name: "requireAggregate", args: 40, pointerBits: []int{4}, @@ -450,8 +447,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?ptr byval\(ptr\) align 8 %pointer.*?load ptr, ptr %pointer.*?"gc-live"\(ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?ptr byval\(%p\.pointerAggregate\) align 8 %value.*?load %p\.pointerAggregate, ptr %value.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten amd64 IR does not match %q", pattern) @@ -459,10 +456,15 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri } runLLVMABICommand(t, rewrittenIR, opt, "-load-pass-plugin="+plugin, "-passes=verify", "-disable-output", "-") + // Match the production pipeline before checking ordinary statepoint spills. + // Loaded pointers are not reused from byval homes in this change. + optimizedGoallcIR := goallcArchive + ".opt.ll" + runLLVMABICommand(t, nil, opt, "-passes=default", "-S", goallcIR, + "-o", optimizedGoallcIR) machineIR := runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, "-stop-after=prolog-epilog", - "-o", "-", goallcIR) + "-o", "-", optimizedGoallcIR) morestackName, ok := goobj.BuiltinSymbolName("runtime.morestack_noctxt", 0) if !ok { t.Fatal("runtime.morestack_noctxt ABI0 is absent from the builtin table") @@ -476,10 +478,10 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri `STATEPOINT -[0-9]+,[^\n]*\$rsp, 0,[^\n]*\$rsp, 8,`, }, "p.liveScalarStackArgument": { - `STATEPOINT -[0-9]+,[^\n]*\$rsp, 72,`, + `STATEPOINT -[0-9]+,[^\n]*\$rsp, 0,`, }, "p.liveAggregateStackArgument": { - `STATEPOINT -[0-9]+,[^\n]*\$rsp, 56,[^\n]*\$rsp, 72,`, + `STATEPOINT -[0-9]+,[^\n]*\$rsp, 0,[^\n]*\$rsp, 8,`, }, } for name, patterns := range machinePatterns { @@ -498,7 +500,7 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri } runLLVMABICommand(t, nil, llc, "-load-pass-plugin="+plugin, - "-filetype=obj", goallcIR, "-o", goallcObject) + "-filetype=obj", optimizedGoallcIR, "-o", goallcObject) // Check the instructions from the GoObj artifact used below, rather than // llc's diagnostic assembly-text output. goallcDisassembly := runLLVMABICommand(t, nil, goTool, "tool", "objdump", goallcObject) @@ -506,8 +508,8 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri `(?s)TEXT p\.initializedPointerResult.*?MOVQ\s+AX, 0x48\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+CX, 0x40\(BP\)`, `(?s)TEXT p\.partiallyInitializedAggregateResult.*?MOVQ\s+AX, 0x50\(BP\)`, - `(?s)TEXT p\.liveScalarStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), AX`, - `(?s)TEXT p\.liveAggregateStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x48\(BP\), BX.*?MOVQ\s+0x38\(BP\), AX`, + `(?s)TEXT p\.liveScalarStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0\(SP\), AX`, + `(?s)TEXT p\.liveAggregateStackArgument.*?R_CALL:p\.safepoint.*?MOVQ\s+0x8\(SP\), BX.*?MOVQ\s+0\(SP\), AX`, } { if !regexp.MustCompile(pattern).Match(goallcDisassembly) { t.Fatalf("GoALLC amd64 object disassembly does not match %q", pattern) @@ -549,17 +551,17 @@ func runLLVMAMD64ArgsPointerMapDifferentialTest(t *testing.T, gorootTestDir stri }, { name: "liveScalarStackArgument", args: 136, entryBits: []int{7}, - goallcLocals: 8, goallcArgs: [][]int{{7}}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, -1}, - nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, -1}, - nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 0}, + goallcLocals: 16, goallcArgs: [][]int{{7}, nil}, goallcMaps: [][]int{nil, {1}}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, -1}, + nativeQueries: []int32{0, -1}, goallcQueries: []int32{1, -1}, + nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 16, 8, 0}, }, { name: "liveAggregateStackArgument", args: 136, entryBits: []int{5, 7}, - goallcLocals: 8, goallcArgs: [][]int{{5, 7}}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, -1}, - nativeQueries: []int32{0, -1}, goallcQueries: []int32{0, -1}, - nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 0}, + goallcLocals: 24, goallcArgs: [][]int{{5, 7}, nil}, goallcMaps: [][]int{nil, {1, 2}}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, -1}, + nativeQueries: []int32{0, -1}, goallcQueries: []int32{1, -1}, + nativePCSP: []int32{0, 8, 0}, goallcPCSP: []int32{0, 8, 24, 8, 0}, }, } for _, tc := range cases { @@ -641,8 +643,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-goallc-pass-plugin-emit-ir", "-filetype=null", "-o", "-", goallcIR) for _, pattern := range []string{ - `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?"gc-live"\(ptr %pointer\).*?gc\.relocate`, - `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal ptr @p\.liveScalarStackArgument.*?ptr byval\(ptr\) align 8 %pointer.*?load ptr, ptr %pointer.*?"gc-live"\(ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, + `(?s)define goabiinternal \{ ptr, ptr \} @p\.liveAggregateStackArgument.*?ptr byval\(%p\.pointerAggregate\) align 8 %value.*?load %p\.pointerAggregate, ptr %value.*?"gc-live"\(ptr %[[:alnum:]$._-]+, ptr %[[:alnum:]$._-]+\).*?gc\.relocate`, } { if !regexp.MustCompile(pattern).Match(rewrittenIR) { t.Fatalf("rewritten source IR does not match %q", pattern) @@ -658,8 +660,8 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p "-load-pass-plugin="+plugin, "-stop-after=finalize-isel", "-o", "-", optimizedGoallcIR) for _, pattern := range []string{ - `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?isImmutable:\s+false.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0`, - `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?id:\s+2.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+\[\].*?STATEPOINT[^\n]*%fixed-stack\.1[^\n]*%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.0.*?LDRXui\s+%fixed-stack\.1`, + `(?s)name:\s+p\.liveScalarStackArgument.*?fixedStack:.*?offset:\s+8.*?isAliased:\s+true.*?stack:\s+.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+`, + `(?s)name:\s+p\.liveAggregateStackArgument.*?fixedStack:.*?offset:\s+8.*?size:\s+24.*?isAliased:\s+true.*?stack:\s+.*?size:\s+8.*?size:\s+8.*?LDRXui\s+%fixed-stack\.[0-9]+.*?LDRXui\s+%fixed-stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STRXui[^\n]*%stack\.[0-9]+.*?STATEPOINT[^\n]*%stack\.[0-9]+[^\n]*%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+.*?LDRXui\s+%stack\.[0-9]+`, } { if !regexp.MustCompile(pattern).Match(machineIR) { t.Fatalf("source MIR does not match %q", pattern) @@ -704,19 +706,19 @@ func runLLVMABIArgsPointerMapSourceTest(t *testing.T, gorootTestDir, llc, opt, p }, { name: "liveScalarStackArgument", args: 136, entryBits: []int{0}, - nativeLocals: 8, goallcLocals: 8, - nativeArgs: [][]int{{0}, nil}, goallcArgs: [][]int{{0}}, - nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, -1}, - nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, + nativeLocals: 8, goallcLocals: 24, + nativeArgs: [][]int{{0}, nil}, goallcArgs: [][]int{{0}, nil}, + nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {1}}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, -1}, + nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 1}, }, { name: "liveAggregateStackArgument", args: 136, entryBits: []int{0, 2}, - nativeLocals: 8, goallcLocals: 8, - nativeArgs: [][]int{{0, 2}, nil}, goallcArgs: [][]int{{0, 2}}, - nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil}, - nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 0, -1}, - nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 0}, + nativeLocals: 8, goallcLocals: 24, + nativeArgs: [][]int{{0, 2}, nil}, goallcArgs: [][]int{{0, 2}, nil}, + nativeMaps: [][]int{nil, nil}, goallcMaps: [][]int{nil, {0, 1}}, + nativePCData: []int32{-1, 0, -1}, goallcPCData: []int32{-1, 1, -1}, + nativeQueries: []int32{0, -1}, goallcQueries: []int32{-1, 1}, }, } for _, tc := range cases { diff --git a/src/cmd/internal/testdir/llvm_alloca_test.go b/src/cmd/internal/testdir/llvm_alloca_test.go index 04e7850acf708c..0c2b17f19f62d4 100644 --- a/src/cmd/internal/testdir/llvm_alloca_test.go +++ b/src/cmd/internal/testdir/llvm_alloca_test.go @@ -92,29 +92,40 @@ func runLLVMAllocaStatepointTest(t *testing.T, gorootTestDir string) { parameterInputFunction := llvmAllocaIRFunction(t, inputIR, "p.parameterAcrossSafepoints") for _, pattern := range []string{ - `define goabiinternal void @p\.parameterAcrossSafepoints\(%p\.pointerLocal %value\)`, - `alloca %p\.pointerLocal, align 8`, - `call void @llvm\.lifetime\.start\.p0\(ptr %v[0-9]+\)`, - `store %p\.pointerLocal %value, ptr %v[0-9]+, align 8`, + `define goabiinternal void @p\.parameterAcrossSafepoints\(ptr byval\(%p\.pointerLocal\) align 8 %value\)`, + `call goabiinternal void @p\.mutateLocal\(ptr %value, i8 0\)`, + `call goabiinternal void @p\.safepoint\(\)`, + `call goabiinternal void @p\.mutateLocal\(ptr %value, i8 1\)`, } { if !regexp.MustCompile(pattern).Match(parameterInputFunction) { t.Fatalf("input parameter-home IR does not match %q\n%s", pattern, parameterInputFunction) } } + for _, forbidden := range []string{"alloca %p.pointerLocal", "llvm.lifetime.start", "store %p.pointerLocal"} { + if bytes.Contains(parameterInputFunction, []byte(forbidden)) { + t.Fatalf("input parameter-home IR retained %q\n%s", + forbidden, parameterInputFunction) + } + } stackParameterInputFunction := llvmAllocaIRFunction(t, inputIR, "p.stackParameterAcrossSafepoints") for _, pattern := range []string{ - `define goabiinternal void @p\.stackParameterAcrossSafepoints\(\[2 x ptr\] %value\)`, - `alloca \[2 x ptr\], align 8`, - `call void @llvm\.lifetime\.start\.p0\(ptr %v[0-9]+\)`, - `store \[2 x ptr\] %value, ptr %v[0-9]+, align 8`, + `define goabiinternal void @p\.stackParameterAcrossSafepoints\(ptr byval\(\[2 x ptr\]\) align 8 %value\)`, + `call goabiinternal void @p\.mutatePointerArray\(ptr %value\)`, + `call goabiinternal void @p\.safepoint\(\)`, } { if !regexp.MustCompile(pattern).Match(stackParameterInputFunction) { t.Fatalf("input stack-parameter-home IR does not match %q\n%s", pattern, stackParameterInputFunction) } } + for _, forbidden := range []string{"alloca [2 x ptr]", "llvm.lifetime.start", "store [2 x ptr]"} { + if bytes.Contains(stackParameterInputFunction, []byte(forbidden)) { + t.Fatalf("input stack-parameter-home IR retained %q\n%s", + forbidden, stackParameterInputFunction) + } + } // LLVM compilation runs before native register allocation turns // OpKeepAlive of a stack address into OpVarLive. Preserve the value in the diff --git a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir index 6d71b5d8562480..5c8412cecf4250 100644 --- a/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir +++ b/src/cmd/internal/testdir/testdata/llvm_args_pointer_maps.mir @@ -33,6 +33,8 @@ frameInfo: stackSize: 0 maxAlignment: 8 hasCalls: true + goABIStackArgsSize: 0 + goABIArgSize: 16 machineFunctionInfo: hasRedZone: false stackSizeZPR: 0 diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index 214205fe1df5c5..d570c3791d0419 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -349,6 +349,32 @@ if(BUILD_TESTING) "${CMAKE_CURRENT_SOURCE_DIR}/testdata/supported-param-attrs.ll" ) + goallc_add_ir_filecheck_test( + GoALLCStatepoints.TypedByVal + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/typed-byval.ll" + IR + ) + goallc_add_filecheck_test( + GoALLCStatepoints.TypedByValMIR + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/typed-byval.ll" + MIR + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -verify-machineinstrs + -stop-after=finalize-isel + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/typed-byval.ll" + ) + add_test( + NAME GoALLCStatepoints.TypedByValGoObj + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -verify-machineinstrs + -filetype=obj + -o "${CMAKE_CURRENT_BINARY_DIR}/typed-byval.goobj" + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/typed-byval.ll" + ) goallc_add_ir_filecheck_test( GoALLCStatepoints.CallOnlyPointersNotLive "${CMAKE_CURRENT_SOURCE_DIR}/testdata/indirect-callee.ll" @@ -516,6 +542,17 @@ if(BUILD_TESTING) DEPENDS GoALLCStatepoints.GoObj ) + goallc_add_filecheck_test( + GoALLCStatepoints.TypedByValObjView + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/typed-byval.ll" + OBJVIEW + "${GOALLC_OBJVIEW_EXECUTABLE}" -json + "${CMAKE_CURRENT_BINARY_DIR}/typed-byval.goobj" + ) + set_tests_properties(GoALLCStatepoints.TypedByValObjView PROPERTIES + DEPENDS GoALLCStatepoints.TypedByValGoObj + ) + goallc_add_filecheck_test( GoALLCStatepoints.DeferEdgeObjView "${CMAKE_CURRENT_SOURCE_DIR}/testdata/defer-edge.ll" diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 132c94886399e6..8ff50fa766f7c9 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -13,6 +13,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/BinaryFormat/GoObj.h" +#include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/GoCallingConv.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" @@ -118,6 +120,15 @@ struct PointerAllocaRecord { bool ActivityUnclear = false; }; +struct PointerByValRecord { + Argument *Base; + bool NeedsStackObject; + uint64_t ByteSize; + uint64_t Alignment; + uint64_t BitCount; + SmallVector BitmapWords; +}; + struct OpenDeferInfo { AllocaInst *Bits = nullptr; AllocaInst *Slots = nullptr; @@ -801,10 +812,10 @@ Expected> collectOpenDeferInfo(Function &F) { // Return true when the optimized IR can make the address observable outside // compiler-controlled direct accesses. This is deliberately a structural // post-optimization decision: the frontend Addrtaken bit is provenance, not a -// promise that prevents SROA or forces a surviving alloca to remain a stack -// object. -bool allocaNeedsStackObject(AllocaInst &Alloca) { - SmallVector Worklist{&Alloca}; +// promise that prevents SROA or forces a surviving frame address to remain a +// stack object. +bool addressNeedsStackObject(Value &Base) { + SmallVector Worklist{&Base}; SmallPtrSet Seen; while (!Worklist.empty()) { Value *Address = Worklist.pop_back_val(); @@ -1245,6 +1256,8 @@ Error collectPointerAllocas( const DataLayout &DL = F.getDataLayout(); for (Instruction &I : instructions(F)) { auto *Alloca = dyn_cast(&I); + if (Alloca && isSingleByValCallCarrier(*Alloca, DL)) + continue; if (!Alloca || !containsPointer(Alloca->getAllocatedType())) continue; auto *ArraySize = dyn_cast(Alloca->getArraySize()); @@ -1303,7 +1316,7 @@ Error collectPointerAllocas( // open-defer state. A matching gc-live base still makes GoObj expand this // layout into LocalsPointerMaps; an unmatched callsite follows the same // StackObject rule as every other address-observable alloca. - bool NeedsStackObject = allocaNeedsStackObject(*Alloca); + bool NeedsStackObject = addressNeedsStackObject(*Alloca); PointerAllocas.push_back({Alloca, NeedsStackObject, *DeferResult, IsOpenDeferSlot, ByteSize, Alloca->getAlign().value(), BitCount, @@ -1316,6 +1329,62 @@ Error collectPointerAllocas( return Error::success(); } +Error collectPointerByVals(Function &F, + SmallVectorImpl &Records) { + if (!isGoCallingConv(F.getCallingConv())) + return Error::success(); + + const DataLayout &DL = F.getDataLayout(); + uint64_t PointerSize = DL.getPointerSize(0); + for (Argument &Arg : F.args()) { + if (!Arg.hasByValAttr()) + continue; + Type *StorageType = Arg.getParamByValType(); + if (!StorageType || !containsPointer(StorageType)) + continue; + + TypeSize AllocationSize = DL.getTypeAllocSize(StorageType); + if (AllocationSize.isScalable()) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints do not support scalable byval parameter " + "layouts"); + Align Alignment = + Arg.getParamAlign().value_or(DL.getABITypeAlign(StorageType)); + uint64_t ByteSize = AllocationSize.getFixedValue(); + if (!PointerSize || !ByteSize || ByteSize % PointerSize != 0 || + Alignment < DL.getABITypeAlign(StorageType)) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require pointer-aligned fixed byval parameter " + "layouts"); + + SmallVector Leaves; + SmallVector Path; + if (Error Err = + enumeratePointerAllocaLeaves(StorageType, DL, Path, 0, Leaves)) + return std::move(Err); + uint64_t BitCount = ByteSize / PointerSize; + SmallVector BitmapWords((BitCount + 63) / 64, 0); + for (const PointerAllocaLeaf &Leaf : Leaves) { + if (Leaf.Offset % PointerSize != 0 || Leaf.Offset >= ByteSize) + return createStringError( + std::errc::not_supported, + "GoALLC statepoint byval pointer slot is not pointer-aligned"); + uint64_t Bit = Leaf.Offset / PointerSize; + uint64_t Mask = uint64_t(1) << (Bit % 64); + if (BitmapWords[Bit / 64] & Mask) + return createStringError( + std::errc::invalid_argument, + "GoALLC statepoint byval pointer slots overlap"); + BitmapWords[Bit / 64] |= Mask; + } + Records.push_back({&Arg, addressNeedsStackObject(Arg), ByteSize, + Alignment.value(), BitCount, std::move(BitmapWords)}); + } + return Error::success(); +} + Error validateSafepoint(const SafepointRecord &Record) { const CallInst &Call = *Record.Call; if (Call.isInlineAsm()) @@ -1332,14 +1401,22 @@ Error validateSafepoint(const SafepointRecord &Record) { std::errc::not_supported, "GoALLC statepoints only support a single deopt call operand bundle"); for (unsigned I = 0; I != Call.arg_size(); ++I) { + if (Call.paramHasAttr(I, Attribute::ByVal) && + (!isGoCallingConv(Call.getCallingConv()) || + !Call.getArgOperand(I)->getType()->isPointerTy() || + !Call.getParamByValType(I) || !Call.getParamAlign(I))) + return createStringError( + std::errc::not_supported, + "GoALLC statepoints require typed, aligned byval only on Go calls"); for (Attribute Attr : Call.getAttributes().getParamAttrs(I)) { // These non-ABI attributes remain valid after LLVM's generic // RewriteStatepointsForGC pass and are natively accepted by both the // statepoint verifier and SelectionDAG call lowering. O2 commonly // infers them on otherwise ordinary runtime calls. Keep ABI-affecting - // attributes fail closed except for nest, whose Go closure ABI lowering - // is covered separately. + // attributes fail closed except for nest and typed byval, whose Go ABI + // lowering is covered separately. if (!Attr.hasAttribute(Attribute::Nest) && + !Attr.hasAttribute(Attribute::ByVal) && !Attr.hasAttribute(Attribute::Captures) && !Attr.hasAttribute(Attribute::ReadNone) && !Attr.hasAttribute(Attribute::ReadOnly) && @@ -1363,8 +1440,9 @@ Error validateSafepoint(const SafepointRecord &Record) { void appendAllocaPtrMapDeoptOperands( IRBuilder<> &Builder, ArrayRef Allocas, + ArrayRef ByVals, SmallVectorImpl &Deopt) { - if (Allocas.empty()) + if (Allocas.empty() && ByVals.empty()) return; // ProtocolLength covers BEGIN through END, but not the trailing duplicate // length. The envelope itself therefore contributes BEGIN, length, @@ -1372,27 +1450,39 @@ void appendAllocaPtrMapDeoptOperands( uint64_t ProtocolLength = 4; for (const PointerAllocaRecord *Alloca : Allocas) ProtocolLength += 10 + Alloca->BitmapWords.size(); + for (const PointerByValRecord *ByVal : ByVals) + ProtocolLength += 10 + ByVal->BitmapWords.size(); auto AppendConstant = [&](uint64_t Value) { Deopt.push_back(ConstantInt::get(Builder.getInt64Ty(), Value)); }; AppendConstant(GoObj::AllocaPtrMapBeginMagic); AppendConstant(ProtocolLength); - AppendConstant(Allocas.size()); - for (const PointerAllocaRecord *Alloca : Allocas) { + AppendConstant(Allocas.size() + ByVals.size()); + auto AppendRecord = [&](Value *Base, uint64_t ByteSize, uint64_t Alignment, + uint64_t BitCount, ArrayRef BitmapWords) { AppendConstant(GoObj::AllocaPtrMapRecordTag); - AppendConstant(10 + Alloca->BitmapWords.size()); - Deopt.push_back(Alloca->Alloca); - AppendConstant(0); // First contract version describes the whole alloca. - AppendConstant(Alloca->ByteSize); - AppendConstant(Alloca->Alignment); - AppendConstant(Alloca->Alloca->getDataLayout().getPointerSize(0)); - AppendConstant(Alloca->BitCount); + AppendConstant(10 + BitmapWords.size()); + Deopt.push_back(Base); + AppendConstant(0); // First contract version describes the whole object. + AppendConstant(ByteSize); + AppendConstant(Alignment); + AppendConstant( + Builder.GetInsertBlock()->getModule()->getDataLayout().getPointerSize( + 0)); + AppendConstant(BitCount); AppendConstant(GoObj::AllocaPtrMapBitmapWordBits); - AppendConstant(Alloca->BitmapWords.size()); - for (uint64_t Word : Alloca->BitmapWords) + AppendConstant(BitmapWords.size()); + for (uint64_t Word : BitmapWords) AppendConstant(Word); + }; + for (const PointerAllocaRecord *Alloca : Allocas) { + AppendRecord(Alloca->Alloca, Alloca->ByteSize, Alloca->Alignment, + Alloca->BitCount, Alloca->BitmapWords); } + for (const PointerByValRecord *ByVal : ByVals) + AppendRecord(ByVal->Base, ByVal->ByteSize, ByVal->Alignment, + ByVal->BitCount, ByVal->BitmapWords); AppendConstant(GoObj::AllocaPtrMapEndMagic); AppendConstant(ProtocolLength); } @@ -1417,6 +1507,7 @@ void appendOpenDeferDeoptOperands(IRBuilder<> &Builder, Error rewriteCall(SafepointRecord &Record, ArrayRef PointerAllocas, + ArrayRef PointerByVals, const std::optional &OpenDefer) { CallInst *Call = Record.Call; @@ -1433,7 +1524,8 @@ Error rewriteCall(SafepointRecord &Record, // Keep the open-defer envelope before the alloca ptrmap envelope. The latter // deliberately remains the final self-describing suffix for compatibility. appendOpenDeferDeoptOperands(Builder, OpenDefer, Deopt); - appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, Deopt); + appendAllocaPtrMapDeoptOperands(Builder, PointerAllocas, PointerByVals, + Deopt); Record.Statepoint = Builder.CreateGCStatepointCall( Record.ID, 0, Callee, CallArgs, Deopt.empty() ? std::nullopt @@ -1515,14 +1607,19 @@ Value *rematerializeAddress(Value *Address, Value *Base, Value *RelocatedBase, void repairRelocationSSA(Function &F, DominatorTree &DT, ArrayRef Records) { - // Each ordinary relocated pointer and each rematerialized alloca-derived - // address is a new reaching definition of its original SSA value. + // Each ordinary relocated pointer and each rematerialized fixed-object + // derived address is a new reaching definition of its original SSA value. + // Static allocas and typed byval arguments are themselves fixed frame + // addresses: SelectionDAG rematerializes either from its frame index at each + // use, so replacing the original IR value with a gc.relocate chain would + // turn later statepoints into ordinary pointer spills. MapVector> Definitions; for (const SafepointRecord &Record : Records) { for (CallInst *RelocateCall : Record.Relocates) { auto *Relocate = cast(RelocateCall); Value *Original = Relocate->getDerivedPtr(); - if (!isa(Original)) + auto *Arg = dyn_cast(Original); + if (!isa(Original) && !(Arg && Arg->hasByValAttr())) Definitions[Original].push_back(RelocateCall); } @@ -1658,6 +1755,9 @@ Error rewriteFunction(Function &F) { SmallVector PointerAllocas; if (Error Err = collectPointerAllocas(F, OpenDefer, PointerAllocas)) return Err; + SmallVector PointerByVals; + if (Error Err = collectPointerByVals(F, PointerByVals)) + return Err; if (Error Err = scalarizeLivePointerAggregates(F)) return Err; @@ -1749,7 +1849,18 @@ Error rewriteFunction(Function &F) { if (IsActive || Alloca.NeedsStackObject) AllocaRecords.push_back(&Alloca); } - if (Error Err = rewriteCall(Record, AllocaRecords, OpenDefer)) + SmallVector ByValRecords; + for (const PointerByValRecord &ByVal : PointerByVals) { + // A typed byval parameter is a caller-initialized fixed Go argument + // object. Standard SSA liveness decides when its contents contribute to + // this call's ArgsPointerMaps. If its address is observable, carry the + // layout at every call so GoObj can also infer the function-level + // StackObject from an unmatched gc-live base. + bool IsActive = Record.Live.contains(ByVal.Base); + if (IsActive || ByVal.NeedsStackObject) + ByValRecords.push_back(&ByVal); + } + if (Error Err = rewriteCall(Record, AllocaRecords, ByValRecords, OpenDefer)) return Err; } eraseOriginalCalls(Records); diff --git a/src/cmd/llvmplugin/testdata/aarch64-frame.ll b/src/cmd/llvmplugin/testdata/aarch64-frame.ll index 148506f6ca6135..5cf02f8d66cb97 100644 --- a/src/cmd/llvmplugin/testdata/aarch64-frame.ll +++ b/src/cmd/llvmplugin/testdata/aarch64-frame.ll @@ -98,11 +98,13 @@ entry: ret ptr %result } -define goabi0 ptr @"aarch64_abi0_pointer_result"(ptr %pointer) #0 gc "goallc" { +define goabi0 ptr @"aarch64_abi0_pointer_result"( + ptr byval(ptr) align 8 %pointer.home) #0 gc "goallc" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 + %pointer = load ptr, ptr %pointer.home, align 8 ret ptr %pointer } @@ -111,11 +113,12 @@ define goabiinternal ptr @aarch64_stack_pointer_arg( i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, - ptr %pointer) #0 gc "goallc" { + ptr byval(ptr) align 8 %pointer.home) #0 gc "goallc" { entry: %buf = alloca [8192 x i8], align 16 %slot = getelementptr inbounds [8192 x i8], ptr %buf, i64 0, i64 8191 store volatile i8 1, ptr %slot, align 1 + %pointer = load ptr, ptr %pointer.home, align 8 ret ptr %pointer } @@ -130,7 +133,7 @@ entry: ; The i64 register argument's home starts beyond the 8-byte scaled-uimm12 ; limit (32760), forcing the frameless morestack path to materialize SP+32776. define goabiinternal i64 @aarch64_large_arg_home( - [4096 x i64] %stackarg, i64 %regarg) #0 gc "goallc" { + ptr byval([4096 x i64]) align 8 %stackarg, i64 %regarg) #0 gc "goallc" { entry: call goabiinternal void @"runtime.GC"() ret i64 %regarg diff --git a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll index e89c46dbebca71..8beec560cc90fb 100644 --- a/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll +++ b/src/cmd/llvmplugin/testdata/alloca-pointer-roots.ll @@ -257,16 +257,14 @@ entry: } define goabiinternal void @argument_aggregate_home_address_across_calls( - %nested %value) gc "goallc" { + ptr byval(%nested) align 8 %value.home) gc "goallc" { entry: - ; The split aggregate parameter still has one complete fixed home and one + ; Typed byval is the complete incoming Go ABI home. It needs no second + ; alloca or parameter copy, while its pointer layout still contributes one ; argp-relative StackObject covering all ABI pieces and padding. - %slot = alloca %nested, align 8 - call void @llvm.lifetime.start.p0(i64 48, ptr %slot) - store %nested %value, ptr %slot, align 8 - call goabiinternal void @observe_stack_address(ptr %slot) + call goabiinternal void @observe_stack_address(ptr %value.home) call goabiinternal void @safepoint() - call goabiinternal void @observe_stack_address(ptr %slot) + call goabiinternal void @observe_stack_address(ptr %value.home) ret void } diff --git a/src/cmd/llvmplugin/testdata/typed-byval.ll b/src/cmd/llvmplugin/testdata/typed-byval.ll new file mode 100644 index 00000000000000..153cb86a4e75db --- /dev/null +++ b/src/cmd/llvmplugin/testdata/typed-byval.ll @@ -0,0 +1,155 @@ +target triple = "x86_64-unknown-linux-goobj" + +%pointer_pair = type { ptr, ptr } +@pointer = external global i8 + +; IR-LABEL: define goabiinternal void @pure_ssa_call_slot( +; A pure SSA value needs an addressable source for byval. The shared CodeGen +; predicate proves that this alloca lives only until the one byval call, so it +; is not modeled as a caller-local GC object. +; IR: call goabiinternal token {{.*}} @llvm.experimental.gc.statepoint +; IR-SAME: ptr byval(ptr) align 8 %argument.home.address +; IR-SAME: i32 0, i32 0) + +; MIR-LABEL: name: pure_ssa_call_slot +; The initializing store is forwarded directly into the outgoing byval slot; +; no local source object survives instruction selection. +; MIR: stack: [] +; MIR: ADJCALLSTACKDOWN64 80 +; MIR: MOV64mr %{{[0-9]+}}, 1, $noreg, 0, $noreg +; MIR: STATEPOINT {{.*}}@consume_pointer + +; IR-LABEL: define goabiinternal void @intervening_call_slot( +; A carrier initialized before another call cannot be mapped onto a future +; outgoing argument area. It remains an ordinary pointer-containing stack +; object at both safepoints. +; IR: %argument.home = alloca ptr, align 8 +; IR: elementtype(void ()) @safepoint +; IR: "deopt"({{.*}}ptr %argument.home +; IR: elementtype(void (i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr)) @consume_pointer +; IR: "deopt"({{.*}}ptr %argument.home + +; MIR-LABEL: name: intervening_call_slot +; MIR: stack: +; MIR-NEXT: - { id: 0, name: argument.home, type: default, offset: 0, size: 8 +; MIR: STATEPOINT {{.*}}@safepoint +; MIR: STATEPOINT {{.*}}@consume_pointer + +; IR-LABEL: define goabiinternal ptr @incoming_pointer_home( +; A pointer value loaded from the incoming byval home follows the generic +; statepoint spill path. The home is not reused as a spill for that value. +; IR: %value = load ptr, ptr %value.home, align 8 +; IR-NOT: "deopt"( +; IR: "gc-live"(ptr %value) + +; MIR-LABEL: name: incoming_pointer_home +; MIR: fixedStack: +; MIR: - { id: [[POINTER_OBJECT_HOME:[0-9]+]], type: default, offset: 0, size: 8 +; MIR: stack: +; MIR-NEXT: - { id: [[POINTER_SPILL:[0-9]+]], name: '', type: default, offset: 0, size: 8 +; MIR: MOV64mr %stack.[[POINTER_SPILL]] +; MIR: STATEPOINT +; MIR-SAME: 1, 8, %stack.[[POINTER_SPILL]], 0 +; MIR: MOV64rm %stack.[[POINTER_SPILL]] + +; IR-LABEL: define goabiinternal ptr @incoming_aggregate_home( +; IR-NOT: "deopt"( +; IR: "gc-live"(ptr %second) + +; MIR-LABEL: name: incoming_aggregate_home +; MIR: stack: +; MIR-NEXT: - { id: [[AGGREGATE_SPILL:[0-9]+]], name: '', type: default, offset: 0, size: 8 +; MIR: MOV64mr %stack.[[AGGREGATE_SPILL]] +; MIR: STATEPOINT +; MIR-SAME: 1, 8, %stack.[[AGGREGATE_SPILL]], 0 +; MIR: MOV64rm %stack.[[AGGREGATE_SPILL]] + +; IR-LABEL: define goabiinternal ptr @passed_to_mutator_incoming_pointer_home( +; Taking the address of a Go parameter and passing it on makes the incoming +; byval home address-observable. The object's pointer layout is retained, while +; the separately loaded pointer still follows the generic spill path. +; IR: %value = load ptr, ptr %value.home, align 8 +; IR: elementtype(void (ptr)) @mutate +; IR: "deopt"({{.*}}ptr %value.home +; IR-SAME: "gc-live"(ptr %value) + +; MIR-LABEL: name: passed_to_mutator_incoming_pointer_home +; MIR: stack: +; MIR-NEXT: - { id: 0, name: '', type: default, offset: 0, size: 8 +; MIR: MOV64mr %stack.0 +; MIR: STATEPOINT +; MIR-SAME: @mutate +; MIR-SAME: 1, 8, %stack.0, 0 +; MIR: STATEPOINT +; MIR-SAME: @safepoint +; MIR-SAME: 1, 8, %stack.0, 0 + +; OBJVIEW-LABEL: "name": "incoming_pointer_home" +; OBJVIEW-NOT: "kind": "stack_objects" +; OBJVIEW-LABEL: "name": "incoming_aggregate_home" +; OBJVIEW-NOT: "kind": "stack_objects" +; OBJVIEW-LABEL: "name": "passed_to_mutator_incoming_pointer_home" +; OBJVIEW: "kind": "stack_objects" +; OBJVIEW: "offset": 0 +; OBJVIEW: "size": 8 +; OBJVIEW: "ptr_bytes": 8 + +declare goabiinternal void @consume_pointer( + i64, i64, i64, i64, i64, i64, i64, i64, i64, + ptr byval(ptr) align 8) + +declare goabiinternal void @safepoint() + +declare goabiinternal void @mutate(ptr) + +define goabiinternal void @pure_ssa_call_slot(ptr %argument) #0 gc "goallc" { +entry: + %argument.home = alloca ptr, align 8 + store ptr %argument, ptr %argument.home, align 8 + call goabiinternal void @consume_pointer( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, + ptr byval(ptr) align 8 %argument.home) + ret void +} + +define goabiinternal void @intervening_call_slot() #0 gc "goallc" { +entry: + %argument.home = alloca ptr, align 8 + store ptr @pointer, ptr %argument.home, align 8 + call goabiinternal void @safepoint() + call goabiinternal void @consume_pointer( + i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, + ptr byval(ptr) align 8 %argument.home) + ret void +} + +define goabiinternal ptr @incoming_pointer_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, + i64 %a7, i64 %a8, ptr byval(ptr) align 8 %value.home) #0 gc "goallc" { +entry: + %value = load ptr, ptr %value.home, align 8 + call goabiinternal void @safepoint() + ret ptr %value +} + +define goabiinternal ptr @incoming_aggregate_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, + i64 %a7, i64 %a8, ptr byval(%pointer_pair) align 8 %value.home) #0 gc "goallc" { +entry: + %second.home = getelementptr inbounds %pointer_pair, ptr %value.home, i32 0, i32 1 + %second = load ptr, ptr %second.home, align 8 + call goabiinternal void @safepoint() + ret ptr %second +} + +define goabiinternal ptr @passed_to_mutator_incoming_pointer_home( + i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, + i64 %a7, i64 %a8, ptr byval(ptr) align 8 %value.home) #0 gc "goallc" { +entry: + %value = load ptr, ptr %value.home, align 8 + call goabiinternal void @mutate(ptr %value.home) + call goabiinternal void @safepoint() + ret ptr %value +} + +attributes #0 = { "frame-pointer"="non-leaf" } diff --git a/test/codegen/_cgo_llvm_unsafe_args.go b/test/codegen/_cgo_llvm_unsafe_args.go index 4cec88cf2d522e..e92d819c02264d 100644 --- a/test/codegen/_cgo_llvm_unsafe_args.go +++ b/test/codegen/_cgo_llvm_unsafe_args.go @@ -10,34 +10,52 @@ package codegen func llvmCgoUnsafeSink(*uintptr) // LLVM-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-SAME: i64 %p, i64 %q) #[[NOINLINE:[0-9]+]] gc "goallc" +// LLVM-SAME: ptr byval(i64) align 8 %p, ptr byval(i64) align 8 %q) #[[NOINLINE:[0-9]+]] gc "goallc" // LLVM-NOT: alloca // LLVM: [[FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-NOT: llvm.addressofreturnaddress // LLVM-NOT: llvm.sponentry // LLVM: [[Q:%.*]] = getelementptr i8, ptr [[FRAME]], i64 8 // LLVM: [[RESULT:%.*]] = getelementptr i8, ptr [[FRAME]], i64 16 -// LLVM: store i64 %p, ptr [[FRAME]] -// LLVM: store i64 %q, ptr [[Q]] +// LLVM: [[P_VALUE:%.*]] = load i64, ptr %p +// LLVM: store i64 [[P_VALUE]], ptr [[FRAME]] +// LLVM: [[Q_VALUE:%.*]] = load i64, ptr %q +// LLVM: store i64 [[Q_VALUE]], ptr [[Q]] // LLVM: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[FRAME]]) // LLVM: {{%.*}} = load i64, ptr [[RESULT]] -// LLVM: attributes #[[NOINLINE]] = { {{.*}}noinline // LLVM-OPT-LABEL: define goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( -// LLVM-OPT-SAME: i64 %p, i64 %q) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" +// LLVM-OPT-SAME: ptr{{.*}}byval(i64) align 8{{.*}} %p, ptr{{.*}}byval(i64) align 8{{.*}} %q) {{.*}}#[[OPT_NOINLINE:[0-9]+]] gc "goallc" // LLVM-OPT-NOT: alloca // LLVM-OPT: [[OPT_FRAME:%.*]] = {{.*}}call ptr @llvm.go.abi0.frame() // LLVM-OPT-NOT: llvm.addressofreturnaddress // LLVM-OPT-NOT: llvm.sponentry // LLVM-OPT: [[OPT_Q:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 8 // LLVM-OPT: [[OPT_RESULT:%.*]] = getelementptr i8, ptr [[OPT_FRAME]], i64 16 -// LLVM-OPT: store i64 %p, ptr [[OPT_FRAME]] -// LLVM-OPT: store i64 %q, ptr [[OPT_Q]] +// LLVM-OPT: [[OPT_P_VALUE:%.*]] = load i64, ptr %p +// LLVM-OPT: store i64 [[OPT_P_VALUE]], ptr [[OPT_FRAME]] +// LLVM-OPT: [[OPT_Q_VALUE:%.*]] = load i64, ptr %q +// LLVM-OPT: store i64 [[OPT_Q_VALUE]], ptr [[OPT_Q]] // LLVM-OPT: {{.*}}call goabiinternal void @codegen.llvmCgoUnsafeSink(ptr{{.*}} [[OPT_FRAME]]) // LLVM-OPT: {{%.*}} = load i64, ptr [[OPT_RESULT]] -// LLVM-OPT: attributes #[[OPT_NOINLINE]] = { {{.*}}noinline // //go:cgo_unsafe_args func llvmCgoUnsafeFrame(p, q uintptr) (r uintptr) { llvmCgoUnsafeSink(&p) return } + +// LLVM-LABEL: define goabiinternal i64 @codegen.llvmCgoUnsafeCall() +// LLVM: store i64 1, ptr [[CALL_P:%[^, ]+]] +// LLVM: store i64 2, ptr [[CALL_Q:%[^, ]+]] +// LLVM: {{%.*}} = {{(tail )?}}call goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( +// LLVM-SAME: ptr byval(i64) align 8 [[CALL_P]], ptr byval(i64) align 8 [[CALL_Q]]) +// LLVM: attributes #[[NOINLINE]] = { {{.*}}noinline +// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmCgoUnsafeCall() +// LLVM-OPT: store i64 1, ptr [[OPT_CALL_P:%[^, ]+]] +// LLVM-OPT: store i64 2, ptr [[OPT_CALL_Q:%[^, ]+]] +// LLVM-OPT: {{%.*}} = {{(tail )?}}call goabi0 i64 @"codegen.llvmCgoUnsafeFrame"( +// LLVM-OPT-SAME: ptr {{.*}}byval(i64) align 8{{.*}} [[OPT_CALL_P]], ptr {{.*}}byval(i64) align 8{{.*}} [[OPT_CALL_Q]]) +// LLVM-OPT: attributes #[[OPT_NOINLINE]] = { {{.*}}noinline +func llvmCgoUnsafeCall() uintptr { + return llvmCgoUnsafeFrame(1, 2) +} diff --git a/test/codegen/issue25378.go b/test/codegen/issue25378.go index 9a704fc40f2800..cd02fff6facf9a 100644 --- a/test/codegen/issue25378.go +++ b/test/codegen/issue25378.go @@ -7,11 +7,11 @@ package codegen // LLVM-DAG: @codegen.wsp = global <{ [256 x i8] }> {{.*}}, section ".noptrdata" -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16([2 x i16] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgUint16(ptr byval([2 x i16]) align 2 // LLVM-DAG: zext i16 {{%.*}} to i64 // LLVM-DAG: icmp ult i64 {{%.*}}, 256 // LLVM-DAG: getelementptr i8, ptr @codegen.wsp, i64 -// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte([2 x i8] +// LLVM-DAG: define goabiinternal i8 @codegen.zeroExtArgByte(ptr byval([2 x i8]) align 1 // LLVM-DAG: zext i8 {{%.*}} to i64 var wsp = [256]bool{ diff --git a/test/codegen/llvm_argument_memory_home.go b/test/codegen/llvm_argument_memory_home.go index 8fa68f1d76e74e..5ff0b9cf2f04e2 100644 --- a/test/codegen/llvm_argument_memory_home.go +++ b/test/codegen/llvm_argument_memory_home.go @@ -12,12 +12,6 @@ type llvmArgumentStrings3 struct { type llvmArgumentStringArray [2]string -// A register-assigned parameter that Go SSA can use directly remains an LLVM -// SSA value and does not acquire a memory home. -func llvmDirectRegisterArgument(x int) int { - return x + 1 -} - // llvmArgumentStrings3 fits wholly in the ABIInternal integer-register budget // but is too large for Go SSA's aggregate-value limit. LLVM gives only this // memory-backed parameter a complete local home instead of reconstructing its @@ -42,25 +36,57 @@ func llvmRegisterArgumentMemoryHome(x llvmArgumentStrings3) int { return len(x.a) + len(x.b) + len(x.c) } -// Non-trivial arrays are assigned wholly to the ABI stack. Their Go SSA -// LocalAddr uses the same local-home initialization instead of reading an -// uninitialized alloca. +// Non-trivial arrays are assigned wholly to the ABI stack. Typed byval exposes +// that incoming Go parameter copy directly, so LocalAddr needs no second local +// home or aggregate reconstruction. // -// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) -// LLVM: [[STACK_HOME:%.*]] = alloca [2 x { ptr, i64 }], align 8 -// LLVM: store [2 x { ptr, i64 }] %x, ptr [[STACK_HOME]], align 8 +// LLVM-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr byval([2 x { ptr, i64 }]) align 8 %x) +// LLVM-NOT: alloca +// LLVM: getelementptr i8, ptr %x, i64 0 +// LLVM: getelementptr i8, ptr %x, i64 16 +// LLVM: load { ptr, i64 } +// LLVM: load { ptr, i64 } // LLVM: ret i64 // -// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome([2 x { ptr, i64 }] %x) +// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr{{.*}}byval([2 x { ptr, i64 }]) align 8{{.*}} %x) // LLVM-OPT-NOT: alloca -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 0 -// LLVM-OPT: extractvalue [2 x { ptr, i64 }] %x, 1 +// LLVM-OPT: getelementptr {{.*}}ptr %x, i64 8 +// LLVM-OPT: load i64 +// LLVM-OPT: getelementptr {{.*}}ptr %x, i64 24 +// LLVM-OPT: load i64 // LLVM-OPT: ret i64 // +//go:noinline +func llvmStackArgumentMemoryHome(x llvmArgumentStringArray) int { + return len(x[0]) + len(x[1]) +} + +// A stack-assigned value that already resides in memory is the byval source +// directly. The frontend must not load the complete aggregate and materialize +// a second temporary object before the call. +// +// LLVM-LABEL: define goabiinternal i64 @codegen.llvmForwardStackArgumentMemory(ptr byval([2 x { ptr, i64 }]) align 8 %x) +// LLVM-NOT: alloca +// LLVM: call goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr byval([2 x { ptr, i64 }]) align 8 %x) +// LLVM: ret i64 +// +// LLVM-OPT-LABEL: define goabiinternal i64 @codegen.llvmForwardStackArgumentMemory(ptr{{.*}}byval([2 x { ptr, i64 }]) align 8{{.*}} %x) +// LLVM-OPT-NOT: alloca +// LLVM-OPT: call goabiinternal i64 @codegen.llvmStackArgumentMemoryHome(ptr{{.*}}byval([2 x { ptr, i64 }]) align 8{{.*}} %x) +// LLVM-OPT: ret i64 +// +//go:noinline +func llvmForwardStackArgumentMemory(x llvmArgumentStringArray) int { + return llvmStackArgumentMemoryHome(x) +} + +// A register-assigned parameter that Go SSA can use directly remains an LLVM +// SSA value and does not acquire a memory home. +// // LLVM-LABEL: define goabiinternal i64 @codegen.llvmDirectRegisterArgument(i64 %x) // LLVM-NOT: alloca // LLVM: add i64 %x, 1 // LLVM: ret i64 -func llvmStackArgumentMemoryHome(x llvmArgumentStringArray) int { - return len(x[0]) + len(x[1]) +func llvmDirectRegisterArgument(x int) int { + return x + 1 } diff --git a/test/codegen/llvm_direct_iface.go b/test/codegen/llvm_direct_iface.go index 1d29e761a6dc8c..dff353c3e55e5b 100644 --- a/test/codegen/llvm_direct_iface.go +++ b/test/codegen/llvm_direct_iface.go @@ -30,11 +30,13 @@ func llvmDirectIfaceSink(llvmDirectIfaceNested) {} // the ordinary ABI call path. // // LLVM-LABEL: define goabiinternal void @codegen.llvmDirectIfaceCall( +// LLVM: [[SLOT:%.*]] = alloca %codegen.llvmDirectIfaceNested, align 8 // LLVM: [[DATA:%.*]] = extractvalue { ptr, ptr } %x, 1 // LLVM: [[LEAF:%.*]] = insertvalue %codegen.llvmDirectIfaceLeaf undef, ptr [[DATA]], 0 // LLVM: [[ARRAY:%.*]] = insertvalue [1 x %codegen.llvmDirectIfaceLeaf] undef, %codegen.llvmDirectIfaceLeaf [[LEAF]], 0 // LLVM: [[NESTED:%.*]] = insertvalue %codegen.llvmDirectIfaceNested {{.*}}, [1 x %codegen.llvmDirectIfaceLeaf] [[ARRAY]], 2 -// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(%codegen.llvmDirectIfaceNested [[NESTED]]) +// LLVM: store %codegen.llvmDirectIfaceNested [[NESTED]], ptr [[SLOT]], align 8 +// LLVM: call goabiinternal void @codegen.llvmDirectIfaceSink(ptr byval(%codegen.llvmDirectIfaceNested) align 8 [[SLOT]]) func llvmDirectIfaceCall(x any) { switch x := x.(type) { case llvmDirectIfaceNested: