diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 8af89c854d18c2..49193ed94192f4 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -69,6 +69,7 @@ const goObjMarkerRelocMD = "goobj.marker_reloc" const goObjSymbolNameMD = "goobj.symbol.name" const llvmFramePointerAttr = "frame-pointer" const llvmFramePointerNonLeaf = "non-leaf" +const llvmTargetCPUAttr = "target-cpu" // Keep fixed-size memmoves within the store expansion limits of the supported // LLVM targets. Larger moves must use runtime.memmove rather than a libc symbol, @@ -337,6 +338,54 @@ func (lfc *LLVMFuncContext) llvmTernaryIntrinsic(v *Value, name string) llvm.Val return lfc.b.CreateCall(sig, fn, []llvm.Value{x, y, z}, v.String()) } +func (lfc *LLVMFuncContext) llvmRoundIntrinsic(v *Value, genericName string, amd64Mode uint64) llvm.Value { + if lfc.F.Config.arch != "amd64" || buildcfg.GOAMD64 >= 2 { + return lfc.llvmUnaryIntrinsic(v, genericName) + } + + // At GOAMD64=v1, Go SSA places these operations behind + // runtime.x86HasSSE41. Preserve that path-sensitive contract in the + // intrinsic instead of enabling SSE4.1 for the whole function. At v2 and + // above, the function target-cpu guarantees SSE4.1 and the generic LLVM + // intrinsic exposes the usual optimization opportunities. + x := lfc.GenLV(v.Args[0]) + want := getLLVMType(v.Type) + if x.Type() != want || want.TypeKind() != llvm.DoubleTypeKind { + v.Fatalf("%s has incompatible LLVM operand and result types", v.Op) + } + i32 := GlobalCtxt.Int32Type() + sig := llvm.FunctionType(want, []llvm.Type{want, i32}, false) + fn := getOrInsertLLVMIntrinsic("llvm.x86.go.sse41.round.f64", sig) + mode := llvm.ConstInt(i32, amd64Mode, false) + return lfc.b.CreateCall(sig, fn, []llvm.Value{x, mode}, v.String()) +} + +func (lfc *LLVMFuncContext) llvmFMA(v *Value) llvm.Value { + if lfc.F.Config.arch != "amd64" || buildcfg.GOAMD64 >= 3 { + return lfc.llvmTernaryIntrinsic(v, "llvm.fma.f64") + } + + // At GOAMD64=v1 and v2, the AMD64 SSA control flow has already guarded this + // operation with runtime.x86HasFMA. GOAMD64=v3 and above instead use the + // generic intrinsic under a function target-cpu that guarantees FMA. + return lfc.llvmTernaryIntrinsic(v, "llvm.x86.go.fma.f64") +} + +func llvmTargetCPU(arch string) string { + if arch != "amd64" { + return "" + } + switch buildcfg.GOAMD64 { + case 1: + return "x86-64" + case 2, 3, 4: + return fmt.Sprintf("x86-64-v%d", buildcfg.GOAMD64) + default: + base.Fatalf("LLVM target CPU is not configured for GOAMD64=v%d", buildcfg.GOAMD64) + return "" + } +} + func (lfc *LLVMFuncContext) buildPureTuple(v *Value, values ...llvm.Value) llvm.Value { resultType := getLLVMType(v.Type) if resultType.TypeKind() != llvm.StructTypeKind { @@ -1732,15 +1781,15 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { case OpAbs: lVal = lfc.llvmUnaryIntrinsic(v, "llvm.fabs.f64") case OpFloor: - lVal = lfc.llvmUnaryIntrinsic(v, "llvm.floor.f64") + lVal = lfc.llvmRoundIntrinsic(v, "llvm.floor.f64", 1) case OpCeil: - lVal = lfc.llvmUnaryIntrinsic(v, "llvm.ceil.f64") + lVal = lfc.llvmRoundIntrinsic(v, "llvm.ceil.f64", 2) case OpTrunc: - lVal = lfc.llvmUnaryIntrinsic(v, "llvm.trunc.f64") + lVal = lfc.llvmRoundIntrinsic(v, "llvm.trunc.f64", 3) case OpRound: lVal = lfc.llvmUnaryIntrinsic(v, "llvm.round.f64") case OpRoundToEven: - lVal = lfc.llvmUnaryIntrinsic(v, "llvm.roundeven.f64") + lVal = lfc.llvmRoundIntrinsic(v, "llvm.roundeven.f64", 0) case OpMin64F: lVal = lfc.llvmBinaryIntrinsic(v, "llvm.minimum.f64") case OpMin32F: @@ -1750,7 +1799,7 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { case OpMax32F: lVal = lfc.llvmBinaryIntrinsic(v, "llvm.maximum.f32") case OpFMA: - lVal = lfc.llvmTernaryIntrinsic(v, "llvm.fma.f64") + lVal = lfc.llvmFMA(v) case OpEq64, OpEq32, OpEq16, OpEq8, OpEqB: lVal = lfc.goBool(lfc.b.CreateICmp(llvm.IntEQ, arg0(), arg1(), v.String()+".i1"), v.String()) case OpEqPtr: @@ -2287,6 +2336,12 @@ func LLVMCompile(f *Func) { f.fe.Fatalf(f.Entry.Pos, "duplicate LLVM definition for %s", f.OwnAux.Fn.Name) } FCtxt.LF.SetGC(goGCStrategy) + if cpu := llvmTargetCPU(f.Config.arch); cpu != "" { + // GOAMD64 levels are the standard x86-64 microarchitecture levels. + // Make the required instruction set visible to both LLVM optimization + // and instruction selection without selecting a host-specific CPU. + FCtxt.LF.AddTargetDependentFunctionAttr(llvmTargetCPUAttr, cpu) + } // Go has already made its source-level inlining decision before LLVM // lowering. Preserve both explicit //go:noinline boundaries and the // frontend's implicit no-inline rules for functions containing defer or diff --git a/test/codegen/llvm_math_intrinsics.go b/test/codegen/llvm_math_intrinsics.go index cc298b7a37916d..4d929df1fd9b48 100644 --- a/test/codegen/llvm_math_intrinsics.go +++ b/test/codegen/llvm_math_intrinsics.go @@ -8,7 +8,8 @@ package codegen import "math" -// LLVM-DAG: call double @llvm.fma.f64(double %x, double %y, double %z) +// LLVM-ARM64-DAG: call double @llvm.fma.f64(double %x, double %y, double %z) +// LLVM-AMD64-DAG: call double @llvm.x86.go.fma.f64(double %x, double %y, double %z) func llvmFMA64(x, y, z float64) float64 { return math.FMA(x, y, z) } diff --git a/test/codegen/llvm_trunc.go b/test/codegen/llvm_trunc.go index bd11dc604e93a3..79d7f271edf5f2 100644 --- a/test/codegen/llvm_trunc.go +++ b/test/codegen/llvm_trunc.go @@ -9,21 +9,26 @@ package codegen import "math" // LLVM-DAG: define goabiinternal double @codegen.llvmTrunc64(double %x) -// LLVM-DAG: call double @llvm.trunc.f64(double %x) +// LLVM-ARM64-DAG: call double @llvm.trunc.f64(double %x) +// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 3) // LLVM-OPT-DAG: define goabiinternal double @codegen.llvmTrunc64(double %x) -// LLVM-OPT-DAG: call double @llvm.trunc.f64(double %x) +// LLVM-OPT-ARM64-DAG: call double @llvm.trunc.f64(double %x) +// LLVM-OPT-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 3) +// LLVM-AMD64-DAG: "target-cpu"="x86-64" func llvmTrunc64(x float64) float64 { return math.Trunc(x) } // LLVM-DAG: define goabiinternal double @codegen.llvmCeil64(double %x) -// LLVM-DAG: call double @llvm.ceil.f64(double %x) +// LLVM-ARM64-DAG: call double @llvm.ceil.f64(double %x) +// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 2) func llvmCeil64(x float64) float64 { return math.Ceil(x) } // LLVM-DAG: define goabiinternal double @codegen.llvmFloor64(double %x) -// LLVM-DAG: call double @llvm.floor.f64(double %x) +// LLVM-ARM64-DAG: call double @llvm.floor.f64(double %x) +// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 1) func llvmFloor64(x float64) float64 { return math.Floor(x) } @@ -35,7 +40,8 @@ func llvmRound64(x float64) float64 { } // LLVM-DAG: define goabiinternal double @codegen.llvmRoundToEven64(double %x) -// LLVM-DAG: call double @llvm.roundeven.f64(double %x) +// LLVM-ARM64-DAG: call double @llvm.roundeven.f64(double %x) +// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 0) func llvmRoundToEven64(x float64) float64 { return math.RoundToEven(x) } diff --git a/test/llvm_tests.json b/test/llvm_tests.json index 75ba704ca73f6c..d01ef361aeb32e 100644 --- a/test/llvm_tests.json +++ b/test/llvm_tests.json @@ -81,8 +81,7 @@ "platform_graylist": { "linux/amd64": { "codegen/llvm_abs.go": "the native AMD64 frontend does not form OpAbs for math.Abs", - "codegen/mathbits.go": "the AMD64 frontend forms unsupported Div128u while compiling the full source file", - "codegen/llvm_trunc.go": "GOAMD64=v1 guards OpTrunc with unsupported HasCPUFeature lowering" + "codegen/mathbits.go": "the AMD64 frontend forms unsupported Div128u while compiling the full source file" } } }, @@ -1172,31 +1171,20 @@ "abi/method_wrapper.go": "x86 GoObj traceback does not yet unwind the panic path through the method wrapper", "append.go": "Linux/amd64 statepoint lowering rejects vectors inside live pointer aggregates", "convert5.go": "Linux/amd64 float-to-integer overflow results differ from the test expectations", - "fixedbugs/bug491.go": "Linux/amd64 link fails because the generated memset relocation has no definition", - "fixedbugs/issue23522.go": "x86 dynamic SSE4.1 floor selection lowers the guarded LLVM intrinsic to an unavailable external libcall", - "fixedbugs/issue2615.go": "Linux/amd64 link fails because the generated memset relocation has no definition", "fixedbugs/issue34968.go": "Linux/amd64 GoObj generation reports an unexpected file-local symbol index", "fixedbugs/issue43570.go": "x86 GoObj panic traceback still reports an unknown caller PC", "fixedbugs/issue45045.go": "x86 statepoint liveness does not yet retain the second map key through GC", "fixedbugs/issue51101.go": "x86 large-frame recursion still reaches morestack below the current stack bound", - "fixedbugs/issue68322.go": "x86 dynamic SSE4.1 truncation lowers the guarded LLVM intrinsic to an unavailable external libcall", "fixedbugs/issue80188.go": "arm64 R30 async-preemption liveness regression has no qualified x86 register-allocation equivalent", "fixedbugs/issue9604b.go": "Linux/amd64 runoutput generation emits duplicate declarations", "gcgort.go": "x86 GoObj pcsp and traceback metadata do not yet unwind all generated goroutine wrappers during stack growth", "genmeth1.go": "Linux/amd64 statepoint lowering rejects vectors inside live pointer aggregates", "inline_caller.go": "Linux/amd64 inline caller line information reports llvm-ir:1", "inline_callers.go": "Linux/amd64 runtime.Callers output is missing expected inline frames", - "llvm_float_intrinsics.go": "x86 dynamic SSE4.1 and FMA intrinsics lower to unavailable external libc and libm calls", "maymorestack.go": "Linux/amd64 execution does not invoke the configured mayMoreStack hook", "reflectmethod2.go": "Linux/amd64 stack growth reports an invalid pointer in the reflected-method frame", "reflectmethod6.go": "Linux/amd64 stack growth reports an invalid pointer in the reflected-method frame", - "fixedbugs/issue15281.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "fixedbugs/issue24488.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "fixedbugs/issue24491b.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "fixedbugs/issue44823.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "fixedbugs/issue46725.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "ddd.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", - "typeparam/list2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified" + "fixedbugs/issue15281.go": "Linux/amd64 allocation placement differs from the test expectations" }, "linux/arm64": { "append.go": "Linux/arm64 statepoint lowering rejects vectors inside live pointer aggregates",