Skip to content

cmd/compile: preserve guarded AMD64 FP intrinsics in LLVM #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhouguangyuan0718 wants to merge 1 commit into
base: go1.27.master
Choose a base branch
from
Open

cmd/compile: preserve guarded AMD64 FP intrinsics in LLVM #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 60 additions & 5 deletions src/cmd/compile/internal/ssa/ssa2llvm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ const goObjMarkerRelocMD = "goobj.marker_reloc"
const goObjSymbolNameMD = "goobj.symbol.name"
const llvmFramePointerAttr = "frame-pointer"
const llvmFramePointerNonLeaf = "non-leaf"
const llvmTargetCPUAttr = "target-cpu"

// Keep fixed-size memmoves within the store expansion limits of the supported
// LLVM targets. Larger moves must use runtime.memmove rather than a libc symbol,
Expand Down Expand Up @@ -337,6 +338,54 @@ func (lfc *LLVMFuncContext) llvmTernaryIntrinsic(v *Value, name string) llvm.Val
return lfc.b.CreateCall(sig, fn, []llvm.Value{x, y, z}, v.String())
}

func (lfc *LLVMFuncContext) llvmRoundIntrinsic(v *Value, genericName string, amd64Mode uint64) llvm.Value {
if lfc.F.Config.arch != "amd64" || buildcfg.GOAMD64 >= 2 {
return lfc.llvmUnaryIntrinsic(v, genericName)
}

// At GOAMD64=v1, Go SSA places these operations behind
// runtime.x86HasSSE41. Preserve that path-sensitive contract in the
// intrinsic instead of enabling SSE4.1 for the whole function. At v2 and
// above, the function target-cpu guarantees SSE4.1 and the generic LLVM
// intrinsic exposes the usual optimization opportunities.
x := lfc.GenLV(v.Args[0])
want := getLLVMType(v.Type)
if x.Type() != want || want.TypeKind() != llvm.DoubleTypeKind {
v.Fatalf("%s has incompatible LLVM operand and result types", v.Op)
}
i32 := GlobalCtxt.Int32Type()
sig := llvm.FunctionType(want, []llvm.Type{want, i32}, false)
fn := getOrInsertLLVMIntrinsic("llvm.x86.go.sse41.round.f64", sig)
mode := llvm.ConstInt(i32, amd64Mode, false)
return lfc.b.CreateCall(sig, fn, []llvm.Value{x, mode}, v.String())
}

func (lfc *LLVMFuncContext) llvmFMA(v *Value) llvm.Value {
if lfc.F.Config.arch != "amd64" || buildcfg.GOAMD64 >= 3 {
return lfc.llvmTernaryIntrinsic(v, "llvm.fma.f64")
}

// At GOAMD64=v1 and v2, the AMD64 SSA control flow has already guarded this
// operation with runtime.x86HasFMA. GOAMD64=v3 and above instead use the
// generic intrinsic under a function target-cpu that guarantees FMA.
return lfc.llvmTernaryIntrinsic(v, "llvm.x86.go.fma.f64")
}

func llvmTargetCPU(arch string) string {
if arch != "amd64" {
return ""
}
switch buildcfg.GOAMD64 {
case 1:
return "x86-64"
case 2, 3, 4:
return fmt.Sprintf("x86-64-v%d", buildcfg.GOAMD64)
default:
base.Fatalf("LLVM target CPU is not configured for GOAMD64=v%d", buildcfg.GOAMD64)
return ""
}
}

func (lfc *LLVMFuncContext) buildPureTuple(v *Value, values ...llvm.Value) llvm.Value {
resultType := getLLVMType(v.Type)
if resultType.TypeKind() != llvm.StructTypeKind {
Expand Down Expand Up @@ -1732,15 +1781,15 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value {
case OpAbs:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.fabs.f64")
case OpFloor:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.floor.f64")
lVal = lfc.llvmRoundIntrinsic(v, "llvm.floor.f64", 1)
case OpCeil:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.ceil.f64")
lVal = lfc.llvmRoundIntrinsic(v, "llvm.ceil.f64", 2)
case OpTrunc:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.trunc.f64")
lVal = lfc.llvmRoundIntrinsic(v, "llvm.trunc.f64", 3)
case OpRound:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.round.f64")
case OpRoundToEven:
lVal = lfc.llvmUnaryIntrinsic(v, "llvm.roundeven.f64")
lVal = lfc.llvmRoundIntrinsic(v, "llvm.roundeven.f64", 0)
case OpMin64F:
lVal = lfc.llvmBinaryIntrinsic(v, "llvm.minimum.f64")
case OpMin32F:
Expand All @@ -1750,7 +1799,7 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value {
case OpMax32F:
lVal = lfc.llvmBinaryIntrinsic(v, "llvm.maximum.f32")
case OpFMA:
lVal = lfc.llvmTernaryIntrinsic(v, "llvm.fma.f64")
lVal = lfc.llvmFMA(v)
case OpEq64, OpEq32, OpEq16, OpEq8, OpEqB:
lVal = lfc.goBool(lfc.b.CreateICmp(llvm.IntEQ, arg0(), arg1(), v.String()+".i1"), v.String())
case OpEqPtr:
Expand Down Expand Up @@ -2287,6 +2336,12 @@ func LLVMCompile(f *Func) {
f.fe.Fatalf(f.Entry.Pos, "duplicate LLVM definition for %s", f.OwnAux.Fn.Name)
}
FCtxt.LF.SetGC(goGCStrategy)
if cpu := llvmTargetCPU(f.Config.arch); cpu != "" {
// GOAMD64 levels are the standard x86-64 microarchitecture levels.
// Make the required instruction set visible to both LLVM optimization
// and instruction selection without selecting a host-specific CPU.
FCtxt.LF.AddTargetDependentFunctionAttr(llvmTargetCPUAttr, cpu)
}
// Go has already made its source-level inlining decision before LLVM
// lowering. Preserve both explicit //go:noinline boundaries and the
// frontend's implicit no-inline rules for functions containing defer or
Expand Down
3 changes: 2 additions & 1 deletion test/codegen/llvm_math_intrinsics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ package codegen

import "math"

// LLVM-DAG: call double @llvm.fma.f64(double %x, double %y, double %z)
// LLVM-ARM64-DAG: call double @llvm.fma.f64(double %x, double %y, double %z)
// LLVM-AMD64-DAG: call double @llvm.x86.go.fma.f64(double %x, double %y, double %z)
func llvmFMA64(x, y, z float64) float64 {
return math.FMA(x, y, z)
}
Expand Down
16 changes: 11 additions & 5 deletions test/codegen/llvm_trunc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,39 @@
import "math"

// LLVM-DAG: define goabiinternal double @codegen.llvmTrunc64(double %x)
// LLVM-DAG: call double @llvm.trunc.f64(double %x)
// LLVM-ARM64-DAG: call double @llvm.trunc.f64(double %x)
// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 3)
// LLVM-OPT-DAG: define goabiinternal double @codegen.llvmTrunc64(double %x)
// LLVM-OPT-DAG: call double @llvm.trunc.f64(double %x)
// LLVM-OPT-ARM64-DAG: call double @llvm.trunc.f64(double %x)
// LLVM-OPT-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 3)
// LLVM-AMD64-DAG: "target-cpu"="x86-64"
func llvmTrunc64(x float64) float64 {
return math.Trunc(x)
}

// LLVM-DAG: define goabiinternal double @codegen.llvmCeil64(double %x)
// LLVM-DAG: call double @llvm.ceil.f64(double %x)
// LLVM-ARM64-DAG: call double @llvm.ceil.f64(double %x)
// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 2)
func llvmCeil64(x float64) float64 {
return math.Ceil(x)
}

// LLVM-DAG: define goabiinternal double @codegen.llvmFloor64(double %x)
// LLVM-DAG: call double @llvm.floor.f64(double %x)
// LLVM-ARM64-DAG: call double @llvm.floor.f64(double %x)
// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 1)
func llvmFloor64(x float64) float64 {
return math.Floor(x)
}

// LLVM-DAG: define goabiinternal double @codegen.llvmRound64(double %x)
// LLVM-DAG: call double @llvm.round.f64(double %x)

Check failure on line 37 in test/codegen/llvm_trunc.go

View workflow job for this annotation

GitHub Actions / linux-amd64 

error: LLVM-DAG: expected string not found in input
func llvmRound64(x float64) float64 {
return math.Round(x)
}

// LLVM-DAG: define goabiinternal double @codegen.llvmRoundToEven64(double %x)
// LLVM-DAG: call double @llvm.roundeven.f64(double %x)
// LLVM-ARM64-DAG: call double @llvm.roundeven.f64(double %x)
// LLVM-AMD64-DAG: call double @llvm.x86.go.sse41.round.f64(double %x, i32 0)
func llvmRoundToEven64(x float64) float64 {
return math.RoundToEven(x)
}
16 changes: 2 additions & 14 deletions test/llvm_tests.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,7 @@
"platform_graylist": {
"linux/amd64": {
"codegen/llvm_abs.go": "the native AMD64 frontend does not form OpAbs for math.Abs",
"codegen/mathbits.go": "the AMD64 frontend forms unsupported Div128u while compiling the full source file",
"codegen/llvm_trunc.go": "GOAMD64=v1 guards OpTrunc with unsupported HasCPUFeature lowering"
"codegen/mathbits.go": "the AMD64 frontend forms unsupported Div128u while compiling the full source file"
}
}
},
Expand Down Expand Up @@ -1172,31 +1171,20 @@
"abi/method_wrapper.go": "x86 GoObj traceback does not yet unwind the panic path through the method wrapper",
"append.go": "Linux/amd64 statepoint lowering rejects vectors inside live pointer aggregates",
"convert5.go": "Linux/amd64 float-to-integer overflow results differ from the test expectations",
"fixedbugs/bug491.go": "Linux/amd64 link fails because the generated memset relocation has no definition",
"fixedbugs/issue23522.go": "x86 dynamic SSE4.1 floor selection lowers the guarded LLVM intrinsic to an unavailable external libcall",
"fixedbugs/issue2615.go": "Linux/amd64 link fails because the generated memset relocation has no definition",
"fixedbugs/issue34968.go": "Linux/amd64 GoObj generation reports an unexpected file-local symbol index",
"fixedbugs/issue43570.go": "x86 GoObj panic traceback still reports an unknown caller PC",
"fixedbugs/issue45045.go": "x86 statepoint liveness does not yet retain the second map key through GC",
"fixedbugs/issue51101.go": "x86 large-frame recursion still reaches morestack below the current stack bound",
"fixedbugs/issue68322.go": "x86 dynamic SSE4.1 truncation lowers the guarded LLVM intrinsic to an unavailable external libcall",
"fixedbugs/issue80188.go": "arm64 R30 async-preemption liveness regression has no qualified x86 register-allocation equivalent",
"fixedbugs/issue9604b.go": "Linux/amd64 runoutput generation emits duplicate declarations",
"gcgort.go": "x86 GoObj pcsp and traceback metadata do not yet unwind all generated goroutine wrappers during stack growth",
"genmeth1.go": "Linux/amd64 statepoint lowering rejects vectors inside live pointer aggregates",
"inline_caller.go": "Linux/amd64 inline caller line information reports llvm-ir:1",
"inline_callers.go": "Linux/amd64 runtime.Callers output is missing expected inline frames",
"llvm_float_intrinsics.go": "x86 dynamic SSE4.1 and FMA intrinsics lower to unavailable external libc and libm calls",
"maymorestack.go": "Linux/amd64 execution does not invoke the configured mayMoreStack hook",
"reflectmethod2.go": "Linux/amd64 stack growth reports an invalid pointer in the reflected-method frame",
"reflectmethod6.go": "Linux/amd64 stack growth reports an invalid pointer in the reflected-method frame",
"fixedbugs/issue15281.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"fixedbugs/issue24488.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"fixedbugs/issue24491b.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"fixedbugs/issue44823.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"fixedbugs/issue46725.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"ddd.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
"typeparam/list2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified"
"fixedbugs/issue15281.go": "Linux/amd64 allocation placement differs from the test expectations"
},
"linux/arm64": {
"append.go": "Linux/arm64 statepoint lowering rejects vectors inside live pointer aggregates",
Expand Down
Loading