diff --git a/src/cmd/compile/internal/ssa/ssa2llvm.go b/src/cmd/compile/internal/ssa/ssa2llvm.go index 33705585d697e6..38f58311ce738b 100644 --- a/src/cmd/compile/internal/ssa/ssa2llvm.go +++ b/src/cmd/compile/internal/ssa/ssa2llvm.go @@ -445,6 +445,73 @@ func (lfc *LLVMFuncContext) bitLen(v *Value) llvm.Value { } } +func (lfc *LLVMFuncContext) trailingZeros(v *Value) llvm.Value { + x := lfc.GenLV(v.Args[0]) + if x.Type().TypeKind() != llvm.IntegerTypeKind { + v.Fatalf("%s has a non-integer LLVM operand", v.Op) + } + bits := x.Type().IntTypeWidth() + if bits != 8 && bits != 16 && bits != 32 && bits != 64 { + v.Fatalf("%s has unsupported operand width %d", v.Op, bits) + } + + i1 := GlobalCtxt.Int1Type() + sig := llvm.FunctionType(x.Type(), []llvm.Type{x.Type(), i1}, false) + fn := getOrInsertLLVMIntrinsic("llvm.cttz.i"+fmt.Sprint(bits), sig) + isZeroPoison := v.Op == OpCtz8NonZero || v.Op == OpCtz16NonZero || + v.Op == OpCtz32NonZero || v.Op == OpCtz64NonZero + zeroPoisonFlag := uint64(0) + if isZeroPoison { + zeroPoisonFlag = 1 + } + trailing := lfc.b.CreateCall(sig, fn, []llvm.Value{ + x, + llvm.ConstInt(i1, zeroPoisonFlag, false), + }, v.String()+".trailing") + + want := getLLVMType(v.Type) + if want.TypeKind() != llvm.IntegerTypeKind { + v.Fatalf("%s has a non-integer LLVM result", v.Op) + } + switch { + case bits < want.IntTypeWidth(): + return lfc.b.CreateZExt(trailing, want, v.String()) + case bits > want.IntTypeWidth(): + return lfc.b.CreateTrunc(trailing, want, v.String()) + default: + trailing.SetName(v.String()) + return trailing + } +} + +func (lfc *LLVMFuncContext) populationCount(v *Value) llvm.Value { + x := lfc.GenLV(v.Args[0]) + if x.Type().TypeKind() != llvm.IntegerTypeKind { + v.Fatalf("%s has a non-integer LLVM operand", v.Op) + } + bits := x.Type().IntTypeWidth() + if bits != 8 && bits != 16 && bits != 32 && bits != 64 { + v.Fatalf("%s has unsupported operand width %d", v.Op, bits) + } + + sig := llvm.FunctionType(x.Type(), []llvm.Type{x.Type()}, false) + fn := getOrInsertLLVMIntrinsic("llvm.ctpop.i"+fmt.Sprint(bits), sig) + count := lfc.b.CreateCall(sig, fn, []llvm.Value{x}, v.String()+".count") + want := getLLVMType(v.Type) + if want.TypeKind() != llvm.IntegerTypeKind { + v.Fatalf("%s has a non-integer LLVM result", v.Op) + } + switch { + case bits < want.IntTypeWidth(): + return lfc.b.CreateZExt(count, want, v.String()) + case bits > want.IntTypeWidth(): + return lfc.b.CreateTrunc(count, want, v.String()) + default: + count.SetName(v.String()) + return count + } +} + func (lfc *LLVMFuncContext) byteSwap(v *Value) llvm.Value { x := lfc.GenLV(v.Args[0]) if x.Type().TypeKind() != llvm.IntegerTypeKind { @@ -1722,6 +1789,11 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value { lVal = lfc.llvmUnaryIntrinsic(v, "llvm.bitreverse.i"+fmt.Sprint(resultType.IntTypeWidth())) case OpBitLen64, OpBitLen32, OpBitLen16, OpBitLen8: lVal = lfc.bitLen(v) + case OpCtz64, OpCtz32, OpCtz16, OpCtz8, + OpCtz64NonZero, OpCtz32NonZero, OpCtz16NonZero, OpCtz8NonZero: + lVal = lfc.trailingZeros(v) + case OpPopCount64, OpPopCount32, OpPopCount16, OpPopCount8: + lVal = lfc.populationCount(v) case OpCondSelect: x, y := arg0(), arg1() if x.Type() != y.Type() || x.Type() != getLLVMType(v.Type) { diff --git a/src/cmd/llvmplugin/CMakeLists.txt b/src/cmd/llvmplugin/CMakeLists.txt index fd7ffcbf568f3f..c1bcc0ab26ed7f 100644 --- a/src/cmd/llvmplugin/CMakeLists.txt +++ b/src/cmd/llvmplugin/CMakeLists.txt @@ -125,6 +125,23 @@ if(BUILD_TESTING) "go-stack-growth-statepoint" ) + add_test( + NAME GoALLCStatepoints.ZeroLengthPointerArray + COMMAND + "${GOALLC_LLC_EXECUTABLE}" + "-load-pass-plugin=$" + -goallc-pass-plugin-emit-ir + -filetype=null + -o - + "${CMAKE_CURRENT_SOURCE_DIR}/testdata/alloca-zero-pointer-array.ll" + ) + set_tests_properties(GoALLCStatepoints.ZeroLengthPointerArray PROPERTIES + PASS_REGULAR_EXPRESSION + "llvm.experimental.gc.statepoint" + FAIL_REGULAR_EXPRESSION + "pointer-aligned fixed alloca layouts" + ) + add_test( NAME GoALLCStatepoints.GCLeafMarkers COMMAND diff --git a/src/cmd/llvmplugin/GoALLCStatepoints.cpp b/src/cmd/llvmplugin/GoALLCStatepoints.cpp index 9d8d253fc763a1..5499260b31cff6 100644 --- a/src/cmd/llvmplugin/GoALLCStatepoints.cpp +++ b/src/cmd/llvmplugin/GoALLCStatepoints.cpp @@ -132,7 +132,7 @@ bool containsPointer(Type *Ty) { if (auto *ST = dyn_cast(Ty)) return llvm::any_of(ST->elements(), containsPointer); if (auto *AT = dyn_cast(Ty)) - return containsPointer(AT->getElementType()); + return AT->getNumElements() != 0 && containsPointer(AT->getElementType()); if (auto *VT = dyn_cast(Ty)) return containsPointer(VT->getElementType()); return false; diff --git a/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll b/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll new file mode 100644 index 00000000000000..bf0ae1fbeb7131 --- /dev/null +++ b/src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll @@ -0,0 +1,11 @@ +target triple = "x86_64-unknown-linux-goobj" + +declare goabiinternal void @safepoint() + +define goabiinternal void @zero_length_pointer_array() + "go-stack-growth-statepoint" gc "goallc" { +entry: + %slot = alloca [0 x ptr], align 8 + call goabiinternal void @safepoint() + ret void +} diff --git a/test/codegen/llvm_bitops_select.go b/test/codegen/llvm_bitops_select.go index 08b8ccdcfe200a..61557e72bef16b 100644 --- a/test/codegen/llvm_bitops_select.go +++ b/test/codegen/llvm_bitops_select.go @@ -39,6 +39,55 @@ func llvmBitLen32(x uint32) int { return bits.Len32(x) } +// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros64(i64 %x) +// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %x, i1 false) +func llvmTrailingZeros64(x uint64) int { + return bits.TrailingZeros64(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros32(i32 %x) +// LLVM-DAG: call i32 @llvm.cttz.i32(i32 %x, i1 false) +// LLVM-DAG: zext i32 %{{.*}} to i64 +func llvmTrailingZeros32(x uint32) int { + return bits.TrailingZeros32(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros16(i16 %x) +// LLVM-DAG: call i16 @llvm.cttz.i16(i16 %x, i1 false) +// LLVM-DAG: zext i16 %{{.*}} to i64 +func llvmTrailingZeros16(x uint16) int { + return bits.TrailingZeros16(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros8(i8 %x) +// LLVM-DAG: call i8 @llvm.cttz.i8(i8 %x, i1 false) +// LLVM-DAG: zext i8 %{{.*}} to i64 +func llvmTrailingZeros8(x uint8) int { + return bits.TrailingZeros8(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros64NonZero(i64 %x) +// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %x, i1 true) +func llvmTrailingZeros64NonZero(x uint64) int { + if x == 0 { + return -1 + } + return bits.TrailingZeros64(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmPopulationCount64(i64 %x) +// LLVM-DAG: call i64 @llvm.ctpop.i64(i64 %x) +func llvmPopulationCount64(x uint64) int { + return bits.OnesCount64(x) +} + +// LLVM-DAG: define goabiinternal i64 @codegen.llvmPopulationCount16(i16 %x) +// LLVM-DAG: call i16 @llvm.ctpop.i16(i16 %x) +// LLVM-DAG: zext i16 %{{.*}} to i64 +func llvmPopulationCount16(x uint16) int { + return bits.OnesCount16(x) +} + // LLVM-DAG: define goabiinternal i64 @codegen.llvmCondSelect(i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) // LLVM-DAG: select i1 %{{.*}}, i64 %{{.*}}, i64 %{{.*}} func llvmCondSelect(cond, x, y int) int { diff --git a/test/codegen/mathbits.go b/test/codegen/mathbits.go index 4d02bfb96c240b..e61755f6bcd15b 100644 --- a/test/codegen/mathbits.go +++ b/test/codegen/mathbits.go @@ -6,6 +6,9 @@ package codegen +// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %{{.*}}, i1 false) +// LLVM-DAG: call i16 @llvm.ctpop.i16(i16 %{{.*}}) + import ( "math/bits" "unsafe" diff --git a/test/llvm_float_intrinsics.go b/test/llvm_float_intrinsics.go index c324d95e426834..8d09da0306a608 100644 --- a/test/llvm_float_intrinsics.go +++ b/test/llvm_float_intrinsics.go @@ -83,6 +83,46 @@ func bitLen32(x uint32) int { return bits.Len32(x) } +//go:noinline +func trailingZeros64(x uint64) int { + return bits.TrailingZeros64(x) +} + +//go:noinline +func trailingZeros32(x uint32) int { + return bits.TrailingZeros32(x) +} + +//go:noinline +func trailingZeros16(x uint16) int { + return bits.TrailingZeros16(x) +} + +//go:noinline +func trailingZeros8(x uint8) int { + return bits.TrailingZeros8(x) +} + +//go:noinline +func populationCount64(x uint64) int { + return bits.OnesCount64(x) +} + +//go:noinline +func populationCount32(x uint32) int { + return bits.OnesCount32(x) +} + +//go:noinline +func populationCount16(x uint16) int { + return bits.OnesCount16(x) +} + +//go:noinline +func populationCount8(x uint8) int { + return bits.OnesCount8(x) +} + //go:noinline func add64Carry(x, y, carry uint64) (uint64, uint64) { return bits.Add64(x, y, carry) @@ -197,6 +237,16 @@ func main() { if bitLen64(0) != 0 || bitLen64(1<<63) != 64 || bitLen32(1<<31) != 32 { panic("bit length semantics") } + if trailingZeros64(0) != 64 || trailingZeros64(1<<63) != 63 || + trailingZeros32(0) != 32 || trailingZeros32(1<<31) != 31 || + trailingZeros16(0) != 16 || trailingZeros16(1<<15) != 15 || + trailingZeros8(0) != 8 || trailingZeros8(1<<7) != 7 { + panic("trailing zero semantics") + } + if populationCount64(^uint64(0)) != 64 || populationCount32(0xf0f0) != 8 || + populationCount16(0xaaaa) != 8 || populationCount8(0xf3) != 6 { + panic("population count semantics") + } if sum, carry := add64Carry(^uint64(0), 0, 1); sum != 0 || carry != 1 { panic("add carry semantics") } diff --git a/test/llvm_tests.json b/test/llvm_tests.json index b6fad16f2d5d64..87bfee3578551e 100644 --- a/test/llvm_tests.json +++ b/test/llvm_tests.json @@ -71,7 +71,8 @@ "codegen/structs.go": "typed struct initialization and zeroing", "codegen/type_descriptor.go": "compiler-owned runtime type descriptor data and GoObj metadata", "codegen/type_descriptor_kinds.go": "all runtime type descriptor layouts and variable descriptor data", - "codegen/type_descriptor_methods.go": "uncommon type data, concrete method tables, wrappers, and R_METHODOFF" + "codegen/type_descriptor_methods.go": "uncommon type data, concrete method tables, wrappers, and R_METHODOFF", + "codegen/mathbits.go": "trailing-zero and population-count intrinsics plus complete integer bit-operation lowering" }, "blacklist": { "codegen/*": "LLVM lowering support has not reached this complete source file yet" @@ -1152,6 +1153,13 @@ "typeparam/mdempsky/16.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "typeparam/settable.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", "unsafebuiltins.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution", + "cmp.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution", + "convert4.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution", + "ddd.go": "zero-length pointer-array stack layout qualified through LLVM compile, GoObj link, and execution", + "fixedbugs/issue16095.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution", + "fixedbugs/issue32288.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution", + "nilptr2.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution", + "typeparam/list2.go": "zero-length pointer-array stack layout qualified through LLVM compile, GoObj link, and execution", "zerodivide.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution" }, "blacklist": { @@ -1272,6 +1280,13 @@ "typeparam/mdempsky/16.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "typeparam/settable.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "unsafebuiltins.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "cmp.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "convert4.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "ddd.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "fixedbugs/issue16095.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "fixedbugs/issue32288.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "nilptr2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", + "typeparam/list2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified", "zerodivide.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified" } }