cmd: unblock LLVM bit operations and stack layouts

src/cmd/compile/internal/ssa/ssa2llvm.go

@@ -445,6 +445,73 @@ func (lfc *LLVMFuncContext) bitLen(v *Value) llvm.Value {
 	}
 }
 
+func (lfc *LLVMFuncContext) trailingZeros(v *Value) llvm.Value {
+	x := lfc.GenLV(v.Args[0])
+	if x.Type().TypeKind() != llvm.IntegerTypeKind {
+		v.Fatalf("%s has a non-integer LLVM operand", v.Op)
+	}
+	bits := x.Type().IntTypeWidth()
+	if bits != 8 && bits != 16 && bits != 32 && bits != 64 {
+		v.Fatalf("%s has unsupported operand width %d", v.Op, bits)
+	}
+
+	i1 := GlobalCtxt.Int1Type()
+	sig := llvm.FunctionType(x.Type(), []llvm.Type{x.Type(), i1}, false)
+	fn := getOrInsertLLVMIntrinsic("llvm.cttz.i"+fmt.Sprint(bits), sig)
+	isZeroPoison := v.Op == OpCtz8NonZero || v.Op == OpCtz16NonZero ||
+		v.Op == OpCtz32NonZero || v.Op == OpCtz64NonZero
+	zeroPoisonFlag := uint64(0)
+	if isZeroPoison {
+		zeroPoisonFlag = 1
+	}
+	trailing := lfc.b.CreateCall(sig, fn, []llvm.Value{
+		x,
+		llvm.ConstInt(i1, zeroPoisonFlag, false),
+	}, v.String()+".trailing")
+
+	want := getLLVMType(v.Type)
+	if want.TypeKind() != llvm.IntegerTypeKind {
+		v.Fatalf("%s has a non-integer LLVM result", v.Op)
+	}
+	switch {
+	case bits < want.IntTypeWidth():
+		return lfc.b.CreateZExt(trailing, want, v.String())
+	case bits > want.IntTypeWidth():
+		return lfc.b.CreateTrunc(trailing, want, v.String())
+	default:
+		trailing.SetName(v.String())
+		return trailing
+	}
+}
+
+func (lfc *LLVMFuncContext) populationCount(v *Value) llvm.Value {
+	x := lfc.GenLV(v.Args[0])
+	if x.Type().TypeKind() != llvm.IntegerTypeKind {
+		v.Fatalf("%s has a non-integer LLVM operand", v.Op)
+	}
+	bits := x.Type().IntTypeWidth()
+	if bits != 8 && bits != 16 && bits != 32 && bits != 64 {
+		v.Fatalf("%s has unsupported operand width %d", v.Op, bits)
+	}
+
+	sig := llvm.FunctionType(x.Type(), []llvm.Type{x.Type()}, false)
+	fn := getOrInsertLLVMIntrinsic("llvm.ctpop.i"+fmt.Sprint(bits), sig)
+	count := lfc.b.CreateCall(sig, fn, []llvm.Value{x}, v.String()+".count")
+	want := getLLVMType(v.Type)
+	if want.TypeKind() != llvm.IntegerTypeKind {
+		v.Fatalf("%s has a non-integer LLVM result", v.Op)
+	}
+	switch {
+	case bits < want.IntTypeWidth():
+		return lfc.b.CreateZExt(count, want, v.String())
+	case bits > want.IntTypeWidth():
+		return lfc.b.CreateTrunc(count, want, v.String())
+	default:
+		count.SetName(v.String())
+		return count
+	}
+}
+
 func (lfc *LLVMFuncContext) byteSwap(v *Value) llvm.Value {
 	x := lfc.GenLV(v.Args[0])
 	if x.Type().TypeKind() != llvm.IntegerTypeKind {
@@ -1722,6 +1789,11 @@ func (lfc *LLVMFuncContext) GenLV(v *Value) llvm.Value {
 		lVal = lfc.llvmUnaryIntrinsic(v, "llvm.bitreverse.i"+fmt.Sprint(resultType.IntTypeWidth()))
 	case OpBitLen64, OpBitLen32, OpBitLen16, OpBitLen8:
 		lVal = lfc.bitLen(v)
+	case OpCtz64, OpCtz32, OpCtz16, OpCtz8,
+		OpCtz64NonZero, OpCtz32NonZero, OpCtz16NonZero, OpCtz8NonZero:
+		lVal = lfc.trailingZeros(v)
+	case OpPopCount64, OpPopCount32, OpPopCount16, OpPopCount8:
+		lVal = lfc.populationCount(v)
 	case OpCondSelect:
 		x, y := arg0(), arg1()
 		if x.Type() != y.Type() || x.Type() != getLLVMType(v.Type) {

src/cmd/llvmplugin/CMakeLists.txt

@@ -125,6 +125,23 @@ if(BUILD_TESTING)
       "go-stack-growth-statepoint"
   )
 
+  add_test(
+    NAME GoALLCStatepoints.ZeroLengthPointerArray
+    COMMAND
+      "${GOALLC_LLC_EXECUTABLE}"
+      "-load-pass-plugin=$<TARGET_FILE:GoALLCStatepoints>"
+      -goallc-pass-plugin-emit-ir
+      -filetype=null
+      -o -
+      "${CMAKE_CURRENT_SOURCE_DIR}/testdata/alloca-zero-pointer-array.ll"
+  )
+  set_tests_properties(GoALLCStatepoints.ZeroLengthPointerArray PROPERTIES
+    PASS_REGULAR_EXPRESSION
+      "llvm.experimental.gc.statepoint"
+    FAIL_REGULAR_EXPRESSION
+      "pointer-aligned fixed alloca layouts"
+  )
+
   add_test(
     NAME GoALLCStatepoints.GCLeafMarkers
     COMMAND

src/cmd/llvmplugin/GoALLCStatepoints.cpp

@@ -132,7 +132,7 @@ bool containsPointer(Type *Ty) {
   if (auto *ST = dyn_cast<StructType>(Ty))
     return llvm::any_of(ST->elements(), containsPointer);
   if (auto *AT = dyn_cast<ArrayType>(Ty))
-    return containsPointer(AT->getElementType());
+    return AT->getNumElements() != 0 && containsPointer(AT->getElementType());
   if (auto *VT = dyn_cast<VectorType>(Ty))
     return containsPointer(VT->getElementType());
   return false;

src/cmd/llvmplugin/testdata/alloca-zero-pointer-array.ll

@@ -0,0 +1,11 @@
+target triple = "x86_64-unknown-linux-goobj"
+
+declare goabiinternal void @safepoint()
+
+define goabiinternal void @zero_length_pointer_array()
+    "go-stack-growth-statepoint" gc "goallc" {
+entry:
+  %slot = alloca [0 x ptr], align 8
+  call goabiinternal void @safepoint()
+  ret void
+}

test/codegen/llvm_bitops_select.go

@@ -39,6 +39,55 @@ func llvmBitLen32(x uint32) int {
 	return bits.Len32(x)
 }
 
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros64(i64 %x)
+// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %x, i1 false)
+func llvmTrailingZeros64(x uint64) int {
+	return bits.TrailingZeros64(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros32(i32 %x)
+// LLVM-DAG: call i32 @llvm.cttz.i32(i32 %x, i1 false)
+// LLVM-DAG: zext i32 %{{.*}} to i64
+func llvmTrailingZeros32(x uint32) int {
+	return bits.TrailingZeros32(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros16(i16 %x)
+// LLVM-DAG: call i16 @llvm.cttz.i16(i16 %x, i1 false)
+// LLVM-DAG: zext i16 %{{.*}} to i64
+func llvmTrailingZeros16(x uint16) int {
+	return bits.TrailingZeros16(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros8(i8 %x)
+// LLVM-DAG: call i8 @llvm.cttz.i8(i8 %x, i1 false)
+// LLVM-DAG: zext i8 %{{.*}} to i64
+func llvmTrailingZeros8(x uint8) int {
+	return bits.TrailingZeros8(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmTrailingZeros64NonZero(i64 %x)
+// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %x, i1 true)
+func llvmTrailingZeros64NonZero(x uint64) int {
+	if x == 0 {
+		return -1
+	}
+	return bits.TrailingZeros64(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmPopulationCount64(i64 %x)
+// LLVM-DAG: call i64 @llvm.ctpop.i64(i64 %x)
+func llvmPopulationCount64(x uint64) int {
+	return bits.OnesCount64(x)
+}
+
+// LLVM-DAG: define goabiinternal i64 @codegen.llvmPopulationCount16(i16 %x)
+// LLVM-DAG: call i16 @llvm.ctpop.i16(i16 %x)
+// LLVM-DAG: zext i16 %{{.*}} to i64
+func llvmPopulationCount16(x uint16) int {
+	return bits.OnesCount16(x)
+}
+
 // LLVM-DAG: define goabiinternal i64 @codegen.llvmCondSelect(i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}})
 // LLVM-DAG: select i1 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}
 func llvmCondSelect(cond, x, y int) int {

test/codegen/mathbits.go

@@ -6,6 +6,9 @@
 
 package codegen
 
+// LLVM-DAG: call i64 @llvm.cttz.i64(i64 %{{.*}}, i1 false)
+// LLVM-DAG: call i16 @llvm.ctpop.i16(i16 %{{.*}})
+
 import (
 	"math/bits"
 	"unsafe"

test/llvm_float_intrinsics.go

@@ -83,6 +83,46 @@ func bitLen32(x uint32) int {
 	return bits.Len32(x)
 }
 
+//go:noinline
+func trailingZeros64(x uint64) int {
+	return bits.TrailingZeros64(x)
+}
+
+//go:noinline
+func trailingZeros32(x uint32) int {
+	return bits.TrailingZeros32(x)
+}
+
+//go:noinline
+func trailingZeros16(x uint16) int {
+	return bits.TrailingZeros16(x)
+}
+
+//go:noinline
+func trailingZeros8(x uint8) int {
+	return bits.TrailingZeros8(x)
+}
+
+//go:noinline
+func populationCount64(x uint64) int {
+	return bits.OnesCount64(x)
+}
+
+//go:noinline
+func populationCount32(x uint32) int {
+	return bits.OnesCount32(x)
+}
+
+//go:noinline
+func populationCount16(x uint16) int {
+	return bits.OnesCount16(x)
+}
+
+//go:noinline
+func populationCount8(x uint8) int {
+	return bits.OnesCount8(x)
+}
+
 //go:noinline
 func add64Carry(x, y, carry uint64) (uint64, uint64) {
 	return bits.Add64(x, y, carry)
@@ -197,6 +237,16 @@ func main() {
 	if bitLen64(0) != 0 || bitLen64(1<<63) != 64 || bitLen32(1<<31) != 32 {
 		panic("bit length semantics")
 	}
+	if trailingZeros64(0) != 64 || trailingZeros64(1<<63) != 63 ||
+		trailingZeros32(0) != 32 || trailingZeros32(1<<31) != 31 ||
+		trailingZeros16(0) != 16 || trailingZeros16(1<<15) != 15 ||
+		trailingZeros8(0) != 8 || trailingZeros8(1<<7) != 7 {
+		panic("trailing zero semantics")
+	}
+	if populationCount64(^uint64(0)) != 64 || populationCount32(0xf0f0) != 8 ||
+		populationCount16(0xaaaa) != 8 || populationCount8(0xf3) != 6 {
+		panic("population count semantics")
+	}
 	if sum, carry := add64Carry(^uint64(0), 0, 1); sum != 0 || carry != 1 {
 		panic("add carry semantics")
 	}

test/llvm_tests.json

@@ -71,7 +71,8 @@
       "codegen/structs.go": "typed struct initialization and zeroing",
       "codegen/type_descriptor.go": "compiler-owned runtime type descriptor data and GoObj metadata",
       "codegen/type_descriptor_kinds.go": "all runtime type descriptor layouts and variable descriptor data",
-      "codegen/type_descriptor_methods.go": "uncommon type data, concrete method tables, wrappers, and R_METHODOFF"
+      "codegen/type_descriptor_methods.go": "uncommon type data, concrete method tables, wrappers, and R_METHODOFF",
+      "codegen/mathbits.go": "trailing-zero and population-count intrinsics plus complete integer bit-operation lowering"
     },
     "blacklist": {
       "codegen/*": "LLVM lowering support has not reached this complete source file yet"
@@ -1152,6 +1153,13 @@
       "typeparam/mdempsky/16.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "typeparam/settable.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
       "unsafebuiltins.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution",
+      "cmp.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution",
+      "convert4.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution",
+      "ddd.go": "zero-length pointer-array stack layout qualified through LLVM compile, GoObj link, and execution",
+      "fixedbugs/issue16095.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution",
+      "fixedbugs/issue32288.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution",
+      "nilptr2.go": "current statepoint payload qualified through LLVM compile, GoObj link, and execution",
+      "typeparam/list2.go": "zero-length pointer-array stack layout qualified through LLVM compile, GoObj link, and execution",
       "zerodivide.go": "Darwin/arm64 blacklist probe qualified through LLVM compile, GoObj link, and execution"
     },
     "blacklist": {
@@ -1272,6 +1280,13 @@
         "typeparam/mdempsky/16.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "typeparam/settable.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "unsafebuiltins.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "cmp.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "convert4.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "ddd.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "fixedbugs/issue16095.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "fixedbugs/issue32288.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "nilptr2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
+        "typeparam/list2.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified",
         "zerodivide.go": "qualified on Darwin/arm64 only; Linux/amd64 runtime not yet verified"
       }
     }