From 0e01a56373e0a9be17001cc2cf8de104e4e0b04d Mon Sep 17 00:00:00 2001 From: bogdan-petkovic Date: Thu, 24 Sep 2026 03:38:22 -0500 Subject: [PATCH 1/3] [EXTERNAL] Be more careful about using expandDivRem24 --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 25 +- .../CodeGen/AMDGPU/GlobalISel/udiv.i32.ll | 86 ++- .../CodeGen/AMDGPU/GlobalISel/udiv.i64.ll | 94 +-- .../CodeGen/AMDGPU/GlobalISel/urem.i32.ll | 90 +-- .../CodeGen/AMDGPU/GlobalISel/urem.i64.ll | 98 ++-- .../AMDGPU/amdgpu-codegenprepare-idiv.ll | 245 ++++++++ .../llvm/test/CodeGen/AMDGPU/udiv.ll | 208 ++++--- .../llvm/test/CodeGen/AMDGPU/udiv64.ll | 553 ++++++++++++------ .../llvm/test/CodeGen/AMDGPU/udivrem24.ll | 286 +++++---- .../llvm/test/CodeGen/AMDGPU/urem64.ll | 402 ++++++++----- 10 files changed, 1438 insertions(+), 649 deletions(-) diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 1bcfc1da3b..c3fb24bc83 100644 --- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -1067,7 +1067,18 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder, Value *Den, bool IsDiv, bool IsSigned) const { unsigned DivBits = getDivNumBits(I, Num, Den, 24, IsSigned); - if (DivBits > 24) + + // v_rcp_f32(float(X)) can have an error of 1 ulp. + // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly + // when abs(Y)>0x800000. + // For example, + // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. + // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. + // + // Note that for DivBits==24 && IsSigned, Y is in the range + // [-0x800000:0x7FFFFF]. abs(Y) is at most + // 0x800000 so it cannot hit this issue. + if (DivBits > (IsSigned ? 24 : 23)) return nullptr; return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned); } @@ -1358,7 +1369,17 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder, return nullptr; Value *Narrowed = nullptr; - if (NumDivBits <= 24) { + // v_rcp_f32(float(X)) can have an error of 1 ulp. + // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly + // when abs(Y)>0x800000. + // For example, + // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. + // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. + // + // Note that for NumDivBits==24 && IsSigned, Y is in the range + // [-0x800000:0x7FFFFF]. abs(Y) is at most + // 0x800000 so it cannot hit this issue. + if (NumDivBits <= (IsSigned ? 24 : 23)) { Narrowed = expandDivRem24Impl(Builder, I, Num, Den, NumDivBits, IsDiv, IsSigned); } else if (NumDivBits <= 32) { diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll index 13e8e12118..b308bdf42f 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll @@ -431,17 +431,25 @@ define i32 @v_udiv_i32_24bit(i32 %num, i32 %den) { ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 -; CGP-NEXT: v_rcp_f32_e32 v2, v1 -; CGP-NEXT: v_mul_f32_e32 v2, v0, v2 -; CGP-NEXT: v_trunc_f32_e32 v2, v2 -; CGP-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 +; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 +; CGP-NEXT: v_rcp_f32_e32 v2, v2 +; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 -; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 +; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 +; CGP-NEXT: v_mul_lo_u32 v3, v2, v1 +; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc +; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and i32 %num, 16777215 %den.mask = and i32 %den, 16777215 @@ -504,28 +512,44 @@ define <2 x i32> @v_udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) { ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 ; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v3 -; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3 -; CGP-NEXT: v_rcp_f32_e32 v4, v2 -; CGP-NEXT: v_rcp_f32_e32 v5, v3 -; CGP-NEXT: v_mul_f32_e32 v4, v0, v4 -; CGP-NEXT: v_mul_f32_e32 v5, v1, v5 -; CGP-NEXT: v_trunc_f32_e32 v4, v4 -; CGP-NEXT: v_trunc_f32_e32 v5, v5 -; CGP-NEXT: v_fma_f32 v0, -v4, v2, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 +; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 +; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 +; CGP-NEXT: v_rcp_f32_e32 v4, v4 +; CGP-NEXT: v_rcp_f32_e32 v6, v6 +; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 -; CGP-NEXT: v_fma_f32 v1, -v5, v3, v1 -; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v2 -; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, v3 -; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 +; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 +; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 +; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 +; CGP-NEXT: v_mul_hi_u32 v4, v0, v4 +; CGP-NEXT: v_mul_hi_u32 v5, v1, v5 +; CGP-NEXT: v_mul_lo_u32 v6, v4, v2 +; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4 +; CGP-NEXT: v_mul_lo_u32 v8, v5, v3 +; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc +; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2 +; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3 +; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5] +; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc +; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4 +; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] +; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and <2 x i32> %num, %den.mask = and <2 x i32> %den, diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll index b6fa328393..63f24a2de6 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll @@ -1878,17 +1878,25 @@ define i64 @v_udiv_i64_24bit(i64 %num, i64 %den) { ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 -; CGP-NEXT: v_rcp_f32_e32 v2, v1 -; CGP-NEXT: v_mul_f32_e32 v2, v0, v2 -; CGP-NEXT: v_trunc_f32_e32 v2, v2 -; CGP-NEXT: v_mad_f32 v0, -v2, v1, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 +; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 +; CGP-NEXT: v_rcp_f32_e32 v2, v2 +; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 -; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 +; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 +; CGP-NEXT: v_mul_lo_u32 v3, v2, v1 +; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc +; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; CGP-NEXT: v_mov_b32_e32 v1, 0 ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and i64 %num, 16777215 @@ -2132,31 +2140,51 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) { ; CGP: ; %bb.0: ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4 -; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 +; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 +; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v4 ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v6 -; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3 -; CGP-NEXT: v_rcp_f32_e32 v4, v1 -; CGP-NEXT: v_rcp_f32_e32 v5, v3 -; CGP-NEXT: v_mul_f32_e32 v4, v0, v4 -; CGP-NEXT: v_mul_f32_e32 v5, v2, v5 -; CGP-NEXT: v_trunc_f32_e32 v4, v4 -; CGP-NEXT: v_trunc_f32_e32 v5, v5 -; CGP-NEXT: v_mad_f32 v0, -v4, v1, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 +; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 +; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 +; CGP-NEXT: v_rcp_f32_e32 v4, v4 +; CGP-NEXT: v_rcp_f32_e32 v6, v6 +; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 -; CGP-NEXT: v_mad_f32 v2, -v5, v3, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 -; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 -; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v1 +; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 +; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 +; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 +; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 +; CGP-NEXT: v_mul_hi_u32 v6, v0, v4 +; CGP-NEXT: v_mul_lo_u32 v4, 0, v4 +; CGP-NEXT: v_mul_hi_u32 v7, v1, v5 +; CGP-NEXT: v_mul_lo_u32 v5, 0, v5 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5 +; CGP-NEXT: v_mul_lo_u32 v6, v4, v2 +; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4 +; CGP-NEXT: v_mul_lo_u32 v8, v5, v3 +; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc +; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2 +; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3 +; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5] +; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc +; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4 +; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] +; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc ; CGP-NEXT: v_mov_b32_e32 v1, 0 ; CGP-NEXT: v_mov_b32_e32 v3, 0 ; CGP-NEXT: s_setpc_b64 s[30:31] diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll index 4fb58ca491..de1e382a5d 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll @@ -409,19 +409,23 @@ define i32 @v_urem_i32_24bit(i32 %num, i32 %den) { ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v2, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1 -; CGP-NEXT: v_rcp_f32_e32 v4, v3 -; CGP-NEXT: v_mul_f32_e32 v4, v2, v4 -; CGP-NEXT: v_trunc_f32_e32 v4, v4 -; CGP-NEXT: v_fma_f32 v2, -v4, v3, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 -; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; CGP-NEXT: v_mul_lo_u32 v1, v2, v1 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 +; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 +; CGP-NEXT: v_rcp_f32_e32 v2, v2 +; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 +; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 +; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 +; CGP-NEXT: v_mul_lo_u32 v2, v2, v1 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and i32 %num, 16777215 %den.mask = and i32 %den, 16777215 @@ -480,32 +484,40 @@ define <2 x i32> @v_urem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) { ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 ; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v3 -; CGP-NEXT: v_cvt_f32_u32_e32 v4, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v5, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v6, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v7, v3 -; CGP-NEXT: v_rcp_f32_e32 v8, v5 -; CGP-NEXT: v_rcp_f32_e32 v9, v7 -; CGP-NEXT: v_mul_f32_e32 v8, v4, v8 -; CGP-NEXT: v_mul_f32_e32 v9, v6, v9 -; CGP-NEXT: v_trunc_f32_e32 v8, v8 -; CGP-NEXT: v_trunc_f32_e32 v9, v9 -; CGP-NEXT: v_fma_f32 v4, -v8, v5, v4 -; CGP-NEXT: v_cvt_u32_f32_e32 v8, v8 -; CGP-NEXT: v_fma_f32 v6, -v9, v7, v6 -; CGP-NEXT: v_cvt_u32_f32_e32 v9, v9 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, v5 -; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5] -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v6|, v7 -; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5 -; CGP-NEXT: v_mul_lo_u32 v2, v4, v2 -; CGP-NEXT: v_mul_lo_u32 v3, v5, v3 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 +; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 +; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 +; CGP-NEXT: v_rcp_f32_e32 v4, v4 +; CGP-NEXT: v_rcp_f32_e32 v6, v6 +; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 +; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 +; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 +; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 +; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 +; CGP-NEXT: v_mul_hi_u32 v4, v0, v4 +; CGP-NEXT: v_mul_hi_u32 v5, v1, v5 +; CGP-NEXT: v_mul_lo_u32 v4, v4, v2 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v3 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v5 +; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and <2 x i32> %num, %den.mask = and <2 x i32> %den, diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll index 9d7cb130c4..0c8d8590fe 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll @@ -1941,19 +1941,23 @@ define i64 @v_urem_i64_24bit(i64 %num, i64 %den) { ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v2, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1 -; CGP-NEXT: v_rcp_f32_e32 v4, v3 -; CGP-NEXT: v_mul_f32_e32 v4, v2, v4 -; CGP-NEXT: v_trunc_f32_e32 v4, v4 -; CGP-NEXT: v_mad_f32 v2, -v4, v3, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 -; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; CGP-NEXT: v_mul_lo_u32 v1, v2, v1 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 +; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 +; CGP-NEXT: v_rcp_f32_e32 v2, v2 +; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 +; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 +; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 +; CGP-NEXT: v_mul_lo_u32 v2, v2, v1 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; CGP-NEXT: v_mov_b32_e32 v1, 0 ; CGP-NEXT: s_setpc_b64 s[30:31] %num.mask = and i64 %num, 16777215 @@ -2193,35 +2197,47 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) { ; CGP: ; %bb.0: ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4 -; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 +; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 +; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v4 ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v6 -; CGP-NEXT: v_cvt_f32_u32_e32 v4, v0 -; CGP-NEXT: v_cvt_f32_u32_e32 v5, v1 -; CGP-NEXT: v_cvt_f32_u32_e32 v6, v2 -; CGP-NEXT: v_cvt_f32_u32_e32 v7, v3 -; CGP-NEXT: v_rcp_f32_e32 v8, v5 -; CGP-NEXT: v_rcp_f32_e32 v9, v7 -; CGP-NEXT: v_mul_f32_e32 v8, v4, v8 -; CGP-NEXT: v_mul_f32_e32 v9, v6, v9 -; CGP-NEXT: v_trunc_f32_e32 v8, v8 -; CGP-NEXT: v_trunc_f32_e32 v9, v9 -; CGP-NEXT: v_mad_f32 v4, -v8, v5, v4 -; CGP-NEXT: v_cvt_u32_f32_e32 v8, v8 -; CGP-NEXT: v_mad_f32 v6, -v9, v7, v6 -; CGP-NEXT: v_cvt_u32_f32_e32 v9, v9 -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, v5 -; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5] -; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v6|, v7 -; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5] -; CGP-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5 -; CGP-NEXT: v_mul_lo_u32 v1, v4, v1 -; CGP-NEXT: v_mul_lo_u32 v3, v5, v3 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v2, v3 -; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v1 +; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 +; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 +; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 +; CGP-NEXT: v_rcp_f32_e32 v4, v4 +; CGP-NEXT: v_rcp_f32_e32 v6, v6 +; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 +; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 +; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 +; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 +; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 +; CGP-NEXT: v_mul_hi_u32 v6, v0, v4 +; CGP-NEXT: v_mul_lo_u32 v4, 0, v4 +; CGP-NEXT: v_mul_hi_u32 v7, v1, v5 +; CGP-NEXT: v_mul_lo_u32 v5, 0, v5 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5 +; CGP-NEXT: v_mul_lo_u32 v4, v4, v2 +; CGP-NEXT: v_mul_lo_u32 v5, v5, v3 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v5 +; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 +; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 +; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc +; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 +; CGP-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc ; CGP-NEXT: v_mov_b32_e32 v1, 0 ; CGP-NEXT: v_mov_b32_e32 v3, 0 ; CGP-NEXT: s_setpc_b64 s[30:31] diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll index 1659ca62a0..a23295b617 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -10134,3 +10134,248 @@ entry: %B = srem <2 x i64> zeroinitializer, zeroinitializer ret <2 x i64> %B } + +; 23-bit sdiv, can use expandDivRem24Impl +define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { +; GFX6-LABEL: sdiv23: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] +; GFX6-NEXT: s_bfe_i32 s5, s5, 0x170000 +; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 +; GFX6-NEXT: s_bfe_i32 s4, s4, 0x170000 +; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 +; GFX6-NEXT: s_xor_b32 s4, s4, s5 +; GFX6-NEXT: v_rcp_f32_e32 v2, v0 +; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +; GFX6-NEXT: s_or_b32 s6, s4, 1 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 +; GFX6-NEXT: v_trunc_f32_e32 v2, v2 +; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 +; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| +; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec +; GFX6-NEXT: s_cselect_b32 s4, s6, 0 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 +; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 +; GFX6-NEXT: s_waitcnt expcnt(0) +; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 +; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 +; GFX6-NEXT: s_endpgm +; +; GFX9-LABEL: sdiv23: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_bfe_i32 s3, s3, 0x170000 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 +; GFX9-NEXT: s_bfe_i32 s2, s2, 0x170000 +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 +; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: v_rcp_f32_e32 v3, v0 +; GFX9-NEXT: s_ashr_i32 s2, s2, 30 +; GFX9-NEXT: s_or_b32 s4, s2, 1 +; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 +; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 +; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| +; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec +; GFX9-NEXT: s_cselect_b32 s2, s4, 0 +; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 +; GFX9-NEXT: global_store_short v1, v0, s[0:1] +; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 +; GFX9-NEXT: s_endpgm + %r = sdiv i23 %x, %y + store i23 %r, ptr addrspace(1) %out + ret void +} + +; 24-bit sdiv, can use expandDivRem24Impl +define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { +; GFX6-LABEL: sdiv24: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] +; GFX6-NEXT: s_bfe_i32 s5, s5, 0x180000 +; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 +; GFX6-NEXT: s_bfe_i32 s4, s4, 0x180000 +; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 +; GFX6-NEXT: s_xor_b32 s4, s4, s5 +; GFX6-NEXT: v_rcp_f32_e32 v2, v0 +; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +; GFX6-NEXT: s_or_b32 s6, s4, 1 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 +; GFX6-NEXT: v_trunc_f32_e32 v2, v2 +; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 +; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| +; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec +; GFX6-NEXT: s_cselect_b32 s4, s6, 0 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 +; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 +; GFX6-NEXT: s_waitcnt expcnt(0) +; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 +; GFX6-NEXT: s_endpgm +; +; GFX9-LABEL: sdiv24: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_bfe_i32 s3, s3, 0x180000 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 +; GFX9-NEXT: s_bfe_i32 s2, s2, 0x180000 +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 +; GFX9-NEXT: s_xor_b32 s2, s2, s3 +; GFX9-NEXT: v_rcp_f32_e32 v3, v0 +; GFX9-NEXT: s_ashr_i32 s2, s2, 30 +; GFX9-NEXT: s_or_b32 s4, s2, 1 +; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 +; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 +; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| +; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec +; GFX9-NEXT: s_cselect_b32 s2, s4, 0 +; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 +; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 +; GFX9-NEXT: global_store_short v1, v0, s[0:1] +; GFX9-NEXT: s_endpgm + %r = sdiv i24 %x, %y + store i24 %r, ptr addrspace(1) %out + ret void +} + +; 23-bit udiv, can use expandDivRem24Impl +define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { +; GFX6-LABEL: udiv23: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_and_b32 s3, s3, 0x7fffff +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_rcp_f32_e32 v2, v0 +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 +; GFX6-NEXT: v_trunc_f32_e32 v2, v2 +; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2 +; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 +; GFX6-NEXT: s_waitcnt expcnt(0) +; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 +; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 +; GFX6-NEXT: s_endpgm +; +; GFX9-LABEL: udiv23: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX9-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2 +; GFX9-NEXT: v_rcp_f32_e32 v2, v0 +; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2 +; GFX9-NEXT: v_trunc_f32_e32 v2, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc +; GFX9-NEXT: global_store_short v3, v0, s[0:1] +; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; GFX9-NEXT: global_store_byte_d16_hi v3, v0, s[0:1] offset:2 +; GFX9-NEXT: s_endpgm + %r = udiv i23 %x, %y + store i23 %r, ptr addrspace(1) %out + ret void +} + +; 24-bit udiv, cannot use expandDivRem24Impl +define amdgpu_kernel void @udiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { +; GFX6-LABEL: udiv24: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_and_b32 s4, s3, 0xffffff +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX6-NEXT: s_sub_i32 s3, 0, s4 +; GFX6-NEXT: s_and_b32 s5, s2, 0xffffff +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_rcp_f32_e32 v0, v0 +; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 +; GFX6-NEXT: v_readfirstlane_b32 s6, v0 +; GFX6-NEXT: s_mul_i32 s6, s6, s4 +; GFX6-NEXT: s_sub_i32 s5, s5, s6 +; GFX6-NEXT: s_sub_i32 s6, s5, s4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: s_cselect_b32 s5, s6, s5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 +; GFX6-NEXT: s_waitcnt expcnt(0) +; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 +; GFX6-NEXT: s_endpgm +; +; GFX9-LABEL: udiv24: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_and_b32 s3, s3, 0xffffff +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX9-NEXT: s_sub_i32 s4, 0, s3 +; GFX9-NEXT: s_and_b32 s2, s2, 0xffffff +; GFX9-NEXT: v_rcp_f32_e32 v0, v0 +; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX9-NEXT: v_readfirstlane_b32 s5, v0 +; GFX9-NEXT: s_mul_i32 s4, s4, s5 +; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4 +; GFX9-NEXT: s_add_i32 s5, s5, s4 +; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5 +; GFX9-NEXT: s_mul_i32 s5, s4, s3 +; GFX9-NEXT: s_sub_i32 s2, s2, s5 +; GFX9-NEXT: s_add_i32 s6, s4, 1 +; GFX9-NEXT: s_sub_i32 s5, s2, s3 +; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_cselect_b32 s4, s6, s4 +; GFX9-NEXT: s_cselect_b32 s2, s5, s2 +; GFX9-NEXT: s_add_i32 s5, s4, 1 +; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_cselect_b32 s2, s5, s4 +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 +; GFX9-NEXT: global_store_short v1, v0, s[0:1] +; GFX9-NEXT: s_endpgm + %r = udiv i24 %x, %y + store i24 %r, ptr addrspace(1) %out + ret void +} diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll index c0918dd78b..b3a64b81ec 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll @@ -1876,76 +1876,96 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %in) { ; SI-LABEL: v_udiv_i24: ; SI: ; %bb.0: -; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 -; SI-NEXT: s_mov_b32 s10, s6 -; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_mov_b32 s8, s2 -; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: s_mov_b32 s8, s6 +; SI-NEXT: s_mov_b32 s9, s7 ; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 ; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 ; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 ; SI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 -; SI-NEXT: s_mov_b32 s4, s0 -; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 ; SI-NEXT: s_waitcnt vmcnt(3) ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; SI-NEXT: s_waitcnt vmcnt(2) ; SI-NEXT: v_or_b32_e32 v0, v1, v0 -; SI-NEXT: v_cvt_f32_u32_e32 v0, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v1, v0 +; SI-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 ; SI-NEXT: s_waitcnt vmcnt(1) -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_rcp_f32_e32 v1, v1 ; SI-NEXT: s_waitcnt vmcnt(0) -; SI-NEXT: v_or_b32_e32 v1, v3, v1 -; SI-NEXT: v_cvt_f32_u32_e32 v1, v1 -; SI-NEXT: v_rcp_f32_e32 v2, v0 -; SI-NEXT: v_mul_f32_e32 v2, v1, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; SI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; SI-NEXT: v_or_b32_e32 v2, v3, v2 +; SI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; SI-NEXT: v_cvt_u32_f32_e32 v1, v1 +; SI-NEXT: v_mul_lo_u32 v4, v4, v1 +; SI-NEXT: v_mul_hi_u32 v4, v1, v4 +; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; SI-NEXT: v_mul_hi_u32 v1, v2, v1 +; SI-NEXT: v_mul_lo_u32 v3, v1, v0 +; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v1 +; SI-NEXT: v_sub_i32_e32 v2, vcc, v2, v3 +; SI-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; SI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: v_udiv_i24: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; VI-NEXT: s_mov_b32 s7, 0xf000 -; VI-NEXT: s_mov_b32 s6, -1 -; VI-NEXT: s_mov_b32 s10, s6 -; VI-NEXT: s_mov_b32 s11, s7 +; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: s_mov_b32 s2, -1 +; VI-NEXT: s_mov_b32 s10, s2 +; VI-NEXT: s_mov_b32 s11, s3 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_mov_b32 s8, s2 -; VI-NEXT: s_mov_b32 s9, s3 +; VI-NEXT: s_mov_b32 s8, s6 +; VI-NEXT: s_mov_b32 s9, s7 ; VI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 ; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 ; VI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 ; VI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 -; VI-NEXT: s_mov_b32 s4, s0 -; VI-NEXT: s_mov_b32 s5, s1 +; VI-NEXT: s_mov_b32 s0, s4 +; VI-NEXT: s_mov_b32 s1, s5 ; VI-NEXT: s_waitcnt vmcnt(3) ; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; VI-NEXT: s_waitcnt vmcnt(2) ; VI-NEXT: v_or_b32_e32 v0, v1, v0 -; VI-NEXT: v_cvt_f32_u32_e32 v0, v0 +; VI-NEXT: v_cvt_f32_u32_e32 v1, v0 +; VI-NEXT: v_sub_u32_e32 v4, vcc, 0, v0 ; VI-NEXT: s_waitcnt vmcnt(1) -; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; VI-NEXT: v_rcp_f32_e32 v1, v1 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_or_b32_e32 v1, v3, v1 -; VI-NEXT: v_cvt_f32_u32_e32 v1, v1 -; VI-NEXT: v_rcp_f32_e32 v2, v0 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; VI-NEXT: v_or_b32_e32 v2, v3, v2 +; VI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; VI-NEXT: v_cvt_u32_f32_e32 v1, v1 +; VI-NEXT: v_mul_lo_u32 v4, v4, v1 +; VI-NEXT: v_mul_hi_u32 v4, v1, v4 +; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v4 +; VI-NEXT: v_mul_hi_u32 v1, v2, v1 +; VI-NEXT: v_mul_lo_u32 v3, v1, v0 +; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v1 +; VI-NEXT: v_sub_u32_e32 v2, vcc, v2, v3 +; VI-NEXT: v_sub_u32_e32 v3, vcc, v2, v0 +; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; VI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc +; VI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v1 +; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; VI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; ; GCN-LABEL: v_udiv_i24: @@ -1958,40 +1978,50 @@ define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: s_add_u32 s4, s2, 4 ; GCN-NEXT: s_addc_u32 s5, s3, 0 ; GCN-NEXT: s_add_u32 s6, s2, 2 +; GCN-NEXT: v_mov_b32_e32 v0, s4 ; GCN-NEXT: s_addc_u32 s7, s3, 0 -; GCN-NEXT: v_mov_b32_e32 v0, s6 -; GCN-NEXT: v_mov_b32_e32 v1, s7 -; GCN-NEXT: s_add_u32 s6, s2, 6 -; GCN-NEXT: s_addc_u32 s7, s3, 0 +; GCN-NEXT: v_mov_b32_e32 v1, s5 +; GCN-NEXT: s_add_u32 s4, s2, 6 +; GCN-NEXT: s_addc_u32 s5, s3, 0 +; GCN-NEXT: v_mov_b32_e32 v2, s4 +; GCN-NEXT: v_mov_b32_e32 v3, s5 +; GCN-NEXT: flat_load_ubyte v4, v[2:3] +; GCN-NEXT: flat_load_ushort v5, v[0:1] ; GCN-NEXT: v_mov_b32_e32 v2, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s2 ; GCN-NEXT: v_mov_b32_e32 v3, s7 -; GCN-NEXT: v_mov_b32_e32 v4, s4 -; GCN-NEXT: v_mov_b32_e32 v5, s5 -; GCN-NEXT: flat_load_ubyte v6, v[2:3] -; GCN-NEXT: flat_load_ushort v4, v[4:5] -; GCN-NEXT: v_mov_b32_e32 v2, s2 -; GCN-NEXT: v_mov_b32_e32 v3, s3 -; GCN-NEXT: flat_load_ubyte v0, v[0:1] -; GCN-NEXT: flat_load_ushort v1, v[2:3] +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: flat_load_ubyte v2, v[2:3] +; GCN-NEXT: flat_load_ushort v0, v[0:1] ; GCN-NEXT: s_waitcnt vmcnt(3) -; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v6 +; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v4 ; GCN-NEXT: s_waitcnt vmcnt(2) -; GCN-NEXT: v_or_b32_e32 v2, v4, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GCN-NEXT: v_or_b32_e32 v3, v5, v1 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v3 +; GCN-NEXT: v_sub_u32_e32 v4, vcc, 0, v3 ; GCN-NEXT: s_waitcnt vmcnt(1) -; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_or_b32_e32 v0, v1, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 -; GCN-NEXT: v_rcp_f32_e32 v4, v2 +; GCN-NEXT: v_or_b32_e32 v2, v0, v2 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v4, v4, v1 +; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 +; GCN-NEXT: v_add_u32_e32 v0, vcc, v1, v4 +; GCN-NEXT: v_mul_hi_u32 v4, v2, v0 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 -; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4 -; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc +; GCN-NEXT: v_mul_lo_u32 v5, v4, v3 +; GCN-NEXT: v_add_u32_e32 v6, vcc, 1, v4 +; GCN-NEXT: v_sub_u32_e32 v2, vcc, v2, v5 +; GCN-NEXT: v_sub_u32_e32 v5, vcc, v2, v3 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 +; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc +; GCN-NEXT: v_add_u32_e32 v5, vcc, 1, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 +; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc ; GCN-NEXT: v_and_b32_e32 v2, 0xffffff, v2 ; GCN-NEXT: flat_store_dword v[0:1], v2 ; GCN-NEXT: s_endpgm @@ -2007,23 +2037,39 @@ define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1030-NEXT: global_load_ubyte v3, v0, s[2:3] offset:2 ; GFX1030-NEXT: global_load_ushort v4, v0, s[2:3] ; GFX1030-NEXT: s_waitcnt vmcnt(3) -; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s2, v1 ; GFX1030-NEXT: s_waitcnt vmcnt(2) -; GFX1030-NEXT: v_or_b32_e32 v1, v2, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s3, v2 ; GFX1030-NEXT: s_waitcnt vmcnt(1) -; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s4, v3 ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_or_b32_e32 v2, v4, v2 -; GFX1030-NEXT: v_rcp_f32_e32 v3, v1 -; GFX1030-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1030-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX1030-NEXT: v_trunc_f32_e32 v3, v3 -; GFX1030-NEXT: v_fma_f32 v2, -v3, v1, v2 -; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v1 -; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo -; GFX1030-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s5, v4 +; GFX1030-NEXT: s_lshl_b32 s2, s2, 16 +; GFX1030-NEXT: s_or_b32 s2, s3, s2 +; GFX1030-NEXT: s_lshl_b32 s4, s4, 16 +; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, s2 +; GFX1030-NEXT: s_sub_i32 s6, 0, s2 +; GFX1030-NEXT: s_or_b32 s4, s5, s4 +; GFX1030-NEXT: v_rcp_f32_e32 v1, v1 +; GFX1030-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s3, v1 +; GFX1030-NEXT: s_mul_i32 s6, s6, s3 +; GFX1030-NEXT: s_mul_hi_u32 s6, s3, s6 +; GFX1030-NEXT: s_add_i32 s3, s3, s6 +; GFX1030-NEXT: s_mul_hi_u32 s3, s4, s3 +; GFX1030-NEXT: s_mul_i32 s5, s3, s2 +; GFX1030-NEXT: s_sub_i32 s4, s4, s5 +; GFX1030-NEXT: s_add_i32 s5, s3, 1 +; GFX1030-NEXT: s_sub_i32 s6, s4, s2 +; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 +; GFX1030-NEXT: s_cselect_b32 s3, s5, s3 +; GFX1030-NEXT: s_cselect_b32 s4, s6, s4 +; GFX1030-NEXT: s_add_i32 s5, s3, 1 +; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 +; GFX1030-NEXT: s_cselect_b32 s2, s5, s3 +; GFX1030-NEXT: s_and_b32 s2, s2, 0xffffff +; GFX1030-NEXT: v_mov_b32_e32 v1, s2 ; GFX1030-NEXT: global_store_dword v0, v1, s[0:1] ; GFX1030-NEXT: s_endpgm ; diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll index edd84a5f09..8653a43a00 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll @@ -396,51 +396,77 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) { define amdgpu_kernel void @s_test_udiv24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_udiv24_64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dword s6, s[4:5], 0xe -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_lshr_b32 s8, s0, 8 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-NEXT: s_sub_i32 s0, 0, s8 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s2, s6, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 ; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-NEXT: s_mul_i32 s0, s0, s8 +; GCN-NEXT: s_sub_i32 s0, s2, s0 +; GCN-NEXT: s_sub_i32 s1, s0, s8 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_udiv24_64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s6, -1 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_lshr_b32 s8, s0, 8 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s2, s6, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 ; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 +; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr i64 %x, 40 @@ -455,36 +481,54 @@ define i64 @v_test_udiv24_i64(i64 %x, i64 %y) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v3 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ; GCN-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-NEXT: v_mul_hi_u32 v2, v2, v1 +; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv24_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v3 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v2 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %1 = lshr i64 %x, 40 %2 = lshr i64 %y, 40 @@ -714,57 +758,83 @@ define amdgpu_kernel void @s_test_udiv23_i64(ptr addrspace(1) %out, i64 %x, i64 define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48 %y) { ; GCN-LABEL: s_test_udiv24_i48: ; GCN: ; %bb.0: +; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_and_b32 s1, s1, 0xffff +; GCN-NEXT: s_and_b32 s0, s0, 0xff000000 +; GCN-NEXT: s_lshr_b64 s[6:7], s[0:1], 24 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s6 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd -; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_sub_i32 s4, 0, s6 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_and_b32 s3, s3, 0xffff -; GCN-NEXT: s_and_b32 s5, s5, 0xffff -; GCN-NEXT: s_and_b32 s4, s4, 0xff000000 -; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], 24 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 ; GCN-NEXT: s_and_b32 s2, s2, 0xff000000 -; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-NEXT: s_lshr_b64 s[4:5], s[2:3], 24 ; GCN-NEXT: s_mov_b32 s3, 0xf000 ; GCN-NEXT: s_mov_b32 s2, -1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; GCN-NEXT: buffer_store_short v3, off, s[0:3], 0 offset:4 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s4, v0 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-NEXT: s_mul_i32 s5, s5, s6 +; GCN-NEXT: s_sub_i32 s4, s4, s5 +; GCN-NEXT: s_sub_i32 s5, s4, s6 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s4, s6 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-NEXT: s_cselect_b32 s4, s5, s4 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s4, s6 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-NEXT: buffer_store_short v1, off, s[0:3], 0 offset:4 ; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_udiv24_i48: ; GCN-IR: ; %bb.0: +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_and_b32 s1, s1, 0xffff +; GCN-IR-NEXT: s_and_b32 s0, s0, 0xff000000 +; GCN-IR-NEXT: s_lshr_b64 s[6:7], s[0:1], 24 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s6 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd -; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 +; GCN-IR-NEXT: s_sub_i32 s4, 0, s6 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_and_b32 s3, s3, 0xffff -; GCN-IR-NEXT: s_and_b32 s5, s5, 0xffff -; GCN-IR-NEXT: s_and_b32 s4, s4, 0xff000000 -; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[4:5], 24 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 ; GCN-IR-NEXT: s_and_b32 s2, s2, 0xff000000 -; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[2:3], 24 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s2, -1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v4, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; GCN-IR-NEXT: buffer_store_short v3, off, s[0:3], 0 offset:4 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s4, v0 +; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-IR-NEXT: s_mul_i32 s5, s5, s6 +; GCN-IR-NEXT: s_sub_i32 s4, s4, s5 +; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-IR-NEXT: s_cselect_b32 s4, s5, s4 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-IR-NEXT: buffer_store_short v1, off, s[0:3], 0 offset:4 ; GCN-IR-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr i48 %x, 24 @@ -1440,42 +1510,66 @@ define amdgpu_kernel void @s_test_udiv24_k_num_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_udiv24_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_mov_b32 s4, 0x41c00000 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GCN-NEXT: s_lshr_b32 s4, s3, 8 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-NEXT: s_sub_i32 s2, 0, s4 ; GCN-NEXT: s_mov_b32 s3, 0xf000 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-NEXT: s_mov_b32 s2, -1 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-NEXT: s_mul_i32 s5, s5, s4 +; GCN-NEXT: s_sub_i32 s5, 24, s5 +; GCN-NEXT: s_sub_i32 s6, s5, s4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_udiv24_k_num_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GCN-IR-NEXT: s_lshr_b32 s4, s3, 8 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ; GCN-IR-NEXT: s_mov_b32 s2, -1 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 +; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 +; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = lshr i64 %x, 40 @@ -1488,44 +1582,56 @@ define amdgpu_kernel void @s_test_udiv24_k_den_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_udiv24_k_den_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 ; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_lshr_b32 s0, s3, 8 +; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s2 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mulk_i32 s1, 0x5b7f +; GCN-NEXT: s_sub_i32 s0, s0, s1 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: s_min_u32 s0, s1, s0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_udiv24_k_den_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 ; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_lshr_b32 s0, s3, 8 +; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s2 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mulk_i32 s1, 0x5b7f +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: s_min_u32 s0, s1, s0 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = lshr i64 %x, 40 @@ -1539,34 +1645,52 @@ define i64 @v_test_udiv24_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-NEXT: v_mul_u32_u24_e32 v2, v1, v0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 24, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv24_k_num_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v1, v0 +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 24, v2 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = udiv i64 24, %x.shr @@ -1578,34 +1702,52 @@ define i64 @v_test_udiv24_pow2_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 0x47000000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-NEXT: v_mul_u32_u24_e32 v2, v1, v0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0x8000, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv24_pow2_k_num_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x47000000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v1, v0 +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0x8000, v2 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = udiv i64 32768, %x.shr @@ -1621,19 +1763,78 @@ define i64 @v_test_udiv24_pow2_k_den_i64(i64 %x) { ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv24_pow2_k_den_i64: -; GCN-IR: ; %bb.0: +; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38000000, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v0, -v1, s4, v0 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4 -; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 8, v1 +; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4 +; GCN-IR-NEXT: v_add_i32_e64 v0, s[4:5], 32, v0 +; GCN-IR-NEXT: s_flbit_i32_b32 s4, 0 +; GCN-IR-NEXT: v_min_u32_e32 v6, s4, v0 +; GCN-IR-NEXT: s_mov_b32 s6, 0 +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v6 +; GCN-IR-NEXT: v_mov_b32_e32 v5, s6 +; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] +; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] +; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] +; GCN-IR-NEXT: v_mov_b32_e32 v1, s6 +; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] +; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], vcc +; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GCN-IR-NEXT: s_cbranch_execz .LBB17_6 +; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 +; GCN-IR-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 63, v2 +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[4:5], v0 +; GCN-IR-NEXT: v_mov_b32_e32 v2, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 +; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execz .LBB17_5 +; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader +; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 0xffffffcf, v6 +; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7 +; GCN-IR-NEXT: v_addc_u32_e64 v9, s[8:9], 0, -1, vcc +; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 +; GCN-IR-NEXT: v_mov_b32_e32 v6, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 +; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff +; GCN-IR-NEXT: .LBB17_3: ; %udiv-do-while +; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 31, v1 +; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s10, v4 +; GCN-IR-NEXT: v_subb_u32_e32 v2, vcc, 0, v5, vcc +; GCN-IR-NEXT: v_or_b32_e32 v0, v6, v0 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v6, 31, v2 +; GCN-IR-NEXT: v_and_b32_e32 v2, 1, v6 +; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6 +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6 +; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc +; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 1, v8 +; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1 +; GCN-IR-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc +; GCN-IR-NEXT: v_mov_b32_e32 v7, v3 +; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] +; GCN-IR-NEXT: v_mov_b32_e32 v6, v2 +; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execnz .LBB17_3 +; GCN-IR-NEXT: ; %bb.4: ; %Flow +; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: .LBB17_5: ; %Flow4 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 +; GCN-IR-NEXT: v_or_b32_e32 v1, v3, v1 +; GCN-IR-NEXT: v_or_b32_e32 v0, v2, v0 +; GCN-IR-NEXT: .LBB17_6: ; %Flow5 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = udiv i64 %x.shr, 32768 diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll index 4d9ece813c..3fecc3882b 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll @@ -620,16 +620,28 @@ define amdgpu_kernel void @udiv24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0xffffff ; SI-NEXT: s_and_b32 s5, s5, 0xffffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -639,20 +651,32 @@ define amdgpu_kernel void @udiv24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0xffffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0xffffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -714,16 +738,28 @@ define amdgpu_kernel void @no_udiv24_u23_u24_i32(ptr addrspace(1) %out, ptr addr ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff ; SI-NEXT: s_and_b32 s5, s5, 0xffffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -733,20 +769,32 @@ define amdgpu_kernel void @no_udiv24_u23_u24_i32(ptr addrspace(1) %out, ptr addr ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0xffffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0x7fffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -808,16 +856,28 @@ define amdgpu_kernel void @no_udiv24_u24_u23_i32(ptr addrspace(1) %out, ptr addr ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0xffffff ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -827,20 +887,32 @@ define amdgpu_kernel void @no_udiv24_u24_u23_i32(ptr addrspace(1) %out, ptr addr ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0x7fffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0xffffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -1470,22 +1542,29 @@ define amdgpu_kernel void @urem24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 ; SI-NEXT: s_mov_b32 s3, 0xf000 -; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s6, s4, 0xffffff -; SI-NEXT: s_and_b32 s7, s5, 0xffffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s6 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s7 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_mul_lo_u32 v0, v0, s5 -; SI-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 -; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; SI-NEXT: s_and_b32 s2, s4, 0xffffff +; SI-NEXT: s_and_b32 s4, s5, 0xffffff +; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; SI-NEXT: s_sub_i32 s5, 0, s4 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s5, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s2, v0 +; SI-NEXT: v_readfirstlane_b32 s5, v0 +; SI-NEXT: s_mul_i32 s5, s5, s4 +; SI-NEXT: s_sub_i32 s2, s2, s5 +; SI-NEXT: s_sub_i32 s5, s2, s4 +; SI-NEXT: s_cmp_ge_u32 s2, s4 +; SI-NEXT: s_cselect_b32 s2, s5, s2 +; SI-NEXT: s_sub_i32 s5, s2, s4 +; SI-NEXT: s_cmp_ge_u32 s2, s4 +; SI-NEXT: s_cselect_b32 s4, s5, s2 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s4 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -1493,24 +1572,31 @@ define amdgpu_kernel void @urem24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 -; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s2, s5, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s2 -; VI-NEXT: s_and_b32 s2, s4, 0xffffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +; VI-NEXT: s_and_b32 s4, s3, 0xffffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0xffffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_rcp_f32_e32 v2, v0 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_mul_lo_u32 v0, v0, s5 -; VI-NEXT: v_sub_u32_e32 v0, vcc, s4, v0 -; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b32 s4, s6, s5 +; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll index 3bb489c654..7840b861ab 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll @@ -632,52 +632,72 @@ define amdgpu_kernel void @s_test_urem31_v2i64(ptr addrspace(1) %out, <2 x i64> define amdgpu_kernel void @s_test_urem24_i64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_urem24_i64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dword s6, s[4:5], 0xe +; GCN-NEXT: s_load_dword s0, s[4:5], 0xe +; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_lshr_b32 s8, s0, 8 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-NEXT: s_sub_i32 s0, 0, s8 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s2, -1 -; GCN-NEXT: s_lshr_b32 s4, s6, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GCN-NEXT: s_lshr_b32 s5, s3, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s5 -; GCN-NEXT: s_mov_b32 s3, 0xf000 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: s_lshr_b32 s2, s3, 8 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 +; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s5, v0 -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; GCN-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-NEXT: s_mul_i32 s0, s0, s8 +; GCN-NEXT: s_sub_i32 s0, s2, s0 +; GCN-NEXT: s_sub_i32 s1, s0, s8 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-NEXT: s_sub_i32 s1, s0, s8 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem24_i64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe +; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe +; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s6, -1 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_lshr_b32 s8, s0, 8 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_mov_b32 s2, -1 -; GCN-IR-NEXT: s_lshr_b32 s4, s6, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GCN-IR-NEXT: s_lshr_b32 s5, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s5 -; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 +; GCN-IR-NEXT: s_mov_b32 s5, s1 ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s5, v0 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 +; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 +; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr i64 %x, 40 %2 = lshr i64 %y, 40 @@ -1385,46 +1405,60 @@ define amdgpu_kernel void @s_test_urem24_k_num_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_urem24_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_mov_b32 s5, 0x41c00000 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_lshr_b32 s4, s3, 8 ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-NEXT: s_sub_i32 s2, 0, s4 ; GCN-NEXT: s_mov_b32 s3, 0xf000 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v1, -v1, v0, s5 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-NEXT: s_mov_b32 s2, -1 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GCN-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-NEXT: s_mul_i32 s5, s5, s4 +; GCN-NEXT: s_sub_i32 s5, 24, s5 +; GCN-NEXT: s_sub_i32 s6, s5, s4 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-NEXT: s_sub_i32 s6, s5, s4 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b32 s4, s6, s5 +; GCN-NEXT: v_mov_b32_e32 v0, s4 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem24_k_num_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_mov_b32 s5, 0x41c00000 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_lshr_b32 s4, s3, 8 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s5 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-IR-NEXT: s_mov_b32 s2, -1 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 +; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 +; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b32 s4, s6, s5 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = lshr i64 %x, 40 @@ -1437,50 +1471,46 @@ define amdgpu_kernel void @s_test_urem24_k_den_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_urem24_k_den_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_movk_i32 s4, 0x5b7f +; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GCN-NEXT: s_mov_b32 s3, 0x46b6fe00 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-NEXT: v_mad_f32 v0, -v1, s3, v0 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s3 ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GCN-NEXT: s_lshr_b32 s0, s3, 8 +; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mulk_i32 s1, 0x5b7f +; GCN-NEXT: s_sub_i32 s0, s0, s1 +; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-NEXT: s_min_u32 s0, s1, s0 +; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-NEXT: s_min_u32 s0, s1, s0 +; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem24_k_den_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_movk_i32 s4, 0x5b7f +; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GCN-IR-NEXT: s_mov_b32 s3, 0x46b6fe00 -; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mad_f32 v0, -v1, s3, v0 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s3 ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GCN-IR-NEXT: s_lshr_b32 s0, s3, 8 +; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-IR-NEXT: s_mov_b32 s5, s1 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mulk_i32 s1, 0x5b7f +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 +; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-IR-NEXT: s_min_u32 s0, s1, s0 +; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 +; GCN-IR-NEXT: s_min_u32 s0, s1, s0 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = lshr i64 %x, 40 @@ -1495,18 +1525,23 @@ define i64 @v_test_urem24_k_num_i64(i64 %x) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-NEXT: v_rcp_f32_e32 v2, v1 -; GCN-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v2, -v2, v1, s4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_urem24_k_num_i64: @@ -1514,18 +1549,23 @@ define i64 @v_test_urem24_k_num_i64(i64 %x) { ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v2, -v2, v1, s4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = urem i64 24, %x.shr @@ -1538,18 +1578,23 @@ define i64 @v_test_urem24_pow2_k_num_i64(i64 %x) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-NEXT: v_rcp_f32_e32 v2, v1 -; GCN-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v2, -v2, v1, s4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 -; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_urem24_pow2_k_num_i64: @@ -1557,18 +1602,23 @@ define i64 @v_test_urem24_pow2_k_num_i64(i64 %x) { ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v2, -v2, v1, s4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = urem i64 32768, %x.shr @@ -1584,21 +1634,81 @@ define i64 @v_test_urem24_pow2_k_den_i64(i64 %x) { ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_urem24_pow2_k_den_i64: -; GCN-IR: ; %bb.0: +; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x38000000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, s4, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, s4 -; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_lshlrev_b32_e32 v1, 15, v1 +; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 +; GCN-IR-NEXT: v_add_i32_e64 v2, s[4:5], 32, v2 +; GCN-IR-NEXT: s_flbit_i32_b32 s4, 0 +; GCN-IR-NEXT: v_min_u32_e32 v8, s4, v2 +; GCN-IR-NEXT: s_mov_b32 s6, 0 +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v8 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s6 +; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] +; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] +; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] +; GCN-IR-NEXT: v_mov_b32_e32 v5, s6 +; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] +; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], vcc +; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GCN-IR-NEXT: s_cbranch_execz .LBB14_6 +; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 +; GCN-IR-NEXT: v_add_i32_e32 v6, vcc, 1, v2 +; GCN-IR-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2 +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[0:1], v2 +; GCN-IR-NEXT: v_mov_b32_e32 v4, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 +; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execz .LBB14_5 +; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader +; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 0xffffffcf, v8 +; GCN-IR-NEXT: v_addc_u32_e64 v10, s[8:9], 0, -1, vcc +; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 +; GCN-IR-NEXT: v_mov_b32_e32 v8, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v9, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 +; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff +; GCN-IR-NEXT: .LBB14_3: ; %udiv-do-while +; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-IR-NEXT: v_lshl_b64 v[6:7], v[6:7], 1 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3 +; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4 +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, s10, v6 +; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4 +; GCN-IR-NEXT: v_and_b32_e32 v4, 1, v8 +; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8 +; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8 +; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1 +; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3 +; GCN-IR-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc +; GCN-IR-NEXT: v_mov_b32_e32 v9, v5 +; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] +; GCN-IR-NEXT: v_mov_b32_e32 v8, v4 +; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execnz .LBB14_3 +; GCN-IR-NEXT: ; %bb.4: ; %Flow +; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: .LBB14_5: ; %Flow4 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-IR-NEXT: v_lshl_b64 v[1:2], v[2:3], 1 +; GCN-IR-NEXT: v_or_b32_e32 v5, v5, v2 +; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v1 +; GCN-IR-NEXT: .LBB14_6: ; %Flow5 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] +; GCN-IR-NEXT: v_lshl_b64 v[1:2], v[4:5], 15 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 -; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v2, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = lshr i64 %x, 40 %result = urem i64 %x.shr, 32768 From e64e00413f7ed618eef181795a598a40c39f7256 Mon Sep 17 00:00:00 2001 From: bogdan-petkovic Date: Thu, 24 Sep 2026 03:38:23 -0500 Subject: [PATCH 2/3] [EXTERNAL] Avoid errors with 23-bit division and remainder. --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 80 +- .../AMDGPU/amdgpu-codegenprepare-idiv.ll | 190 +++- .../llvm/test/CodeGen/AMDGPU/sdiv.ll | 135 ++- .../llvm/test/CodeGen/AMDGPU/sdiv64.ll | 814 ++++++++++++------ .../llvm/test/CodeGen/AMDGPU/sdivrem24.ll | 18 +- .../llvm/test/CodeGen/AMDGPU/srem64.ll | 705 ++++++++------- .../llvm/test/CodeGen/AMDGPU/udiv.ll | 208 +++-- .../llvm/test/CodeGen/AMDGPU/udiv64.ll | 82 +- .../llvm/test/CodeGen/AMDGPU/udivrem24.ll | 210 +++-- .../llvm/test/CodeGen/AMDGPU/urem64.ll | 128 +-- 10 files changed, 1620 insertions(+), 950 deletions(-) diff --git a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index c3fb24bc83..62c7610445 100644 --- a/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -181,14 +181,15 @@ class AMDGPUCodeGenPrepareImpl unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den, unsigned MaxDivBits, bool Signed) const; - /// Expands 24 bit div or rem. - Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I, - Value *Num, Value *Den, - bool IsDiv, bool IsSigned) const; + /// Expands div or rem by using floating-point operations. + /// Operands must be in the range [-0x400000,0x3FFFFF] + Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I, + Value *Num, Value *Den, bool IsDiv, + bool IsSigned) const; - Value *expandDivRem24Impl(IRBuilder<> &Builder, BinaryOperator &I, - Value *Num, Value *Den, unsigned NumBits, - bool IsDiv, bool IsSigned) const; + Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I, + Value *Num, Value *Den, unsigned NumBits, + bool IsDiv, bool IsSigned) const; /// Expands 32 bit div or rem. Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I, @@ -1060,32 +1061,39 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num, return DivBits; } -// The fractional part of a float is enough to accurately represent up to -// a 24-bit signed integer. -Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder, - BinaryOperator &I, Value *Num, - Value *Den, bool IsDiv, - bool IsSigned) const { - unsigned DivBits = getDivNumBits(I, Num, Den, 24, IsSigned); +Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder, + BinaryOperator &I, + Value *Num, Value *Den, + bool IsDiv, + bool IsSigned) const { + unsigned DivBits = getDivNumBits(I, Num, Den, 23, IsSigned); - // v_rcp_f32(float(X)) can have an error of 1 ulp. - // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly - // when abs(Y)>0x800000. - // For example, - // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. - // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. - // - // Note that for DivBits==24 && IsSigned, Y is in the range - // [-0x800000:0x7FFFFF]. abs(Y) is at most - // 0x800000 so it cannot hit this issue. - if (DivBits > (IsSigned ? 24 : 23)) + if (DivBits > (IsSigned ? 23 : 22)) return nullptr; - return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned); + return expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits, IsDiv, + IsSigned); } -Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl( +Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl( IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den, unsigned DivBits, bool IsDiv, bool IsSigned) const { + + // v_rcp_f32(float(X)) can have an error of 1 ulp. + // This would cause incorrect calculation of Y/X if: + // Y = (0x7FFFFF/X)*(X-0)-1 + // were allowed. + // + // For example, + // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059. + // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257. + // + // Thus, we conservatively restrict expandDivRemToFloatImpl to + // [-0x40000,0x3FFFFF] for IsSigned + // [0x000000,0x3FFFFF] for !IsSigned. + assert(DivBits <= (IsSigned ? 23 : 22) && + "abs(Num) must be <= than 0x40000 for expandDivRemToFloatImpl to work " + "correctly"); + Type *I32Ty = Builder.getInt32Ty(); Num = Builder.CreateTrunc(Num, I32Ty); Den = Builder.CreateTrunc(Den, I32Ty); @@ -1260,7 +1268,7 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder, } } - if (Value *Res = expandDivRem24(Builder, I, X, Y, IsDiv, IsSigned)) { + if (Value *Res = expandDivRemToFloat(Builder, I, X, Y, IsDiv, IsSigned)) { return IsSigned ? Builder.CreateSExtOrTrunc(Res, Ty) : Builder.CreateZExtOrTrunc(Res, Ty); } @@ -1369,19 +1377,9 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder, return nullptr; Value *Narrowed = nullptr; - // v_rcp_f32(float(X)) can have an error of 1 ulp. - // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly - // when abs(Y)>0x800000. - // For example, - // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. - // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. - // - // Note that for NumDivBits==24 && IsSigned, Y is in the range - // [-0x800000:0x7FFFFF]. abs(Y) is at most - // 0x800000 so it cannot hit this issue. - if (NumDivBits <= (IsSigned ? 24 : 23)) { - Narrowed = expandDivRem24Impl(Builder, I, Num, Den, NumDivBits, - IsDiv, IsSigned); + if (NumDivBits <= (IsSigned ? 23 : 22)) { + Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, NumDivBits, IsDiv, + IsSigned); } else if (NumDivBits <= 32) { Narrowed = expandDivRem32(Builder, I, Num, Den); } diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll index a23295b617..df10ca9a88 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -10135,7 +10135,7 @@ entry: ret <2 x i64> %B } -; 23-bit sdiv, can use expandDivRem24Impl +; 23-bit sdiv, can use expandDivRemToFloatImpl define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ; GFX6-LABEL: sdiv23: ; GFX6: ; %bb.0: @@ -10196,31 +10196,44 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ret void } -; 24-bit sdiv, can use expandDivRem24Impl +; 24-bit sdiv, cannot use expandDivRemToFloatImpl define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ; GFX6-LABEL: sdiv24: ; GFX6: ; %bb.0: ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] -; GFX6-NEXT: s_bfe_i32 s5, s5, 0x180000 -; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 -; GFX6-NEXT: s_bfe_i32 s4, s4, 0x180000 -; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 -; GFX6-NEXT: s_xor_b32 s4, s4, s5 -; GFX6-NEXT: v_rcp_f32_e32 v2, v0 -; GFX6-NEXT: s_ashr_i32 s4, s4, 30 -; GFX6-NEXT: s_or_b32 s6, s4, 1 -; GFX6-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 -; GFX6-NEXT: v_trunc_f32_e32 v2, v2 -; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| -; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec -; GFX6-NEXT: s_cselect_b32 s4, s6, 0 +; GFX6-NEXT: s_bfe_i32 s4, s3, 0x180000 +; GFX6-NEXT: s_abs_i32 s5, s4 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5 +; GFX6-NEXT: s_sub_i32 s3, 0, s5 +; GFX6-NEXT: s_bfe_i32 s6, s2, 0x180000 +; GFX6-NEXT: s_abs_i32 s7, s6 +; GFX6-NEXT: v_rcp_f32_e32 v0, v0 +; GFX6-NEXT: s_xor_b32 s4, s6, s4 +; GFX6-NEXT: s_ashr_i32 s4, s4, 31 ; GFX6-NEXT: s_mov_b32 s2, -1 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 +; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0 +; GFX6-NEXT: v_readfirstlane_b32 s6, v0 +; GFX6-NEXT: s_mul_i32 s6, s6, s5 +; GFX6-NEXT: s_sub_i32 s6, s7, s6 +; GFX6-NEXT: s_sub_i32 s7, s6, s5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s6, s5 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: s_cselect_b32 s6, s7, s6 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s6, s5 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0 ; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 @@ -10233,21 +10246,34 @@ define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_bfe_i32 s3, s3, 0x180000 -; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 +; GFX9-NEXT: s_abs_i32 s4, s3 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX9-NEXT: s_sub_i32 s5, 0, s4 ; GFX9-NEXT: s_bfe_i32 s2, s2, 0x180000 -; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 +; GFX9-NEXT: s_xor_b32 s3, s2, s3 +; GFX9-NEXT: v_rcp_f32_e32 v0, v0 +; GFX9-NEXT: s_abs_i32 s2, s2 +; GFX9-NEXT: s_ashr_i32 s3, s3, 31 +; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX9-NEXT: v_readfirstlane_b32 s6, v0 +; GFX9-NEXT: s_mul_i32 s5, s5, s6 +; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5 +; GFX9-NEXT: s_add_i32 s6, s6, s5 +; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6 +; GFX9-NEXT: s_mul_i32 s6, s5, s4 +; GFX9-NEXT: s_sub_i32 s2, s2, s6 +; GFX9-NEXT: s_add_i32 s7, s5, 1 +; GFX9-NEXT: s_sub_i32 s6, s2, s4 +; GFX9-NEXT: s_cmp_ge_u32 s2, s4 +; GFX9-NEXT: s_cselect_b32 s5, s7, s5 +; GFX9-NEXT: s_cselect_b32 s2, s6, s2 +; GFX9-NEXT: s_add_i32 s6, s5, 1 +; GFX9-NEXT: s_cmp_ge_u32 s2, s4 +; GFX9-NEXT: s_cselect_b32 s2, s6, s5 ; GFX9-NEXT: s_xor_b32 s2, s2, s3 -; GFX9-NEXT: v_rcp_f32_e32 v3, v0 -; GFX9-NEXT: s_ashr_i32 s2, s2, 30 -; GFX9-NEXT: s_or_b32 s4, s2, 1 -; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 -; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| -; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec -; GFX9-NEXT: s_cselect_b32 s2, s4, 0 -; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 +; GFX9-NEXT: s_sub_i32 s2, s2, s3 +; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 ; GFX9-NEXT: global_store_short v1, v0, s[0:1] ; GFX9-NEXT: s_endpgm @@ -10256,15 +10282,15 @@ define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ret void } -; 23-bit udiv, can use expandDivRem24Impl -define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { -; GFX6-LABEL: udiv23: +; 22-bit udiv, can use expandDivRemToFloatImpl +define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) { +; GFX6-LABEL: udiv22: ; GFX6: ; %bb.0: ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_and_b32 s3, s3, 0x7fffff +; GFX6-NEXT: s_and_b32 s3, s3, 0x3fffff ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX6-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX6-NEXT: s_and_b32 s2, s2, 0x3fffff ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: v_rcp_f32_e32 v2, v0 @@ -10277,18 +10303,18 @@ define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 +; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 6 ; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ; GFX6-NEXT: s_endpgm ; -; GFX9-LABEL: udiv23: +; GFX9-LABEL: udiv22: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v3, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff +; GFX9-NEXT: s_and_b32 s3, s3, 0x3fffff ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX9-NEXT: s_and_b32 s2, s2, 0x3fffff ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2 ; GFX9-NEXT: v_rcp_f32_e32 v2, v0 ; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2 @@ -10298,15 +10324,91 @@ define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc ; GFX9-NEXT: global_store_short v3, v0, s[0:1] -; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 0x3fffff, v0 ; GFX9-NEXT: global_store_byte_d16_hi v3, v0, s[0:1] offset:2 +; GFX9-NEXT: s_endpgm + %r = udiv i22 %x, %y + store i22 %r, ptr addrspace(1) %out + ret void +} + +; 23-bit udiv, cannot use expandDivRemToFloatImpl +define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { +; GFX6-LABEL: udiv23: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_and_b32 s4, s3, 0x7fffff +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GFX6-NEXT: s_sub_i32 s3, 0, s4 +; GFX6-NEXT: s_and_b32 s5, s2, 0x7fffff +; GFX6-NEXT: s_mov_b32 s2, -1 +; GFX6-NEXT: v_rcp_f32_e32 v0, v0 +; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 +; GFX6-NEXT: v_readfirstlane_b32 s6, v0 +; GFX6-NEXT: s_mul_i32 s6, s6, s4 +; GFX6-NEXT: s_sub_i32 s5, s5, s6 +; GFX6-NEXT: s_sub_i32 s6, s5, s4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: s_cselect_b32 s5, s6, s5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GFX6-NEXT: s_cmp_ge_u32 s5, s4 +; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 +; GFX6-NEXT: s_waitcnt expcnt(0) +; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 +; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 +; GFX6-NEXT: s_endpgm +; +; GFX9-LABEL: udiv23: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX9-NEXT: s_sub_i32 s4, 0, s3 +; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX9-NEXT: v_rcp_f32_e32 v0, v0 +; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX9-NEXT: v_readfirstlane_b32 s5, v0 +; GFX9-NEXT: s_mul_i32 s4, s4, s5 +; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4 +; GFX9-NEXT: s_add_i32 s5, s5, s4 +; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5 +; GFX9-NEXT: s_mul_i32 s5, s4, s3 +; GFX9-NEXT: s_sub_i32 s2, s2, s5 +; GFX9-NEXT: s_add_i32 s6, s4, 1 +; GFX9-NEXT: s_sub_i32 s5, s2, s3 +; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_cselect_b32 s4, s6, s4 +; GFX9-NEXT: s_cselect_b32 s2, s5, s2 +; GFX9-NEXT: s_add_i32 s5, s4, 1 +; GFX9-NEXT: s_cmp_ge_u32 s2, s3 +; GFX9-NEXT: s_cselect_b32 s2, s5, s4 +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX9-NEXT: global_store_short v1, v0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 ; GFX9-NEXT: s_endpgm %r = udiv i23 %x, %y store i23 %r, ptr addrspace(1) %out ret void } -; 24-bit udiv, cannot use expandDivRem24Impl +; 24-bit udiv, cannot use expandDivRemToFloatImpl define amdgpu_kernel void @udiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ; GFX6-LABEL: udiv24: ; GFX6: ; %bb.0: diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv.ll index b690879dab..b082c59f84 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv.ll @@ -1801,23 +1801,37 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GCN-NEXT: s_waitcnt vmcnt(2) ; GCN-NEXT: v_or_b32_e32 v1, v1, v4 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 +; GCN-NEXT: v_max_i32_e32 v1, v1, v4 +; GCN-NEXT: v_cvt_f32_u32_e32 v4, v1 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v1 ; GCN-NEXT: s_waitcnt vmcnt(1) -; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GCN-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GCN-NEXT: v_rcp_f32_e32 v4, v4 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_or_b32_e32 v3, v3, v4 -; GCN-NEXT: v_cvt_f32_i32_e32 v3, v3 -; GCN-NEXT: v_rcp_f32_e32 v4, v1 +; GCN-NEXT: v_or_b32_e32 v3, v3, v6 +; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v3 +; GCN-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 +; GCN-NEXT: v_max_i32_e32 v3, v3, v6 ; GCN-NEXT: v_xor_b32_e32 v0, v2, v0 -; GCN-NEXT: v_ashrrev_i32_e32 v0, 30, v0 -; GCN-NEXT: v_or_b32_e32 v0, 1, v0 -; GCN-NEXT: v_mul_f32_e32 v2, v3, v4 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v3, -v2, v1, v3 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v1| -; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 +; GCN-NEXT: v_mul_lo_u32 v5, v5, v4 +; GCN-NEXT: v_mul_hi_u32 v5, v4, v5 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; GCN-NEXT: v_mul_hi_u32 v4, v3, v4 +; GCN-NEXT: v_mul_lo_u32 v2, v4, v1 +; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v4 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v3, v2 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v1 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc +; GCN-NEXT: v_xor_b32_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 ; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GCN-NEXT: s_endpgm @@ -1842,23 +1856,37 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; TONGA-NEXT: s_waitcnt vmcnt(2) ; TONGA-NEXT: v_or_b32_e32 v1, v1, v4 -; TONGA-NEXT: v_cvt_f32_i32_e32 v1, v1 +; TONGA-NEXT: v_sub_u32_e32 v4, vcc, 0, v1 +; TONGA-NEXT: v_max_i32_e32 v1, v1, v4 +; TONGA-NEXT: v_cvt_f32_u32_e32 v4, v1 +; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v1 ; TONGA-NEXT: s_waitcnt vmcnt(1) -; TONGA-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; TONGA-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; TONGA-NEXT: v_rcp_f32_e32 v4, v4 ; TONGA-NEXT: s_waitcnt vmcnt(0) -; TONGA-NEXT: v_or_b32_e32 v3, v3, v4 -; TONGA-NEXT: v_cvt_f32_i32_e32 v3, v3 -; TONGA-NEXT: v_rcp_f32_e32 v4, v1 +; TONGA-NEXT: v_or_b32_e32 v3, v3, v6 +; TONGA-NEXT: v_sub_u32_e32 v6, vcc, 0, v3 +; TONGA-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 +; TONGA-NEXT: v_cvt_u32_f32_e32 v4, v4 +; TONGA-NEXT: v_max_i32_e32 v3, v3, v6 ; TONGA-NEXT: v_xor_b32_e32 v0, v2, v0 -; TONGA-NEXT: v_ashrrev_i32_e32 v0, 30, v0 -; TONGA-NEXT: v_or_b32_e32 v0, 1, v0 -; TONGA-NEXT: v_mul_f32_e32 v2, v3, v4 -; TONGA-NEXT: v_trunc_f32_e32 v2, v2 -; TONGA-NEXT: v_mad_f32 v3, -v2, v1, v3 -; TONGA-NEXT: v_cvt_i32_f32_e32 v2, v2 -; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v1| -; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc -; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v2 +; TONGA-NEXT: v_ashrrev_i32_e32 v0, 31, v0 +; TONGA-NEXT: v_mul_lo_u32 v5, v5, v4 +; TONGA-NEXT: v_mul_hi_u32 v5, v4, v5 +; TONGA-NEXT: v_add_u32_e32 v4, vcc, v4, v5 +; TONGA-NEXT: v_mul_hi_u32 v4, v3, v4 +; TONGA-NEXT: v_mul_lo_u32 v2, v4, v1 +; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v4 +; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v3, v2 +; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v2, v1 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; TONGA-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc +; TONGA-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; TONGA-NEXT: v_add_u32_e32 v3, vcc, 1, v4 +; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 +; TONGA-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc +; TONGA-NEXT: v_xor_b32_e32 v1, v1, v0 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v1, v0 ; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 24 ; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; TONGA-NEXT: s_endpgm @@ -1878,28 +1906,45 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: buffer_load_sbyte v2, off, s[4:7], 0 offset:2 ; GFX9-NEXT: buffer_load_ushort v3, off, s[4:7], 0 ; GFX9-NEXT: s_mov_b32 s0, s8 -; GFX9-NEXT: s_mov_b32 s1, s9 ; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX9-NEXT: v_readfirstlane_b32 s1, v0 ; GFX9-NEXT: s_waitcnt vmcnt(2) -; GFX9-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX9-NEXT: v_readfirstlane_b32 s4, v1 +; GFX9-NEXT: s_lshl_b32 s1, s1, 16 +; GFX9-NEXT: s_or_b32 s1, s4, s1 +; GFX9-NEXT: s_abs_i32 s4, s1 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_readfirstlane_b32 s5, v2 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_or_b32_e32 v3, v3, v4 -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3 -; GFX9-NEXT: v_rcp_f32_e32 v4, v1 +; GFX9-NEXT: v_readfirstlane_b32 s6, v3 +; GFX9-NEXT: s_lshl_b32 s5, s5, 16 +; GFX9-NEXT: v_rcp_f32_e32 v1, v1 +; GFX9-NEXT: s_or_b32 s5, s6, s5 +; GFX9-NEXT: s_sub_i32 s6, 0, s4 +; GFX9-NEXT: s_abs_i32 s5, s5 +; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 -; GFX9-NEXT: v_ashrrev_i32_e32 v0, 30, v0 -; GFX9-NEXT: v_or_b32_e32 v0, 1, v0 -; GFX9-NEXT: v_mul_f32_e32 v2, v3, v4 -; GFX9-NEXT: v_trunc_f32_e32 v2, v2 -; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v2 -; GFX9-NEXT: v_mad_f32 v2, -v2, v1, v3 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v1| -; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc -; GFX9-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v0 +; GFX9-NEXT: s_mov_b32 s1, s9 +; GFX9-NEXT: v_readfirstlane_b32 s7, v1 +; GFX9-NEXT: s_mul_i32 s6, s6, s7 +; GFX9-NEXT: s_mul_hi_u32 s6, s7, s6 +; GFX9-NEXT: s_add_i32 s7, s7, s6 +; GFX9-NEXT: s_mul_hi_u32 s6, s5, s7 +; GFX9-NEXT: s_mul_i32 s7, s6, s4 +; GFX9-NEXT: s_sub_i32 s5, s5, s7 +; GFX9-NEXT: s_add_i32 s8, s6, 1 +; GFX9-NEXT: s_sub_i32 s7, s5, s4 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 +; GFX9-NEXT: s_cselect_b32 s6, s8, s6 +; GFX9-NEXT: s_cselect_b32 s5, s7, s5 +; GFX9-NEXT: s_add_i32 s7, s6, 1 +; GFX9-NEXT: s_cmp_ge_u32 s5, s4 +; GFX9-NEXT: s_cselect_b32 s4, s7, s6 +; GFX9-NEXT: v_xor_b32_e32 v1, s4, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, v1, v0 ; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX9-NEXT: s_endpgm diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll index 68466abf31..a39d3fb10a 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -480,63 +480,89 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) { define amdgpu_kernel void @s_test_sdiv24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_sdiv24_64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_load_dword s2, s[4:5], 0xe +; GCN-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i32 s0, s2, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 +; GCN-NEXT: s_ashr_i32 s8, s0, 8 +; GCN-NEXT: s_abs_i32 s9, s8 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_sub_i32 s2, 0, s9 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_ashr_i32 s0, s3, 8 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_ashr_i32 s1, s3, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, s1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: s_xor_b32 s0, s1, s0 -; GCN-NEXT: s_ashr_i32 s0, s0, 30 -; GCN-NEXT: s_or_b32 s2, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s2, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-NEXT: s_abs_i32 s2, s0 +; GCN-NEXT: s_xor_b32 s0, s0, s8 +; GCN-NEXT: s_ashr_i32 s0, s0, 31 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s3, s1, s9 +; GCN-NEXT: s_sub_i32 s2, s2, s3 +; GCN-NEXT: s_add_i32 s8, s1, 1 +; GCN-NEXT: s_sub_i32 s3, s2, s9 +; GCN-NEXT: s_cmp_ge_u32 s2, s9 +; GCN-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-NEXT: s_add_i32 s3, s1, 1 +; GCN-NEXT: s_cmp_ge_u32 s2, s9 +; GCN-NEXT: s_cselect_b32 s1, s3, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: v_mov_b32_e32 v1, s1 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_sdiv24_64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe +; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i32 s0, s2, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 +; GCN-IR-NEXT: s_ashr_i32 s8, s0, 8 +; GCN-IR-NEXT: s_abs_i32 s9, s8 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_sub_i32 s2, 0, s9 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_ashr_i32 s1, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: s_xor_b32 s0, s1, s0 -; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 -; GCN-IR-NEXT: s_or_b32 s2, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-IR-NEXT: s_abs_i32 s2, s0 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s8 +; GCN-IR-NEXT: s_ashr_i32 s0, s0, 31 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s3, s1, s9 +; GCN-IR-NEXT: s_sub_i32 s2, s2, s3 +; GCN-IR-NEXT: s_add_i32 s8, s1, 1 +; GCN-IR-NEXT: s_sub_i32 s3, s2, s9 +; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 +; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-IR-NEXT: s_add_i32 s3, s1, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 +; GCN-IR-NEXT: s_cselect_b32 s1, s3, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = ashr i64 %x, 40 @@ -958,92 +984,146 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(ptr addrspace(1) %out, <2 x i64> ; GCN-LABEL: s_test_sdiv24_v2i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd -; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 -; GCN-NEXT: s_mov_b32 s3, 0xf000 -; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i32 s4, s13, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s4 -; GCN-NEXT: s_ashr_i32 s5, s9, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, s5 -; GCN-NEXT: s_xor_b32 s4, s5, s4 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: s_ashr_i32 s4, s4, 30 +; GCN-NEXT: s_ashr_i32 s0, s13, 8 +; GCN-NEXT: s_abs_i32 s1, s0 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s1 +; GCN-NEXT: s_sub_i32 s2, 0, s1 ; GCN-NEXT: s_ashr_i32 s6, s11, 8 ; GCN-NEXT: s_ashr_i32 s7, s15, 8 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: s_or_b32 s8, s4, 1 -; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| -; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec -; GCN-NEXT: s_cselect_b32 s4, s8, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v2 -; GCN-NEXT: v_cvt_f32_i32_e32 v2, s7 -; GCN-NEXT: v_cvt_f32_i32_e32 v3, s6 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-NEXT: s_ashr_i32 s2, s9, 8 +; GCN-NEXT: s_abs_i32 s3, s2 +; GCN-NEXT: s_xor_b32 s0, s2, s0 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: s_ashr_i32 s8, s0, 31 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-NEXT: s_mul_i32 s2, s0, s1 +; GCN-NEXT: s_sub_i32 s2, s3, s2 +; GCN-NEXT: s_add_i32 s9, s0, 1 +; GCN-NEXT: s_sub_i32 s3, s2, s1 +; GCN-NEXT: s_cmp_ge_u32 s2, s1 +; GCN-NEXT: s_cselect_b32 s0, s9, s0 +; GCN-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-NEXT: s_add_i32 s3, s0, 1 +; GCN-NEXT: s_cmp_ge_u32 s2, s1 +; GCN-NEXT: s_cselect_b32 s9, s3, s0 +; GCN-NEXT: s_abs_i32 s10, s7 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s10 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +; GCN-NEXT: s_sub_i32 s4, 0, s10 +; GCN-NEXT: s_abs_i32 s5, s6 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: s_mov_b32 s3, 0xf000 +; GCN-NEXT: s_mov_b32 s2, -1 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 ; GCN-NEXT: s_xor_b32 s4, s6, s7 -; GCN-NEXT: s_ashr_i32 s4, s4, 30 -; GCN-NEXT: v_rcp_f32_e32 v4, v2 -; GCN-NEXT: s_or_b32 s6, s4, 1 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2| -; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec -; GCN-NEXT: s_cselect_b32 s4, s6, 0 -; GCN-NEXT: v_add_i32_e32 v2, vcc, s4, v4 -; GCN-NEXT: v_bfe_i32 v2, v2, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v2 +; GCN-NEXT: s_xor_b32 s6, s9, s8 +; GCN-NEXT: s_sub_i32 s6, s6, s8 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: s_ashr_i32 s7, s6, 31 +; GCN-NEXT: s_ashr_i32 s4, s4, 31 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v2, s5, v0 +; GCN-NEXT: v_mov_b32_e32 v0, s6 +; GCN-NEXT: v_mov_b32_e32 v1, s7 +; GCN-NEXT: v_readfirstlane_b32 s6, v2 +; GCN-NEXT: s_mul_i32 s7, s6, s10 +; GCN-NEXT: s_sub_i32 s5, s5, s7 +; GCN-NEXT: s_add_i32 s8, s6, 1 +; GCN-NEXT: s_sub_i32 s7, s5, s10 +; GCN-NEXT: s_cmp_ge_u32 s5, s10 +; GCN-NEXT: s_cselect_b32 s6, s8, s6 +; GCN-NEXT: s_cselect_b32 s5, s7, s5 +; GCN-NEXT: s_add_i32 s7, s6, 1 +; GCN-NEXT: s_cmp_ge_u32 s5, s10 +; GCN-NEXT: s_cselect_b32 s5, s7, s6 +; GCN-NEXT: s_xor_b32 s5, s5, s4 +; GCN-NEXT: s_sub_i32 s4, s5, s4 +; GCN-NEXT: s_ashr_i32 s5, s4, 31 +; GCN-NEXT: v_mov_b32_e32 v2, s4 +; GCN-NEXT: v_mov_b32_e32 v3, s5 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_sdiv24_v2i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd -; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 -; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 -; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i32 s4, s13, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s4 -; GCN-IR-NEXT: s_ashr_i32 s5, s9, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s5 -; GCN-IR-NEXT: s_xor_b32 s4, s5, s4 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 +; GCN-IR-NEXT: s_ashr_i32 s0, s13, 8 +; GCN-IR-NEXT: s_abs_i32 s1, s0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s1 +; GCN-IR-NEXT: s_sub_i32 s2, 0, s1 ; GCN-IR-NEXT: s_ashr_i32 s6, s11, 8 ; GCN-IR-NEXT: s_ashr_i32 s7, s15, 8 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: s_or_b32 s8, s4, 1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| -; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], exec -; GCN-IR-NEXT: s_cselect_b32 s4, s8, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s4, v2 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, s7 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v3, s6 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-IR-NEXT: s_ashr_i32 s2, s9, 8 +; GCN-IR-NEXT: s_abs_i32 s3, s2 +; GCN-IR-NEXT: s_xor_b32 s0, s2, s0 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: s_ashr_i32 s8, s0, 31 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-IR-NEXT: s_mul_i32 s2, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s3, s2 +; GCN-IR-NEXT: s_add_i32 s9, s0, 1 +; GCN-IR-NEXT: s_sub_i32 s3, s2, s1 +; GCN-IR-NEXT: s_cmp_ge_u32 s2, s1 +; GCN-IR-NEXT: s_cselect_b32 s0, s9, s0 +; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-IR-NEXT: s_add_i32 s3, s0, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s2, s1 +; GCN-IR-NEXT: s_cselect_b32 s9, s3, s0 +; GCN-IR-NEXT: s_abs_i32 s10, s7 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s10 +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +; GCN-IR-NEXT: s_sub_i32 s4, 0, s10 +; GCN-IR-NEXT: s_abs_i32 s5, s6 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s2, -1 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 ; GCN-IR-NEXT: s_xor_b32 s4, s6, s7 -; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 -; GCN-IR-NEXT: v_rcp_f32_e32 v4, v2 -; GCN-IR-NEXT: s_or_b32 s6, s4, 1 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-IR-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2| -; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], exec -; GCN-IR-NEXT: s_cselect_b32 s4, s6, 0 -; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, s4, v4 -; GCN-IR-NEXT: v_bfe_i32 v2, v2, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v3, 31, v2 +; GCN-IR-NEXT: s_xor_b32 s6, s9, s8 +; GCN-IR-NEXT: s_sub_i32 s6, s6, s8 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: s_ashr_i32 s7, s6, 31 +; GCN-IR-NEXT: s_ashr_i32 s4, s4, 31 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, s5, v0 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s6 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s7 +; GCN-IR-NEXT: v_readfirstlane_b32 s6, v2 +; GCN-IR-NEXT: s_mul_i32 s7, s6, s10 +; GCN-IR-NEXT: s_sub_i32 s5, s5, s7 +; GCN-IR-NEXT: s_add_i32 s8, s6, 1 +; GCN-IR-NEXT: s_sub_i32 s7, s5, s10 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s10 +; GCN-IR-NEXT: s_cselect_b32 s6, s8, s6 +; GCN-IR-NEXT: s_cselect_b32 s5, s7, s5 +; GCN-IR-NEXT: s_add_i32 s7, s6, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s10 +; GCN-IR-NEXT: s_cselect_b32 s5, s7, s6 +; GCN-IR-NEXT: s_xor_b32 s5, s5, s4 +; GCN-IR-NEXT: s_sub_i32 s4, s5, s4 +; GCN-IR-NEXT: s_ashr_i32 s5, s4, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v2, s4 +; GCN-IR-NEXT: v_mov_b32_e32 v3, s5 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %1 = ashr <2 x i64> %x, @@ -1056,32 +1136,46 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(ptr addrspace(1) %out, <2 x i64> define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %y) { ; GCN-LABEL: s_test_sdiv24_48: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd +; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: s_sext_i32_i16 s9, s9 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_lshr_b64 s[0:1], s[8:9], 24 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 +; GCN-NEXT: s_sext_i32_i16 s1, s1 +; GCN-NEXT: s_lshr_b64 s[8:9], s[0:1], 24 +; GCN-NEXT: s_abs_i32 s9, s8 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_sub_i32 s4, 0, s9 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sext_i32_i16 s3, s3 ; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: s_xor_b32 s0, s2, s0 -; GCN-NEXT: s_ashr_i32 s0, s0, 30 -; GCN-NEXT: s_or_b32 s2, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s2, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_xor_b32 s1, s2, s8 +; GCN-NEXT: s_ashr_i32 s1, s1, 31 +; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_abs_i32 s0, s2 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mul_i32 s2, s2, s9 +; GCN-NEXT: s_sub_i32 s0, s0, s2 +; GCN-NEXT: s_sub_i32 s2, s0, s9 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: s_cselect_b32 s0, s2, s0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: v_xor_b32_e32 v0, s1, v0 +; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GCN-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 @@ -1089,32 +1183,46 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 % ; ; GCN-IR-LABEL: s_test_sdiv24_48: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: s_sext_i32_i16 s9, s9 -; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_lshr_b64 s[0:1], s[8:9], 24 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 +; GCN-IR-NEXT: s_sext_i32_i16 s1, s1 +; GCN-IR-NEXT: s_lshr_b64 s[8:9], s[0:1], 24 +; GCN-IR-NEXT: s_abs_i32 s9, s8 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_sub_i32 s4, 0, s9 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sext_i32_i16 s3, s3 ; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: s_xor_b32 s0, s2, s0 -; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 -; GCN-IR-NEXT: s_or_b32 s2, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: s_mov_b32 s5, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s2, s8 +; GCN-IR-NEXT: s_ashr_i32 s1, s1, 31 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_abs_i32 s0, s2 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mul_i32 s2, s2, s9 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s2 +; GCN-IR-NEXT: s_sub_i32 s2, s0, s9 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: s_cselect_b32 s0, s2, s0 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v0, s1, v0 +; GCN-IR-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GCN-IR-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 @@ -1830,23 +1938,35 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(ptr addrspace(1) %out, i64 %x ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-NEXT: s_mov_b32 s2, 0x41c00000 +; GCN-NEXT: s_abs_i32 s2, s2 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GCN-NEXT: s_sub_i32 s4, 0, s2 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 ; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_ashr_i32 s0, s3, 31 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_or_b32 s3, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_mad_f32 v2, -v1, v0, s2 -; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v1 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s3, s1, s2 +; GCN-NEXT: s_sub_i32 s3, 24, s3 +; GCN-NEXT: s_add_i32 s8, s1, 1 +; GCN-NEXT: s_sub_i32 s9, s3, s2 +; GCN-NEXT: s_cmp_ge_u32 s3, s2 +; GCN-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-NEXT: s_cselect_b32 s3, s9, s3 +; GCN-NEXT: s_add_i32 s8, s1, 1 +; GCN-NEXT: s_cmp_ge_u32 s3, s2 +; GCN-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: v_mov_b32_e32 v1, s1 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -1857,23 +1977,35 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(ptr addrspace(1) %out, i64 %x ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-IR-NEXT: s_mov_b32 s2, 0x41c00000 +; GCN-IR-NEXT: s_abs_i32 s2, s2 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GCN-IR-NEXT: s_sub_i32 s4, 0, s2 +; GCN-IR-NEXT: s_mov_b32 s5, s1 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 ; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_or_b32 s3, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_mad_f32 v2, -v1, v0, s2 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v1 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s3, s1, s2 +; GCN-IR-NEXT: s_sub_i32 s3, 24, s3 +; GCN-IR-NEXT: s_add_i32 s8, s1, 1 +; GCN-IR-NEXT: s_sub_i32 s9, s3, s2 +; GCN-IR-NEXT: s_cmp_ge_u32 s3, s2 +; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-IR-NEXT: s_cselect_b32 s3, s9, s3 +; GCN-IR-NEXT: s_add_i32 s8, s1, 1 +; GCN-IR-NEXT: s_cmp_ge_u32 s3, s2 +; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = ashr i64 %x, 40 @@ -1886,52 +2018,64 @@ define amdgpu_kernel void @s_test_sdiv24_k_den_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_sdiv24_k_den_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 +; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: s_ashr_i32 s0, s3, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 -; GCN-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-NEXT: s_abs_i32 s0, s0 +; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_or_b32 s3, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v1 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-NEXT: s_ashr_i32 s1, s3, 31 +; GCN-NEXT: s_mul_i32 s3, s2, 0x5b7f +; GCN-NEXT: s_sub_i32 s0, s0, s3 +; GCN-NEXT: s_add_i32 s3, s2, 1 +; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-NEXT: s_add_i32 s3, s0, 0xffffa481 +; GCN-NEXT: s_min_u32 s0, s3, s0 +; GCN-NEXT: s_add_i32 s3, s2, 1 +; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-NEXT: s_xor_b32 s0, s0, s1 +; GCN-NEXT: s_sub_i32 s0, s0, s1 +; GCN-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-NEXT: v_mov_b32_e32 v0, s0 +; GCN-NEXT: v_mov_b32_e32 v1, s1 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_sdiv24_k_den_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 +; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 -; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-IR-NEXT: s_abs_i32 s0, s0 +; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_or_b32 s3, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v1 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: s_ashr_i32 s1, s3, 31 +; GCN-IR-NEXT: s_mul_i32 s3, s2, 0x5b7f +; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 +; GCN-IR-NEXT: s_add_i32 s3, s2, 1 +; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 +; GCN-IR-NEXT: s_add_i32 s3, s0, 0xffffa481 +; GCN-IR-NEXT: s_min_u32 s0, s3, s0 +; GCN-IR-NEXT: s_add_i32 s3, s2, 1 +; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e +; GCN-IR-NEXT: s_cselect_b32 s0, s3, s2 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 +; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = ashr i64 %x, 40 @@ -1945,19 +2089,30 @@ define i64 @v_test_sdiv24_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x41c00000 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v3, -v2, v0, s4 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| -; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -1965,19 +2120,30 @@ define i64 @v_test_sdiv24_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v3, -v2, v0, s4 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| -; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 @@ -1990,19 +2156,31 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x47000000 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 +; GCN-NEXT: s_mov_b32 s4, 0x8000 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v3, -v2, v0, s4 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| -; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 +; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -2010,19 +2188,31 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 +; GCN-IR-NEXT: s_mov_b32 s4, 0x8000 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v3, -v2, v0, s4 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| -; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 17, v2 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 +; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 @@ -2043,22 +2233,86 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) { ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_sdiv24_pow2_k_den_i64: -; GCN-IR: ; %bb.0: +; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v1 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x38000000, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v0, -v2, s4, v0 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4 -; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v8 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v8, v8 +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v0, v8 +; GCN-IR-NEXT: v_subb_u32_e32 v5, vcc, v1, v8, vcc +; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4 +; GCN-IR-NEXT: v_add_i32_e64 v0, s[4:5], 32, v0 +; GCN-IR-NEXT: v_ffbh_u32_e32 v1, v5 +; GCN-IR-NEXT: v_min_u32_e32 v6, v0, v1 +; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 48, v6 +; GCN-IR-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5] +; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] +; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[0:1] +; GCN-IR-NEXT: v_mov_b32_e32 v9, v8 +; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[0:1] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1 +; GCN-IR-NEXT: v_cndmask_b32_e64 v3, v5, 0, s[4:5] +; GCN-IR-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] +; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc +; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GCN-IR-NEXT: s_cbranch_execz .LBB18_6 +; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 +; GCN-IR-NEXT: v_add_i32_e32 v7, vcc, 1, v0 +; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 63, v0 +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[4:5], v0 +; GCN-IR-NEXT: v_mov_b32_e32 v2, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 +; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execz .LBB18_5 +; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader +; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6 +; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7 +; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc +; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 +; GCN-IR-NEXT: v_mov_b32_e32 v6, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 +; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff +; GCN-IR-NEXT: .LBB18_3: ; %udiv-do-while +; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 31, v1 +; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s10, v4 +; GCN-IR-NEXT: v_subb_u32_e32 v2, vcc, 0, v5, vcc +; GCN-IR-NEXT: v_or_b32_e32 v0, v6, v0 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v6, 31, v2 +; GCN-IR-NEXT: v_and_b32_e32 v2, 1, v6 +; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6 +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6 +; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc +; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 1, v10 +; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1 +; GCN-IR-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc +; GCN-IR-NEXT: v_mov_b32_e32 v7, v3 +; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] +; GCN-IR-NEXT: v_mov_b32_e32 v6, v2 +; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execnz .LBB18_3 +; GCN-IR-NEXT: ; %bb.4: ; %Flow +; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: .LBB18_5: ; %Flow4 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 +; GCN-IR-NEXT: v_or_b32_e32 v3, v3, v1 +; GCN-IR-NEXT: v_or_b32_e32 v2, v2, v0 +; GCN-IR-NEXT: .LBB18_6: ; %Flow5 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] +; GCN-IR-NEXT: v_xor_b32_e32 v0, v2, v8 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v9 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v8 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 %result = sdiv i64 %x.shr, 32768 diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdivrem24.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdivrem24.ll index abfb56a582..4750f17b1a 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdivrem24.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/sdivrem24.ll @@ -41,10 +41,7 @@ define amdgpu_kernel void @sdiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i } ; FUNC-LABEL: {{^}}sdiv24_i32: -; SI: v_cvt_f32_i32 -; SI: v_cvt_f32_i32 -; SI: v_rcp_f32 -; SI: v_cvt_i32_f32 +; SI-NOT: v_cvt_i32_f32 ; EG: INT_TO_FLT ; EG-DAG: INT_TO_FLT @@ -156,10 +153,7 @@ define amdgpu_kernel void @srem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i } ; FUNC-LABEL: {{^}}srem24_i32: -; SI: v_cvt_f32_i32 -; SI: v_cvt_f32_i32 -; SI: v_rcp_f32 -; SI: v_cvt_i32_f32 +; SI-NOT: v_cvt_i32_f32 ; EG: INT_TO_FLT ; EG-DAG: INT_TO_FLT @@ -269,9 +263,7 @@ define amdgpu_kernel void @no_srem25_i25_i24_i32(ptr addrspace(1) %out, ptr addr } ; FUNC-LABEL: {{^}}srem25_i24_i11_i32: -; SI: v_cvt_f32_i32 -; SI: v_rcp_f32 -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24 +; SI-NOT: v_cvt_f32_i32 ; EG: INT_TO_FLT ; EG: RECIP_IEEE @@ -289,9 +281,7 @@ define amdgpu_kernel void @srem25_i24_i11_i32(ptr addrspace(1) %out, ptr addrspa } ; FUNC-LABEL: {{^}}srem25_i11_i24_i32: -; SI: v_cvt_f32_i32 -; SI: v_rcp_f32 -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24 +; SI-NOT: v_cvt_f32_i32 ; EG: INT_TO_FLT ; EG: RECIP_IEEE diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll index 82196b73b6..f3fdfaadc8 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/srem64.ll @@ -546,34 +546,39 @@ define amdgpu_kernel void @s_test_srem23_64(ptr addrspace(1) %out, i64 %x, i64 % define amdgpu_kernel void @s_test_srem24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_srem24_64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_load_dword s2, s[4:5], 0xe +; GCN-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: s_ashr_i32 s3, s3, 8 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i32 s2, s2, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, s3 -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: s_xor_b32 s0, s3, s2 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: s_ashr_i32 s0, s0, 30 +; GCN-NEXT: s_ashr_i32 s0, s0, 8 +; GCN-NEXT: s_abs_i32 s8, s0 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_sub_i32 s2, 0, s8 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_or_b32 s8, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s8, 0 -; GCN-NEXT: v_readfirstlane_b32 s1, v2 -; GCN-NEXT: s_add_i32 s0, s1, s0 -; GCN-NEXT: s_mul_i32 s0, s0, s2 -; GCN-NEXT: s_sub_i32 s0, s3, s0 -; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-NEXT: s_ashr_i32 s2, s3, 8 +; GCN-NEXT: s_abs_i32 s2, s2 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s1, s1, s8 +; GCN-NEXT: s_sub_i32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -582,34 +587,39 @@ define amdgpu_kernel void @s_test_srem24_64(ptr addrspace(1) %out, i64 %x, i64 % ; ; GCN-IR-LABEL: s_test_srem24_64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe +; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: s_ashr_i32 s3, s3, 8 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i32 s2, s2, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s3 -; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: s_xor_b32 s0, s3, s2 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 +; GCN-IR-NEXT: s_ashr_i32 s0, s0, 8 +; GCN-IR-NEXT: s_abs_i32 s8, s0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_sub_i32 s2, 0, s8 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: s_or_b32 s8, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s8, 0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v2 -; GCN-IR-NEXT: s_add_i32 s0, s1, s0 -; GCN-IR-NEXT: s_mul_i32 s0, s0, s2 -; GCN-IR-NEXT: s_sub_i32 s0, s3, s0 -; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 +; GCN-IR-NEXT: s_abs_i32 s2, s2 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 +; GCN-IR-NEXT: s_sub_i32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -627,23 +637,31 @@ define i64 @v_test_srem24_64(i64 %x, i64 %y) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v3 -; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 8, v1 -; GCN-NEXT: v_cvt_f32_i32_e32 v3, v1 -; GCN-NEXT: v_xor_b32_e32 v5, v1, v0 -; GCN-NEXT: v_rcp_f32_e32 v4, v2 -; GCN-NEXT: v_ashrrev_i32_e32 v5, 30, v5 -; GCN-NEXT: v_or_b32_e32 v5, 1, v5 -; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| -; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v0, v2, v0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 +; GCN-NEXT: v_ashrrev_i32_e32 v4, 8, v1 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v4 +; GCN-NEXT: v_max_i32_e32 v4, v5, v4 +; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-NEXT: v_mul_hi_u32 v2, v4, v2 +; GCN-NEXT: v_mul_u32_u24_e32 v2, v2, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -651,23 +669,31 @@ define i64 @v_test_srem24_64(i64 %x, i64 %y) { ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v3 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v3, v1 -; GCN-IR-NEXT: v_xor_b32_e32 v5, v1, v0 -; GCN-IR-NEXT: v_rcp_f32_e32 v4, v2 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v5, 30, v5 -; GCN-IR-NEXT: v_or_b32_e32 v5, 1, v5 -; GCN-IR-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-IR-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| -; GCN-IR-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc -; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v2, v0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v4, 8, v1 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v5, vcc, 0, v4 +; GCN-IR-NEXT: v_max_i32_e32 v4, v5, v4 +; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v4, v2 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v4, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %1 = ashr i64 %x, 40 @@ -1205,72 +1231,92 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %y) { ; GCN-LABEL: s_test_srem24_48: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_sext_i32_i16 s9, s9 -; GCN-NEXT: s_lshr_b64 s[4:5], s[8:9], 24 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s4 +; GCN-NEXT: s_sext_i32_i16 s1, s1 +; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], 24 +; GCN-NEXT: s_abs_i32 s8, s0 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_sub_i32 s4, 0, s8 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_sext_i32_i16 s3, s3 ; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-NEXT: s_xor_b32 s3, s2, s4 -; GCN-NEXT: s_ashr_i32 s3, s3, 30 -; GCN-NEXT: s_or_b32 s3, s3, 1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-NEXT: v_cmp_ge_f32_e64 s[8:9], |v1|, |v0| -; GCN-NEXT: s_and_b64 s[8:9], s[8:9], exec -; GCN-NEXT: s_cselect_b32 s3, s3, 0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, s3, v2 -; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: s_abs_i32 s3, s2 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_ashr_i32 s0, s2, 31 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-NEXT: s_mul_i32 s1, s1, s8 +; GCN-NEXT: s_sub_i32 s1, s3, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s8 +; GCN-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_xor_b32 s1, s1, s0 +; GCN-NEXT: s_sub_i32 s0, s1, s0 +; GCN-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 -; GCN-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 +; GCN-NEXT: s_waitcnt expcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s1 +; GCN-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_srem24_48: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_sext_i32_i16 s9, s9 -; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[8:9], 24 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s4 +; GCN-IR-NEXT: s_sext_i32_i16 s1, s1 +; GCN-IR-NEXT: s_lshr_b64 s[0:1], s[0:1], 24 +; GCN-IR-NEXT: s_abs_i32 s8, s0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_sub_i32 s4, 0, s8 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_sext_i32_i16 s3, s3 ; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 -; GCN-IR-NEXT: s_xor_b32 s3, s2, s4 -; GCN-IR-NEXT: s_ashr_i32 s3, s3, 30 -; GCN-IR-NEXT: s_or_b32 s3, s3, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[8:9], |v1|, |v0| -; GCN-IR-NEXT: s_and_b64 s[8:9], s[8:9], exec -; GCN-IR-NEXT: s_cselect_b32 s3, s3, 0 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s3, v2 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 -; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: s_abs_i32 s3, s2 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 +; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: s_ashr_i32 s0, s2, 31 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 +; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 +; GCN-IR-NEXT: s_sub_i32 s1, s3, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 +; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 +; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: buffer_store_dword v0, off, s[4:7], 0 -; GCN-IR-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 +; GCN-IR-NEXT: s_waitcnt expcnt(0) +; GCN-IR-NEXT: v_mov_b32_e32 v0, s1 +; GCN-IR-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 ; GCN-IR-NEXT: s_endpgm %1 = ashr i48 %x, 24 %2 = ashr i48 %y, 24 @@ -1982,65 +2028,65 @@ define amdgpu_kernel void @s_test_srem24_k_num_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_srem24_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_mov_b32 s8, 0x41c00000 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i32 s2, s3, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: s_ashr_i32 s0, s3, 31 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-NEXT: s_or_b32 s3, s0, 1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_mad_f32 v2, -v1, v0, s8 -; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-NEXT: v_readfirstlane_b32 s1, v1 -; GCN-NEXT: s_add_i32 s0, s1, s0 -; GCN-NEXT: s_mul_i32 s0, s0, s2 -; GCN-NEXT: s_sub_i32 s0, 24, s0 -; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 -; GCN-NEXT: s_ashr_i32 s1, s0, 31 -; GCN-NEXT: v_mov_b32_e32 v0, s0 -; GCN-NEXT: v_mov_b32_e32 v1, s1 -; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GCN-NEXT: s_abs_i32 s4, s2 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-NEXT: s_sub_i32 s2, 0, s4 +; GCN-NEXT: s_mov_b32 s3, 0xf000 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-NEXT: s_mov_b32 s2, -1 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-NEXT: s_mul_i32 s5, s5, s4 +; GCN-NEXT: s_sub_i32 s5, 24, s5 +; GCN-NEXT: s_sub_i32 s6, s5, s4 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-NEXT: s_sub_i32 s6, s5, s4 +; GCN-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-NEXT: s_cselect_b32 s4, s6, s5 +; GCN-NEXT: s_ashr_i32 s5, s4, 31 +; GCN-NEXT: v_mov_b32_e32 v0, s4 +; GCN-NEXT: v_mov_b32_e32 v1, s5 +; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_srem24_k_num_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_mov_b32 s8, 0x41c00000 -; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 -; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 -; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 -; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 -; GCN-IR-NEXT: s_or_b32 s3, s0, 1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_mad_f32 v2, -v1, v0, s8 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v1 -; GCN-IR-NEXT: s_add_i32 s0, s1, s0 -; GCN-IR-NEXT: s_mul_i32 s0, s0, s2 -; GCN-IR-NEXT: s_sub_i32 s0, 24, s0 -; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 -; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 -; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 -; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 -; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GCN-IR-NEXT: s_abs_i32 s4, s2 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 +; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 +; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 +; GCN-IR-NEXT: s_mov_b32 s2, -1 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 +; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 +; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 +; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 +; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 +; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 +; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 +; GCN-IR-NEXT: s_cselect_b32 s4, s6, s5 +; GCN-IR-NEXT: s_ashr_i32 s5, s4, 31 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 +; GCN-IR-NEXT: v_mov_b32_e32 v1, s5 +; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %x.shr = ashr i64 %x, 40 %result = srem i64 24, %x.shr @@ -2052,28 +2098,25 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(ptr addrspace(1) %out, i64 %x ; GCN-LABEL: s_test_srem24_k_den_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 +; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: s_ashr_i32 s8, s3, 8 -; GCN-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-NEXT: s_ashr_i32 s0, s3, 8 +; GCN-NEXT: s_abs_i32 s0, s0 +; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mulk_i32 s2, 0x5b7f +; GCN-NEXT: s_sub_i32 s0, s0, s2 +; GCN-NEXT: s_add_i32 s2, s0, 0xffffa481 +; GCN-NEXT: s_min_u32 s0, s2, s0 +; GCN-NEXT: s_add_i32 s2, s0, 0xffffa481 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-NEXT: s_or_b32 s3, s0, 1 -; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-NEXT: v_readfirstlane_b32 s1, v1 -; GCN-NEXT: s_add_i32 s0, s1, s0 -; GCN-NEXT: s_mulk_i32 s0, 0x5b7f -; GCN-NEXT: s_sub_i32 s0, s8, s0 -; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 +; GCN-NEXT: s_ashr_i32 s1, s3, 31 +; GCN-NEXT: s_min_u32 s0, s2, s0 +; GCN-NEXT: s_xor_b32 s0, s0, s1 +; GCN-NEXT: s_sub_i32 s0, s0, s1 ; GCN-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 @@ -2083,28 +2126,25 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(ptr addrspace(1) %out, i64 %x ; GCN-IR-LABEL: s_test_srem24_k_den_i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 +; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: s_ashr_i32 s8, s3, 8 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: s_mov_b32 s4, s0 -; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 +; GCN-IR-NEXT: s_abs_i32 s0, s0 +; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mulk_i32 s2, 0x5b7f +; GCN-IR-NEXT: s_sub_i32 s0, s0, s2 +; GCN-IR-NEXT: s_add_i32 s2, s0, 0xffffa481 +; GCN-IR-NEXT: s_min_u32 s0, s2, s0 +; GCN-IR-NEXT: s_add_i32 s2, s0, 0xffffa481 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 -; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 -; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GCN-IR-NEXT: s_or_b32 s3, s0, 1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 -; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 -; GCN-IR-NEXT: v_readfirstlane_b32 s1, v1 -; GCN-IR-NEXT: s_add_i32 s0, s1, s0 -; GCN-IR-NEXT: s_mulk_i32 s0, 0x5b7f -; GCN-IR-NEXT: s_sub_i32 s0, s8, s0 -; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 +; GCN-IR-NEXT: s_ashr_i32 s1, s3, 31 +; GCN-IR-NEXT: s_min_u32 s0, s2, s0 +; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 +; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 @@ -2121,21 +2161,25 @@ define i64 @v_test_srem24_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-NEXT: v_rcp_f32_e32 v3, v2 -; GCN-NEXT: v_mul_f32_e32 v3, 0x41c00000, v3 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mad_f32 v4, -v3, v2, s4 -; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v0, v1, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -2143,21 +2187,25 @@ define i64 @v_test_srem24_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_rcp_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x41c00000, v3 -; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-IR-NEXT: v_mad_f32 v4, -v3, v2, s4 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| -; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 +; GCN-IR-NEXT: v_max_i32_e32 v0, v1, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 @@ -2170,21 +2218,25 @@ define i64 @v_test_srem24_pow2_k_num_i64(i64 %x) { ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-NEXT: v_rcp_f32_e32 v3, v2 -; GCN-NEXT: v_mul_f32_e32 v3, 0x47000000, v3 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mad_f32 v4, -v3, v2, s4 -; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 +; GCN-NEXT: v_max_i32_e32 v0, v1, v0 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -2192,21 +2244,25 @@ define i64 @v_test_srem24_pow2_k_num_i64(i64 %x) { ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_rcp_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x47000000, v3 -; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-IR-NEXT: v_mad_f32 v4, -v3, v2, s4 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| -; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 +; GCN-IR-NEXT: v_max_i32_e32 v0, v1, v0 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 +; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 +; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 +; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 @@ -2229,24 +2285,89 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) { ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_srem24_pow2_k_den_i64: -; GCN-IR: ; %bb.0: +; GCN-IR: ; %bb.0: ; %_udiv-special-cases ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-IR-NEXT: v_ashrrev_i32_e32 v10, 31, v1 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 -; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 -; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x38000000, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-IR-NEXT: v_mad_f32 v2, -v3, s4, v2 -; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, s4 -; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-IR-NEXT: v_lshlrev_b32_e32 v1, 15, v1 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v10, v10 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc +; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 +; GCN-IR-NEXT: v_add_i32_e64 v2, s[4:5], 32, v2 +; GCN-IR-NEXT: v_ffbh_u32_e32 v3, v1 +; GCN-IR-NEXT: v_min_u32_e32 v8, v2, v3 +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v8 +; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] +; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] +; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] +; GCN-IR-NEXT: v_mov_b32_e32 v11, v10 +; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] +; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1 +; GCN-IR-NEXT: v_cndmask_b32_e64 v5, v1, 0, s[4:5] +; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5] +; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc +; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; GCN-IR-NEXT: s_cbranch_execz .LBB18_6 +; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 +; GCN-IR-NEXT: v_add_i32_e32 v6, vcc, 1, v2 +; GCN-IR-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2 +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[0:1], v2 +; GCN-IR-NEXT: v_mov_b32_e32 v4, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 +; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 +; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] +; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execz .LBB18_5 +; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader +; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8 +; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6 +; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc +; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 +; GCN-IR-NEXT: v_mov_b32_e32 v8, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v9, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 +; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff +; GCN-IR-NEXT: .LBB18_3: ; %udiv-do-while +; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-IR-NEXT: v_lshl_b64 v[6:7], v[6:7], 1 +; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3 +; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4 +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, s10, v6 +; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4 +; GCN-IR-NEXT: v_and_b32_e32 v4, 1, v8 +; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8 +; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8 +; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v12 +; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3 +; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc +; GCN-IR-NEXT: v_mov_b32_e32 v9, v5 +; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] +; GCN-IR-NEXT: v_mov_b32_e32 v8, v4 +; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: s_cbranch_execnz .LBB18_3 +; GCN-IR-NEXT: ; %bb.4: ; %Flow +; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] +; GCN-IR-NEXT: .LBB18_5: ; %Flow4 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 +; GCN-IR-NEXT: v_or_b32_e32 v5, v5, v3 +; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 +; GCN-IR-NEXT: .LBB18_6: ; %Flow5 +; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] +; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[4:5], 15 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10 +; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v11 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %x.shr = ashr i64 %x, 40 %result = srem i64 %x.shr, 32768 diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll index b3a64b81ec..dbcf04f3cf 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv.ll @@ -1676,76 +1676,96 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %in) { ; SI-LABEL: v_udiv_i23: ; SI: ; %bb.0: -; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 -; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 -; SI-NEXT: s_mov_b32 s10, s6 -; SI-NEXT: s_mov_b32 s11, s7 +; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: s_mov_b32 s10, s2 +; SI-NEXT: s_mov_b32 s11, s3 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_mov_b32 s8, s2 -; SI-NEXT: s_mov_b32 s9, s3 +; SI-NEXT: s_mov_b32 s8, s6 +; SI-NEXT: s_mov_b32 s9, s7 ; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 ; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 ; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 ; SI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 -; SI-NEXT: s_mov_b32 s4, s0 -; SI-NEXT: s_mov_b32 s5, s1 +; SI-NEXT: s_mov_b32 s0, s4 +; SI-NEXT: s_mov_b32 s1, s5 ; SI-NEXT: s_waitcnt vmcnt(3) ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; SI-NEXT: s_waitcnt vmcnt(2) ; SI-NEXT: v_or_b32_e32 v0, v1, v0 -; SI-NEXT: v_cvt_f32_u32_e32 v0, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v1, v0 +; SI-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 ; SI-NEXT: s_waitcnt vmcnt(1) -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_rcp_f32_e32 v1, v1 ; SI-NEXT: s_waitcnt vmcnt(0) -; SI-NEXT: v_or_b32_e32 v1, v3, v1 -; SI-NEXT: v_cvt_f32_u32_e32 v1, v1 -; SI-NEXT: v_rcp_f32_e32 v2, v0 -; SI-NEXT: v_mul_f32_e32 v2, v1, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; SI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; SI-NEXT: v_or_b32_e32 v2, v3, v2 +; SI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; SI-NEXT: v_cvt_u32_f32_e32 v1, v1 +; SI-NEXT: v_mul_lo_u32 v4, v4, v1 +; SI-NEXT: v_mul_hi_u32 v4, v1, v4 +; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; SI-NEXT: v_mul_hi_u32 v1, v2, v1 +; SI-NEXT: v_mul_lo_u32 v3, v1, v0 +; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v1 +; SI-NEXT: v_sub_i32_e32 v2, vcc, v2, v3 +; SI-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 +; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v1 +; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; SI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 -; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: v_udiv_i23: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; VI-NEXT: s_mov_b32 s7, 0xf000 -; VI-NEXT: s_mov_b32 s6, -1 -; VI-NEXT: s_mov_b32 s10, s6 -; VI-NEXT: s_mov_b32 s11, s7 +; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: s_mov_b32 s2, -1 +; VI-NEXT: s_mov_b32 s10, s2 +; VI-NEXT: s_mov_b32 s11, s3 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_mov_b32 s8, s2 -; VI-NEXT: s_mov_b32 s9, s3 +; VI-NEXT: s_mov_b32 s8, s6 +; VI-NEXT: s_mov_b32 s9, s7 ; VI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 ; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 ; VI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 ; VI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 -; VI-NEXT: s_mov_b32 s4, s0 -; VI-NEXT: s_mov_b32 s5, s1 +; VI-NEXT: s_mov_b32 s0, s4 +; VI-NEXT: s_mov_b32 s1, s5 ; VI-NEXT: s_waitcnt vmcnt(3) ; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; VI-NEXT: s_waitcnt vmcnt(2) ; VI-NEXT: v_or_b32_e32 v0, v1, v0 -; VI-NEXT: v_cvt_f32_u32_e32 v0, v0 +; VI-NEXT: v_cvt_f32_u32_e32 v1, v0 +; VI-NEXT: v_sub_u32_e32 v4, vcc, 0, v0 ; VI-NEXT: s_waitcnt vmcnt(1) -; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; VI-NEXT: v_rcp_f32_e32 v1, v1 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_or_b32_e32 v1, v3, v1 -; VI-NEXT: v_cvt_f32_u32_e32 v1, v1 -; VI-NEXT: v_rcp_f32_e32 v2, v0 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +; VI-NEXT: v_or_b32_e32 v2, v3, v2 +; VI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; VI-NEXT: v_cvt_u32_f32_e32 v1, v1 +; VI-NEXT: v_mul_lo_u32 v4, v4, v1 +; VI-NEXT: v_mul_hi_u32 v4, v1, v4 +; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v4 +; VI-NEXT: v_mul_hi_u32 v1, v2, v1 +; VI-NEXT: v_mul_lo_u32 v3, v1, v0 +; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v1 +; VI-NEXT: v_sub_u32_e32 v2, vcc, v2, v3 +; VI-NEXT: v_sub_u32_e32 v3, vcc, v2, v0 +; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; VI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc +; VI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v1 +; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 +; VI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc ; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 -; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; ; GCN-LABEL: v_udiv_i23: @@ -1758,40 +1778,50 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i ; GCN-NEXT: s_add_u32 s4, s2, 4 ; GCN-NEXT: s_addc_u32 s5, s3, 0 ; GCN-NEXT: s_add_u32 s6, s2, 2 +; GCN-NEXT: v_mov_b32_e32 v0, s4 ; GCN-NEXT: s_addc_u32 s7, s3, 0 -; GCN-NEXT: v_mov_b32_e32 v0, s6 -; GCN-NEXT: v_mov_b32_e32 v1, s7 -; GCN-NEXT: s_add_u32 s6, s2, 6 -; GCN-NEXT: s_addc_u32 s7, s3, 0 +; GCN-NEXT: v_mov_b32_e32 v1, s5 +; GCN-NEXT: s_add_u32 s4, s2, 6 +; GCN-NEXT: s_addc_u32 s5, s3, 0 +; GCN-NEXT: v_mov_b32_e32 v2, s4 +; GCN-NEXT: v_mov_b32_e32 v3, s5 +; GCN-NEXT: flat_load_ubyte v4, v[2:3] +; GCN-NEXT: flat_load_ushort v5, v[0:1] ; GCN-NEXT: v_mov_b32_e32 v2, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s2 ; GCN-NEXT: v_mov_b32_e32 v3, s7 -; GCN-NEXT: v_mov_b32_e32 v4, s4 -; GCN-NEXT: v_mov_b32_e32 v5, s5 -; GCN-NEXT: flat_load_ubyte v6, v[2:3] -; GCN-NEXT: flat_load_ushort v4, v[4:5] -; GCN-NEXT: v_mov_b32_e32 v2, s2 -; GCN-NEXT: v_mov_b32_e32 v3, s3 -; GCN-NEXT: flat_load_ubyte v0, v[0:1] -; GCN-NEXT: flat_load_ushort v1, v[2:3] +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: flat_load_ubyte v2, v[2:3] +; GCN-NEXT: flat_load_ushort v0, v[0:1] ; GCN-NEXT: s_waitcnt vmcnt(3) -; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v6 +; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v4 ; GCN-NEXT: s_waitcnt vmcnt(2) -; GCN-NEXT: v_or_b32_e32 v2, v4, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GCN-NEXT: v_or_b32_e32 v3, v5, v1 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v3 +; GCN-NEXT: v_sub_u32_e32 v4, vcc, 0, v3 ; GCN-NEXT: s_waitcnt vmcnt(1) -; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GCN-NEXT: v_rcp_f32_e32 v1, v1 ; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_or_b32_e32 v0, v1, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 -; GCN-NEXT: v_rcp_f32_e32 v4, v2 +; GCN-NEXT: v_or_b32_e32 v2, v0, v2 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_mul_lo_u32 v4, v4, v1 +; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 +; GCN-NEXT: v_add_u32_e32 v0, vcc, v1, v4 +; GCN-NEXT: v_mul_hi_u32 v4, v2, v0 ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 -; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4 -; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc +; GCN-NEXT: v_mul_lo_u32 v5, v4, v3 +; GCN-NEXT: v_add_u32_e32 v6, vcc, 1, v4 +; GCN-NEXT: v_sub_u32_e32 v2, vcc, v2, v5 +; GCN-NEXT: v_sub_u32_e32 v5, vcc, v2, v3 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 +; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc +; GCN-NEXT: v_add_u32_e32 v5, vcc, 1, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 +; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc ; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v2 ; GCN-NEXT: flat_store_dword v[0:1], v2 ; GCN-NEXT: s_endpgm @@ -1807,23 +1837,39 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1030-NEXT: global_load_ubyte v3, v0, s[2:3] offset:2 ; GFX1030-NEXT: global_load_ushort v4, v0, s[2:3] ; GFX1030-NEXT: s_waitcnt vmcnt(3) -; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s2, v1 ; GFX1030-NEXT: s_waitcnt vmcnt(2) -; GFX1030-NEXT: v_or_b32_e32 v1, v2, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s3, v2 ; GFX1030-NEXT: s_waitcnt vmcnt(1) -; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s4, v3 ; GFX1030-NEXT: s_waitcnt vmcnt(0) -; GFX1030-NEXT: v_or_b32_e32 v2, v4, v2 -; GFX1030-NEXT: v_rcp_f32_e32 v3, v1 -; GFX1030-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1030-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX1030-NEXT: v_trunc_f32_e32 v3, v3 -; GFX1030-NEXT: v_fma_f32 v2, -v3, v1, v2 -; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v1 -; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo -; GFX1030-NEXT: v_and_b32_e32 v1, 0x7fffff, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s5, v4 +; GFX1030-NEXT: s_lshl_b32 s2, s2, 16 +; GFX1030-NEXT: s_or_b32 s2, s3, s2 +; GFX1030-NEXT: s_lshl_b32 s4, s4, 16 +; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, s2 +; GFX1030-NEXT: s_sub_i32 s6, 0, s2 +; GFX1030-NEXT: s_or_b32 s4, s5, s4 +; GFX1030-NEXT: v_rcp_f32_e32 v1, v1 +; GFX1030-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 +; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX1030-NEXT: v_readfirstlane_b32 s3, v1 +; GFX1030-NEXT: s_mul_i32 s6, s6, s3 +; GFX1030-NEXT: s_mul_hi_u32 s6, s3, s6 +; GFX1030-NEXT: s_add_i32 s3, s3, s6 +; GFX1030-NEXT: s_mul_hi_u32 s3, s4, s3 +; GFX1030-NEXT: s_mul_i32 s5, s3, s2 +; GFX1030-NEXT: s_sub_i32 s4, s4, s5 +; GFX1030-NEXT: s_add_i32 s5, s3, 1 +; GFX1030-NEXT: s_sub_i32 s6, s4, s2 +; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 +; GFX1030-NEXT: s_cselect_b32 s3, s5, s3 +; GFX1030-NEXT: s_cselect_b32 s4, s6, s4 +; GFX1030-NEXT: s_add_i32 s5, s3, 1 +; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 +; GFX1030-NEXT: s_cselect_b32 s2, s5, s3 +; GFX1030-NEXT: s_and_b32 s2, s2, 0x7fffff +; GFX1030-NEXT: v_mov_b32_e32 v1, s2 ; GFX1030-NEXT: global_store_dword v0, v1, s[0:1] ; GFX1030-NEXT: s_endpgm ; diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll index 8653a43a00..97e0937760 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udiv64.ll @@ -701,51 +701,77 @@ define amdgpu_kernel void @s_test_udiv31_i64(ptr addrspace(1) %out, i64 %x, i64 define amdgpu_kernel void @s_test_udiv23_i64(ptr addrspace(1) %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_udiv23_i64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dword s6, s[4:5], 0xe -; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_lshr_b32 s8, s0, 9 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-NEXT: s_sub_i32 s0, 0, s8 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s2, s6, 9 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-NEXT: s_lshr_b32 s2, s3, 9 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_rcp_f32_e32 v2, v0 ; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 ; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-NEXT: s_mul_i32 s0, s0, s8 +; GCN-NEXT: s_sub_i32 s0, s2, s0 +; GCN-NEXT: s_sub_i32 s1, s0, s8 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_udiv23_i64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s6, -1 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +; GCN-IR-NEXT: s_lshr_b32 s8, s0, 9 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 +; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 +; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s2, s6, 9 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-IR-NEXT: s_lshr_b32 s2, s3, 9 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 -; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 ; GCN-IR-NEXT: s_mov_b32 s4, s0 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 +; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 +; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 +; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 +; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr i64 %x, 41 diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll index 3fecc3882b..15d5ef7232 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/udivrem24.ll @@ -529,16 +529,28 @@ define amdgpu_kernel void @udiv23_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -548,20 +560,32 @@ define amdgpu_kernel void @udiv23_i32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0x7fffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0x7fffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -1986,16 +2010,28 @@ define amdgpu_kernel void @test_udiv24_u16_u23_i32(ptr addrspace(1) %out, ptr ad ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0xffff ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -2005,20 +2041,32 @@ define amdgpu_kernel void @test_udiv24_u16_u23_i32(ptr addrspace(1) %out, ptr ad ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0xffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0x7fffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0xffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -2077,16 +2125,28 @@ define amdgpu_kernel void @test_udiv24_u23_u16_i32(ptr addrspace(1) %out, ptr ad ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff ; SI-NEXT: s_and_b32 s5, s5, 0xffff -; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 -; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 -; SI-NEXT: v_rcp_f32_e32 v2, v1 -; SI-NEXT: v_mul_f32_e32 v2, v0, v2 -; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 -; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc -; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 +; SI-NEXT: s_sub_i32 s6, 0, s5 +; SI-NEXT: v_rcp_f32_e32 v0, v0 +; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; SI-NEXT: v_mul_lo_u32 v1, s6, v0 +; SI-NEXT: v_mul_hi_u32 v1, v0, v1 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; SI-NEXT: v_mul_hi_u32 v0, s4, v0 +; SI-NEXT: v_readfirstlane_b32 s6, v0 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_mul_i32 s6, s6, s5 +; SI-NEXT: s_sub_i32 s4, s4, s6 +; SI-NEXT: s_sub_i32 s6, s4, s5 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; SI-NEXT: s_cselect_b32 s4, s6, s4 +; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 +; SI-NEXT: s_cmp_ge_u32 s4, s5 +; SI-NEXT: s_cselect_b64 vcc, -1, 0 +; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -2096,20 +2156,32 @@ define amdgpu_kernel void @test_udiv24_u23_u16_i32(ptr addrspace(1) %out, ptr ad ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s3, 0xffff -; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 -; VI-NEXT: s_and_b32 s2, s2, 0x7fffff -; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: v_rcp_f32_e32 v2, v0 +; VI-NEXT: s_and_b32 s4, s3, 0xffff +; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 +; VI-NEXT: s_sub_i32 s3, 0, s4 +; VI-NEXT: s_and_b32 s5, s2, 0x7fffff ; VI-NEXT: s_mov_b32 s2, -1 -; VI-NEXT: v_mul_f32_e32 v2, v1, v2 -; VI-NEXT: v_trunc_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 -; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 -; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc -; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +; VI-NEXT: v_rcp_f32_e32 v0, v0 +; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 +; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 +; VI-NEXT: v_mul_lo_u32 v1, s3, v0 +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: v_mul_hi_u32 v1, v0, v1 +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_mul_hi_u32 v0, s5, v0 +; VI-NEXT: v_readfirstlane_b32 s6, v0 +; VI-NEXT: s_mul_i32 s6, s6, s4 +; VI-NEXT: s_sub_i32 s5, s5, s6 +; VI-NEXT: s_sub_i32 s6, s5, s4 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; VI-NEXT: s_cselect_b32 s5, s6, s5 +; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 +; VI-NEXT: s_cmp_ge_u32 s5, s4 +; VI-NEXT: s_cselect_b64 vcc, -1, 0 +; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; diff --git a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll index 7840b861ab..0952013401 100644 --- a/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll +++ b/external/llvm-project/llvm/test/CodeGen/AMDGPU/urem64.ll @@ -710,36 +710,45 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 ; GCN-LABEL: s_test_urem23_64_v2i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd -; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ; GCN-NEXT: s_mov_b32 s3, 0xf000 -; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s6, s13, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GCN-NEXT: s_lshr_b32 s4, s15, 9 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s4 -; GCN-NEXT: s_lshr_b32 s5, s11, 9 +; GCN-NEXT: s_lshr_b32 s0, s13, 1 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s0 +; GCN-NEXT: s_sub_i32 s1, 0, s0 +; GCN-NEXT: s_lshr_b32 s6, s15, 9 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, s6 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, s5 -; GCN-NEXT: v_rcp_f32_e32 v3, v1 -; GCN-NEXT: s_sub_i32 s8, 0, s6 +; GCN-NEXT: s_lshr_b32 s7, s11, 9 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_f32_e32 v3, v2, v3 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mad_f32 v2, -v3, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v4, s8, v0 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-NEXT: s_lshr_b32 s7, s9, 1 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v1, v1, s4 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v0, v0, s7 -; GCN-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 -; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v1 -; GCN-NEXT: v_readfirstlane_b32 s4, v0 +; GCN-NEXT: v_mul_lo_u32 v1, s1, v0 +; GCN-NEXT: s_lshr_b32 s1, s9, 1 +; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-NEXT: v_mul_hi_u32 v0, v0, s1 +; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-NEXT: s_mul_i32 s2, s2, s0 +; GCN-NEXT: s_sub_i32 s1, s1, s2 +; GCN-NEXT: s_sub_i32 s2, s1, s0 +; GCN-NEXT: s_cmp_ge_u32 s1, s0 +; GCN-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-NEXT: s_sub_i32 s2, s1, s0 +; GCN-NEXT: s_cmp_ge_u32 s1, s0 +; GCN-NEXT: s_cselect_b32 s8, s2, s1 +; GCN-NEXT: s_sub_i32 s0, 0, s6 +; GCN-NEXT: v_mul_lo_u32 v0, s0, v1 +; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +; GCN-NEXT: s_mov_b32 s2, -1 +; GCN-NEXT: v_mul_hi_u32 v0, v1, v0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0 +; GCN-NEXT: v_mul_hi_u32 v2, v0, s7 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: v_mov_b32_e32 v0, s8 +; GCN-NEXT: v_mov_b32_e32 v3, v1 +; GCN-NEXT: v_readfirstlane_b32 s4, v2 ; GCN-NEXT: s_mul_i32 s4, s4, s6 ; GCN-NEXT: s_sub_i32 s4, s7, s4 ; GCN-NEXT: s_sub_i32 s5, s4, s6 @@ -748,45 +757,53 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 ; GCN-NEXT: s_sub_i32 s5, s4, s6 ; GCN-NEXT: s_cmp_ge_u32 s4, s6 ; GCN-NEXT: s_cselect_b32 s4, s5, s4 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: v_mov_b32_e32 v0, s4 -; GCN-NEXT: v_mov_b32_e32 v3, v1 +; GCN-NEXT: v_mov_b32_e32 v2, s4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem23_64_v2i64: ; GCN-IR: ; %bb.0: ; GCN-IR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd -; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 -; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s6, s13, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GCN-IR-NEXT: s_lshr_b32 s4, s15, 9 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s4 -; GCN-IR-NEXT: s_lshr_b32 s5, s11, 9 +; GCN-IR-NEXT: s_lshr_b32 s0, s13, 1 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s0 +; GCN-IR-NEXT: s_sub_i32 s1, 0, s0 +; GCN-IR-NEXT: s_lshr_b32 s6, s15, 9 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s6 ; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s5 -; GCN-IR-NEXT: v_rcp_f32_e32 v3, v1 -; GCN-IR-NEXT: s_sub_i32 s8, 0, s6 +; GCN-IR-NEXT: s_lshr_b32 s7, s11, 9 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-IR-NEXT: v_mul_f32_e32 v3, v2, v3 -; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-IR-NEXT: v_mad_f32 v2, -v3, v1, v2 -; GCN-IR-NEXT: v_mul_lo_u32 v4, s8, v0 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 -; GCN-IR-NEXT: s_lshr_b32 s7, s9, 1 -; GCN-IR-NEXT: v_mul_hi_u32 v4, v0, v4 -; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, s4 -; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s7 -; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 -; GCN-IR-NEXT: v_and_b32_e32 v2, 0x7fffff, v1 -; GCN-IR-NEXT: v_readfirstlane_b32 s4, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v1, s1, v0 +; GCN-IR-NEXT: s_lshr_b32 s1, s9, 1 +; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s1 +; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 +; GCN-IR-NEXT: s_mul_i32 s2, s2, s0 +; GCN-IR-NEXT: s_sub_i32 s1, s1, s2 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s0 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s0 +; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s2, s1, s0 +; GCN-IR-NEXT: s_cmp_ge_u32 s1, s0 +; GCN-IR-NEXT: s_cselect_b32 s8, s2, s1 +; GCN-IR-NEXT: s_sub_i32 s0, 0, s6 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s0, v1 +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +; GCN-IR-NEXT: s_mov_b32 s2, -1 +; GCN-IR-NEXT: v_mul_hi_u32 v0, v1, v0 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v1, v0 +; GCN-IR-NEXT: v_mul_hi_u32 v2, v0, s7 +; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +; GCN-IR-NEXT: v_mov_b32_e32 v0, s8 +; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 +; GCN-IR-NEXT: v_readfirstlane_b32 s4, v2 ; GCN-IR-NEXT: s_mul_i32 s4, s4, s6 ; GCN-IR-NEXT: s_sub_i32 s4, s7, s4 ; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 @@ -795,9 +812,8 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 ; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 ; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 ; GCN-IR-NEXT: s_cselect_b32 s4, s5, s4 -; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 -; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 +; GCN-IR-NEXT: v_mov_b32_e32 v2, s4 +; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr <2 x i64> %x, From 05fc5d792c86b31f382316e371274c09962996a9 Mon Sep 17 00:00:00 2001 From: bogdan-petkovic Date: Thu, 24 Sep 2026 03:38:23 -0500 Subject: [PATCH 3/3] Internal patch records and regression test for the float-based div/rem cherry-picks Co-authored-by: Cursor --- llvm-patches/llvm-patch-content.txt | 104 + llvm-patches/patch201186.patch | 2582 +++++++++++++ llvm-patches/patch202753.patch | 3224 +++++++++++++++++ .../rock-gemm-large-grid-float-divrem.mlir | 22 + 4 files changed, 5932 insertions(+) create mode 100644 llvm-patches/patch201186.patch create mode 100644 llvm-patches/patch202753.patch create mode 100644 mlir/test/fusion/pr-e2e/rock-gemm-large-grid-float-divrem.mlir diff --git a/llvm-patches/llvm-patch-content.txt b/llvm-patches/llvm-patch-content.txt index f34fbd3dbe..c64e4ff54e 100644 --- a/llvm-patches/llvm-patch-content.txt +++ b/llvm-patches/llvm-patch-content.txt @@ -754,3 +754,107 @@ rocmlir-driver test named above provides the equivalent coverage. Drop this patch on the next LLVM/Triton bump that includes upstream commit 8a9c0755eb6c4f00a12bc128b7d4c9d8d0d08667. + +---------------------------------------------------------------------- +patch201186.patch +---------------------------------------------------------------------- +- Upstream commit : 4fdb108b8e203e7fc7ff3ea73b7fc5d009a36954 +- Upstream PR : https://github.com/llvm/llvm-project/pull/201186 +- Upstream title : [AMDGPU] Be more careful about using expandDivRem24 +- File Fixed : llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp + llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll + llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll + llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll + llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll + llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll + llvm/test/CodeGen/AMDGPU/udiv.ll + llvm/test/CodeGen/AMDGPU/udiv64.ll + llvm/test/CodeGen/AMDGPU/udivrem24.ll + llvm/test/CodeGen/AMDGPU/urem64.ll +- Contents : Restricts the float-based expansion of 32-bit integer + div/rem to 23 bits for unsigned operands, keeping 24 for + signed where the magnitude cannot reach 0x800000, and + updates the affected LIT expectations. + +Symptom without the patch +------------------------- +GEMM kernels silently return wrong results, and intermittently die with a memory +access fault, once the grid grows past roughly 8.3 million workgroups. The +weekly hit this on gfx90a for a 9216x1536 f16 GEMM with a 1x1 tile: 238592 of +the 14155776 output elements were wrong on every run, and roughly one run in +twelve died with a memory access fault. 55 of the 6466 problem/tile pairs in the +tier1 gemm quick tuning space were exposed. + +Root cause +---------- +makeGroupedGridLayout derives m_block from the grid-wide block id as bid % +thisMBlocksPerGroup. That divisor is a runtime value, so AMDGPUCodeGenPrepare +replaced the remainder with expandDivRem24Impl once it proved both operands fit +in 24 bits. That algorithm computes trunc(float(Num) * rcp(float(Den))) and +corrects only for an underestimated quotient, so the 1 ulp error of v_rcp_f32 +leaves the quotient one too large once the dividend exceeds 0x800000. The +remainder then goes negative, is masked back to 24 bits, and m_block lands far +outside the tensor. Buffer descriptors set num_records to 0x7ffffffe so that the +0x80000000 disabled-lane sentinel still falls out of range, which also means the +hardware does not clamp the access. + +CI coverage +----------- +mlir/test/fusion/pr-e2e/rock-gemm-large-grid-float-divrem.mlir runs the +9216x1536 f16 GEMM above with its 1x1 tile and gridGroupSize=11, the group size +the heuristic picks on gfx90a, pinned so the test does not depend on the CU +count. It runs under ROCMLIR_DRIVER_PR_E2E_TEST_ENABLED and verifies the full +output; without this patch about 238000 of the 14155776 elements come out wrong +on gfx942, so a future LLVM bump that loses the fix will fail there. The LIT +updates carried by this patch are not run here, since the vendored LLVM is +built as libraries only. + +Drop this patch on the next LLVM/Triton bump that includes upstream commit +4fdb108b8e203e7fc7ff3ea73b7fc5d009a36954. + +---------------------------------------------------------------------- +patch202753.patch +---------------------------------------------------------------------- +- Upstream commit : 077ec06128e27e3dce2b97563dab703f89024ddf +- Upstream PR : https://github.com/llvm/llvm-project/pull/202753 +- Upstream title : [AMDGPU] Avoid errors with 23-bit division and remainder. +- File Fixed : llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp + llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll + llvm/test/CodeGen/AMDGPU/sdiv.ll + llvm/test/CodeGen/AMDGPU/sdiv64.ll + llvm/test/CodeGen/AMDGPU/sdivrem24.ll + llvm/test/CodeGen/AMDGPU/srem64.ll + llvm/test/CodeGen/AMDGPU/udiv.ll + llvm/test/CodeGen/AMDGPU/udiv64.ll + llvm/test/CodeGen/AMDGPU/udivrem24.ll + llvm/test/CodeGen/AMDGPU/urem64.ll +- Contents : Tightens the bound from patch201186 further, to 22 bits + for unsigned and 23 for signed, renames expandDivRem24 to + expandDivRemToFloat since the bound is no longer 24 bits, + and asserts the supported range in the implementation. + +Symptom without the patch +------------------------- +The same wrong quotients as patch201186, for dividends that still fit in 23 +bits. Sweeping every divisor below 2^24, the earliest dividend where the float +expansion is inexact is 8290592, which is below 0x800000 and therefore still +allowed by patch201186 on its own. + +Root cause +---------- +The 1 ulp error of v_rcp_f32 pushes trunc(float(Num) * rcp(float(Den))) over an +integer boundary well before the dividend reaches 0x800000, so a 23-bit bound is +still too generous. Upstream settled on 22 bits for unsigned operands, which +leaves enough headroom for the reciprocal error at every divisor. + +CI coverage +----------- +None downstream. The test named under patch201186 only fails for dividends +above 0x800000, which patch201186 already excludes, so it does not tell this +patch apart. The LIT updates carried by this patch cover the narrower bound but +are not run here, since the vendored LLVM is built as libraries only. + +Applies on top of patch201186.patch and does not apply to a tree without it. + +Drop this patch on the next LLVM/Triton bump that includes upstream commit +077ec06128e27e3dce2b97563dab703f89024ddf. diff --git a/llvm-patches/patch201186.patch b/llvm-patches/patch201186.patch new file mode 100644 index 0000000000..d3262f5560 --- /dev/null +++ b/llvm-patches/patch201186.patch @@ -0,0 +1,2582 @@ +diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +index fa401fbfa77cc5..c5501236edb5b9 100644 +--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp ++++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +@@ -1062,7 +1062,18 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder, + Value *Den, bool IsDiv, + bool IsSigned) const { + unsigned DivBits = getDivNumBits(I, Num, Den, 24, IsSigned); +- if (DivBits > 24) ++ ++ // v_rcp_f32(float(X)) can have an error of 1 ulp. ++ // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly ++ // when abs(Y)>0x800000. ++ // For example, ++ // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. ++ // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. ++ // ++ // Note that for DivBits==24 && IsSigned, Y is in the range ++ // [-0x800000:0x7FFFFF]. abs(Y) is at most ++ // 0x800000 so it cannot hit this issue. ++ if (DivBits > (IsSigned ? 24 : 23)) + return nullptr; + return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned); + } +@@ -1353,7 +1364,17 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder, + return nullptr; + + Value *Narrowed = nullptr; +- if (NumDivBits <= 24) { ++ // v_rcp_f32(float(X)) can have an error of 1 ulp. ++ // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly ++ // when abs(Y)>0x800000. ++ // For example, ++ // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. ++ // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. ++ // ++ // Note that for NumDivBits==24 && IsSigned, Y is in the range ++ // [-0x800000:0x7FFFFF]. abs(Y) is at most ++ // 0x800000 so it cannot hit this issue. ++ if (NumDivBits <= (IsSigned ? 24 : 23)) { + Narrowed = expandDivRem24Impl(Builder, I, Num, Den, NumDivBits, + IsDiv, IsSigned); + } else if (NumDivBits <= 32) { +diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll +index 13e8e121187759..b308bdf42f6d22 100644 +--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll ++++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll +@@ -431,17 +431,25 @@ define i32 @v_udiv_i32_24bit(i32 %num, i32 %den) { + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; CGP-NEXT: v_rcp_f32_e32 v2, v1 +-; CGP-NEXT: v_mul_f32_e32 v2, v0, v2 +-; CGP-NEXT: v_trunc_f32_e32 v2, v2 +-; CGP-NEXT: v_fma_f32 v0, -v2, v1, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 ++; CGP-NEXT: v_rcp_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 + ; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 +-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 ++; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 ++; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 ++; CGP-NEXT: v_mul_lo_u32 v3, v2, v1 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ++; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and i32 %num, 16777215 + %den.mask = and i32 %den, 16777215 +@@ -504,28 +512,44 @@ define <2 x i32> @v_udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) { + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 + ; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 + ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v3 +-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3 +-; CGP-NEXT: v_rcp_f32_e32 v4, v2 +-; CGP-NEXT: v_rcp_f32_e32 v5, v3 +-; CGP-NEXT: v_mul_f32_e32 v4, v0, v4 +-; CGP-NEXT: v_mul_f32_e32 v5, v1, v5 +-; CGP-NEXT: v_trunc_f32_e32 v4, v4 +-; CGP-NEXT: v_trunc_f32_e32 v5, v5 +-; CGP-NEXT: v_fma_f32 v0, -v4, v2, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 ++; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 ++; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 ++; CGP-NEXT: v_rcp_f32_e32 v4, v4 ++; CGP-NEXT: v_rcp_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 + ; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +-; CGP-NEXT: v_fma_f32 v1, -v5, v3, v1 +-; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v2 +-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, v3 +-; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +-; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 ++; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 ++; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 ++; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 ++; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 ++; CGP-NEXT: v_mul_hi_u32 v4, v0, v4 ++; CGP-NEXT: v_mul_hi_u32 v5, v1, v5 ++; CGP-NEXT: v_mul_lo_u32 v6, v4, v2 ++; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4 ++; CGP-NEXT: v_mul_lo_u32 v8, v5, v3 ++; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 ++; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc ++; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2 ++; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3 ++; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5] ++; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc ++; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4 ++; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] ++; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and <2 x i32> %num, + %den.mask = and <2 x i32> %den, +diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll +index 74015b29f88711..713e02b711961c 100644 +--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll ++++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll +@@ -1919,17 +1919,25 @@ define i64 @v_udiv_i64_24bit(i64 %num, i64 %den) { + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; CGP-NEXT: v_rcp_f32_e32 v2, v1 +-; CGP-NEXT: v_mul_f32_e32 v2, v0, v2 +-; CGP-NEXT: v_trunc_f32_e32 v2, v2 +-; CGP-NEXT: v_mad_f32 v0, -v2, v1, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 ++; CGP-NEXT: v_rcp_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 + ; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 +-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 ++; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 ++; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 ++; CGP-NEXT: v_mul_lo_u32 v3, v2, v1 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ++; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc + ; CGP-NEXT: v_mov_b32_e32 v1, 0 + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and i64 %num, 16777215 +@@ -2173,31 +2181,51 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) { + ; CGP: ; %bb.0: + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4 +-; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 ++; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 ++; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v4 + ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v6 +-; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3 +-; CGP-NEXT: v_rcp_f32_e32 v4, v1 +-; CGP-NEXT: v_rcp_f32_e32 v5, v3 +-; CGP-NEXT: v_mul_f32_e32 v4, v0, v4 +-; CGP-NEXT: v_mul_f32_e32 v5, v2, v5 +-; CGP-NEXT: v_trunc_f32_e32 v4, v4 +-; CGP-NEXT: v_trunc_f32_e32 v5, v5 +-; CGP-NEXT: v_mad_f32 v0, -v4, v1, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 ++; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 ++; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 ++; CGP-NEXT: v_rcp_f32_e32 v4, v4 ++; CGP-NEXT: v_rcp_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 + ; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +-; CGP-NEXT: v_mad_f32 v2, -v5, v3, v2 +-; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1 +-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5] +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 +-; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +-; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v1 ++; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 ++; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 ++; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 ++; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 ++; CGP-NEXT: v_mul_hi_u32 v6, v0, v4 ++; CGP-NEXT: v_mul_lo_u32 v4, 0, v4 ++; CGP-NEXT: v_mul_hi_u32 v7, v1, v5 ++; CGP-NEXT: v_mul_lo_u32 v5, 0, v5 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5 ++; CGP-NEXT: v_mul_lo_u32 v6, v4, v2 ++; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4 ++; CGP-NEXT: v_mul_lo_u32 v8, v5, v3 ++; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 ++; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc ++; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2 ++; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3 ++; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5] ++; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc ++; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4 ++; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5] ++; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc + ; CGP-NEXT: v_mov_b32_e32 v1, 0 + ; CGP-NEXT: v_mov_b32_e32 v3, 0 + ; CGP-NEXT: s_setpc_b64 s[30:31] +diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll +index 4fb58ca491b569..de1e382a5db8b3 100644 +--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll ++++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll +@@ -409,19 +409,23 @@ define i32 @v_urem_i32_24bit(i32 %num, i32 %den) { + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v2, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1 +-; CGP-NEXT: v_rcp_f32_e32 v4, v3 +-; CGP-NEXT: v_mul_f32_e32 v4, v2, v4 +-; CGP-NEXT: v_trunc_f32_e32 v4, v4 +-; CGP-NEXT: v_fma_f32 v2, -v4, v3, v2 +-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 +-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +-; CGP-NEXT: v_mul_lo_u32 v1, v2, v1 +-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 ++; CGP-NEXT: v_rcp_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 ++; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 ++; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 ++; CGP-NEXT: v_mul_lo_u32 v2, v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and i32 %num, 16777215 + %den.mask = and i32 %den, 16777215 +@@ -480,32 +484,40 @@ define <2 x i32> @v_urem_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) { + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 + ; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 + ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v3 +-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v6, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v7, v3 +-; CGP-NEXT: v_rcp_f32_e32 v8, v5 +-; CGP-NEXT: v_rcp_f32_e32 v9, v7 +-; CGP-NEXT: v_mul_f32_e32 v8, v4, v8 +-; CGP-NEXT: v_mul_f32_e32 v9, v6, v9 +-; CGP-NEXT: v_trunc_f32_e32 v8, v8 +-; CGP-NEXT: v_trunc_f32_e32 v9, v9 +-; CGP-NEXT: v_fma_f32 v4, -v8, v5, v4 +-; CGP-NEXT: v_cvt_u32_f32_e32 v8, v8 +-; CGP-NEXT: v_fma_f32 v6, -v9, v7, v6 +-; CGP-NEXT: v_cvt_u32_f32_e32 v9, v9 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, v5 +-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5] +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v6|, v7 +-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v4, vcc, v8, v4 +-; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5 +-; CGP-NEXT: v_mul_lo_u32 v2, v4, v2 +-; CGP-NEXT: v_mul_lo_u32 v3, v5, v3 +-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1 ++; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 ++; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 ++; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 ++; CGP-NEXT: v_rcp_f32_e32 v4, v4 ++; CGP-NEXT: v_rcp_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ++; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 ++; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 ++; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 ++; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 ++; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 ++; CGP-NEXT: v_mul_hi_u32 v4, v0, v4 ++; CGP-NEXT: v_mul_hi_u32 v5, v1, v5 ++; CGP-NEXT: v_mul_lo_u32 v4, v4, v2 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v3 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 ++; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v5 ++; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ++; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and <2 x i32> %num, + %den.mask = and <2 x i32> %den, +diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll +index d8f33d469f1ace..da247d20ab2ecc 100644 +--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll ++++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll +@@ -1984,19 +1984,23 @@ define i64 @v_urem_i64_24bit(i64 %num, i64 %den) { + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v2, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v1 +-; CGP-NEXT: v_rcp_f32_e32 v4, v3 +-; CGP-NEXT: v_mul_f32_e32 v4, v2, v4 +-; CGP-NEXT: v_trunc_f32_e32 v4, v4 +-; CGP-NEXT: v_mad_f32 v2, -v4, v3, v2 +-; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3 +-; CGP-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +-; CGP-NEXT: v_mul_lo_u32 v1, v2, v1 +-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 ++; CGP-NEXT: v_rcp_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; CGP-NEXT: v_mul_lo_u32 v3, v3, v2 ++; CGP-NEXT: v_mul_hi_u32 v3, v2, v3 ++; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; CGP-NEXT: v_mul_hi_u32 v2, v0, v2 ++; CGP-NEXT: v_mul_lo_u32 v2, v2, v1 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v1 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc + ; CGP-NEXT: v_mov_b32_e32 v1, 0 + ; CGP-NEXT: s_setpc_b64 s[30:31] + %num.mask = and i64 %num, 16777215 +@@ -2236,35 +2240,47 @@ define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) { + ; CGP: ; %bb.0: + ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4 +-; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2 ++; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2 ++; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v4 + ; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v6 +-; CGP-NEXT: v_cvt_f32_u32_e32 v4, v0 +-; CGP-NEXT: v_cvt_f32_u32_e32 v5, v1 +-; CGP-NEXT: v_cvt_f32_u32_e32 v6, v2 +-; CGP-NEXT: v_cvt_f32_u32_e32 v7, v3 +-; CGP-NEXT: v_rcp_f32_e32 v8, v5 +-; CGP-NEXT: v_rcp_f32_e32 v9, v7 +-; CGP-NEXT: v_mul_f32_e32 v8, v4, v8 +-; CGP-NEXT: v_mul_f32_e32 v9, v6, v9 +-; CGP-NEXT: v_trunc_f32_e32 v8, v8 +-; CGP-NEXT: v_trunc_f32_e32 v9, v9 +-; CGP-NEXT: v_mad_f32 v4, -v8, v5, v4 +-; CGP-NEXT: v_cvt_u32_f32_e32 v8, v8 +-; CGP-NEXT: v_mad_f32 v6, -v9, v7, v6 +-; CGP-NEXT: v_cvt_u32_f32_e32 v9, v9 +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, v5 +-; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5] +-; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v6|, v7 +-; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5] +-; CGP-NEXT: v_add_i32_e32 v4, vcc, v8, v4 +-; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5 +-; CGP-NEXT: v_mul_lo_u32 v1, v4, v1 +-; CGP-NEXT: v_mul_lo_u32 v3, v5, v3 +-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v2, v3 +-; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v1 ++; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2 ++; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3 ++; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3 ++; CGP-NEXT: v_rcp_f32_e32 v4, v4 ++; CGP-NEXT: v_rcp_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6 ++; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4 ++; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v4 ++; CGP-NEXT: v_mul_lo_u32 v7, v7, v6 ++; CGP-NEXT: v_mul_hi_u32 v5, v4, v5 ++; CGP-NEXT: v_mul_hi_u32 v7, v6, v7 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7 ++; CGP-NEXT: v_mul_hi_u32 v6, v0, v4 ++; CGP-NEXT: v_mul_lo_u32 v4, 0, v4 ++; CGP-NEXT: v_mul_hi_u32 v7, v1, v5 ++; CGP-NEXT: v_mul_lo_u32 v5, 0, v5 ++; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4 ++; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5 ++; CGP-NEXT: v_mul_lo_u32 v4, v4, v2 ++; CGP-NEXT: v_mul_lo_u32 v5, v5, v3 ++; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 ++; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v5 ++; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ++; CGP-NEXT: v_sub_i32_e32 v4, vcc, v0, v2 ++; CGP-NEXT: v_sub_i32_e32 v5, vcc, v1, v3 ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ++; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ++; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 ++; CGP-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc + ; CGP-NEXT: v_mov_b32_e32 v1, 0 + ; CGP-NEXT: v_mov_b32_e32 v3, 0 + ; CGP-NEXT: s_setpc_b64 s[30:31] +diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +index 1659ca62a0516a..a23295b6177e41 100644 +--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll ++++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +@@ -10134,3 +10134,248 @@ entry: + %B = srem <2 x i64> zeroinitializer, zeroinitializer + ret <2 x i64> %B + } ++ ++; 23-bit sdiv, can use expandDivRem24Impl ++define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ++; GFX6-LABEL: sdiv23: ++; GFX6: ; %bb.0: ++; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GFX6-NEXT: s_waitcnt lgkmcnt(0) ++; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] ++; GFX6-NEXT: s_bfe_i32 s5, s5, 0x170000 ++; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 ++; GFX6-NEXT: s_bfe_i32 s4, s4, 0x170000 ++; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 ++; GFX6-NEXT: s_xor_b32 s4, s4, s5 ++; GFX6-NEXT: v_rcp_f32_e32 v2, v0 ++; GFX6-NEXT: s_ashr_i32 s4, s4, 30 ++; GFX6-NEXT: s_or_b32 s6, s4, 1 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ++; GFX6-NEXT: v_trunc_f32_e32 v2, v2 ++; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ++; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 ++; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| ++; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec ++; GFX6-NEXT: s_cselect_b32 s4, s6, 0 ++; GFX6-NEXT: s_mov_b32 s2, -1 ++; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 ++; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ++; GFX6-NEXT: s_waitcnt expcnt(0) ++; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 ++; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ++; GFX6-NEXT: s_endpgm ++; ++; GFX9-LABEL: sdiv23: ++; GFX9: ; %bb.0: ++; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ++; GFX9-NEXT: v_mov_b32_e32 v1, 0 ++; GFX9-NEXT: s_waitcnt lgkmcnt(0) ++; GFX9-NEXT: s_bfe_i32 s3, s3, 0x170000 ++; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 ++; GFX9-NEXT: s_bfe_i32 s2, s2, 0x170000 ++; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 ++; GFX9-NEXT: s_xor_b32 s2, s2, s3 ++; GFX9-NEXT: v_rcp_f32_e32 v3, v0 ++; GFX9-NEXT: s_ashr_i32 s2, s2, 30 ++; GFX9-NEXT: s_or_b32 s4, s2, 1 ++; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 ++; GFX9-NEXT: v_trunc_f32_e32 v3, v3 ++; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 ++; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 ++; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| ++; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec ++; GFX9-NEXT: s_cselect_b32 s2, s4, 0 ++; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 ++; GFX9-NEXT: global_store_short v1, v0, s[0:1] ++; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 ++; GFX9-NEXT: s_endpgm ++ %r = sdiv i23 %x, %y ++ store i23 %r, ptr addrspace(1) %out ++ ret void ++} ++ ++; 24-bit sdiv, can use expandDivRem24Impl ++define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ++; GFX6-LABEL: sdiv24: ++; GFX6: ; %bb.0: ++; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GFX6-NEXT: s_waitcnt lgkmcnt(0) ++; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] ++; GFX6-NEXT: s_bfe_i32 s5, s5, 0x180000 ++; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 ++; GFX6-NEXT: s_bfe_i32 s4, s4, 0x180000 ++; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 ++; GFX6-NEXT: s_xor_b32 s4, s4, s5 ++; GFX6-NEXT: v_rcp_f32_e32 v2, v0 ++; GFX6-NEXT: s_ashr_i32 s4, s4, 30 ++; GFX6-NEXT: s_or_b32 s6, s4, 1 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ++; GFX6-NEXT: v_trunc_f32_e32 v2, v2 ++; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ++; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 ++; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| ++; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec ++; GFX6-NEXT: s_cselect_b32 s4, s6, 0 ++; GFX6-NEXT: s_mov_b32 s2, -1 ++; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 ++; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ++; GFX6-NEXT: s_waitcnt expcnt(0) ++; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ++; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ++; GFX6-NEXT: s_endpgm ++; ++; GFX9-LABEL: sdiv24: ++; GFX9: ; %bb.0: ++; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ++; GFX9-NEXT: v_mov_b32_e32 v1, 0 ++; GFX9-NEXT: s_waitcnt lgkmcnt(0) ++; GFX9-NEXT: s_bfe_i32 s3, s3, 0x180000 ++; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 ++; GFX9-NEXT: s_bfe_i32 s2, s2, 0x180000 ++; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 ++; GFX9-NEXT: s_xor_b32 s2, s2, s3 ++; GFX9-NEXT: v_rcp_f32_e32 v3, v0 ++; GFX9-NEXT: s_ashr_i32 s2, s2, 30 ++; GFX9-NEXT: s_or_b32 s4, s2, 1 ++; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 ++; GFX9-NEXT: v_trunc_f32_e32 v3, v3 ++; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 ++; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 ++; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| ++; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec ++; GFX9-NEXT: s_cselect_b32 s2, s4, 0 ++; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 ++; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 ++; GFX9-NEXT: global_store_short v1, v0, s[0:1] ++; GFX9-NEXT: s_endpgm ++ %r = sdiv i24 %x, %y ++ store i24 %r, ptr addrspace(1) %out ++ ret void ++} ++ ++; 23-bit udiv, can use expandDivRem24Impl ++define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ++; GFX6-LABEL: udiv23: ++; GFX6: ; %bb.0: ++; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GFX6-NEXT: s_waitcnt lgkmcnt(0) ++; GFX6-NEXT: s_and_b32 s3, s3, 0x7fffff ++; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 ++; GFX6-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_rcp_f32_e32 v2, v0 ++; GFX6-NEXT: s_mov_b32 s2, -1 ++; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ++; GFX6-NEXT: v_trunc_f32_e32 v2, v2 ++; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2 ++; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ++; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ++; GFX6-NEXT: s_waitcnt expcnt(0) ++; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 ++; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ++; GFX6-NEXT: s_endpgm ++; ++; GFX9-LABEL: udiv23: ++; GFX9: ; %bb.0: ++; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ++; GFX9-NEXT: v_mov_b32_e32 v3, 0 ++; GFX9-NEXT: s_waitcnt lgkmcnt(0) ++; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff ++; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ++; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2 ++; GFX9-NEXT: v_rcp_f32_e32 v2, v0 ++; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2 ++; GFX9-NEXT: v_trunc_f32_e32 v2, v2 ++; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2 ++; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1 ++; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc ++; GFX9-NEXT: global_store_short v3, v0, s[0:1] ++; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; GFX9-NEXT: global_store_byte_d16_hi v3, v0, s[0:1] offset:2 ++; GFX9-NEXT: s_endpgm ++ %r = udiv i23 %x, %y ++ store i23 %r, ptr addrspace(1) %out ++ ret void ++} ++ ++; 24-bit udiv, cannot use expandDivRem24Impl ++define amdgpu_kernel void @udiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { ++; GFX6-LABEL: udiv24: ++; GFX6: ; %bb.0: ++; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GFX6-NEXT: s_waitcnt lgkmcnt(0) ++; GFX6-NEXT: s_and_b32 s4, s3, 0xffffff ++; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GFX6-NEXT: s_sub_i32 s3, 0, s4 ++; GFX6-NEXT: s_and_b32 s5, s2, 0xffffff ++; GFX6-NEXT: s_mov_b32 s2, -1 ++; GFX6-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 ++; GFX6-NEXT: v_readfirstlane_b32 s6, v0 ++; GFX6-NEXT: s_mul_i32 s6, s6, s4 ++; GFX6-NEXT: s_sub_i32 s5, s5, s6 ++; GFX6-NEXT: s_sub_i32 s6, s5, s4 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: s_cselect_b32 s5, s6, s5 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ++; GFX6-NEXT: s_waitcnt expcnt(0) ++; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ++; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ++; GFX6-NEXT: s_endpgm ++; ++; GFX9-LABEL: udiv24: ++; GFX9: ; %bb.0: ++; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ++; GFX9-NEXT: v_mov_b32_e32 v1, 0 ++; GFX9-NEXT: s_waitcnt lgkmcnt(0) ++; GFX9-NEXT: s_and_b32 s3, s3, 0xffffff ++; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ++; GFX9-NEXT: s_sub_i32 s4, 0, s3 ++; GFX9-NEXT: s_and_b32 s2, s2, 0xffffff ++; GFX9-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX9-NEXT: v_readfirstlane_b32 s5, v0 ++; GFX9-NEXT: s_mul_i32 s4, s4, s5 ++; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4 ++; GFX9-NEXT: s_add_i32 s5, s5, s4 ++; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5 ++; GFX9-NEXT: s_mul_i32 s5, s4, s3 ++; GFX9-NEXT: s_sub_i32 s2, s2, s5 ++; GFX9-NEXT: s_add_i32 s6, s4, 1 ++; GFX9-NEXT: s_sub_i32 s5, s2, s3 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s3 ++; GFX9-NEXT: s_cselect_b32 s4, s6, s4 ++; GFX9-NEXT: s_cselect_b32 s2, s5, s2 ++; GFX9-NEXT: s_add_i32 s5, s4, 1 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s3 ++; GFX9-NEXT: s_cselect_b32 s2, s5, s4 ++; GFX9-NEXT: v_mov_b32_e32 v0, s2 ++; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 ++; GFX9-NEXT: global_store_short v1, v0, s[0:1] ++; GFX9-NEXT: s_endpgm ++ %r = udiv i24 %x, %y ++ store i24 %r, ptr addrspace(1) %out ++ ret void ++} +diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll +index 50b863f9741387..859a65f88800d0 100644 +--- a/llvm/test/CodeGen/AMDGPU/udiv.ll ++++ b/llvm/test/CodeGen/AMDGPU/udiv.ll +@@ -1876,76 +1876,96 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i + define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %in) { + ; SI-LABEL: v_udiv_i24: + ; SI: ; %bb.0: +-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; SI-NEXT: s_mov_b32 s7, 0xf000 +-; SI-NEXT: s_mov_b32 s6, -1 +-; SI-NEXT: s_mov_b32 s10, s6 +-; SI-NEXT: s_mov_b32 s11, s7 ++; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9 ++; SI-NEXT: s_mov_b32 s3, 0xf000 ++; SI-NEXT: s_mov_b32 s2, -1 ++; SI-NEXT: s_mov_b32 s10, s2 ++; SI-NEXT: s_mov_b32 s11, s3 + ; SI-NEXT: s_waitcnt lgkmcnt(0) +-; SI-NEXT: s_mov_b32 s8, s2 +-; SI-NEXT: s_mov_b32 s9, s3 ++; SI-NEXT: s_mov_b32 s8, s6 ++; SI-NEXT: s_mov_b32 s9, s7 + ; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 + ; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 + ; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 + ; SI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 +-; SI-NEXT: s_mov_b32 s4, s0 +-; SI-NEXT: s_mov_b32 s5, s1 ++; SI-NEXT: s_mov_b32 s0, s4 ++; SI-NEXT: s_mov_b32 s1, s5 + ; SI-NEXT: s_waitcnt vmcnt(3) + ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 + ; SI-NEXT: s_waitcnt vmcnt(2) + ; SI-NEXT: v_or_b32_e32 v0, v1, v0 +-; SI-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; SI-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 + ; SI-NEXT: s_waitcnt vmcnt(1) +-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ++; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; SI-NEXT: v_rcp_f32_e32 v1, v1 + ; SI-NEXT: s_waitcnt vmcnt(0) +-; SI-NEXT: v_or_b32_e32 v1, v3, v1 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; SI-NEXT: v_rcp_f32_e32 v2, v0 +-; SI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; SI-NEXT: v_or_b32_e32 v2, v3, v2 ++; SI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; SI-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; SI-NEXT: v_mul_lo_u32 v4, v4, v1 ++; SI-NEXT: v_mul_hi_u32 v4, v1, v4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v4 ++; SI-NEXT: v_mul_hi_u32 v1, v2, v1 ++; SI-NEXT: v_mul_lo_u32 v3, v1, v0 ++; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v1 ++; SI-NEXT: v_sub_i32_e32 v2, vcc, v2, v3 ++; SI-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ++; SI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ++; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; + ; VI-LABEL: v_udiv_i24: + ; VI: ; %bb.0: +-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +-; VI-NEXT: s_mov_b32 s7, 0xf000 +-; VI-NEXT: s_mov_b32 s6, -1 +-; VI-NEXT: s_mov_b32 s10, s6 +-; VI-NEXT: s_mov_b32 s11, s7 ++; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: s_mov_b32 s2, -1 ++; VI-NEXT: s_mov_b32 s10, s2 ++; VI-NEXT: s_mov_b32 s11, s3 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_mov_b32 s8, s2 +-; VI-NEXT: s_mov_b32 s9, s3 ++; VI-NEXT: s_mov_b32 s8, s6 ++; VI-NEXT: s_mov_b32 s9, s7 + ; VI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 + ; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 + ; VI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 + ; VI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 +-; VI-NEXT: s_mov_b32 s4, s0 +-; VI-NEXT: s_mov_b32 s5, s1 ++; VI-NEXT: s_mov_b32 s0, s4 ++; VI-NEXT: s_mov_b32 s1, s5 + ; VI-NEXT: s_waitcnt vmcnt(3) + ; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 + ; VI-NEXT: s_waitcnt vmcnt(2) + ; VI-NEXT: v_or_b32_e32 v0, v1, v0 +-; VI-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; VI-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; VI-NEXT: v_sub_u32_e32 v4, vcc, 0, v0 + ; VI-NEXT: s_waitcnt vmcnt(1) +-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ++; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; VI-NEXT: v_rcp_f32_e32 v1, v1 + ; VI-NEXT: s_waitcnt vmcnt(0) +-; VI-NEXT: v_or_b32_e32 v1, v3, v1 +-; VI-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; VI-NEXT: v_or_b32_e32 v2, v3, v2 ++; VI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; VI-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; VI-NEXT: v_mul_lo_u32 v4, v4, v1 ++; VI-NEXT: v_mul_hi_u32 v4, v1, v4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v4 ++; VI-NEXT: v_mul_hi_u32 v1, v2, v1 ++; VI-NEXT: v_mul_lo_u32 v3, v1, v0 ++; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v1 ++; VI-NEXT: v_sub_u32_e32 v2, vcc, v2, v3 ++; VI-NEXT: v_sub_u32_e32 v3, vcc, v2, v0 ++; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; VI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ++; VI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v1 ++; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ++; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; + ; GCN-LABEL: v_udiv_i24: +@@ -1958,40 +1978,50 @@ define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GCN-NEXT: s_add_u32 s4, s2, 4 + ; GCN-NEXT: s_addc_u32 s5, s3, 0 + ; GCN-NEXT: s_add_u32 s6, s2, 2 ++; GCN-NEXT: v_mov_b32_e32 v0, s4 + ; GCN-NEXT: s_addc_u32 s7, s3, 0 +-; GCN-NEXT: v_mov_b32_e32 v0, s6 +-; GCN-NEXT: v_mov_b32_e32 v1, s7 +-; GCN-NEXT: s_add_u32 s6, s2, 6 +-; GCN-NEXT: s_addc_u32 s7, s3, 0 ++; GCN-NEXT: v_mov_b32_e32 v1, s5 ++; GCN-NEXT: s_add_u32 s4, s2, 6 ++; GCN-NEXT: s_addc_u32 s5, s3, 0 ++; GCN-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-NEXT: v_mov_b32_e32 v3, s5 ++; GCN-NEXT: flat_load_ubyte v4, v[2:3] ++; GCN-NEXT: flat_load_ushort v5, v[0:1] + ; GCN-NEXT: v_mov_b32_e32 v2, s6 ++; GCN-NEXT: v_mov_b32_e32 v0, s2 + ; GCN-NEXT: v_mov_b32_e32 v3, s7 +-; GCN-NEXT: v_mov_b32_e32 v4, s4 +-; GCN-NEXT: v_mov_b32_e32 v5, s5 +-; GCN-NEXT: flat_load_ubyte v6, v[2:3] +-; GCN-NEXT: flat_load_ushort v4, v[4:5] +-; GCN-NEXT: v_mov_b32_e32 v2, s2 +-; GCN-NEXT: v_mov_b32_e32 v3, s3 +-; GCN-NEXT: flat_load_ubyte v0, v[0:1] +-; GCN-NEXT: flat_load_ushort v1, v[2:3] ++; GCN-NEXT: v_mov_b32_e32 v1, s3 ++; GCN-NEXT: flat_load_ubyte v2, v[2:3] ++; GCN-NEXT: flat_load_ushort v0, v[0:1] + ; GCN-NEXT: s_waitcnt vmcnt(3) +-; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v6 ++; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v4 + ; GCN-NEXT: s_waitcnt vmcnt(2) +-; GCN-NEXT: v_or_b32_e32 v2, v4, v2 +-; GCN-NEXT: v_cvt_f32_u32_e32 v2, v2 ++; GCN-NEXT: v_or_b32_e32 v3, v5, v1 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v3 ++; GCN-NEXT: v_sub_u32_e32 v4, vcc, 0, v3 + ; GCN-NEXT: s_waitcnt vmcnt(1) +-; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ++; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 + ; GCN-NEXT: s_waitcnt vmcnt(0) +-; GCN-NEXT: v_or_b32_e32 v0, v1, v0 +-; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 +-; GCN-NEXT: v_rcp_f32_e32 v4, v2 ++; GCN-NEXT: v_or_b32_e32 v2, v0, v2 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v4, v4, v1 ++; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 ++; GCN-NEXT: v_add_u32_e32 v0, vcc, v1, v4 ++; GCN-NEXT: v_mul_hi_u32 v4, v2, v0 + ; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: v_mov_b32_e32 v1, s1 +-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4 +-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 +-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc ++; GCN-NEXT: v_mul_lo_u32 v5, v4, v3 ++; GCN-NEXT: v_add_u32_e32 v6, vcc, 1, v4 ++; GCN-NEXT: v_sub_u32_e32 v2, vcc, v2, v5 ++; GCN-NEXT: v_sub_u32_e32 v5, vcc, v2, v3 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 ++; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc ++; GCN-NEXT: v_add_u32_e32 v5, vcc, 1, v4 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc + ; GCN-NEXT: v_and_b32_e32 v2, 0xffffff, v2 + ; GCN-NEXT: flat_store_dword v[0:1], v2 + ; GCN-NEXT: s_endpgm +@@ -2007,23 +2037,39 @@ define amdgpu_kernel void @v_udiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GFX1030-NEXT: global_load_ubyte v3, v0, s[2:3] offset:2 + ; GFX1030-NEXT: global_load_ushort v4, v0, s[2:3] + ; GFX1030-NEXT: s_waitcnt vmcnt(3) +-; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s2, v1 + ; GFX1030-NEXT: s_waitcnt vmcnt(2) +-; GFX1030-NEXT: v_or_b32_e32 v1, v2, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s3, v2 + ; GFX1030-NEXT: s_waitcnt vmcnt(1) +-; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 16, v3 +-; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s4, v3 + ; GFX1030-NEXT: s_waitcnt vmcnt(0) +-; GFX1030-NEXT: v_or_b32_e32 v2, v4, v2 +-; GFX1030-NEXT: v_rcp_f32_e32 v3, v1 +-; GFX1030-NEXT: v_cvt_f32_u32_e32 v2, v2 +-; GFX1030-NEXT: v_mul_f32_e32 v3, v2, v3 +-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3 +-; GFX1030-NEXT: v_fma_f32 v2, -v3, v1, v2 +-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3 +-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v1 +-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo +-; GFX1030-NEXT: v_and_b32_e32 v1, 0xffffff, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s5, v4 ++; GFX1030-NEXT: s_lshl_b32 s2, s2, 16 ++; GFX1030-NEXT: s_or_b32 s2, s3, s2 ++; GFX1030-NEXT: s_lshl_b32 s4, s4, 16 ++; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, s2 ++; GFX1030-NEXT: s_sub_i32 s6, 0, s2 ++; GFX1030-NEXT: s_or_b32 s4, s5, s4 ++; GFX1030-NEXT: v_rcp_f32_e32 v1, v1 ++; GFX1030-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s3, v1 ++; GFX1030-NEXT: s_mul_i32 s6, s6, s3 ++; GFX1030-NEXT: s_mul_hi_u32 s6, s3, s6 ++; GFX1030-NEXT: s_add_i32 s3, s3, s6 ++; GFX1030-NEXT: s_mul_hi_u32 s3, s4, s3 ++; GFX1030-NEXT: s_mul_i32 s5, s3, s2 ++; GFX1030-NEXT: s_sub_i32 s4, s4, s5 ++; GFX1030-NEXT: s_add_i32 s5, s3, 1 ++; GFX1030-NEXT: s_sub_i32 s6, s4, s2 ++; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 ++; GFX1030-NEXT: s_cselect_b32 s3, s5, s3 ++; GFX1030-NEXT: s_cselect_b32 s4, s6, s4 ++; GFX1030-NEXT: s_add_i32 s5, s3, 1 ++; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 ++; GFX1030-NEXT: s_cselect_b32 s2, s5, s3 ++; GFX1030-NEXT: s_and_b32 s2, s2, 0xffffff ++; GFX1030-NEXT: v_mov_b32_e32 v1, s2 + ; GFX1030-NEXT: global_store_dword v0, v1, s[0:1] + ; GFX1030-NEXT: s_endpgm + ; +diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll +index edd84a5f09e5e7..8653a43a00c6a4 100644 +--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll ++++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll +@@ -396,51 +396,77 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) { + define amdgpu_kernel void @s_test_udiv24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { + ; GCN-LABEL: s_test_udiv24_64: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dword s6, s[4:5], 0xe +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-NEXT: s_mov_b32 s6, -1 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_lshr_b32 s8, s0, 8 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s2, s6, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 + ; GCN-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 + ; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_udiv24_64: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-IR-NEXT: s_mov_b32 s6, -1 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_lshr_b32 s8, s0, 8 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s2, s6, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 + ; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 + ; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = lshr i64 %x, 40 +@@ -455,36 +481,54 @@ define i64 @v_test_udiv24_i64(i64 %x, i64 %y) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v3 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 + ; GCN-NEXT: v_lshrrev_b32_e32 v1, 8, v1 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-NEXT: v_mul_hi_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v2 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_udiv24_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v3 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v2 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %1 = lshr i64 %x, 40 + %2 = lshr i64 %y, 40 +@@ -714,57 +758,83 @@ define amdgpu_kernel void @s_test_udiv23_i64(ptr addrspace(1) %out, i64 %x, i64 + define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48 %y) { + ; GCN-LABEL: s_test_udiv24_i48: + ; GCN: ; %bb.0: ++; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_and_b32 s1, s1, 0xffff ++; GCN-NEXT: s_and_b32 s0, s0, 0xff000000 ++; GCN-NEXT: s_lshr_b64 s[6:7], s[0:1], 24 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s6 + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd +-; GCN-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-NEXT: s_sub_i32 s4, 0, s6 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_and_b32 s3, s3, 0xffff +-; GCN-NEXT: s_and_b32 s5, s5, 0xffff +-; GCN-NEXT: s_and_b32 s4, s4, 0xff000000 +-; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], 24 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 + ; GCN-NEXT: s_and_b32 s2, s2, 0xff000000 +-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-NEXT: s_lshr_b64 s[4:5], s[2:3], 24 + ; GCN-NEXT: s_mov_b32 s3, 0xf000 + ; GCN-NEXT: s_mov_b32 s2, -1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v4, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; GCN-NEXT: buffer_store_short v3, off, s[0:3], 0 offset:4 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s4, v0 ++; GCN-NEXT: v_mov_b32_e32 v1, 0 ++; GCN-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-NEXT: s_mul_i32 s5, s5, s6 ++; GCN-NEXT: s_sub_i32 s4, s4, s5 ++; GCN-NEXT: s_sub_i32 s5, s4, s6 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s4, s6 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; GCN-NEXT: s_cselect_b32 s4, s5, s4 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s4, s6 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; GCN-NEXT: buffer_store_short v1, off, s[0:3], 0 offset:4 + ; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_udiv24_i48: + ; GCN-IR: ; %bb.0: ++; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_and_b32 s1, s1, 0xffff ++; GCN-IR-NEXT: s_and_b32 s0, s0, 0xff000000 ++; GCN-IR-NEXT: s_lshr_b64 s[6:7], s[0:1], 24 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s6 + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd +-; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-IR-NEXT: s_sub_i32 s4, 0, s6 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_and_b32 s3, s3, 0xffff +-; GCN-IR-NEXT: s_and_b32 s5, s5, 0xffff +-; GCN-IR-NEXT: s_and_b32 s4, s4, 0xff000000 +-; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[4:5], 24 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 + ; GCN-IR-NEXT: s_and_b32 s2, s2, 0xff000000 +-; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[2:3], 24 + ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s2, -1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v4, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; GCN-IR-NEXT: buffer_store_short v3, off, s[0:3], 0 offset:4 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s4, v0 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-IR-NEXT: s_mul_i32 s5, s5, s6 ++; GCN-IR-NEXT: s_sub_i32 s4, s4, s5 ++; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; GCN-IR-NEXT: s_cselect_b32 s4, s5, s4 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ++; GCN-IR-NEXT: buffer_store_short v1, off, s[0:3], 0 offset:4 + ; GCN-IR-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = lshr i48 %x, 24 +@@ -1440,42 +1510,66 @@ define amdgpu_kernel void @s_test_udiv24_k_num_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_udiv24_k_num_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_mov_b32 s4, 0x41c00000 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ++; GCN-NEXT: s_lshr_b32 s4, s3, 8 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-NEXT: s_sub_i32 s2, 0, s4 + ; GCN-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 + ; GCN-NEXT: s_mov_b32 s2, -1 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_udiv24_k_num_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 ++; GCN-IR-NEXT: s_lshr_b32 s4, s3, 8 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 + ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 + ; GCN-IR-NEXT: s_mov_b32 s2, -1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = lshr i64 %x, 40 +@@ -1488,44 +1582,56 @@ define amdgpu_kernel void @s_test_udiv24_k_den_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_udiv24_k_den_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 +-; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 + ; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: s_lshr_b32 s0, s3, 8 ++; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s2 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mulk_i32 s1, 0x5b7f ++; GCN-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: s_min_u32 s0, s1, s0 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_udiv24_k_den_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 +-; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 + ; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: s_lshr_b32 s0, s3, 8 ++; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s2 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mulk_i32 s1, 0x5b7f ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: s_min_u32 s0, s1, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = lshr i64 %x, 40 +@@ -1539,34 +1645,52 @@ define i64 @v_test_udiv24_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-NEXT: v_mul_u32_u24_e32 v2, v1, v0 ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 24, v2 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_udiv24_k_num_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v1, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 24, v2 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = udiv i64 24, %x.shr +@@ -1578,34 +1702,52 @@ define i64 @v_test_udiv24_pow2_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x47000000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-NEXT: v_mul_u32_u24_e32 v2, v1, v0 ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0x8000, v2 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_udiv24_pow2_k_num_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x47000000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v1, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0x8000, v2 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v3, s[4:5], v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = udiv i64 32768, %x.shr +@@ -1621,19 +1763,78 @@ define i64 @v_test_udiv24_pow2_k_den_i64(i64 %x) { + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_udiv24_pow2_k_den_i64: +-; GCN-IR: ; %bb.0: ++; GCN-IR: ; %bb.0: ; %_udiv-special-cases + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +-; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38000000, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v0, -v1, s4, v0 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4 +-; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 8, v1 ++; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4 ++; GCN-IR-NEXT: v_add_i32_e64 v0, s[4:5], 32, v0 ++; GCN-IR-NEXT: s_flbit_i32_b32 s4, 0 ++; GCN-IR-NEXT: v_min_u32_e32 v6, s4, v0 ++; GCN-IR-NEXT: s_mov_b32 s6, 0 ++; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v6 ++; GCN-IR-NEXT: v_mov_b32_e32 v5, s6 ++; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] ++; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] ++; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s6 ++; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ++; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] ++; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ++; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], vcc ++; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ++; GCN-IR-NEXT: s_cbranch_execz .LBB17_6 ++; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 ++; GCN-IR-NEXT: v_add_i32_e32 v7, vcc, 1, v2 ++; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 63, v2 ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[4:5], v0 ++; GCN-IR-NEXT: v_mov_b32_e32 v2, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 ++; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execz .LBB17_5 ++; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader ++; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 0xffffffcf, v6 ++; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7 ++; GCN-IR-NEXT: v_addc_u32_e64 v9, s[8:9], 0, -1, vcc ++; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v6, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff ++; GCN-IR-NEXT: .LBB17_3: ; %udiv-do-while ++; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 ++; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 31, v1 ++; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s10, v4 ++; GCN-IR-NEXT: v_subb_u32_e32 v2, vcc, 0, v5, vcc ++; GCN-IR-NEXT: v_or_b32_e32 v0, v6, v0 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v6, 31, v2 ++; GCN-IR-NEXT: v_and_b32_e32 v2, 1, v6 ++; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6 ++; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6 ++; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 1, v8 ++; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1 ++; GCN-IR-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc ++; GCN-IR-NEXT: v_mov_b32_e32 v7, v3 ++; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] ++; GCN-IR-NEXT: v_mov_b32_e32 v6, v2 ++; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execnz .LBB17_3 ++; GCN-IR-NEXT: ; %bb.4: ; %Flow ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: .LBB17_5: ; %Flow4 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 ++; GCN-IR-NEXT: v_or_b32_e32 v1, v3, v1 ++; GCN-IR-NEXT: v_or_b32_e32 v0, v2, v0 ++; GCN-IR-NEXT: .LBB17_6: ; %Flow5 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = udiv i64 %x.shr, 32768 +diff --git a/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/llvm/test/CodeGen/AMDGPU/udivrem24.ll +index a075418b3bd91c..98948afd9eceda 100644 +--- a/llvm/test/CodeGen/AMDGPU/udivrem24.ll ++++ b/llvm/test/CodeGen/AMDGPU/udivrem24.ll +@@ -620,16 +620,28 @@ define amdgpu_kernel void @udiv24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0xffffff + ; SI-NEXT: s_and_b32 s5, s5, 0xffffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -639,20 +651,32 @@ define amdgpu_kernel void @udiv24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0xffffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0xffffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +@@ -711,16 +735,28 @@ define amdgpu_kernel void @no_udiv24_u23_u24_i32(ptr addrspace(1) %out, ptr addr + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff + ; SI-NEXT: s_and_b32 s5, s5, 0xffffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -730,20 +766,32 @@ define amdgpu_kernel void @no_udiv24_u23_u24_i32(ptr addrspace(1) %out, ptr addr + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0xffffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0x7fffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +@@ -802,16 +850,28 @@ define amdgpu_kernel void @no_udiv24_u24_u23_i32(ptr addrspace(1) %out, ptr addr + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0xffffff + ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -821,20 +881,32 @@ define amdgpu_kernel void @no_udiv24_u24_u23_i32(ptr addrspace(1) %out, ptr addr + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0x7fffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0xffffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +@@ -1461,22 +1533,29 @@ define amdgpu_kernel void @urem24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 + ; SI-NEXT: s_mov_b32 s3, 0xf000 +-; SI-NEXT: s_mov_b32 s2, -1 + ; SI-NEXT: s_waitcnt lgkmcnt(0) +-; SI-NEXT: s_and_b32 s6, s4, 0xffffff +-; SI-NEXT: s_and_b32 s7, s5, 0xffffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s6 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s7 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_mul_lo_u32 v0, v0, s5 +-; SI-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +-; SI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; SI-NEXT: s_and_b32 s2, s4, 0xffffff ++; SI-NEXT: s_and_b32 s4, s5, 0xffffff ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; SI-NEXT: s_sub_i32 s5, 0, s4 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s5, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s2, v0 ++; SI-NEXT: v_readfirstlane_b32 s5, v0 ++; SI-NEXT: s_mul_i32 s5, s5, s4 ++; SI-NEXT: s_sub_i32 s2, s2, s5 ++; SI-NEXT: s_sub_i32 s5, s2, s4 ++; SI-NEXT: s_cmp_ge_u32 s2, s4 ++; SI-NEXT: s_cselect_b32 s2, s5, s2 ++; SI-NEXT: s_sub_i32 s5, s2, s4 ++; SI-NEXT: s_cmp_ge_u32 s2, s4 ++; SI-NEXT: s_cselect_b32 s4, s5, s2 ++; SI-NEXT: s_mov_b32 s2, -1 ++; SI-NEXT: v_mov_b32_e32 v0, s4 + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -1484,24 +1563,31 @@ define amdgpu_kernel void @urem24_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; VI: ; %bb.0: + ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 +-; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s2, s5, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s2 +-; VI-NEXT: s_and_b32 s2, s4, 0xffffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 ++; VI-NEXT: s_and_b32 s4, s3, 0xffffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0xffffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_mul_lo_u32 v0, v0, s5 +-; VI-NEXT: v_sub_u32_e32 v0, vcc, s4, v0 +-; VI-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b32 s4, s6, s5 ++; VI-NEXT: v_mov_b32_e32 v0, s4 + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll +index 3bb489c6545354..7840b861abb9ac 100644 +--- a/llvm/test/CodeGen/AMDGPU/urem64.ll ++++ b/llvm/test/CodeGen/AMDGPU/urem64.ll +@@ -632,52 +632,72 @@ define amdgpu_kernel void @s_test_urem31_v2i64(ptr addrspace(1) %out, <2 x i64> + define amdgpu_kernel void @s_test_urem24_i64(ptr addrspace(1) %out, i64 %x, i64 %y) { + ; GCN-LABEL: s_test_urem24_i64: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dword s6, s[4:5], 0xe ++; GCN-NEXT: s_load_dword s0, s[4:5], 0xe ++; GCN-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-NEXT: s_mov_b32 s6, -1 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_lshr_b32 s8, s0, 8 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_mov_b32 s2, -1 +-; GCN-NEXT: s_lshr_b32 s4, s6, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; GCN-NEXT: s_lshr_b32 s5, s3, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; GCN-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: s_lshr_b32 s2, s3, 8 ++; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 ++; GCN-NEXT: s_mov_b32 s5, s1 + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s5, v0 +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ++; GCN-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_urem24_i64: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe ++; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe ++; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-IR-NEXT: s_mov_b32 s6, -1 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_lshr_b32 s8, s0, 8 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_mov_b32 s2, -1 +-; GCN-IR-NEXT: s_lshr_b32 s4, s6, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; GCN-IR-NEXT: s_lshr_b32 s5, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 ++; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 ++; GCN-IR-NEXT: s_mov_b32 s5, s1 + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s5, v0 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = lshr i64 %x, 40 + %2 = lshr i64 %y, 40 +@@ -1385,46 +1405,60 @@ define amdgpu_kernel void @s_test_urem24_k_num_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_urem24_k_num_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_mov_b32 s5, 0x41c00000 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_mov_b32 s2, -1 + ; GCN-NEXT: s_lshr_b32 s4, s3, 8 + ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-NEXT: s_sub_i32 s2, 0, s4 + ; GCN-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v1, -v1, v0, s5 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-NEXT: s_mov_b32 s2, -1 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; GCN-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b32 s4, s6, s5 ++; GCN-NEXT: v_mov_b32_e32 v0, s4 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_urem24_k_num_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_mov_b32 s5, 0x41c00000 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_mov_b32 s2, -1 + ; GCN-IR-NEXT: s_lshr_b32 s4, s3, 8 + ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 + ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v1, -v1, v0, s5 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-IR-NEXT: s_mov_b32 s2, -1 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b32 s4, s6, s5 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = lshr i64 %x, 40 +@@ -1437,50 +1471,46 @@ define amdgpu_kernel void @s_test_urem24_k_den_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_urem24_k_den_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_movk_i32 s4, 0x5b7f ++; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 +-; GCN-NEXT: s_mov_b32 s3, 0x46b6fe00 +-; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-NEXT: v_mad_f32 v0, -v1, s3, v0 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s3 + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_mov_b32 s4, s0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; GCN-NEXT: s_lshr_b32 s0, s3, 8 ++; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-NEXT: s_mov_b32 s5, s1 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mulk_i32 s1, 0x5b7f ++; GCN-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-NEXT: s_min_u32 s0, s1, s0 ++; GCN-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-NEXT: s_min_u32 s0, s1, s0 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_urem24_k_den_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_movk_i32 s4, 0x5b7f ++; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 +-; GCN-IR-NEXT: s_mov_b32 s3, 0x46b6fe00 +-; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mad_f32 v0, -v1, s3, v0 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s3 + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_mov_b32 s4, s0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 ++; GCN-IR-NEXT: s_lshr_b32 s0, s3, 8 ++; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-IR-NEXT: s_mov_b32 s5, s1 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mulk_i32 s1, 0x5b7f ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-IR-NEXT: s_min_u32 s0, s1, s0 ++; GCN-IR-NEXT: s_add_i32 s1, s0, 0xffffa481 ++; GCN-IR-NEXT: s_min_u32 s0, s1, s0 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = lshr i64 %x, 40 +@@ -1495,18 +1525,23 @@ define i64 @v_test_urem24_k_num_i64(i64 %x) { + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 + ; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-NEXT: v_rcp_f32_e32 v2, v1 +-; GCN-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v2, -v2, v1, s4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_urem24_k_num_i64: +@@ -1514,18 +1549,23 @@ define i64 @v_test_urem24_k_num_i64(i64 %x) { + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 + ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v2, -v2, v1, s4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = urem i64 24, %x.shr +@@ -1538,18 +1578,23 @@ define i64 @v_test_urem24_pow2_k_num_i64(i64 %x) { + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_lshrrev_b32_e32 v0, 8, v1 + ; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-NEXT: v_rcp_f32_e32 v2, v1 +-; GCN-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v2, -v2, v1, s4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +-; GCN-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_urem24_pow2_k_num_i64: +@@ -1557,18 +1602,23 @@ define i64 @v_test_urem24_pow2_k_num_i64(i64 %x) { + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 + ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v2, -v2, v1, s4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = urem i64 32768, %x.shr +@@ -1584,21 +1634,81 @@ define i64 @v_test_urem24_pow2_k_den_i64(i64 %x) { + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_urem24_pow2_k_den_i64: +-; GCN-IR: ; %bb.0: ++; GCN-IR: ; %bb.0: ; %_udiv-special-cases + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x38000000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, s4, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, s4 +-; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_lshlrev_b32_e32 v1, 15, v1 ++; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_add_i32_e64 v2, s[4:5], 32, v2 ++; GCN-IR-NEXT: s_flbit_i32_b32 s4, 0 ++; GCN-IR-NEXT: v_min_u32_e32 v8, s4, v2 ++; GCN-IR-NEXT: s_mov_b32 s6, 0 ++; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v8 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s6 ++; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] ++; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ++; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] ++; GCN-IR-NEXT: v_mov_b32_e32 v5, s6 ++; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ++; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] ++; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ++; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], vcc ++; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ++; GCN-IR-NEXT: s_cbranch_execz .LBB14_6 ++; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 ++; GCN-IR-NEXT: v_add_i32_e32 v6, vcc, 1, v2 ++; GCN-IR-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2 ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[0:1], v2 ++; GCN-IR-NEXT: v_mov_b32_e32 v4, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 ++; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 ++; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execz .LBB14_5 ++; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader ++; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 0xffffffcf, v8 ++; GCN-IR-NEXT: v_addc_u32_e64 v10, s[8:9], 0, -1, vcc ++; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v8, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v9, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 ++; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff ++; GCN-IR-NEXT: .LBB14_3: ; %udiv-do-while ++; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 ++; GCN-IR-NEXT: v_lshl_b64 v[6:7], v[6:7], 1 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3 ++; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4 ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 ++; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, s10, v6 ++; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, 0, v7, vcc ++; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4 ++; GCN-IR-NEXT: v_and_b32_e32 v4, 1, v8 ++; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8 ++; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8 ++; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v1 ++; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3 ++; GCN-IR-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc ++; GCN-IR-NEXT: v_mov_b32_e32 v9, v5 ++; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] ++; GCN-IR-NEXT: v_mov_b32_e32 v8, v4 ++; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execnz .LBB14_3 ++; GCN-IR-NEXT: ; %bb.4: ; %Flow ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: .LBB14_5: ; %Flow4 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] ++; GCN-IR-NEXT: v_lshl_b64 v[1:2], v[2:3], 1 ++; GCN-IR-NEXT: v_or_b32_e32 v5, v5, v2 ++; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v1 ++; GCN-IR-NEXT: .LBB14_6: ; %Flow5 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] ++; GCN-IR-NEXT: v_lshl_b64 v[1:2], v[4:5], 15 + ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +-; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 ++; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v2, vcc + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = lshr i64 %x, 40 + %result = urem i64 %x.shr, 32768 diff --git a/llvm-patches/patch202753.patch b/llvm-patches/patch202753.patch new file mode 100644 index 0000000000..348fc144f3 --- /dev/null +++ b/llvm-patches/patch202753.patch @@ -0,0 +1,3224 @@ +diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +index c5501236edb5b9..394c2326a64ef0 100644 +--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp ++++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +@@ -182,14 +182,15 @@ class AMDGPUCodeGenPrepareImpl + unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den, + unsigned MaxDivBits, bool Signed) const; + +- /// Expands 24 bit div or rem. +- Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I, +- Value *Num, Value *Den, +- bool IsDiv, bool IsSigned) const; ++ /// Expands div or rem by using floating-point operations. ++ /// Operands must be in the range [-0x400000,0x3FFFFF] ++ Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I, ++ Value *Num, Value *Den, bool IsDiv, ++ bool IsSigned) const; + +- Value *expandDivRem24Impl(IRBuilder<> &Builder, BinaryOperator &I, +- Value *Num, Value *Den, unsigned NumBits, +- bool IsDiv, bool IsSigned) const; ++ Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I, ++ Value *Num, Value *Den, unsigned NumBits, ++ bool IsDiv, bool IsSigned) const; + + /// Expands 32 bit div or rem. + Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I, +@@ -1055,32 +1056,39 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num, + return DivBits; + } + +-// The fractional part of a float is enough to accurately represent up to +-// a 24-bit signed integer. +-Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder, +- BinaryOperator &I, Value *Num, +- Value *Den, bool IsDiv, +- bool IsSigned) const { +- unsigned DivBits = getDivNumBits(I, Num, Den, 24, IsSigned); ++Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder, ++ BinaryOperator &I, ++ Value *Num, Value *Den, ++ bool IsDiv, ++ bool IsSigned) const { ++ unsigned DivBits = getDivNumBits(I, Num, Den, 23, IsSigned); + +- // v_rcp_f32(float(X)) can have an error of 1 ulp. +- // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly +- // when abs(Y)>0x800000. +- // For example, +- // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. +- // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. +- // +- // Note that for DivBits==24 && IsSigned, Y is in the range +- // [-0x800000:0x7FFFFF]. abs(Y) is at most +- // 0x800000 so it cannot hit this issue. +- if (DivBits > (IsSigned ? 24 : 23)) ++ if (DivBits > (IsSigned ? 23 : 22)) + return nullptr; +- return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned); ++ return expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits, IsDiv, ++ IsSigned); + } + +-Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl( ++Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl( + IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den, + unsigned DivBits, bool IsDiv, bool IsSigned) const { ++ ++ // v_rcp_f32(float(X)) can have an error of 1 ulp. ++ // This would cause incorrect calculation of Y/X if: ++ // Y = (0x7FFFFF/X)*(X-0)-1 ++ // were allowed. ++ // ++ // For example, ++ // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059. ++ // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257. ++ // ++ // Thus, we conservatively restrict expandDivRemToFloatImpl to ++ // [-0x40000,0x3FFFFF] for IsSigned ++ // [0x000000,0x3FFFFF] for !IsSigned. ++ assert(DivBits <= (IsSigned ? 23 : 22) && ++ "abs(Num) must be <= than 0x40000 for expandDivRemToFloatImpl to work " ++ "correctly"); ++ + Type *I32Ty = Builder.getInt32Ty(); + Num = Builder.CreateTrunc(Num, I32Ty); + Den = Builder.CreateTrunc(Den, I32Ty); +@@ -1255,7 +1263,7 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder, + } + } + +- if (Value *Res = expandDivRem24(Builder, I, X, Y, IsDiv, IsSigned)) { ++ if (Value *Res = expandDivRemToFloat(Builder, I, X, Y, IsDiv, IsSigned)) { + return IsSigned ? Builder.CreateSExtOrTrunc(Res, Ty) : + Builder.CreateZExtOrTrunc(Res, Ty); + } +@@ -1364,19 +1372,9 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder, + return nullptr; + + Value *Narrowed = nullptr; +- // v_rcp_f32(float(X)) can have an error of 1 ulp. +- // This can cause expandDivRem24Impl to sometimes calculate Y/X incorrectly +- // when abs(Y)>0x800000. +- // For example, +- // (0xbf2758/0xbf2759) erroneously produces 1 instead of 0. +- // (0xe3170d/0x000c32) erroneously produces 4767 instead of 4766. +- // +- // Note that for NumDivBits==24 && IsSigned, Y is in the range +- // [-0x800000:0x7FFFFF]. abs(Y) is at most +- // 0x800000 so it cannot hit this issue. +- if (NumDivBits <= (IsSigned ? 24 : 23)) { +- Narrowed = expandDivRem24Impl(Builder, I, Num, Den, NumDivBits, +- IsDiv, IsSigned); ++ if (NumDivBits <= (IsSigned ? 23 : 22)) { ++ Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, NumDivBits, IsDiv, ++ IsSigned); + } else if (NumDivBits <= 32) { + Narrowed = expandDivRem32(Builder, I, Num, Den); + } +diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +index b883f82c0ca27f..322d27934cc6ef 100644 +--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll ++++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +@@ -10135,7 +10135,7 @@ entry: + ret <2 x i64> %B + } + +-; 23-bit sdiv, can use expandDivRem24Impl ++; 23-bit sdiv, can use expandDivRemToFloatImpl + define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { + ; GFX6-LABEL: sdiv23: + ; GFX6: ; %bb.0: +@@ -10196,31 +10196,44 @@ define amdgpu_kernel void @sdiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { + ret void + } + +-; 24-bit sdiv, can use expandDivRem24Impl ++; 24-bit sdiv, cannot use expandDivRemToFloatImpl + define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { + ; GFX6-LABEL: sdiv24: + ; GFX6: ; %bb.0: + ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 + ; GFX6-NEXT: s_waitcnt lgkmcnt(0) +-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3] +-; GFX6-NEXT: s_bfe_i32 s5, s5, 0x180000 +-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5 +-; GFX6-NEXT: s_bfe_i32 s4, s4, 0x180000 +-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 +-; GFX6-NEXT: s_xor_b32 s4, s4, s5 +-; GFX6-NEXT: v_rcp_f32_e32 v2, v0 +-; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +-; GFX6-NEXT: s_or_b32 s6, s4, 1 +-; GFX6-NEXT: s_mov_b32 s3, 0xf000 +-; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GFX6-NEXT: v_trunc_f32_e32 v2, v2 +-; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| +-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec +-; GFX6-NEXT: s_cselect_b32 s4, s6, 0 ++; GFX6-NEXT: s_bfe_i32 s4, s3, 0x180000 ++; GFX6-NEXT: s_abs_i32 s5, s4 ++; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; GFX6-NEXT: s_sub_i32 s3, 0, s5 ++; GFX6-NEXT: s_bfe_i32 s6, s2, 0x180000 ++; GFX6-NEXT: s_abs_i32 s7, s6 ++; GFX6-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX6-NEXT: s_xor_b32 s4, s6, s4 ++; GFX6-NEXT: s_ashr_i32 s4, s4, 31 + ; GFX6-NEXT: s_mov_b32 s2, -1 +-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2 ++; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0 ++; GFX6-NEXT: v_readfirstlane_b32 s6, v0 ++; GFX6-NEXT: s_mul_i32 s6, s6, s5 ++; GFX6-NEXT: s_sub_i32 s6, s7, s6 ++; GFX6-NEXT: s_sub_i32 s7, s6, s5 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s6, s5 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: s_cselect_b32 s6, s7, s6 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s6, s5 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0 ++; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0 + ; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 + ; GFX6-NEXT: s_waitcnt expcnt(0) + ; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +@@ -10233,21 +10246,34 @@ define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { + ; GFX9-NEXT: v_mov_b32_e32 v1, 0 + ; GFX9-NEXT: s_waitcnt lgkmcnt(0) + ; GFX9-NEXT: s_bfe_i32 s3, s3, 0x180000 +-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3 ++; GFX9-NEXT: s_abs_i32 s4, s3 ++; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GFX9-NEXT: s_sub_i32 s5, 0, s4 + ; GFX9-NEXT: s_bfe_i32 s2, s2, 0x180000 +-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2 ++; GFX9-NEXT: s_xor_b32 s3, s2, s3 ++; GFX9-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX9-NEXT: s_abs_i32 s2, s2 ++; GFX9-NEXT: s_ashr_i32 s3, s3, 31 ++; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX9-NEXT: v_readfirstlane_b32 s6, v0 ++; GFX9-NEXT: s_mul_i32 s5, s5, s6 ++; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5 ++; GFX9-NEXT: s_add_i32 s6, s6, s5 ++; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6 ++; GFX9-NEXT: s_mul_i32 s6, s5, s4 ++; GFX9-NEXT: s_sub_i32 s2, s2, s6 ++; GFX9-NEXT: s_add_i32 s7, s5, 1 ++; GFX9-NEXT: s_sub_i32 s6, s2, s4 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s4 ++; GFX9-NEXT: s_cselect_b32 s5, s7, s5 ++; GFX9-NEXT: s_cselect_b32 s2, s6, s2 ++; GFX9-NEXT: s_add_i32 s6, s5, 1 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s4 ++; GFX9-NEXT: s_cselect_b32 s2, s6, s5 + ; GFX9-NEXT: s_xor_b32 s2, s2, s3 +-; GFX9-NEXT: v_rcp_f32_e32 v3, v0 +-; GFX9-NEXT: s_ashr_i32 s2, s2, 30 +-; GFX9-NEXT: s_or_b32 s4, s2, 1 +-; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 +-; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +-; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 +-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| +-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec +-; GFX9-NEXT: s_cselect_b32 s2, s4, 0 +-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 ++; GFX9-NEXT: s_sub_i32 s2, s2, s3 ++; GFX9-NEXT: v_mov_b32_e32 v0, s2 + ; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 + ; GFX9-NEXT: global_store_short v1, v0, s[0:1] + ; GFX9-NEXT: s_endpgm +@@ -10256,15 +10282,15 @@ define amdgpu_kernel void @sdiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { + ret void + } + +-; 23-bit udiv, can use expandDivRem24Impl +-define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { +-; GFX6-LABEL: udiv23: ++; 22-bit udiv, can use expandDivRemToFloatImpl ++define amdgpu_kernel void @udiv22(ptr addrspace(1) %out, i22 %x, i22 %y) { ++; GFX6-LABEL: udiv22: + ; GFX6: ; %bb.0: + ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 + ; GFX6-NEXT: s_waitcnt lgkmcnt(0) +-; GFX6-NEXT: s_and_b32 s3, s3, 0x7fffff ++; GFX6-NEXT: s_and_b32 s3, s3, 0x3fffff + ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; GFX6-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX6-NEXT: s_and_b32 s2, s2, 0x3fffff + ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2 + ; GFX6-NEXT: s_mov_b32 s3, 0xf000 + ; GFX6-NEXT: v_rcp_f32_e32 v2, v0 +@@ -10277,18 +10303,18 @@ define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { + ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc + ; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 + ; GFX6-NEXT: s_waitcnt expcnt(0) +-; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 ++; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 6 + ; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 + ; GFX6-NEXT: s_endpgm + ; +-; GFX9-LABEL: udiv23: ++; GFX9-LABEL: udiv22: + ; GFX9: ; %bb.0: + ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 + ; GFX9-NEXT: v_mov_b32_e32 v3, 0 + ; GFX9-NEXT: s_waitcnt lgkmcnt(0) +-; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff ++; GFX9-NEXT: s_and_b32 s3, s3, 0x3fffff + ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX9-NEXT: s_and_b32 s2, s2, 0x3fffff + ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2 + ; GFX9-NEXT: v_rcp_f32_e32 v2, v0 + ; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2 +@@ -10298,15 +10324,91 @@ define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { + ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 + ; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc + ; GFX9-NEXT: global_store_short v3, v0, s[0:1] +-; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; GFX9-NEXT: v_and_b32_e32 v0, 0x3fffff, v0 + ; GFX9-NEXT: global_store_byte_d16_hi v3, v0, s[0:1] offset:2 ++; GFX9-NEXT: s_endpgm ++ %r = udiv i22 %x, %y ++ store i22 %r, ptr addrspace(1) %out ++ ret void ++} ++ ++; 23-bit udiv, cannot use expandDivRemToFloatImpl ++define amdgpu_kernel void @udiv23(ptr addrspace(1) %out, i23 %x, i23 %y) { ++; GFX6-LABEL: udiv23: ++; GFX6: ; %bb.0: ++; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GFX6-NEXT: s_waitcnt lgkmcnt(0) ++; GFX6-NEXT: s_and_b32 s4, s3, 0x7fffff ++; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GFX6-NEXT: s_sub_i32 s3, 0, s4 ++; GFX6-NEXT: s_and_b32 s5, s2, 0x7fffff ++; GFX6-NEXT: s_mov_b32 s2, -1 ++; GFX6-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX6-NEXT: v_mul_lo_u32 v1, s3, v0 ++; GFX6-NEXT: s_mov_b32 s3, 0xf000 ++; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0 ++; GFX6-NEXT: v_readfirstlane_b32 s6, v0 ++; GFX6-NEXT: s_mul_i32 s6, s6, s4 ++; GFX6-NEXT: s_sub_i32 s5, s5, s6 ++; GFX6-NEXT: s_sub_i32 s6, s5, s4 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: s_cselect_b32 s5, s6, s5 ++; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GFX6-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX6-NEXT: s_cselect_b64 vcc, -1, 0 ++; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0 ++; GFX6-NEXT: s_waitcnt expcnt(0) ++; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 7 ++; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2 ++; GFX6-NEXT: s_endpgm ++; ++; GFX9-LABEL: udiv23: ++; GFX9: ; %bb.0: ++; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ++; GFX9-NEXT: v_mov_b32_e32 v1, 0 ++; GFX9-NEXT: s_waitcnt lgkmcnt(0) ++; GFX9-NEXT: s_and_b32 s3, s3, 0x7fffff ++; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3 ++; GFX9-NEXT: s_sub_i32 s4, 0, s3 ++; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX9-NEXT: v_rcp_f32_e32 v0, v0 ++; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GFX9-NEXT: v_readfirstlane_b32 s5, v0 ++; GFX9-NEXT: s_mul_i32 s4, s4, s5 ++; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4 ++; GFX9-NEXT: s_add_i32 s5, s5, s4 ++; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5 ++; GFX9-NEXT: s_mul_i32 s5, s4, s3 ++; GFX9-NEXT: s_sub_i32 s2, s2, s5 ++; GFX9-NEXT: s_add_i32 s6, s4, 1 ++; GFX9-NEXT: s_sub_i32 s5, s2, s3 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s3 ++; GFX9-NEXT: s_cselect_b32 s4, s6, s4 ++; GFX9-NEXT: s_cselect_b32 s2, s5, s2 ++; GFX9-NEXT: s_add_i32 s5, s4, 1 ++; GFX9-NEXT: s_cmp_ge_u32 s2, s3 ++; GFX9-NEXT: s_cselect_b32 s2, s5, s4 ++; GFX9-NEXT: v_mov_b32_e32 v0, s2 ++; GFX9-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX9-NEXT: global_store_short v1, v0, s[0:1] ++; GFX9-NEXT: v_mov_b32_e32 v0, s2 ++; GFX9-NEXT: global_store_byte_d16_hi v1, v0, s[0:1] offset:2 + ; GFX9-NEXT: s_endpgm + %r = udiv i23 %x, %y + store i23 %r, ptr addrspace(1) %out + ret void + } + +-; 24-bit udiv, cannot use expandDivRem24Impl ++; 24-bit udiv, cannot use expandDivRemToFloatImpl + define amdgpu_kernel void @udiv24(ptr addrspace(1) %out, i24 %x, i24 %y) { + ; GFX6-LABEL: udiv24: + ; GFX6: ; %bb.0: +diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll +index b690879dab99bd..b082c59f84abcf 100644 +--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll ++++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll +@@ -1801,23 +1801,37 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v0 + ; GCN-NEXT: s_waitcnt vmcnt(2) + ; GCN-NEXT: v_or_b32_e32 v1, v1, v4 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, v1 ++; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 ++; GCN-NEXT: v_max_i32_e32 v1, v1, v4 ++; GCN-NEXT: v_cvt_f32_u32_e32 v4, v1 ++; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v1 + ; GCN-NEXT: s_waitcnt vmcnt(1) +-; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ++; GCN-NEXT: v_lshlrev_b32_e32 v6, 16, v2 ++; GCN-NEXT: v_rcp_f32_e32 v4, v4 + ; GCN-NEXT: s_waitcnt vmcnt(0) +-; GCN-NEXT: v_or_b32_e32 v3, v3, v4 +-; GCN-NEXT: v_cvt_f32_i32_e32 v3, v3 +-; GCN-NEXT: v_rcp_f32_e32 v4, v1 ++; GCN-NEXT: v_or_b32_e32 v3, v3, v6 ++; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v3 ++; GCN-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 ++; GCN-NEXT: v_max_i32_e32 v3, v3, v6 + ; GCN-NEXT: v_xor_b32_e32 v0, v2, v0 +-; GCN-NEXT: v_ashrrev_i32_e32 v0, 30, v0 +-; GCN-NEXT: v_or_b32_e32 v0, 1, v0 +-; GCN-NEXT: v_mul_f32_e32 v2, v3, v4 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v3, -v2, v1, v3 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v1| +-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ++; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ++; GCN-NEXT: v_mul_lo_u32 v5, v5, v4 ++; GCN-NEXT: v_mul_hi_u32 v5, v4, v5 ++; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 ++; GCN-NEXT: v_mul_hi_u32 v4, v3, v4 ++; GCN-NEXT: v_mul_lo_u32 v2, v4, v1 ++; GCN-NEXT: v_add_i32_e32 v5, vcc, 1, v4 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v3, v2 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v1 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 ++; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-NEXT: v_add_i32_e32 v3, vcc, 1, v4 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc ++; GCN-NEXT: v_xor_b32_e32 v1, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 + ; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 + ; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; GCN-NEXT: s_endpgm +@@ -1842,23 +1856,37 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i + ; TONGA-NEXT: v_lshlrev_b32_e32 v4, 16, v0 + ; TONGA-NEXT: s_waitcnt vmcnt(2) + ; TONGA-NEXT: v_or_b32_e32 v1, v1, v4 +-; TONGA-NEXT: v_cvt_f32_i32_e32 v1, v1 ++; TONGA-NEXT: v_sub_u32_e32 v4, vcc, 0, v1 ++; TONGA-NEXT: v_max_i32_e32 v1, v1, v4 ++; TONGA-NEXT: v_cvt_f32_u32_e32 v4, v1 ++; TONGA-NEXT: v_sub_u32_e32 v5, vcc, 0, v1 + ; TONGA-NEXT: s_waitcnt vmcnt(1) +-; TONGA-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ++; TONGA-NEXT: v_lshlrev_b32_e32 v6, 16, v2 ++; TONGA-NEXT: v_rcp_f32_e32 v4, v4 + ; TONGA-NEXT: s_waitcnt vmcnt(0) +-; TONGA-NEXT: v_or_b32_e32 v3, v3, v4 +-; TONGA-NEXT: v_cvt_f32_i32_e32 v3, v3 +-; TONGA-NEXT: v_rcp_f32_e32 v4, v1 ++; TONGA-NEXT: v_or_b32_e32 v3, v3, v6 ++; TONGA-NEXT: v_sub_u32_e32 v6, vcc, 0, v3 ++; TONGA-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4 ++; TONGA-NEXT: v_cvt_u32_f32_e32 v4, v4 ++; TONGA-NEXT: v_max_i32_e32 v3, v3, v6 + ; TONGA-NEXT: v_xor_b32_e32 v0, v2, v0 +-; TONGA-NEXT: v_ashrrev_i32_e32 v0, 30, v0 +-; TONGA-NEXT: v_or_b32_e32 v0, 1, v0 +-; TONGA-NEXT: v_mul_f32_e32 v2, v3, v4 +-; TONGA-NEXT: v_trunc_f32_e32 v2, v2 +-; TONGA-NEXT: v_mad_f32 v3, -v2, v1, v3 +-; TONGA-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; TONGA-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v1| +-; TONGA-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +-; TONGA-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ++; TONGA-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ++; TONGA-NEXT: v_mul_lo_u32 v5, v5, v4 ++; TONGA-NEXT: v_mul_hi_u32 v5, v4, v5 ++; TONGA-NEXT: v_add_u32_e32 v4, vcc, v4, v5 ++; TONGA-NEXT: v_mul_hi_u32 v4, v3, v4 ++; TONGA-NEXT: v_mul_lo_u32 v2, v4, v1 ++; TONGA-NEXT: v_add_u32_e32 v5, vcc, 1, v4 ++; TONGA-NEXT: v_sub_u32_e32 v2, vcc, v3, v2 ++; TONGA-NEXT: v_sub_u32_e32 v3, vcc, v2, v1 ++; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 ++; TONGA-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc ++; TONGA-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; TONGA-NEXT: v_add_u32_e32 v3, vcc, 1, v4 ++; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v2, v1 ++; TONGA-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc ++; TONGA-NEXT: v_xor_b32_e32 v1, v1, v0 ++; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v1, v0 + ; TONGA-NEXT: v_bfe_i32 v0, v0, 0, 24 + ; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; TONGA-NEXT: s_endpgm +@@ -1878,28 +1906,45 @@ define amdgpu_kernel void @v_sdiv_i24(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GFX9-NEXT: buffer_load_sbyte v2, off, s[4:7], 0 offset:2 + ; GFX9-NEXT: buffer_load_ushort v3, off, s[4:7], 0 + ; GFX9-NEXT: s_mov_b32 s0, s8 +-; GFX9-NEXT: s_mov_b32 s1, s9 + ; GFX9-NEXT: s_waitcnt vmcnt(3) +-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ++; GFX9-NEXT: v_readfirstlane_b32 s1, v0 + ; GFX9-NEXT: s_waitcnt vmcnt(2) +-; GFX9-NEXT: v_or_b32_e32 v1, v1, v4 +-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1 ++; GFX9-NEXT: v_readfirstlane_b32 s4, v1 ++; GFX9-NEXT: s_lshl_b32 s1, s1, 16 ++; GFX9-NEXT: s_or_b32 s1, s4, s1 ++; GFX9-NEXT: s_abs_i32 s4, s1 ++; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4 + ; GFX9-NEXT: s_waitcnt vmcnt(1) +-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ++; GFX9-NEXT: v_readfirstlane_b32 s5, v2 + ; GFX9-NEXT: s_waitcnt vmcnt(0) +-; GFX9-NEXT: v_or_b32_e32 v3, v3, v4 +-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3 +-; GFX9-NEXT: v_rcp_f32_e32 v4, v1 ++; GFX9-NEXT: v_readfirstlane_b32 s6, v3 ++; GFX9-NEXT: s_lshl_b32 s5, s5, 16 ++; GFX9-NEXT: v_rcp_f32_e32 v1, v1 ++; GFX9-NEXT: s_or_b32 s5, s6, s5 ++; GFX9-NEXT: s_sub_i32 s6, 0, s4 ++; GFX9-NEXT: s_abs_i32 s5, s5 ++; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 + ; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 +-; GFX9-NEXT: v_ashrrev_i32_e32 v0, 30, v0 +-; GFX9-NEXT: v_or_b32_e32 v0, 1, v0 +-; GFX9-NEXT: v_mul_f32_e32 v2, v3, v4 +-; GFX9-NEXT: v_trunc_f32_e32 v2, v2 +-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v2 +-; GFX9-NEXT: v_mad_f32 v2, -v2, v1, v3 +-; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v1| +-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +-; GFX9-NEXT: v_add_u32_e32 v0, v4, v0 ++; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v0 ++; GFX9-NEXT: s_mov_b32 s1, s9 ++; GFX9-NEXT: v_readfirstlane_b32 s7, v1 ++; GFX9-NEXT: s_mul_i32 s6, s6, s7 ++; GFX9-NEXT: s_mul_hi_u32 s6, s7, s6 ++; GFX9-NEXT: s_add_i32 s7, s7, s6 ++; GFX9-NEXT: s_mul_hi_u32 s6, s5, s7 ++; GFX9-NEXT: s_mul_i32 s7, s6, s4 ++; GFX9-NEXT: s_sub_i32 s5, s5, s7 ++; GFX9-NEXT: s_add_i32 s8, s6, 1 ++; GFX9-NEXT: s_sub_i32 s7, s5, s4 ++; GFX9-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX9-NEXT: s_cselect_b32 s6, s8, s6 ++; GFX9-NEXT: s_cselect_b32 s5, s7, s5 ++; GFX9-NEXT: s_add_i32 s7, s6, 1 ++; GFX9-NEXT: s_cmp_ge_u32 s5, s4 ++; GFX9-NEXT: s_cselect_b32 s4, s7, s6 ++; GFX9-NEXT: v_xor_b32_e32 v1, s4, v0 ++; GFX9-NEXT: v_sub_u32_e32 v0, v1, v0 + ; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 24 + ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; GFX9-NEXT: s_endpgm +diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll +index 68466abf31aa0d..a39d3fb10a2fa6 100644 +--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll ++++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll +@@ -480,63 +480,89 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) { + define amdgpu_kernel void @s_test_sdiv24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { + ; GCN-LABEL: s_test_sdiv24_64: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_load_dword s2, s[4:5], 0xe ++; GCN-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: s_mov_b32 s4, s0 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_ashr_i32 s0, s2, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 ++; GCN-NEXT: s_ashr_i32 s8, s0, 8 ++; GCN-NEXT: s_abs_i32 s9, s8 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_sub_i32 s2, 0, s9 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: s_ashr_i32 s0, s3, 8 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: s_ashr_i32 s1, s3, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: s_xor_b32 s0, s1, s0 +-; GCN-NEXT: s_ashr_i32 s0, s0, 30 +-; GCN-NEXT: s_or_b32 s2, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s2, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-NEXT: s_abs_i32 s2, s0 ++; GCN-NEXT: s_xor_b32 s0, s0, s8 ++; GCN-NEXT: s_ashr_i32 s0, s0, 31 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mul_i32 s3, s1, s9 ++; GCN-NEXT: s_sub_i32 s2, s2, s3 ++; GCN-NEXT: s_add_i32 s8, s1, 1 ++; GCN-NEXT: s_sub_i32 s3, s2, s9 ++; GCN-NEXT: s_cmp_ge_u32 s2, s9 ++; GCN-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-NEXT: s_add_i32 s3, s1, 1 ++; GCN-NEXT: s_cmp_ge_u32 s2, s9 ++; GCN-NEXT: s_cselect_b32 s1, s3, s1 ++; GCN-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_sdiv24_64: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe ++; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: s_mov_b32 s4, s0 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_ashr_i32 s0, s2, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 ++; GCN-IR-NEXT: s_ashr_i32 s8, s0, 8 ++; GCN-IR-NEXT: s_abs_i32 s9, s8 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s9 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: s_ashr_i32 s1, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: s_xor_b32 s0, s1, s0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 +-; GCN-IR-NEXT: s_or_b32 s2, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-IR-NEXT: s_abs_i32 s2, s0 ++; GCN-IR-NEXT: s_xor_b32 s0, s0, s8 ++; GCN-IR-NEXT: s_ashr_i32 s0, s0, 31 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mul_i32 s3, s1, s9 ++; GCN-IR-NEXT: s_sub_i32 s2, s2, s3 ++; GCN-IR-NEXT: s_add_i32 s8, s1, 1 ++; GCN-IR-NEXT: s_sub_i32 s3, s2, s9 ++; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 ++; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-IR-NEXT: s_add_i32 s3, s1, 1 ++; GCN-IR-NEXT: s_cmp_ge_u32 s2, s9 ++; GCN-IR-NEXT: s_cselect_b32 s1, s3, s1 ++; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = ashr i64 %x, 40 +@@ -958,92 +984,146 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(ptr addrspace(1) %out, <2 x i64> + ; GCN-LABEL: s_test_sdiv24_v2i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd +-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +-; GCN-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-NEXT: s_mov_b32 s2, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_ashr_i32 s4, s13, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s4 +-; GCN-NEXT: s_ashr_i32 s5, s9, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s5 +-; GCN-NEXT: s_xor_b32 s4, s5, s4 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: s_ashr_i32 s4, s4, 30 ++; GCN-NEXT: s_ashr_i32 s0, s13, 8 ++; GCN-NEXT: s_abs_i32 s1, s0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s1 ++; GCN-NEXT: s_sub_i32 s2, 0, s1 + ; GCN-NEXT: s_ashr_i32 s6, s11, 8 + ; GCN-NEXT: s_ashr_i32 s7, s15, 8 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: s_or_b32 s8, s4, 1 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| +-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec +-; GCN-NEXT: s_cselect_b32 s4, s8, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v2 +-; GCN-NEXT: v_cvt_f32_i32_e32 v2, s7 +-; GCN-NEXT: v_cvt_f32_i32_e32 v3, s6 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-NEXT: s_ashr_i32 s2, s9, 8 ++; GCN-NEXT: s_abs_i32 s3, s2 ++; GCN-NEXT: s_xor_b32 s0, s2, s0 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: s_ashr_i32 s8, s0, 31 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 ++; GCN-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-NEXT: s_mul_i32 s2, s0, s1 ++; GCN-NEXT: s_sub_i32 s2, s3, s2 ++; GCN-NEXT: s_add_i32 s9, s0, 1 ++; GCN-NEXT: s_sub_i32 s3, s2, s1 ++; GCN-NEXT: s_cmp_ge_u32 s2, s1 ++; GCN-NEXT: s_cselect_b32 s0, s9, s0 ++; GCN-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-NEXT: s_add_i32 s3, s0, 1 ++; GCN-NEXT: s_cmp_ge_u32 s2, s1 ++; GCN-NEXT: s_cselect_b32 s9, s3, s0 ++; GCN-NEXT: s_abs_i32 s10, s7 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s10 ++; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ++; GCN-NEXT: s_sub_i32 s4, 0, s10 ++; GCN-NEXT: s_abs_i32 s5, s6 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-NEXT: s_mov_b32 s2, -1 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 + ; GCN-NEXT: s_xor_b32 s4, s6, s7 +-; GCN-NEXT: s_ashr_i32 s4, s4, 30 +-; GCN-NEXT: v_rcp_f32_e32 v4, v2 +-; GCN-NEXT: s_or_b32 s6, s4, 1 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2| +-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec +-; GCN-NEXT: s_cselect_b32 s4, s6, 0 +-; GCN-NEXT: v_add_i32_e32 v2, vcc, s4, v4 +-; GCN-NEXT: v_bfe_i32 v2, v2, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v2 ++; GCN-NEXT: s_xor_b32 s6, s9, s8 ++; GCN-NEXT: s_sub_i32 s6, s6, s8 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: s_ashr_i32 s7, s6, 31 ++; GCN-NEXT: s_ashr_i32 s4, s4, 31 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, s5, v0 ++; GCN-NEXT: v_mov_b32_e32 v0, s6 ++; GCN-NEXT: v_mov_b32_e32 v1, s7 ++; GCN-NEXT: v_readfirstlane_b32 s6, v2 ++; GCN-NEXT: s_mul_i32 s7, s6, s10 ++; GCN-NEXT: s_sub_i32 s5, s5, s7 ++; GCN-NEXT: s_add_i32 s8, s6, 1 ++; GCN-NEXT: s_sub_i32 s7, s5, s10 ++; GCN-NEXT: s_cmp_ge_u32 s5, s10 ++; GCN-NEXT: s_cselect_b32 s6, s8, s6 ++; GCN-NEXT: s_cselect_b32 s5, s7, s5 ++; GCN-NEXT: s_add_i32 s7, s6, 1 ++; GCN-NEXT: s_cmp_ge_u32 s5, s10 ++; GCN-NEXT: s_cselect_b32 s5, s7, s6 ++; GCN-NEXT: s_xor_b32 s5, s5, s4 ++; GCN-NEXT: s_sub_i32 s4, s5, s4 ++; GCN-NEXT: s_ashr_i32 s5, s4, 31 ++; GCN-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-NEXT: v_mov_b32_e32 v3, s5 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_sdiv24_v2i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd +-; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 +-; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-IR-NEXT: s_mov_b32 s2, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_ashr_i32 s4, s13, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s4 +-; GCN-IR-NEXT: s_ashr_i32 s5, s9, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s5 +-; GCN-IR-NEXT: s_xor_b32 s4, s5, s4 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 ++; GCN-IR-NEXT: s_ashr_i32 s0, s13, 8 ++; GCN-IR-NEXT: s_abs_i32 s1, s0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s1 + ; GCN-IR-NEXT: s_ashr_i32 s6, s11, 8 + ; GCN-IR-NEXT: s_ashr_i32 s7, s15, 8 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: s_or_b32 s8, s4, 1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], exec +-; GCN-IR-NEXT: s_cselect_b32 s4, s8, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s4, v2 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, s7 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v3, s6 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-IR-NEXT: s_ashr_i32 s2, s9, 8 ++; GCN-IR-NEXT: s_abs_i32 s3, s2 ++; GCN-IR-NEXT: s_xor_b32 s0, s2, s0 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: s_ashr_i32 s8, s0, 31 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-IR-NEXT: s_mul_i32 s2, s0, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s3, s2 ++; GCN-IR-NEXT: s_add_i32 s9, s0, 1 ++; GCN-IR-NEXT: s_sub_i32 s3, s2, s1 ++; GCN-IR-NEXT: s_cmp_ge_u32 s2, s1 ++; GCN-IR-NEXT: s_cselect_b32 s0, s9, s0 ++; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-IR-NEXT: s_add_i32 s3, s0, 1 ++; GCN-IR-NEXT: s_cmp_ge_u32 s2, s1 ++; GCN-IR-NEXT: s_cselect_b32 s9, s3, s0 ++; GCN-IR-NEXT: s_abs_i32 s10, s7 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s10 ++; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ++; GCN-IR-NEXT: s_sub_i32 s4, 0, s10 ++; GCN-IR-NEXT: s_abs_i32 s5, s6 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-IR-NEXT: s_mov_b32 s2, -1 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 + ; GCN-IR-NEXT: s_xor_b32 s4, s6, s7 +-; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 +-; GCN-IR-NEXT: v_rcp_f32_e32 v4, v2 +-; GCN-IR-NEXT: s_or_b32 s6, s4, 1 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-IR-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v4, v4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2| +-; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], exec +-; GCN-IR-NEXT: s_cselect_b32 s4, s6, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, s4, v4 +-; GCN-IR-NEXT: v_bfe_i32 v2, v2, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v3, 31, v2 ++; GCN-IR-NEXT: s_xor_b32 s6, s9, s8 ++; GCN-IR-NEXT: s_sub_i32 s6, s6, s8 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: s_ashr_i32 s7, s6, 31 ++; GCN-IR-NEXT: s_ashr_i32 s4, s4, 31 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, s5, v0 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s6 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s7 ++; GCN-IR-NEXT: v_readfirstlane_b32 s6, v2 ++; GCN-IR-NEXT: s_mul_i32 s7, s6, s10 ++; GCN-IR-NEXT: s_sub_i32 s5, s5, s7 ++; GCN-IR-NEXT: s_add_i32 s8, s6, 1 ++; GCN-IR-NEXT: s_sub_i32 s7, s5, s10 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s10 ++; GCN-IR-NEXT: s_cselect_b32 s6, s8, s6 ++; GCN-IR-NEXT: s_cselect_b32 s5, s7, s5 ++; GCN-IR-NEXT: s_add_i32 s7, s6, 1 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s10 ++; GCN-IR-NEXT: s_cselect_b32 s5, s7, s6 ++; GCN-IR-NEXT: s_xor_b32 s5, s5, s4 ++; GCN-IR-NEXT: s_sub_i32 s4, s5, s4 ++; GCN-IR-NEXT: s_ashr_i32 s5, s4, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, s5 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = ashr <2 x i64> %x, +@@ -1056,32 +1136,46 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(ptr addrspace(1) %out, <2 x i64> + define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %y) { + ; GCN-LABEL: s_test_sdiv24_48: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd ++; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_mov_b32 s4, s0 +-; GCN-NEXT: s_sext_i32_i16 s9, s9 +-; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: s_lshr_b64 s[0:1], s[8:9], 24 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 ++; GCN-NEXT: s_sext_i32_i16 s1, s1 ++; GCN-NEXT: s_lshr_b64 s[8:9], s[0:1], 24 ++; GCN-NEXT: s_abs_i32 s9, s8 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s9 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_sub_i32 s4, 0, s9 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_sext_i32_i16 s3, s3 + ; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: s_xor_b32 s0, s2, s0 +-; GCN-NEXT: s_ashr_i32 s0, s0, 30 +-; GCN-NEXT: s_or_b32 s2, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s2, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v2 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: s_mov_b32 s5, s1 ++; GCN-NEXT: s_xor_b32 s1, s2, s8 ++; GCN-NEXT: s_ashr_i32 s1, s1, 31 ++; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: s_abs_i32 s0, s2 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s0, v0 ++; GCN-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-NEXT: s_mul_i32 s2, s2, s9 ++; GCN-NEXT: s_sub_i32 s0, s0, s2 ++; GCN-NEXT: s_sub_i32 s2, s0, s9 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s9 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: s_cselect_b32 s0, s2, s0 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s9 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: v_xor_b32_e32 v0, s1, v0 ++; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 + ; GCN-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 +@@ -1089,32 +1183,46 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 % + ; + ; GCN-IR-LABEL: s_test_sdiv24_48: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd ++; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_mov_b32 s4, s0 +-; GCN-IR-NEXT: s_sext_i32_i16 s9, s9 +-; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: s_lshr_b64 s[0:1], s[8:9], 24 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 ++; GCN-IR-NEXT: s_sext_i32_i16 s1, s1 ++; GCN-IR-NEXT: s_lshr_b64 s[8:9], s[0:1], 24 ++; GCN-IR-NEXT: s_abs_i32 s9, s8 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s9 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_sub_i32 s4, 0, s9 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_sext_i32_i16 s3, s3 + ; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: s_xor_b32 s0, s2, s0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 +-; GCN-IR-NEXT: s_or_b32 s2, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s2, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v2 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_mov_b32 s5, s1 ++; GCN-IR-NEXT: s_xor_b32 s1, s2, s8 ++; GCN-IR-NEXT: s_ashr_i32 s1, s1, 31 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: s_abs_i32 s0, s2 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s0, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-IR-NEXT: s_mul_i32 s2, s2, s9 ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s2 ++; GCN-IR-NEXT: s_sub_i32 s2, s0, s9 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: s_cselect_b32 s0, s2, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s9 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: v_xor_b32_e32 v0, s1, v0 ++; GCN-IR-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: buffer_store_dword v0, off, s[4:7], 0 + ; GCN-IR-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 +@@ -1830,23 +1938,35 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(ptr addrspace(1) %out, i64 %x + ; GCN-NEXT: s_mov_b32 s6, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_ashr_i32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-NEXT: s_mov_b32 s2, 0x41c00000 ++; GCN-NEXT: s_abs_i32 s2, s2 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ++; GCN-NEXT: s_sub_i32 s4, 0, s2 ++; GCN-NEXT: s_mov_b32 s5, s1 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 + ; GCN-NEXT: s_mov_b32 s4, s0 + ; GCN-NEXT: s_ashr_i32 s0, s3, 31 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: s_or_b32 s3, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_mad_f32 v2, -v1, v0, s2 +-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v1 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mul_i32 s3, s1, s2 ++; GCN-NEXT: s_sub_i32 s3, 24, s3 ++; GCN-NEXT: s_add_i32 s8, s1, 1 ++; GCN-NEXT: s_sub_i32 s9, s3, s2 ++; GCN-NEXT: s_cmp_ge_u32 s3, s2 ++; GCN-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-NEXT: s_cselect_b32 s3, s9, s3 ++; GCN-NEXT: s_add_i32 s8, s1, 1 ++; GCN-NEXT: s_cmp_ge_u32 s3, s2 ++; GCN-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; +@@ -1857,23 +1977,35 @@ define amdgpu_kernel void @s_test_sdiv24_k_num_i64(ptr addrspace(1) %out, i64 %x + ; GCN-IR-NEXT: s_mov_b32 s6, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-IR-NEXT: s_mov_b32 s2, 0x41c00000 ++; GCN-IR-NEXT: s_abs_i32 s2, s2 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 ++; GCN-IR-NEXT: s_sub_i32 s4, 0, s2 ++; GCN-IR-NEXT: s_mov_b32 s5, s1 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 + ; GCN-IR-NEXT: s_mov_b32 s4, s0 + ; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: s_or_b32 s3, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_mad_f32 v2, -v1, v0, s2 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v1 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mul_i32 s3, s1, s2 ++; GCN-IR-NEXT: s_sub_i32 s3, 24, s3 ++; GCN-IR-NEXT: s_add_i32 s8, s1, 1 ++; GCN-IR-NEXT: s_sub_i32 s9, s3, s2 ++; GCN-IR-NEXT: s_cmp_ge_u32 s3, s2 ++; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-IR-NEXT: s_cselect_b32 s3, s9, s3 ++; GCN-IR-NEXT: s_add_i32 s8, s1, 1 ++; GCN-IR-NEXT: s_cmp_ge_u32 s3, s2 ++; GCN-IR-NEXT: s_cselect_b32 s1, s8, s1 ++; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = ashr i64 %x, 40 +@@ -1886,52 +2018,64 @@ define amdgpu_kernel void @s_test_sdiv24_k_den_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_sdiv24_k_den_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 ++; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_mov_b32 s4, s0 + ; GCN-NEXT: s_ashr_i32 s0, s3, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0 +-; GCN-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-NEXT: s_abs_i32 s0, s0 ++; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-NEXT: v_readfirstlane_b32 s2, v0 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: s_or_b32 s3, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v1 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-NEXT: s_ashr_i32 s1, s3, 31 ++; GCN-NEXT: s_mul_i32 s3, s2, 0x5b7f ++; GCN-NEXT: s_sub_i32 s0, s0, s3 ++; GCN-NEXT: s_add_i32 s3, s2, 1 ++; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-NEXT: s_add_i32 s3, s0, 0xffffa481 ++; GCN-NEXT: s_min_u32 s0, s3, s0 ++; GCN-NEXT: s_add_i32 s3, s2, 1 ++; GCN-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-NEXT: s_cselect_b32 s0, s3, s2 ++; GCN-NEXT: s_xor_b32 s0, s0, s1 ++; GCN-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_sdiv24_k_den_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_mov_b32 s4, s0 + ; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-IR-NEXT: s_abs_i32 s0, s0 ++; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: s_or_b32 s3, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s0, v1 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: s_ashr_i32 s1, s3, 31 ++; GCN-IR-NEXT: s_mul_i32 s3, s2, 0x5b7f ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s3 ++; GCN-IR-NEXT: s_add_i32 s3, s2, 1 ++; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-IR-NEXT: s_cselect_b32 s2, s3, s2 ++; GCN-IR-NEXT: s_add_i32 s3, s0, 0xffffa481 ++; GCN-IR-NEXT: s_min_u32 s0, s3, s0 ++; GCN-IR-NEXT: s_add_i32 s3, s2, 1 ++; GCN-IR-NEXT: s_cmpk_gt_u32 s0, 0x5b7e ++; GCN-IR-NEXT: s_cselect_b32 s0, s3, s2 ++; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 ++; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = ashr i64 %x, 40 +@@ -1945,19 +2089,30 @@ define i64 @v_test_sdiv24_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x41c00000 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v3, -v2, v0, s4 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| +-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +-; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 ++; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ++; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc ++; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; +@@ -1965,19 +2120,30 @@ define i64 @v_test_sdiv24_k_num_i64(i64 %x) { + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x41c00000, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v3, -v2, v0, s4 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| +-; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v2, 24 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 +@@ -1990,19 +2156,31 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x47000000 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ++; GCN-NEXT: s_mov_b32 s4, 0x8000 ++; GCN-NEXT: v_rcp_f32_e32 v2, v2 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v3, -v2, v0, s4 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| +-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +-; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ++; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ++; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc ++; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; +@@ -2010,19 +2188,31 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ++; GCN-IR-NEXT: s_mov_b32 s4, 0x8000 ++; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x47000000, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v3, -v2, v0, s4 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v0| +-; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 +@@ -2043,22 +2233,86 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) { + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_sdiv24_pow2_k_den_i64: +-; GCN-IR: ; %bb.0: ++; GCN-IR: ; %bb.0: ; %_udiv-special-cases + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v1 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x38000000, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v0, -v2, s4, v0 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, s4 +-; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v8 ++; GCN-IR-NEXT: v_xor_b32_e32 v1, v8, v8 ++; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v0, v8 ++; GCN-IR-NEXT: v_subb_u32_e32 v5, vcc, v1, v8, vcc ++; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4 ++; GCN-IR-NEXT: v_add_i32_e64 v0, s[4:5], 32, v0 ++; GCN-IR-NEXT: v_ffbh_u32_e32 v1, v5 ++; GCN-IR-NEXT: v_min_u32_e32 v6, v0, v1 ++; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 48, v6 ++; GCN-IR-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5] ++; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] ++; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[0:1] ++; GCN-IR-NEXT: v_mov_b32_e32 v9, v8 ++; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ++; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[0:1] ++; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ++; GCN-IR-NEXT: v_cndmask_b32_e64 v3, v5, 0, s[4:5] ++; GCN-IR-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5] ++; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc ++; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ++; GCN-IR-NEXT: s_cbranch_execz .LBB18_6 ++; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 ++; GCN-IR-NEXT: v_add_i32_e32 v7, vcc, 1, v0 ++; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v0, s[4:5], 63, v0 ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[4:5], v0 ++; GCN-IR-NEXT: v_mov_b32_e32 v2, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 ++; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execz .LBB18_5 ++; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader ++; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6 ++; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7 ++; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc ++; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v6, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, 0 ++; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff ++; GCN-IR-NEXT: .LBB18_3: ; %udiv-do-while ++; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 ++; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 31, v1 ++; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s10, v4 ++; GCN-IR-NEXT: v_subb_u32_e32 v2, vcc, 0, v5, vcc ++; GCN-IR-NEXT: v_or_b32_e32 v0, v6, v0 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v6, 31, v2 ++; GCN-IR-NEXT: v_and_b32_e32 v2, 1, v6 ++; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6 ++; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6 ++; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 1, v10 ++; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1 ++; GCN-IR-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc ++; GCN-IR-NEXT: v_mov_b32_e32 v7, v3 ++; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] ++; GCN-IR-NEXT: v_mov_b32_e32 v6, v2 ++; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execnz .LBB18_3 ++; GCN-IR-NEXT: ; %bb.4: ; %Flow ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: .LBB18_5: ; %Flow4 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] ++; GCN-IR-NEXT: v_lshl_b64 v[0:1], v[0:1], 1 ++; GCN-IR-NEXT: v_or_b32_e32 v3, v3, v1 ++; GCN-IR-NEXT: v_or_b32_e32 v2, v2, v0 ++; GCN-IR-NEXT: .LBB18_6: ; %Flow5 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v2, v8 ++; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v9 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v8 ++; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 + %result = sdiv i64 %x.shr, 32768 +diff --git a/llvm/test/CodeGen/AMDGPU/sdivrem24.ll b/llvm/test/CodeGen/AMDGPU/sdivrem24.ll +index abfb56a58221f8..4750f17b1a0966 100644 +--- a/llvm/test/CodeGen/AMDGPU/sdivrem24.ll ++++ b/llvm/test/CodeGen/AMDGPU/sdivrem24.ll +@@ -41,10 +41,7 @@ define amdgpu_kernel void @sdiv24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i + } + + ; FUNC-LABEL: {{^}}sdiv24_i32: +-; SI: v_cvt_f32_i32 +-; SI: v_cvt_f32_i32 +-; SI: v_rcp_f32 +-; SI: v_cvt_i32_f32 ++; SI-NOT: v_cvt_i32_f32 + + ; EG: INT_TO_FLT + ; EG-DAG: INT_TO_FLT +@@ -156,10 +153,7 @@ define amdgpu_kernel void @srem24_i16(ptr addrspace(1) %out, ptr addrspace(1) %i + } + + ; FUNC-LABEL: {{^}}srem24_i32: +-; SI: v_cvt_f32_i32 +-; SI: v_cvt_f32_i32 +-; SI: v_rcp_f32 +-; SI: v_cvt_i32_f32 ++; SI-NOT: v_cvt_i32_f32 + + ; EG: INT_TO_FLT + ; EG-DAG: INT_TO_FLT +@@ -269,9 +263,7 @@ define amdgpu_kernel void @no_srem25_i25_i24_i32(ptr addrspace(1) %out, ptr addr + } + + ; FUNC-LABEL: {{^}}srem25_i24_i11_i32: +-; SI: v_cvt_f32_i32 +-; SI: v_rcp_f32 +-; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24 ++; SI-NOT: v_cvt_f32_i32 + + ; EG: INT_TO_FLT + ; EG: RECIP_IEEE +@@ -289,9 +281,7 @@ define amdgpu_kernel void @srem25_i24_i11_i32(ptr addrspace(1) %out, ptr addrspa + } + + ; FUNC-LABEL: {{^}}srem25_i11_i24_i32: +-; SI: v_cvt_f32_i32 +-; SI: v_rcp_f32 +-; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24 ++; SI-NOT: v_cvt_f32_i32 + + ; EG: INT_TO_FLT + ; EG: RECIP_IEEE +diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll +index 82196b73b66e49..f3fdfaadc86ab4 100644 +--- a/llvm/test/CodeGen/AMDGPU/srem64.ll ++++ b/llvm/test/CodeGen/AMDGPU/srem64.ll +@@ -546,34 +546,39 @@ define amdgpu_kernel void @s_test_srem23_64(ptr addrspace(1) %out, i64 %x, i64 % + define amdgpu_kernel void @s_test_srem24_64(ptr addrspace(1) %out, i64 %x, i64 %y) { + ; GCN-LABEL: s_test_srem24_64: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_load_dword s2, s[4:5], 0xe ++; GCN-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: s_ashr_i32 s3, s3, 8 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_ashr_i32 s2, s2, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s3 +-; GCN-NEXT: s_mov_b32 s4, s0 +-; GCN-NEXT: s_xor_b32 s0, s3, s2 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: s_ashr_i32 s0, s0, 30 ++; GCN-NEXT: s_ashr_i32 s0, s0, 8 ++; GCN-NEXT: s_abs_i32 s8, s0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_sub_i32 s2, 0, s8 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: s_or_b32 s8, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s8, 0 +-; GCN-NEXT: v_readfirstlane_b32 s1, v2 +-; GCN-NEXT: s_add_i32 s0, s1, s0 +-; GCN-NEXT: s_mul_i32 s0, s0, s2 +-; GCN-NEXT: s_sub_i32 s0, s3, s0 +-; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 ++; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-NEXT: s_ashr_i32 s2, s3, 8 ++; GCN-NEXT: s_abs_i32 s2, s2 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s2, v0 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mul_i32 s1, s1, s8 ++; GCN-NEXT: s_sub_i32 s1, s2, s1 ++; GCN-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-NEXT: s_sub_i32 s0, s1, s0 + ; GCN-NEXT: s_ashr_i32 s1, s0, 31 + ; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: v_mov_b32_e32 v1, s1 +@@ -582,34 +587,39 @@ define amdgpu_kernel void @s_test_srem24_64(ptr addrspace(1) %out, i64 %x, i64 % + ; + ; GCN-IR-LABEL: s_test_srem24_64: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_load_dword s2, s[4:5], 0xe ++; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: s_ashr_i32 s3, s3, 8 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_ashr_i32 s2, s2, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s3 +-; GCN-IR-NEXT: s_mov_b32 s4, s0 +-; GCN-IR-NEXT: s_xor_b32 s0, s3, s2 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s0, 30 ++; GCN-IR-NEXT: s_ashr_i32 s0, s0, 8 ++; GCN-IR-NEXT: s_abs_i32 s8, s0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s8 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: s_or_b32 s8, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s8, 0 +-; GCN-IR-NEXT: v_readfirstlane_b32 s1, v2 +-; GCN-IR-NEXT: s_add_i32 s0, s1, s0 +-; GCN-IR-NEXT: s_mul_i32 s0, s0, s2 +-; GCN-IR-NEXT: s_sub_i32 s0, s3, s0 +-; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 ++; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 ++; GCN-IR-NEXT: s_abs_i32 s2, s2 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s2, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 ++; GCN-IR-NEXT: s_sub_i32 s1, s2, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 + ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 + ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 +@@ -627,23 +637,31 @@ define i64 @v_test_srem24_64(i64 %x, i64 %y) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v3 +-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 8, v1 +-; GCN-NEXT: v_cvt_f32_i32_e32 v3, v1 +-; GCN-NEXT: v_xor_b32_e32 v5, v1, v0 +-; GCN-NEXT: v_rcp_f32_e32 v4, v2 +-; GCN-NEXT: v_ashrrev_i32_e32 v5, 30, v5 +-; GCN-NEXT: v_or_b32_e32 v5, 1, v5 +-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| +-; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc +-; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +-; GCN-NEXT: v_mul_lo_u32 v0, v2, v0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ++; GCN-NEXT: v_ashrrev_i32_e32 v4, 8, v1 ++; GCN-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v4 ++; GCN-NEXT: v_max_i32_e32 v4, v5, v4 ++; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 ++; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-NEXT: v_mul_hi_u32 v2, v4, v2 ++; GCN-NEXT: v_mul_u32_u24_e32 v2, v2, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v4, v2 ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ++; GCN-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; +@@ -651,23 +669,31 @@ define i64 @v_test_srem24_64(i64 %x, i64 %y) { + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v3 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v3, v1 +-; GCN-IR-NEXT: v_xor_b32_e32 v5, v1, v0 +-; GCN-IR-NEXT: v_rcp_f32_e32 v4, v2 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v5, 30, v5 +-; GCN-IR-NEXT: v_or_b32_e32 v5, 1, v5 +-; GCN-IR-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-IR-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v4, v4 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| +-; GCN-IR-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v2, v0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v1, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v4, 8, v1 ++; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_sub_i32_e32 v5, vcc, 0, v4 ++; GCN-IR-NEXT: v_max_i32_e32 v4, v5, v4 ++; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v3, v2, v3 ++; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v4, v2 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v2, v2, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v4, v2 ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %1 = ashr i64 %x, 40 +@@ -1205,72 +1231,92 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % + define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %y) { + ; GCN-LABEL: s_test_srem24_48: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_sext_i32_i16 s9, s9 +-; GCN-NEXT: s_lshr_b64 s[4:5], s[8:9], 24 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s4 ++; GCN-NEXT: s_sext_i32_i16 s1, s1 ++; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], 24 ++; GCN-NEXT: s_abs_i32 s8, s0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_sub_i32 s4, 0, s8 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_sext_i32_i16 s3, s3 + ; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-NEXT: v_cvt_f32_i32_e32 v1, s2 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-NEXT: s_xor_b32 s3, s2, s4 +-; GCN-NEXT: s_ashr_i32 s3, s3, 30 +-; GCN-NEXT: s_or_b32 s3, s3, 1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[8:9], |v1|, |v0| +-; GCN-NEXT: s_and_b64 s[8:9], s[8:9], exec +-; GCN-NEXT: s_cselect_b32 s3, s3, 0 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, s3, v2 +-; GCN-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: s_abs_i32 s3, s2 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: s_ashr_i32 s0, s2, 31 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 ++; GCN-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-NEXT: s_mul_i32 s1, s1, s8 ++; GCN-NEXT: s_sub_i32 s1, s3, s1 ++; GCN-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 +-; GCN-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 ++; GCN-NEXT: s_waitcnt expcnt(0) ++; GCN-NEXT: v_mov_b32_e32 v0, s1 ++; GCN-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_srem24_48: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_sext_i32_i16 s9, s9 +-; GCN-IR-NEXT: s_lshr_b64 s[4:5], s[8:9], 24 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s4 ++; GCN-IR-NEXT: s_sext_i32_i16 s1, s1 ++; GCN-IR-NEXT: s_lshr_b64 s[0:1], s[0:1], 24 ++; GCN-IR-NEXT: s_abs_i32 s8, s0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_sub_i32 s4, 0, s8 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_sext_i32_i16 s3, s3 + ; GCN-IR-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s2 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 +-; GCN-IR-NEXT: s_xor_b32 s3, s2, s4 +-; GCN-IR-NEXT: s_ashr_i32 s3, s3, 30 +-; GCN-IR-NEXT: s_or_b32 s3, s3, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[8:9], |v1|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[8:9], s[8:9], exec +-; GCN-IR-NEXT: s_cselect_b32 s3, s3, 0 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, s3, v2 +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s4 +-; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: s_abs_i32 s3, s2 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v0 ++; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: s_ashr_i32 s0, s2, 31 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, s3, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s1, v0 ++; GCN-IR-NEXT: s_mul_i32 s1, s1, s8 ++; GCN-IR-NEXT: s_sub_i32 s1, s3, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s8 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s8 ++; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-IR-NEXT: s_xor_b32 s1, s1, s0 ++; GCN-IR-NEXT: s_sub_i32 s0, s1, s0 ++; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-IR-NEXT: buffer_store_dword v0, off, s[4:7], 0 +-; GCN-IR-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 ++; GCN-IR-NEXT: s_waitcnt expcnt(0) ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s1 ++; GCN-IR-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 + ; GCN-IR-NEXT: s_endpgm + %1 = ashr i48 %x, 24 + %2 = ashr i48 %y, 24 +@@ -1982,65 +2028,65 @@ define amdgpu_kernel void @s_test_srem24_k_num_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_srem24_k_num_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_mov_b32 s8, 0x41c00000 +-; GCN-NEXT: s_mov_b32 s7, 0xf000 +-; GCN-NEXT: s_mov_b32 s6, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_ashr_i32 s2, s3, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-NEXT: s_mov_b32 s4, s0 +-; GCN-NEXT: s_ashr_i32 s0, s3, 31 +-; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-NEXT: s_or_b32 s3, s0, 1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_mad_f32 v2, -v1, v0, s8 +-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-NEXT: v_readfirstlane_b32 s1, v1 +-; GCN-NEXT: s_add_i32 s0, s1, s0 +-; GCN-NEXT: s_mul_i32 s0, s0, s2 +-; GCN-NEXT: s_sub_i32 s0, 24, s0 +-; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 +-; GCN-NEXT: s_ashr_i32 s1, s0, 31 +-; GCN-NEXT: v_mov_b32_e32 v0, s0 +-; GCN-NEXT: v_mov_b32_e32 v1, s1 +-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ++; GCN-NEXT: s_abs_i32 s4, s2 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-NEXT: s_sub_i32 s2, 0, s4 ++; GCN-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-NEXT: s_mov_b32 s2, -1 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-NEXT: s_cselect_b32 s4, s6, s5 ++; GCN-NEXT: s_ashr_i32 s5, s4, 31 ++; GCN-NEXT: v_mov_b32_e32 v0, s4 ++; GCN-NEXT: v_mov_b32_e32 v1, s5 ++; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_srem24_k_num_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_mov_b32 s8, 0x41c00000 +-; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 +-; GCN-IR-NEXT: s_mov_b32 s6, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_ashr_i32 s2, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s2 +-; GCN-IR-NEXT: s_mov_b32 s4, s0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 +-; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v1, v0 +-; GCN-IR-NEXT: s_or_b32 s3, s0, 1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x41c00000, v1 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_mad_f32 v2, -v1, v0, s8 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-IR-NEXT: v_readfirstlane_b32 s1, v1 +-; GCN-IR-NEXT: s_add_i32 s0, s1, s0 +-; GCN-IR-NEXT: s_mul_i32 s0, s0, s2 +-; GCN-IR-NEXT: s_sub_i32 s0, 24, s0 +-; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 +-; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 +-; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 +-; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 +-; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ++; GCN-IR-NEXT: s_abs_i32 s4, s2 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; GCN-IR-NEXT: s_sub_i32 s2, 0, s4 ++; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v0 ++; GCN-IR-NEXT: s_mov_b32 s2, -1 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, 24 ++; GCN-IR-NEXT: v_readfirstlane_b32 s5, v0 ++; GCN-IR-NEXT: s_mul_i32 s5, s5, s4 ++; GCN-IR-NEXT: s_sub_i32 s5, 24, s5 ++; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b32 s5, s6, s5 ++; GCN-IR-NEXT: s_sub_i32 s6, s5, s4 ++; GCN-IR-NEXT: s_cmp_ge_u32 s5, s4 ++; GCN-IR-NEXT: s_cselect_b32 s4, s6, s5 ++; GCN-IR-NEXT: s_ashr_i32 s5, s4, 31 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, s5 ++; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %x.shr = ashr i64 %x, 40 + %result = srem i64 24, %x.shr +@@ -2052,28 +2098,25 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(ptr addrspace(1) %out, i64 %x + ; GCN-LABEL: s_test_srem24_k_den_i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_mov_b32 s2, 0x46b6fe00 ++; GCN-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: s_ashr_i32 s8, s3, 8 +-; GCN-NEXT: v_cvt_f32_i32_e32 v0, s8 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: s_mov_b32 s4, s0 +-; GCN-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-NEXT: s_ashr_i32 s0, s3, 8 ++; GCN-NEXT: s_abs_i32 s0, s0 ++; GCN-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-NEXT: s_mulk_i32 s2, 0x5b7f ++; GCN-NEXT: s_sub_i32 s0, s0, s2 ++; GCN-NEXT: s_add_i32 s2, s0, 0xffffa481 ++; GCN-NEXT: s_min_u32 s0, s2, s0 ++; GCN-NEXT: s_add_i32 s2, s0, 0xffffa481 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-NEXT: s_or_b32 s3, s0, 1 +-; GCN-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 +-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-NEXT: v_readfirstlane_b32 s1, v1 +-; GCN-NEXT: s_add_i32 s0, s1, s0 +-; GCN-NEXT: s_mulk_i32 s0, 0x5b7f +-; GCN-NEXT: s_sub_i32 s0, s8, s0 +-; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000 ++; GCN-NEXT: s_ashr_i32 s1, s3, 31 ++; GCN-NEXT: s_min_u32 s0, s2, s0 ++; GCN-NEXT: s_xor_b32 s0, s0, s1 ++; GCN-NEXT: s_sub_i32 s0, s0, s1 + ; GCN-NEXT: s_ashr_i32 s1, s0, 31 + ; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: v_mov_b32_e32 v1, s1 +@@ -2083,28 +2126,25 @@ define amdgpu_kernel void @s_test_srem24_k_den_i64(ptr addrspace(1) %out, i64 %x + ; GCN-IR-LABEL: s_test_srem24_k_den_i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_mov_b32 s2, 0x46b6fe00 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, 0x2cc45 + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 + ; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: s_ashr_i32 s8, s3, 8 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s8 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: s_mov_b32 s4, s0 +-; GCN-IR-NEXT: s_ashr_i32 s0, s3, 31 ++; GCN-IR-NEXT: s_ashr_i32 s0, s3, 8 ++; GCN-IR-NEXT: s_abs_i32 s0, s0 ++; GCN-IR-NEXT: v_mul_hi_u32_u24_e32 v0, s0, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-IR-NEXT: s_mulk_i32 s2, 0x5b7f ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s2 ++; GCN-IR-NEXT: s_add_i32 s2, s0, 0xffffa481 ++; GCN-IR-NEXT: s_min_u32 s0, s2, s0 ++; GCN-IR-NEXT: s_add_i32 s2, s0, 0xffffa481 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x38331158, v0 +-; GCN-IR-NEXT: v_trunc_f32_e32 v1, v1 +-; GCN-IR-NEXT: v_mad_f32 v0, -v1, s2, v0 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v1, v1 +-; GCN-IR-NEXT: s_or_b32 s3, s0, 1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 s[0:1], |v0|, s2 +-; GCN-IR-NEXT: s_and_b64 s[0:1], s[0:1], exec +-; GCN-IR-NEXT: s_cselect_b32 s0, s3, 0 +-; GCN-IR-NEXT: v_readfirstlane_b32 s1, v1 +-; GCN-IR-NEXT: s_add_i32 s0, s1, s0 +-; GCN-IR-NEXT: s_mulk_i32 s0, 0x5b7f +-; GCN-IR-NEXT: s_sub_i32 s0, s8, s0 +-; GCN-IR-NEXT: s_bfe_i32 s0, s0, 0x180000 ++; GCN-IR-NEXT: s_ashr_i32 s1, s3, 31 ++; GCN-IR-NEXT: s_min_u32 s0, s2, s0 ++; GCN-IR-NEXT: s_xor_b32 s0, s0, s1 ++; GCN-IR-NEXT: s_sub_i32 s0, s0, s1 + ; GCN-IR-NEXT: s_ashr_i32 s1, s0, 31 + ; GCN-IR-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-IR-NEXT: v_mov_b32_e32 v1, s1 +@@ -2121,21 +2161,25 @@ define i64 @v_test_srem24_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-NEXT: v_rcp_f32_e32 v3, v2 +-; GCN-NEXT: v_mul_f32_e32 v3, 0x41c00000, v3 +-; GCN-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-NEXT: v_mad_f32 v4, -v3, v2, s4 +-; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| +-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +-; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +-; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 ++; GCN-NEXT: v_max_i32_e32 v0, v1, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; +@@ -2143,21 +2187,25 @@ define i64 @v_test_srem24_k_num_i64(i64 %x) { + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x41c00000 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x41c00000, v3 +-; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_mad_f32 v4, -v3, v2, s4 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| +-; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 ++; GCN-IR-NEXT: v_max_i32_e32 v0, v1, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v1, 24 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 24, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 +@@ -2170,21 +2218,25 @@ define i64 @v_test_srem24_pow2_k_num_i64(i64 %x) { + ; GCN: ; %bb.0: + ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-NEXT: v_rcp_f32_e32 v3, v2 +-; GCN-NEXT: v_mul_f32_e32 v3, 0x47000000, v3 +-; GCN-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-NEXT: v_mad_f32 v4, -v3, v2, s4 +-; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| +-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +-; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +-; GCN-NEXT: v_mul_lo_u32 v0, v1, v0 +-; GCN-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +-; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 ++; GCN-NEXT: v_max_i32_e32 v0, v1, v0 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; +@@ -2192,21 +2244,25 @@ define i64 @v_test_srem24_pow2_k_num_i64(i64 %x) { + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x47000000, v3 +-; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_mad_f32 v4, -v3, v2, s4 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| +-; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +-; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v0 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0, v0 ++; GCN-IR-NEXT: v_max_i32_e32 v0, v1, v0 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, 0, v0 ++; GCN-IR-NEXT: v_rcp_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v1, v2 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v1, 17, v1 ++; GCN-IR-NEXT: v_mul_u32_u24_e32 v1, v1, v0 ++; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, 0x8000, v1 ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ++; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, v1, v0 ++; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v1, v0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 +@@ -2229,24 +2285,89 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) { + ; GCN-NEXT: s_setpc_b64 s[30:31] + ; + ; GCN-IR-LABEL: v_test_srem24_pow2_k_den_i64: +-; GCN-IR: ; %bb.0: ++; GCN-IR: ; %bb.0: ; %_udiv-special-cases + ; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v10, 31, v1 + ; GCN-IR-NEXT: v_ashrrev_i32_e32 v0, 8, v1 +-; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, v0 +-; GCN-IR-NEXT: s_mov_b32 s4, 0x47000000 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +-; GCN-IR-NEXT: v_or_b32_e32 v1, 1, v1 +-; GCN-IR-NEXT: v_mul_f32_e32 v3, 0x38000000, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_mad_f32 v2, -v3, s4, v2 +-; GCN-IR-NEXT: v_cvt_i32_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, s4 +-; GCN-IR-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +-; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +-; GCN-IR-NEXT: v_lshlrev_b32_e32 v1, 15, v1 +-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 +-; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 +-; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10 ++; GCN-IR-NEXT: v_xor_b32_e32 v1, v10, v10 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10 ++; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc ++; GCN-IR-NEXT: v_ffbh_u32_e32 v2, v0 ++; GCN-IR-NEXT: v_add_i32_e64 v2, s[4:5], 32, v2 ++; GCN-IR-NEXT: v_ffbh_u32_e32 v3, v1 ++; GCN-IR-NEXT: v_min_u32_e32 v8, v2, v3 ++; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 48, v8 ++; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5] ++; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] ++; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3] ++; GCN-IR-NEXT: v_mov_b32_e32 v11, v10 ++; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ++; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] ++; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ++; GCN-IR-NEXT: v_cndmask_b32_e64 v5, v1, 0, s[4:5] ++; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5] ++; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc ++; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ++; GCN-IR-NEXT: s_cbranch_execz .LBB18_6 ++; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 ++; GCN-IR-NEXT: v_add_i32_e32 v6, vcc, 1, v2 ++; GCN-IR-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ++; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2 ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[0:1], v2 ++; GCN-IR-NEXT: v_mov_b32_e32 v4, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 ++; GCN-IR-NEXT: s_xor_b64 s[4:5], vcc, -1 ++; GCN-IR-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] ++; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execz .LBB18_5 ++; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader ++; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8 ++; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6 ++; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc ++; GCN-IR-NEXT: s_mov_b64 s[8:9], 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v8, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v9, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 ++; GCN-IR-NEXT: s_movk_i32 s10, 0x7fff ++; GCN-IR-NEXT: .LBB18_3: ; %udiv-do-while ++; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1 ++; GCN-IR-NEXT: v_lshl_b64 v[6:7], v[6:7], 1 ++; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3 ++; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4 ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 ++; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, s10, v6 ++; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, 0, v7, vcc ++; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2 ++; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4 ++; GCN-IR-NEXT: v_and_b32_e32 v4, 1, v8 ++; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8 ++; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8 ++; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc ++; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v12 ++; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3 ++; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc ++; GCN-IR-NEXT: v_mov_b32_e32 v9, v5 ++; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9] ++; GCN-IR-NEXT: v_mov_b32_e32 v8, v4 ++; GCN-IR-NEXT: s_andn2_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: s_cbranch_execnz .LBB18_3 ++; GCN-IR-NEXT: ; %bb.4: ; %Flow ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ++; GCN-IR-NEXT: .LBB18_5: ; %Flow4 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[4:5] ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 ++; GCN-IR-NEXT: v_or_b32_e32 v5, v5, v3 ++; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ++; GCN-IR-NEXT: .LBB18_6: ; %Flow5 ++; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] ++; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[4:5], 15 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 ++; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc ++; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10 ++; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v11 ++; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10 ++; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc + ; GCN-IR-NEXT: s_setpc_b64 s[30:31] + %x.shr = ashr i64 %x, 40 + %result = srem i64 %x.shr, 32768 +diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll +index 859a65f88800d0..17a579c59eeb3d 100644 +--- a/llvm/test/CodeGen/AMDGPU/udiv.ll ++++ b/llvm/test/CodeGen/AMDGPU/udiv.ll +@@ -1676,76 +1676,96 @@ define amdgpu_kernel void @v_udiv_i16(ptr addrspace(1) %out, ptr addrspace(1) %i + define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %in) { + ; SI-LABEL: v_udiv_i23: + ; SI: ; %bb.0: +-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 +-; SI-NEXT: s_mov_b32 s7, 0xf000 +-; SI-NEXT: s_mov_b32 s6, -1 +-; SI-NEXT: s_mov_b32 s10, s6 +-; SI-NEXT: s_mov_b32 s11, s7 ++; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9 ++; SI-NEXT: s_mov_b32 s3, 0xf000 ++; SI-NEXT: s_mov_b32 s2, -1 ++; SI-NEXT: s_mov_b32 s10, s2 ++; SI-NEXT: s_mov_b32 s11, s3 + ; SI-NEXT: s_waitcnt lgkmcnt(0) +-; SI-NEXT: s_mov_b32 s8, s2 +-; SI-NEXT: s_mov_b32 s9, s3 ++; SI-NEXT: s_mov_b32 s8, s6 ++; SI-NEXT: s_mov_b32 s9, s7 + ; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 + ; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 + ; SI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 + ; SI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 +-; SI-NEXT: s_mov_b32 s4, s0 +-; SI-NEXT: s_mov_b32 s5, s1 ++; SI-NEXT: s_mov_b32 s0, s4 ++; SI-NEXT: s_mov_b32 s1, s5 + ; SI-NEXT: s_waitcnt vmcnt(3) + ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 + ; SI-NEXT: s_waitcnt vmcnt(2) + ; SI-NEXT: v_or_b32_e32 v0, v1, v0 +-; SI-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; SI-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 + ; SI-NEXT: s_waitcnt vmcnt(1) +-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ++; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; SI-NEXT: v_rcp_f32_e32 v1, v1 + ; SI-NEXT: s_waitcnt vmcnt(0) +-; SI-NEXT: v_or_b32_e32 v1, v3, v1 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; SI-NEXT: v_rcp_f32_e32 v2, v0 +-; SI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; SI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; SI-NEXT: v_or_b32_e32 v2, v3, v2 ++; SI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; SI-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; SI-NEXT: v_mul_lo_u32 v4, v4, v1 ++; SI-NEXT: v_mul_hi_u32 v4, v1, v4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v4 ++; SI-NEXT: v_mul_hi_u32 v1, v2, v1 ++; SI-NEXT: v_mul_lo_u32 v3, v1, v0 ++; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v1 ++; SI-NEXT: v_sub_i32_e32 v2, vcc, v2, v3 ++; SI-NEXT: v_sub_i32_e32 v3, vcc, v2, v0 ++; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ++; SI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; SI-NEXT: v_add_i32_e32 v3, vcc, 1, v1 ++; SI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ++; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; + ; VI-LABEL: v_udiv_i23: + ; VI: ; %bb.0: +-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +-; VI-NEXT: s_mov_b32 s7, 0xf000 +-; VI-NEXT: s_mov_b32 s6, -1 +-; VI-NEXT: s_mov_b32 s10, s6 +-; VI-NEXT: s_mov_b32 s11, s7 ++; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: s_mov_b32 s2, -1 ++; VI-NEXT: s_mov_b32 s10, s2 ++; VI-NEXT: s_mov_b32 s11, s3 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_mov_b32 s8, s2 +-; VI-NEXT: s_mov_b32 s9, s3 ++; VI-NEXT: s_mov_b32 s8, s6 ++; VI-NEXT: s_mov_b32 s9, s7 + ; VI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:6 + ; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:4 + ; VI-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:2 + ; VI-NEXT: buffer_load_ushort v3, off, s[8:11], 0 +-; VI-NEXT: s_mov_b32 s4, s0 +-; VI-NEXT: s_mov_b32 s5, s1 ++; VI-NEXT: s_mov_b32 s0, s4 ++; VI-NEXT: s_mov_b32 s1, s5 + ; VI-NEXT: s_waitcnt vmcnt(3) + ; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 + ; VI-NEXT: s_waitcnt vmcnt(2) + ; VI-NEXT: v_or_b32_e32 v0, v1, v0 +-; VI-NEXT: v_cvt_f32_u32_e32 v0, v0 ++; VI-NEXT: v_cvt_f32_u32_e32 v1, v0 ++; VI-NEXT: v_sub_u32_e32 v4, vcc, 0, v0 + ; VI-NEXT: s_waitcnt vmcnt(1) +-; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ++; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; VI-NEXT: v_rcp_f32_e32 v1, v1 + ; VI-NEXT: s_waitcnt vmcnt(0) +-; VI-NEXT: v_or_b32_e32 v1, v3, v1 +-; VI-NEXT: v_cvt_f32_u32_e32 v1, v1 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc ++; VI-NEXT: v_or_b32_e32 v2, v3, v2 ++; VI-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; VI-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; VI-NEXT: v_mul_lo_u32 v4, v4, v1 ++; VI-NEXT: v_mul_hi_u32 v4, v1, v4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v4 ++; VI-NEXT: v_mul_hi_u32 v1, v2, v1 ++; VI-NEXT: v_mul_lo_u32 v3, v1, v0 ++; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v1 ++; VI-NEXT: v_sub_u32_e32 v2, vcc, v2, v3 ++; VI-NEXT: v_sub_u32_e32 v3, vcc, v2, v0 ++; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; VI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ++; VI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ++; VI-NEXT: v_add_u32_e32 v3, vcc, 1, v1 ++; VI-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc + ; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 +-; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ++; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; + ; GCN-LABEL: v_udiv_i23: +@@ -1758,40 +1778,50 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GCN-NEXT: s_add_u32 s4, s2, 4 + ; GCN-NEXT: s_addc_u32 s5, s3, 0 + ; GCN-NEXT: s_add_u32 s6, s2, 2 ++; GCN-NEXT: v_mov_b32_e32 v0, s4 + ; GCN-NEXT: s_addc_u32 s7, s3, 0 +-; GCN-NEXT: v_mov_b32_e32 v0, s6 +-; GCN-NEXT: v_mov_b32_e32 v1, s7 +-; GCN-NEXT: s_add_u32 s6, s2, 6 +-; GCN-NEXT: s_addc_u32 s7, s3, 0 ++; GCN-NEXT: v_mov_b32_e32 v1, s5 ++; GCN-NEXT: s_add_u32 s4, s2, 6 ++; GCN-NEXT: s_addc_u32 s5, s3, 0 ++; GCN-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-NEXT: v_mov_b32_e32 v3, s5 ++; GCN-NEXT: flat_load_ubyte v4, v[2:3] ++; GCN-NEXT: flat_load_ushort v5, v[0:1] + ; GCN-NEXT: v_mov_b32_e32 v2, s6 ++; GCN-NEXT: v_mov_b32_e32 v0, s2 + ; GCN-NEXT: v_mov_b32_e32 v3, s7 +-; GCN-NEXT: v_mov_b32_e32 v4, s4 +-; GCN-NEXT: v_mov_b32_e32 v5, s5 +-; GCN-NEXT: flat_load_ubyte v6, v[2:3] +-; GCN-NEXT: flat_load_ushort v4, v[4:5] +-; GCN-NEXT: v_mov_b32_e32 v2, s2 +-; GCN-NEXT: v_mov_b32_e32 v3, s3 +-; GCN-NEXT: flat_load_ubyte v0, v[0:1] +-; GCN-NEXT: flat_load_ushort v1, v[2:3] ++; GCN-NEXT: v_mov_b32_e32 v1, s3 ++; GCN-NEXT: flat_load_ubyte v2, v[2:3] ++; GCN-NEXT: flat_load_ushort v0, v[0:1] + ; GCN-NEXT: s_waitcnt vmcnt(3) +-; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v6 ++; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v4 + ; GCN-NEXT: s_waitcnt vmcnt(2) +-; GCN-NEXT: v_or_b32_e32 v2, v4, v2 +-; GCN-NEXT: v_cvt_f32_u32_e32 v2, v2 ++; GCN-NEXT: v_or_b32_e32 v3, v5, v1 ++; GCN-NEXT: v_cvt_f32_u32_e32 v1, v3 ++; GCN-NEXT: v_sub_u32_e32 v4, vcc, 0, v3 + ; GCN-NEXT: s_waitcnt vmcnt(1) +-; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ++; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ++; GCN-NEXT: v_rcp_f32_e32 v1, v1 + ; GCN-NEXT: s_waitcnt vmcnt(0) +-; GCN-NEXT: v_or_b32_e32 v0, v1, v0 +-; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 +-; GCN-NEXT: v_rcp_f32_e32 v4, v2 ++; GCN-NEXT: v_or_b32_e32 v2, v0, v2 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_mul_lo_u32 v4, v4, v1 ++; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 ++; GCN-NEXT: v_add_u32_e32 v0, vcc, v1, v4 ++; GCN-NEXT: v_mul_hi_u32 v4, v2, v0 + ; GCN-NEXT: v_mov_b32_e32 v0, s0 + ; GCN-NEXT: v_mov_b32_e32 v1, s1 +-; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 +-; GCN-NEXT: v_trunc_f32_e32 v4, v4 +-; GCN-NEXT: v_cvt_u32_f32_e32 v5, v4 +-; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 +-; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc ++; GCN-NEXT: v_mul_lo_u32 v5, v4, v3 ++; GCN-NEXT: v_add_u32_e32 v6, vcc, 1, v4 ++; GCN-NEXT: v_sub_u32_e32 v2, vcc, v2, v5 ++; GCN-NEXT: v_sub_u32_e32 v5, vcc, v2, v3 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 ++; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc ++; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc ++; GCN-NEXT: v_add_u32_e32 v5, vcc, 1, v4 ++; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3 ++; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc + ; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v2 + ; GCN-NEXT: flat_store_dword v[0:1], v2 + ; GCN-NEXT: s_endpgm +@@ -1807,23 +1837,39 @@ define amdgpu_kernel void @v_udiv_i23(ptr addrspace(1) %out, ptr addrspace(1) %i + ; GFX1030-NEXT: global_load_ubyte v3, v0, s[2:3] offset:2 + ; GFX1030-NEXT: global_load_ushort v4, v0, s[2:3] + ; GFX1030-NEXT: s_waitcnt vmcnt(3) +-; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s2, v1 + ; GFX1030-NEXT: s_waitcnt vmcnt(2) +-; GFX1030-NEXT: v_or_b32_e32 v1, v2, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s3, v2 + ; GFX1030-NEXT: s_waitcnt vmcnt(1) +-; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 16, v3 +-; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s4, v3 + ; GFX1030-NEXT: s_waitcnt vmcnt(0) +-; GFX1030-NEXT: v_or_b32_e32 v2, v4, v2 +-; GFX1030-NEXT: v_rcp_f32_e32 v3, v1 +-; GFX1030-NEXT: v_cvt_f32_u32_e32 v2, v2 +-; GFX1030-NEXT: v_mul_f32_e32 v3, v2, v3 +-; GFX1030-NEXT: v_trunc_f32_e32 v3, v3 +-; GFX1030-NEXT: v_fma_f32 v2, -v3, v1, v2 +-; GFX1030-NEXT: v_cvt_u32_f32_e32 v3, v3 +-; GFX1030-NEXT: v_cmp_ge_f32_e64 vcc_lo, |v2|, v1 +-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo +-; GFX1030-NEXT: v_and_b32_e32 v1, 0x7fffff, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s5, v4 ++; GFX1030-NEXT: s_lshl_b32 s2, s2, 16 ++; GFX1030-NEXT: s_or_b32 s2, s3, s2 ++; GFX1030-NEXT: s_lshl_b32 s4, s4, 16 ++; GFX1030-NEXT: v_cvt_f32_u32_e32 v1, s2 ++; GFX1030-NEXT: s_sub_i32 s6, 0, s2 ++; GFX1030-NEXT: s_or_b32 s4, s5, s4 ++; GFX1030-NEXT: v_rcp_f32_e32 v1, v1 ++; GFX1030-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ++; GFX1030-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GFX1030-NEXT: v_readfirstlane_b32 s3, v1 ++; GFX1030-NEXT: s_mul_i32 s6, s6, s3 ++; GFX1030-NEXT: s_mul_hi_u32 s6, s3, s6 ++; GFX1030-NEXT: s_add_i32 s3, s3, s6 ++; GFX1030-NEXT: s_mul_hi_u32 s3, s4, s3 ++; GFX1030-NEXT: s_mul_i32 s5, s3, s2 ++; GFX1030-NEXT: s_sub_i32 s4, s4, s5 ++; GFX1030-NEXT: s_add_i32 s5, s3, 1 ++; GFX1030-NEXT: s_sub_i32 s6, s4, s2 ++; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 ++; GFX1030-NEXT: s_cselect_b32 s3, s5, s3 ++; GFX1030-NEXT: s_cselect_b32 s4, s6, s4 ++; GFX1030-NEXT: s_add_i32 s5, s3, 1 ++; GFX1030-NEXT: s_cmp_ge_u32 s4, s2 ++; GFX1030-NEXT: s_cselect_b32 s2, s5, s3 ++; GFX1030-NEXT: s_and_b32 s2, s2, 0x7fffff ++; GFX1030-NEXT: v_mov_b32_e32 v1, s2 + ; GFX1030-NEXT: global_store_dword v0, v1, s[0:1] + ; GFX1030-NEXT: s_endpgm + ; +diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll +index 8653a43a00c6a4..97e0937760f924 100644 +--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll ++++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll +@@ -701,51 +701,77 @@ define amdgpu_kernel void @s_test_udiv31_i64(ptr addrspace(1) %out, i64 %x, i64 + define amdgpu_kernel void @s_test_udiv23_i64(ptr addrspace(1) %out, i64 %x, i64 %y) { + ; GCN-LABEL: s_test_udiv23_i64: + ; GCN: ; %bb.0: +-; GCN-NEXT: s_load_dword s6, s[4:5], 0xe +-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-NEXT: s_mov_b32 s6, -1 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-NEXT: s_lshr_b32 s8, s0, 9 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s0, v0 ++; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s2, s6, 9 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 + ; GCN-NEXT: s_lshr_b32 s2, s3, 9 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-NEXT: s_mov_b32 s6, -1 +-; GCN-NEXT: v_rcp_f32_e32 v2, v0 + ; GCN-NEXT: s_mov_b32 s4, s0 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, s2 + ; GCN-NEXT: s_mov_b32 s5, s1 +-; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 + ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_udiv23_i64: + ; GCN-IR: ; %bb.0: +-; GCN-IR-NEXT: s_load_dword s6, s[4:5], 0xe +-; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: s_load_dword s0, s[4:5], 0xe + ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ++; GCN-IR-NEXT: s_mov_b32 s6, -1 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) ++; GCN-IR-NEXT: s_lshr_b32 s8, s0, 9 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, 0, s8 ++; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v0 ++; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s2, s6, 9 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s2 + ; GCN-IR-NEXT: s_lshr_b32 s2, s3, 9 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; GCN-IR-NEXT: s_mov_b32 s6, -1 +-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v0 + ; GCN-IR-NEXT: s_mov_b32 s4, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s2 + ; GCN-IR-NEXT: s_mov_b32 s5, s1 +-; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +-; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ++; GCN-IR-NEXT: v_readfirstlane_b32 s0, v0 ++; GCN-IR-NEXT: s_mul_i32 s0, s0, s8 ++; GCN-IR-NEXT: s_sub_i32 s0, s2, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, s0, s8 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; GCN-IR-NEXT: s_cselect_b32 s0, s1, s0 ++; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s0, s8 ++; GCN-IR-NEXT: s_cselect_b64 vcc, -1, 0 ++; GCN-IR-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 + ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = lshr i64 %x, 41 +diff --git a/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/llvm/test/CodeGen/AMDGPU/udivrem24.ll +index 98948afd9eceda..d119f8400acf22 100644 +--- a/llvm/test/CodeGen/AMDGPU/udivrem24.ll ++++ b/llvm/test/CodeGen/AMDGPU/udivrem24.ll +@@ -529,16 +529,28 @@ define amdgpu_kernel void @udiv23_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff + ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -548,20 +560,32 @@ define amdgpu_kernel void @udiv23_i32(ptr addrspace(1) %out, ptr addrspace(1) %i + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0x7fffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0x7fffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +@@ -1978,16 +2002,28 @@ define amdgpu_kernel void @test_udiv24_u16_u23_i32(ptr addrspace(1) %out, ptr ad + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0xffff + ; SI-NEXT: s_and_b32 s5, s5, 0x7fffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -1997,20 +2033,32 @@ define amdgpu_kernel void @test_udiv24_u16_u23_i32(ptr addrspace(1) %out, ptr ad + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0xffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0x7fffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0xffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +@@ -2069,16 +2117,28 @@ define amdgpu_kernel void @test_udiv24_u23_u16_i32(ptr addrspace(1) %out, ptr ad + ; SI-NEXT: s_waitcnt lgkmcnt(0) + ; SI-NEXT: s_and_b32 s4, s4, 0x7fffff + ; SI-NEXT: s_and_b32 s5, s5, 0xffff +-; SI-NEXT: v_cvt_f32_u32_e32 v0, s4 +-; SI-NEXT: v_cvt_f32_u32_e32 v1, s5 +-; SI-NEXT: v_rcp_f32_e32 v2, v1 +-; SI-NEXT: v_mul_f32_e32 v2, v0, v2 +-; SI-NEXT: v_trunc_f32_e32 v2, v2 +-; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +-; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 +-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |v0|, v1 +-; SI-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc +-; SI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; SI-NEXT: v_cvt_f32_u32_e32 v0, s5 ++; SI-NEXT: s_sub_i32 s6, 0, s5 ++; SI-NEXT: v_rcp_f32_e32 v0, v0 ++; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; SI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; SI-NEXT: v_mul_lo_u32 v1, s6, v0 ++; SI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; SI-NEXT: v_mul_hi_u32 v0, s4, v0 ++; SI-NEXT: v_readfirstlane_b32 s6, v0 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_mul_i32 s6, s6, s5 ++; SI-NEXT: s_sub_i32 s4, s4, s6 ++; SI-NEXT: s_sub_i32 s6, s4, s5 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; SI-NEXT: s_cselect_b32 s4, s6, s4 ++; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0 ++; SI-NEXT: s_cmp_ge_u32 s4, s5 ++; SI-NEXT: s_cselect_b64 vcc, -1, 0 ++; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; SI-NEXT: s_endpgm + ; +@@ -2088,20 +2148,32 @@ define amdgpu_kernel void @test_udiv24_u23_u16_i32(ptr addrspace(1) %out, ptr ad + ; VI-NEXT: s_waitcnt lgkmcnt(0) + ; VI-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 + ; VI-NEXT: s_waitcnt lgkmcnt(0) +-; VI-NEXT: s_and_b32 s3, s3, 0xffff +-; VI-NEXT: v_cvt_f32_u32_e32 v0, s3 +-; VI-NEXT: s_and_b32 s2, s2, 0x7fffff +-; VI-NEXT: v_cvt_f32_u32_e32 v1, s2 +-; VI-NEXT: s_mov_b32 s3, 0xf000 +-; VI-NEXT: v_rcp_f32_e32 v2, v0 ++; VI-NEXT: s_and_b32 s4, s3, 0xffff ++; VI-NEXT: v_cvt_f32_u32_e32 v0, s4 ++; VI-NEXT: s_sub_i32 s3, 0, s4 ++; VI-NEXT: s_and_b32 s5, s2, 0x7fffff + ; VI-NEXT: s_mov_b32 s2, -1 +-; VI-NEXT: v_mul_f32_e32 v2, v1, v2 +-; VI-NEXT: v_trunc_f32_e32 v2, v2 +-; VI-NEXT: v_cvt_u32_f32_e32 v3, v2 +-; VI-NEXT: v_mad_f32 v1, -v2, v0, v1 +-; VI-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 +-; VI-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc +-; VI-NEXT: v_and_b32_e32 v0, 0x7fffff, v0 ++; VI-NEXT: v_rcp_f32_e32 v0, v0 ++; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ++; VI-NEXT: v_cvt_u32_f32_e32 v0, v0 ++; VI-NEXT: v_mul_lo_u32 v1, s3, v0 ++; VI-NEXT: s_mov_b32 s3, 0xf000 ++; VI-NEXT: v_mul_hi_u32 v1, v0, v1 ++; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ++; VI-NEXT: v_mul_hi_u32 v0, s5, v0 ++; VI-NEXT: v_readfirstlane_b32 s6, v0 ++; VI-NEXT: s_mul_i32 s6, s6, s4 ++; VI-NEXT: s_sub_i32 s5, s5, s6 ++; VI-NEXT: s_sub_i32 s6, s5, s4 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ++; VI-NEXT: s_cselect_b32 s5, s6, s5 ++; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0 ++; VI-NEXT: s_cmp_ge_u32 s5, s4 ++; VI-NEXT: s_cselect_b64 vcc, -1, 0 ++; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc + ; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 + ; VI-NEXT: s_endpgm + ; +diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll +index 7840b861abb9ac..0952013401892c 100644 +--- a/llvm/test/CodeGen/AMDGPU/urem64.ll ++++ b/llvm/test/CodeGen/AMDGPU/urem64.ll +@@ -710,36 +710,45 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 + ; GCN-LABEL: s_test_urem23_64_v2i64: + ; GCN: ; %bb.0: + ; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd +-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 + ; GCN-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-NEXT: s_mov_b32 s2, -1 + ; GCN-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-NEXT: s_lshr_b32 s6, s13, 1 +-; GCN-NEXT: v_cvt_f32_u32_e32 v0, s6 +-; GCN-NEXT: s_lshr_b32 s4, s15, 9 +-; GCN-NEXT: v_cvt_f32_u32_e32 v1, s4 +-; GCN-NEXT: s_lshr_b32 s5, s11, 9 ++; GCN-NEXT: s_lshr_b32 s0, s13, 1 ++; GCN-NEXT: v_cvt_f32_u32_e32 v0, s0 ++; GCN-NEXT: s_sub_i32 s1, 0, s0 ++; GCN-NEXT: s_lshr_b32 s6, s15, 9 ++; GCN-NEXT: v_cvt_f32_u32_e32 v2, s6 + ; GCN-NEXT: v_rcp_f32_e32 v0, v0 +-; GCN-NEXT: v_cvt_f32_u32_e32 v2, s5 +-; GCN-NEXT: v_rcp_f32_e32 v3, v1 +-; GCN-NEXT: s_sub_i32 s8, 0, s6 ++; GCN-NEXT: s_lshr_b32 s7, s11, 9 ++; GCN-NEXT: v_rcp_f32_e32 v2, v2 + ; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 + ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +-; GCN-NEXT: v_mul_f32_e32 v3, v2, v3 +-; GCN-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-NEXT: v_mad_f32 v2, -v3, v1, v2 +-; GCN-NEXT: v_mul_lo_u32 v4, s8, v0 +-; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 +-; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-NEXT: s_lshr_b32 s7, s9, 1 +-; GCN-NEXT: v_mul_hi_u32 v4, v0, v4 +-; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-NEXT: v_mul_lo_u32 v1, v1, s4 +-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +-; GCN-NEXT: v_mul_hi_u32 v0, v0, s7 +-; GCN-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 +-; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v1 +-; GCN-NEXT: v_readfirstlane_b32 s4, v0 ++; GCN-NEXT: v_mul_lo_u32 v1, s1, v0 ++; GCN-NEXT: s_lshr_b32 s1, s9, 1 ++; GCN-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-NEXT: v_mul_hi_u32 v0, v0, s1 ++; GCN-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 ++; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-NEXT: s_mul_i32 s2, s2, s0 ++; GCN-NEXT: s_sub_i32 s1, s1, s2 ++; GCN-NEXT: s_sub_i32 s2, s1, s0 ++; GCN-NEXT: s_cmp_ge_u32 s1, s0 ++; GCN-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-NEXT: s_sub_i32 s2, s1, s0 ++; GCN-NEXT: s_cmp_ge_u32 s1, s0 ++; GCN-NEXT: s_cselect_b32 s8, s2, s1 ++; GCN-NEXT: s_sub_i32 s0, 0, s6 ++; GCN-NEXT: v_mul_lo_u32 v0, s0, v1 ++; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ++; GCN-NEXT: s_mov_b32 s2, -1 ++; GCN-NEXT: v_mul_hi_u32 v0, v1, v0 ++; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0 ++; GCN-NEXT: v_mul_hi_u32 v2, v0, s7 ++; GCN-NEXT: v_mov_b32_e32 v1, 0 ++; GCN-NEXT: v_mov_b32_e32 v0, s8 ++; GCN-NEXT: v_mov_b32_e32 v3, v1 ++; GCN-NEXT: v_readfirstlane_b32 s4, v2 + ; GCN-NEXT: s_mul_i32 s4, s4, s6 + ; GCN-NEXT: s_sub_i32 s4, s7, s4 + ; GCN-NEXT: s_sub_i32 s5, s4, s6 +@@ -748,45 +757,53 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 + ; GCN-NEXT: s_sub_i32 s5, s4, s6 + ; GCN-NEXT: s_cmp_ge_u32 s4, s6 + ; GCN-NEXT: s_cselect_b32 s4, s5, s4 +-; GCN-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-NEXT: v_mov_b32_e32 v0, s4 +-; GCN-NEXT: v_mov_b32_e32 v3, v1 ++; GCN-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 + ; GCN-NEXT: s_endpgm + ; + ; GCN-IR-LABEL: s_test_urem23_64_v2i64: + ; GCN-IR: ; %bb.0: + ; GCN-IR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd +-; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 + ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +-; GCN-IR-NEXT: s_mov_b32 s2, -1 + ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) +-; GCN-IR-NEXT: s_lshr_b32 s6, s13, 1 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s6 +-; GCN-IR-NEXT: s_lshr_b32 s4, s15, 9 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s4 +-; GCN-IR-NEXT: s_lshr_b32 s5, s11, 9 ++; GCN-IR-NEXT: s_lshr_b32 s0, s13, 1 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, 0, s0 ++; GCN-IR-NEXT: s_lshr_b32 s6, s15, 9 ++; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s6 + ; GCN-IR-NEXT: v_rcp_f32_e32 v0, v0 +-; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s5 +-; GCN-IR-NEXT: v_rcp_f32_e32 v3, v1 +-; GCN-IR-NEXT: s_sub_i32 s8, 0, s6 ++; GCN-IR-NEXT: s_lshr_b32 s7, s11, 9 ++; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 + ; GCN-IR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 + ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v0, v0 +-; GCN-IR-NEXT: v_mul_f32_e32 v3, v2, v3 +-; GCN-IR-NEXT: v_trunc_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_mad_f32 v2, -v3, v1, v2 +-; GCN-IR-NEXT: v_mul_lo_u32 v4, s8, v0 +-; GCN-IR-NEXT: v_cvt_u32_f32_e32 v3, v3 +-; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 +-; GCN-IR-NEXT: s_lshr_b32 s7, s9, 1 +-; GCN-IR-NEXT: v_mul_hi_u32 v4, v0, v4 +-; GCN-IR-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +-; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, s4 +-; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +-; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s7 +-; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 +-; GCN-IR-NEXT: v_and_b32_e32 v2, 0x7fffff, v1 +-; GCN-IR-NEXT: v_readfirstlane_b32 s4, v0 ++; GCN-IR-NEXT: v_mul_lo_u32 v1, s1, v0 ++; GCN-IR-NEXT: s_lshr_b32 s1, s9, 1 ++; GCN-IR-NEXT: v_mul_hi_u32 v1, v0, v1 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v0, s1 ++; GCN-IR-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 ++; GCN-IR-NEXT: v_cvt_u32_f32_e32 v1, v1 ++; GCN-IR-NEXT: v_readfirstlane_b32 s2, v0 ++; GCN-IR-NEXT: s_mul_i32 s2, s2, s0 ++; GCN-IR-NEXT: s_sub_i32 s1, s1, s2 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s0 ++; GCN-IR-NEXT: s_cselect_b32 s1, s2, s1 ++; GCN-IR-NEXT: s_sub_i32 s2, s1, s0 ++; GCN-IR-NEXT: s_cmp_ge_u32 s1, s0 ++; GCN-IR-NEXT: s_cselect_b32 s8, s2, s1 ++; GCN-IR-NEXT: s_sub_i32 s0, 0, s6 ++; GCN-IR-NEXT: v_mul_lo_u32 v0, s0, v1 ++; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ++; GCN-IR-NEXT: s_mov_b32 s2, -1 ++; GCN-IR-NEXT: v_mul_hi_u32 v0, v1, v0 ++; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v1, v0 ++; GCN-IR-NEXT: v_mul_hi_u32 v2, v0, s7 ++; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 ++; GCN-IR-NEXT: v_mov_b32_e32 v0, s8 ++; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 ++; GCN-IR-NEXT: v_readfirstlane_b32 s4, v2 + ; GCN-IR-NEXT: s_mul_i32 s4, s4, s6 + ; GCN-IR-NEXT: s_sub_i32 s4, s7, s4 + ; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 +@@ -795,9 +812,8 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(ptr addrspace(1) %out, <2 x i6 + ; GCN-IR-NEXT: s_sub_i32 s5, s4, s6 + ; GCN-IR-NEXT: s_cmp_ge_u32 s4, s6 + ; GCN-IR-NEXT: s_cselect_b32 s4, s5, s4 +-; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +-; GCN-IR-NEXT: v_mov_b32_e32 v0, s4 +-; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 ++; GCN-IR-NEXT: v_mov_b32_e32 v2, s4 ++; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) + ; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 + ; GCN-IR-NEXT: s_endpgm + %1 = lshr <2 x i64> %x, diff --git a/mlir/test/fusion/pr-e2e/rock-gemm-large-grid-float-divrem.mlir b/mlir/test/fusion/pr-e2e/rock-gemm-large-grid-float-divrem.mlir new file mode 100644 index 0000000000..e4f2ca686b --- /dev/null +++ b/mlir/test/fusion/pr-e2e/rock-gemm-large-grid-float-divrem.mlir @@ -0,0 +1,22 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Regression for the AMDGPU backend's float-based expansion of 32-bit integer +// div/rem (llvm-patches/patch201186.patch). makeGroupedGridLayout computes +// m_block as bid % thisMBlocksPerGroup, with the grid-wide block id as the +// dividend and a runtime divisor. Without the patch AMDGPUCodeGenPrepare +// expands that through trunc(float(bid) * rcp(float(divisor))), which comes +// out one too large for some dividends past roughly 2^23, so m_block lands far +// outside A: wrong results, and occasionally a memory access fault. +// +// A 1x1 tile over 9216x1536 gives 14155776 workgroups. gridGroupSize=11 pins +// the divisor instead of deriving it from the CU and chiplet counts; +// v_rcp_f32(11) lies above 1/11 on gfx90a and gfx942, and without the patch +// about 238000 of the 14155776 output elements come out wrong there. + +// RUN: rocmlir-gen --arch %arch -operation gemm -t f16 -out_datatype f16 -g 1 -m 9216 -k 512 -n 1536 -transA=True -transB=False -transO=False --perf_config=gemm:mPerBlock=1,nPerBlock=1,kPerBlock=32,kpack=1,numCTAs=1,numWaves=2,matrixInstrNonkdim=16,splitKFactor=1,numStages=1,wavesPerEU=0,gridGroupSize=11,useAsyncCopy=-1,useBlockPingpong=-1,useInThreadTranspose=-1,useBufferOps=-1,useBufferAtomics=-1,useReductionLayout=-1,useOptimizeEpilogue=-1,useBf16x3ForF32=-1 -pv \ +// RUN: | rocmlir-driver -c \ +// RUN: | rocm-run \ +// RUN: | FileCheck %s + +// CHECK: [1 1 1]