Skip to content

[AIROCMLIR-1292] [EXTERNAL] Cherry-pick the upstream fix for the float-based 32-bit div/rem expansion #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bogdan-petkovic merged 3 commits into from
Sep 28, 2026
Merged

[AIROCMLIR-1292] [EXTERNAL] Cherry-pick the upstream fix for the float-based 32-bit div/rem expansion #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0e01a56
[EXTERNAL] Be more careful about using expandDivRem24
bogdan-petkovic Sep 24, 2026
e64e004
[EXTERNAL] Avoid errors with 23-bit division and remainder.
bogdan-petkovic Sep 24, 2026
05fc5d7
Internal patch records and regression test for the float-based div/re…
bogdan-petkovic Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 40 additions & 21 deletions external/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,14 +181,15 @@ class AMDGPUCodeGenPrepareImpl
unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
unsigned MaxDivBits, bool Signed) const;

/// Expands 24 bit div or rem.
Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I,
Value *Num, Value *Den,
bool IsDiv, bool IsSigned) const;
/// Expands div or rem by using floating-point operations.
/// Operands must be in the range [-0x400000,0x3FFFFF]
Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I,
Value *Num, Value *Den, bool IsDiv,
bool IsSigned) const;

Value *expandDivRem24Impl(IRBuilder<> &Builder, BinaryOperator &I,
Value *Num, Value *Den, unsigned NumBits,
bool IsDiv, bool IsSigned) const;
Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I,
Value *Num, Value *Den, unsigned NumBits,
bool IsDiv, bool IsSigned) const;

/// Expands 32 bit div or rem.
Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
Expand Down Expand Up @@ -1060,21 +1061,39 @@ unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
return DivBits;
}

// The fractional part of a float is enough to accurately represent up to
// a 24-bit signed integer.
Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder,
BinaryOperator &I, Value *Num,
Value *Den, bool IsDiv,
bool IsSigned) const {
unsigned DivBits = getDivNumBits(I, Num, Den, 24, IsSigned);
if (DivBits > 24)
Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder,
BinaryOperator &I,
Value *Num, Value *Den,
bool IsDiv,
bool IsSigned) const {
unsigned DivBits = getDivNumBits(I, Num, Den, 23, IsSigned);

if (DivBits > (IsSigned ? 23 : 22))
return nullptr;
Comment thread
bogdan-petkovic marked this conversation as resolved.
return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned);
return expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits, IsDiv,
IsSigned);
}

Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl(
Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
unsigned DivBits, bool IsDiv, bool IsSigned) const {

// v_rcp_f32(float(X)) can have an error of 1 ulp.
// This would cause incorrect calculation of Y/X if:
// Y = (0x7FFFFF/X)*(X-0)-1
// were allowed.
//
// For example,
// (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059.
// (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
//
// Thus, we conservatively restrict expandDivRemToFloatImpl to
// [-0x40000,0x3FFFFF] for IsSigned
// [0x000000,0x3FFFFF] for !IsSigned.
assert(DivBits <= (IsSigned ? 23 : 22) &&
"abs(Num) must be <= than 0x40000 for expandDivRemToFloatImpl to work "
"correctly");

Type *I32Ty = Builder.getInt32Ty();
Num = Builder.CreateTrunc(Num, I32Ty);
Den = Builder.CreateTrunc(Den, I32Ty);
Expand Down Expand Up @@ -1249,7 +1268,7 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
}
}

if (Value *Res = expandDivRem24(Builder, I, X, Y, IsDiv, IsSigned)) {
if (Value *Res = expandDivRemToFloat(Builder, I, X, Y, IsDiv, IsSigned)) {
return IsSigned ? Builder.CreateSExtOrTrunc(Res, Ty) :
Builder.CreateZExtOrTrunc(Res, Ty);
}
Expand Down Expand Up @@ -1358,9 +1377,9 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
return nullptr;

Value *Narrowed = nullptr;
if (NumDivBits <= 24) {
Narrowed = expandDivRem24Impl(Builder, I, Num, Den, NumDivBits,
IsDiv, IsSigned);
if (NumDivBits <= (IsSigned ? 23 : 22)) {
Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, NumDivBits, IsDiv,
IsSigned);
} else if (NumDivBits <= 32) {
Narrowed = expandDivRem32(Builder, I, Num, Den);
}
Expand Down
86 changes: 55 additions & 31 deletions external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,17 +431,25 @@ define i32 @v_udiv_i32_24bit(i32 %num, i32 %den) {
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1
; CGP-NEXT: v_rcp_f32_e32 v2, v1
; CGP-NEXT: v_mul_f32_e32 v2, v0, v2
; CGP-NEXT: v_trunc_f32_e32 v2, v2
; CGP-NEXT: v_fma_f32 v0, -v2, v1, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1
; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1
; CGP-NEXT: v_rcp_f32_e32 v2, v2
; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1
; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; CGP-NEXT: v_mul_hi_u32 v2, v0, v2
; CGP-NEXT: v_mul_lo_u32 v3, v2, v1
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; CGP-NEXT: s_setpc_b64 s[30:31]
%num.mask = and i32 %num, 16777215
%den.mask = and i32 %den, 16777215
Expand Down Expand Up @@ -504,28 +512,44 @@ define <2 x i32> @v_udiv_v2i32_24bit(<2 x i32> %num, <2 x i32> %den) {
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1
; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2
; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v3
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2
; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1
; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3
; CGP-NEXT: v_rcp_f32_e32 v4, v2
; CGP-NEXT: v_rcp_f32_e32 v5, v3
; CGP-NEXT: v_mul_f32_e32 v4, v0, v4
; CGP-NEXT: v_mul_f32_e32 v5, v1, v5
; CGP-NEXT: v_trunc_f32_e32 v4, v4
; CGP-NEXT: v_trunc_f32_e32 v5, v5
; CGP-NEXT: v_fma_f32 v0, -v4, v2, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2
; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2
; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3
; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3
; CGP-NEXT: v_rcp_f32_e32 v4, v4
; CGP-NEXT: v_rcp_f32_e32 v6, v6
; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4
; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
; CGP-NEXT: v_fma_f32 v1, -v5, v3, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v2
; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, v3
; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
; CGP-NEXT: v_mul_lo_u32 v5, v5, v4
; CGP-NEXT: v_mul_lo_u32 v7, v7, v6
; CGP-NEXT: v_mul_hi_u32 v5, v4, v5
; CGP-NEXT: v_mul_hi_u32 v7, v6, v7
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7
; CGP-NEXT: v_mul_hi_u32 v4, v0, v4
; CGP-NEXT: v_mul_hi_u32 v5, v1, v5
; CGP-NEXT: v_mul_lo_u32 v6, v4, v2
; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4
; CGP-NEXT: v_mul_lo_u32 v8, v5, v3
; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc
; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2
; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3
; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4
; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
; CGP-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc
; CGP-NEXT: s_setpc_b64 s[30:31]
%num.mask = and <2 x i32> %num, <i32 16777215, i32 16777215>
%den.mask = and <2 x i32> %den, <i32 16777215, i32 16777215>
Expand Down
94 changes: 61 additions & 33 deletions external/llvm-project/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1878,17 +1878,25 @@ define i64 @v_udiv_i64_24bit(i64 %num, i64 %den) {
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1
; CGP-NEXT: v_rcp_f32_e32 v2, v1
; CGP-NEXT: v_mul_f32_e32 v2, v0, v2
; CGP-NEXT: v_trunc_f32_e32 v2, v2
; CGP-NEXT: v_mad_f32 v0, -v2, v1, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v2, v1
; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v1
; CGP-NEXT: v_rcp_f32_e32 v2, v2
; CGP-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; CGP-NEXT: v_cvt_u32_f32_e32 v2, v2
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1
; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; CGP-NEXT: v_add_i32_e32 v0, vcc, v2, v0
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_mul_lo_u32 v3, v3, v2
; CGP-NEXT: v_mul_hi_u32 v3, v2, v3
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; CGP-NEXT: v_mul_hi_u32 v2, v0, v2
; CGP-NEXT: v_mul_lo_u32 v3, v2, v1
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v2
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
; CGP-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; CGP-NEXT: v_mov_b32_e32 v1, 0
; CGP-NEXT: s_setpc_b64 s[30:31]
%num.mask = and i64 %num, 16777215
Expand Down Expand Up @@ -2132,31 +2140,51 @@ define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v4
; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v2
; CGP-NEXT: v_and_b32_e32 v1, 0xffffff, v2
; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v4
; CGP-NEXT: v_and_b32_e32 v3, 0xffffff, v6
; CGP-NEXT: v_cvt_f32_u32_e32 v0, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v1, v1
; CGP-NEXT: v_cvt_f32_u32_e32 v2, v2
; CGP-NEXT: v_cvt_f32_u32_e32 v3, v3
; CGP-NEXT: v_rcp_f32_e32 v4, v1
; CGP-NEXT: v_rcp_f32_e32 v5, v3
; CGP-NEXT: v_mul_f32_e32 v4, v0, v4
; CGP-NEXT: v_mul_f32_e32 v5, v2, v5
; CGP-NEXT: v_trunc_f32_e32 v4, v4
; CGP-NEXT: v_trunc_f32_e32 v5, v5
; CGP-NEXT: v_mad_f32 v0, -v4, v1, v0
; CGP-NEXT: v_cvt_f32_u32_e32 v4, v2
; CGP-NEXT: v_sub_i32_e32 v5, vcc, 0, v2
; CGP-NEXT: v_cvt_f32_u32_e32 v6, v3
; CGP-NEXT: v_sub_i32_e32 v7, vcc, 0, v3
; CGP-NEXT: v_rcp_f32_e32 v4, v4
; CGP-NEXT: v_rcp_f32_e32 v6, v6
; CGP-NEXT: v_mul_f32_e32 v4, 0x4f7ffffe, v4
; CGP-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
; CGP-NEXT: v_cvt_u32_f32_e32 v4, v4
; CGP-NEXT: v_mad_f32 v2, -v5, v3, v2
; CGP-NEXT: v_cvt_u32_f32_e32 v5, v5
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v0|, v1
; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; CGP-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, v3
; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; CGP-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; CGP-NEXT: v_add_i32_e32 v1, vcc, v5, v1
; CGP-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; CGP-NEXT: v_and_b32_e32 v2, 0xffffff, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
; CGP-NEXT: v_mul_lo_u32 v5, v5, v4
; CGP-NEXT: v_mul_lo_u32 v7, v7, v6
; CGP-NEXT: v_mul_hi_u32 v5, v4, v5
; CGP-NEXT: v_mul_hi_u32 v7, v6, v7
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7
; CGP-NEXT: v_mul_hi_u32 v6, v0, v4
; CGP-NEXT: v_mul_lo_u32 v4, 0, v4
; CGP-NEXT: v_mul_hi_u32 v7, v1, v5
; CGP-NEXT: v_mul_lo_u32 v5, 0, v5
; CGP-NEXT: v_add_i32_e32 v4, vcc, v6, v4
; CGP-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; CGP-NEXT: v_mul_lo_u32 v6, v4, v2
; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v4
; CGP-NEXT: v_mul_lo_u32 v8, v5, v3
; CGP-NEXT: v_add_i32_e32 v9, vcc, 1, v5
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc
; CGP-NEXT: v_sub_i32_e64 v6, s[4:5], v0, v2
; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v3
; CGP-NEXT: v_cndmask_b32_e64 v5, v5, v9, s[4:5]
; CGP-NEXT: v_sub_i32_e64 v7, s[6:7], v1, v3
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
; CGP-NEXT: v_add_i32_e32 v6, vcc, 1, v4
; CGP-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v5
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
; CGP-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
; CGP-NEXT: v_mov_b32_e32 v1, 0
; CGP-NEXT: v_mov_b32_e32 v3, 0
; CGP-NEXT: s_setpc_b64 s[30:31]
Expand Down
Loading
Loading