diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 5443f4e509f0..2f074fe61586 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -7543,7 +7543,7 @@ agx_calc_stats(agx_context *ctx, unsigned size, struct agx2_stats *stats) } static bool -agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *_) +agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *g17p) { if (instr->type != nir_instr_type_alu) return false; @@ -7661,6 +7661,11 @@ agx_lower_sincos_impl(struct nir_builder *b, nir_instr *instr, void *data) static bool agx_lower_sincos(nir_shader *shader, unsigned native) { + /* Compute on every target but G17P goes through agx_nir_lower_sincos, the + * exact reduction. Graphics keeps the base sequence. */ + if (native != 1 && mesa_shader_stage_is_compute(shader->info.stage)) + return false; + return nir_shader_lower_instructions(shader, agx_lower_sincos_filter, agx_lower_sincos_impl, (void *)(uintptr_t)native); @@ -8257,6 +8262,11 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics, } while (progress); } + /* int64 lowering and the perspective divide emit fdiv after preprocessing; + * the backend has no fdiv, so lower what is left before it is scheduled. + */ + NIR_PASS(_, nir, agx_nir_lower_fdiv_late); + /* Reassociate before forming preambles because it makes preambles more * effective. Clean up after. On M5, also order the loop-invariant terms of * a chain in a loop first, so accumulations of per-iteration constants @@ -10778,72 +10788,6 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl, offset_out, 0, true, NULL); } -/* - * The hardware frcp instruction is sometimes off by 1 ULP. For correctly - * rounded frcp, a refinement step is required. This routine has been - * exhaustively tested with a modified math_bruteforce. - * - * While Khronos APIs allow 2.5 ULP error for divides, nir_lower_idiv relies on - * correctly rounded frcp. This is therefore load bearing for integer division - * on all APIs. - */ -static nir_def * -libagx_frcp(nir_builder *b, nir_def *x) -{ - nir_def *u = nir_frcp(b, x); - - /* Do 1 Newton-Raphson refinement step. - * - * Define f(u) = xu - 1. Then f(u) = 0 iff u = 1/x. Newton's method gives: - * - * u_2 = u - f(u) / f'(u) = u - (xu - 1) / x - * - * Our original guess is close, so we approximate (1 / x) by u: - * - * u_2 = u - u(xu - 1) = u + u(1 - xu) - * = fma(fma(-x, u, 1), u, u) - */ - nir_def *one = nir_imm_float(b, 1.0); - nir_def *u_2 = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, one), u, u); - - /* If the original value was infinite, frcp will generate the correct zero. - * However, the Newton-Raphson step would multiply 0 * Inf and get a NaN. So - * skip the refinement step for infinite inputs. We do this backwards, - * checking whether the refined result is NaN, since we can implement this - * check in a single fcmpsel instruction. The other case where the refinement - * is NaN is a NaN input, in which skipping refinement is acceptable. - */ - return nir_bcsel(b, nir_fisnan(b, u_2), u, u_2); -} - -static bool -agx_nir_lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) -{ - if (alu->op != nir_op_frcp || !nir_alu_instr_is_exact(alu) || - alu->def.bit_size != 32) - return false; - - /* The refinement makes frcp correctly rounded, which only integer - * division needs: nir_lower_idiv takes the reciprocal of an integer - * converted to float. An application float division keeps the 2.5 ULP - * that every Khronos API allows even when it is `precise`/NoContraction - * (which forbids contraction, not the hardware reciprocal), so do not - * spend two FMAs and a select on it; DXVK marks every operation precise. - * Driver-internal shaders keep the refinement unconditionally. */ - if (!b->shader->info.internal) { - nir_scalar src = nir_scalar_resolved(alu->src[0].src.ssa, - alu->src[0].swizzle[0]); - if (!nir_scalar_is_alu(src) || - (nir_scalar_alu_op(src) != nir_op_u2f32 && - nir_scalar_alu_op(src) != nir_op_i2f32)) - return false; - } - - b->cursor = nir_before_instr(&alu->instr); - nir_def_replace(&alu->def, libagx_frcp(b, nir_ssa_for_alu_src(b, alu, 0))); - return true; -} - static bool agx_nir_lower_g17p_f16_classify(nir_builder *b, nir_alu_instr *alu, void *_data) @@ -10993,6 +10937,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep * FMAs only. Other shaders keep the lowering here. */ if (!g17p || !agx_nir_fma_rte_applies(nir)) NIR_PASS(_, nir, agx_lower_sincos, g17p ? 1 : native_fragment_depth ? 2 : 0); + NIR_PASS(_, nir, agx_nir_lower_sincos); NIR_PASS(_, nir, nir_shader_intrinsics_pass, agx_lower_front_face, nir_metadata_control_flow, NULL); if (g17p) @@ -11018,9 +10963,11 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep NIR_PASS(_, nir, nir_lower_idiv, &idiv_options); - /* Has to run after nir_lower_idiv */ - NIR_PASS(_, nir, nir_shader_alu_pass, agx_nir_lower_fdiv, - nir_metadata_control_flow, NULL); + /* Has to run after nir_lower_idiv, and before the log lowering so the + * reciprocals it builds are not refined twice. + */ + NIR_PASS(_, nir, agx_nir_lower_fdiv); + NIR_PASS(_, nir, agx_nir_lower_log); NIR_PASS(_, nir, nir_opt_deref); NIR_PASS(_, nir, nir_lower_vars_to_ssa); diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h index 8e33177e4365..663d65e8a0a1 100644 --- a/src/asahi/compiler/agx_compile.h +++ b/src/asahi/compiler/agx_compile.h @@ -665,7 +665,6 @@ agx_round_registers(unsigned halfregs) } static const nir_shader_compiler_options agx_nir_options = { - .lower_fdiv = true, .float_mul_add16 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, .float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, .lower_flrp16 = true, diff --git a/src/asahi/compiler/agx_nir.h b/src/asahi/compiler/agx_nir.h index d38cdbb132cd..61746b542887 100644 --- a/src/asahi/compiler/agx_nir.h +++ b/src/asahi/compiler/agx_nir.h @@ -57,6 +57,10 @@ bool agx_nir_lower_word_bounds(struct nir_shader *s); bool agx_nir_share_bounds(struct nir_shader *s); bool agx_nir_lower_atomics(struct nir_shader *s, bool native_fadd); bool agx_nir_opt_canonical_minmax(struct nir_shader *s); +bool agx_nir_lower_fdiv(struct nir_shader *s); +bool agx_nir_lower_fdiv_late(struct nir_shader *s); +bool agx_nir_lower_log(struct nir_shader *s); +bool agx_nir_lower_sincos(struct nir_shader *s); bool agx_nir_lower_texture_early(struct nir_shader *s, bool support_lod_bias); bool agx_nir_lower_g17p_helper_samples(struct nir_shader *s); diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c new file mode 100644 index 000000000000..0dc9e44e464e --- /dev/null +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -0,0 +1,682 @@ +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright 2026 Joshua Warren + * SPDX-License-Identifier: MIT + */ + +#include +#include "compiler/nir/nir.h" +#include "compiler/nir/nir_builder.h" +#include "compiler/nir/nir_builtin_builder.h" +#include "agx_nir.h" + +/* + * AGX transcendental lowering with fp32 rounding contracts. + * + * The hardware gives an approximate rcp (off by 1 ulp), an approximate log2 + * (absolute error only, so relative error explodes near 1) and a sin unit that + * takes an argument in quadrants. Everything here is built from the exact fp32 + * fma, so the sequences are marked no-fast-math to keep nir_opt_algebraic from + * reassociating or contracting them. + */ + +/* The caller keeps x near one so the Newton residual cannot flush. */ +static nir_def * +rcp_normal(nir_builder *b, nir_def *x) +{ + nir_def *u = nir_frcp(b, x); + return nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, + nir_imm_float(b, 1.0)), u, u); +} + +/* Normalize before refinement; exponent restoration must flush the exact + * quotient before rounding, including values just below the normal boundary. */ +static nir_def * +div_rn(nir_builder *b, nir_def *a, nir_def *d) +{ + nir_def *aa = nir_iand_imm(b, a, 0x7fffffff); + nir_def *dd = nir_iand_imm(b, d, 0x7fffffff); + nir_def *ea = nir_ushr_imm(b, aa, 23); + nir_def *ed = nir_ushr_imm(b, dd, 23); + nir_def *ma = nir_ior_imm(b, nir_iand_imm(b, aa, 0x7fffff), 0x3f800000); + nir_def *md = nir_ior_imm(b, nir_iand_imm(b, dd, 0x7fffff), 0x3f800000); + nir_def *y = nir_frcp(b, md); + nir_def *q = nir_fmul(b, ma, y); + nir_def *r = nir_ffma(b, nir_fneg(b, md), q, ma); + q = nir_ffma(b, r, y, q); + nir_def *e = nir_isub(b, ea, ed); + nir_def *out = nir_iadd(b, q, nir_ishl_imm(b, e, 23)); + nir_def *under = nir_ior(b, nir_ilt_imm(b, e, -126), + nir_iand(b, nir_ieq_imm(b, e, -126), nir_ult(b, ma, md))); + out = nir_bcsel(b, under, nir_imm_int(b, 0), out); + nir_def *over = nir_ige_imm(b, nir_iadd(b, e, nir_ushr_imm(b, q, 23)), 255); + out = nir_bcsel(b, over, nir_imm_int(b, 0x7f800000), out); + nir_def *az = nir_ieq_imm(b, ea, 0), *dz = nir_ieq_imm(b, ed, 0); + nir_def *ai = nir_ieq_imm(b, ea, 255), *di = nir_ieq_imm(b, ed, 255); + out = nir_bcsel(b, nir_ior(b, az, di), nir_imm_int(b, 0), out); + out = nir_bcsel(b, nir_ior(b, dz, ai), nir_imm_int(b, 0x7f800000), out); + out = nir_ior(b, out, nir_iand_imm(b, nir_ixor(b, a, d), 0x80000000)); + nir_def *invalid = nir_ior(b, nir_ugt_imm(b, aa, 0x7f800000), + nir_ugt_imm(b, dd, 0x7f800000)); + invalid = nir_ior(b, invalid, nir_ior(b, nir_iand(b, az, dz), nir_iand(b, ai, di))); + return nir_bcsel(b, invalid, nir_imm_int(b, 0x7fc00000), out); +} + +/* + * 1/x, bit for bit div_rn(1, x): one Newton step on the hardware rcp, which + * flushes denormal inputs to +-inf and results below 2^-126 to +-0. Where the + * step is NaN (x zero, denormal, infinite or NaN), the hardware rcp already + * holds the answer, and NaN becomes the canonical NaN. tests/div_window.py + * checks every input. + */ +static nir_def * +rcp_rn(nir_builder *b, nir_def *x) +{ + nir_def *u = nir_frcp(b, x); + nir_def *q = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, + nir_imm_float(b, 1.0)), u, u); + q = nir_bcsel(b, nir_fisnan(b, q), u, q); + return nir_bcsel(b, nir_fisnan(b, q), nir_imm_int(b, 0x7fc00000), q); +} + +static bool +is_one(const nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == 0x3f800000; +} + +static bool +lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *data) +{ + bool fdiv_only = *(bool *)data; + + if (alu->op != nir_op_fdiv && (alu->op != nir_op_frcp || fdiv_only)) + return false; + + if (alu->def.bit_size == 64) + return false; + + /* Graphics shaders keep the hardware reciprocal and the cheap quotient, + * except that integer division's exact reciprocal stays correctly rounded + * here too: nir_lower_idiv relies on it. The exact sequences below are + * for compute (MLX and kernels), where the rounding contract matters. A + * late fdiv (int64 lowering, perspective divide) still has to be lowered, + * because the backend has no fdiv. + */ + if (!mesa_shader_stage_is_compute(b->shader->info.stage)) { + if (alu->op == nir_op_frcp) { + if (alu->def.bit_size != 32 || !nir_alu_instr_is_exact(alu)) + return false; + + /* The refinement only serves integer division: an application + * reciprocal keeps the hardware instruction. Driver-internal + * shaders keep it unconditionally. */ + if (!b->shader->info.internal) { + nir_scalar src = nir_scalar_resolved(alu->src[0].src.ssa, + alu->src[0].swizzle[0]); + if (!nir_scalar_is_alu(src) || + (nir_scalar_alu_op(src) != nir_op_u2f32 && + nir_scalar_alu_op(src) != nir_op_i2f32)) + return false; + } + + /* Same refinement as compute, with the exact float controls scoped + * to these emissions instead of leaking to the rest of the shader. */ + b->cursor = nir_before_instr(&alu->instr); + unsigned saved_fp = b->fp_math_ctrl; + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def_replace(&alu->def, rcp_rn(b, nir_ssa_for_alu_src(b, alu, 0))); + b->fp_math_ctrl = saved_fp; + return true; + } + + b->cursor = nir_before_instr(&alu->instr); + nir_def *q = nir_fmul(b, nir_ssa_for_alu_src(b, alu, 0), + nir_frcp(b, nir_ssa_for_alu_src(b, alu, 1))); + nir_def_replace(&alu->def, q); + return true; + } + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + + nir_def *res; + if (alu->op == nir_op_frcp) { + if (alu->def.bit_size != 32) + return false; + + res = rcp_rn(b, nir_ssa_for_alu_src(b, alu, 0)); + } else { + nir_def *a = nir_ssa_for_alu_src(b, alu, 0); + nir_def *d = nir_ssa_for_alu_src(b, alu, 1); + + if (alu->def.bit_size == 32 && alu->def.num_components == 1 && + is_one(alu, 0)) + res = rcp_rn(b, d); + else if (alu->def.bit_size == 32) + res = div_rn(b, a, d); + else + res = nir_fmul(b, a, nir_frcp(b, d)); + } + + nir_def_replace(&alu->def, res); + return true; +} + +bool +agx_nir_lower_fdiv(nir_shader *s) +{ + bool fdiv_only = false; + return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, + &fdiv_only); +} + +/* + * Passes that run after agx_preprocess_nir still emit fdiv: nir_lower_int64 + * (f2u64/f2i64 divide by 2^32, and lower_fmod turns the frem into a second + * fdiv) and agx_nir_lower_interpolation (perspective divide). Lower only fdiv + * here so the reciprocals built by the first pass and by the log lowering are + * not refined a second time. + */ +bool +agx_nir_lower_fdiv_late(nir_shader *s) +{ + bool fdiv_only = true; + return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, + &fdiv_only); +} + +/* + * Natural log and log2 as double-float (fp32 pair) arithmetic. + * + * x = 2^e m with m in [sqrt(1/2), sqrt(2)). With s = (m - 1) / (m + 1), + * + * log(m) = 2s + 2s^3 (1/3 + s^2/5 + s^4/7 + ...) + * + * s is kept as a pair (s, s_lo) from one correctly rounded division plus the + * fma residual, the cubic term is kept as a pair, and the terms are summed + * with Fast2Sum so the value handed to the final rounding carries about 40 + * bits. That makes the exact cases (powers of two, log(3)) come out correctly + * rounded and everything else faithful, without any table. + */ +struct log_parts { + nir_def *e; /* exponent as float */ + nir_def *s; /* 2s = leading term, as a pair (s, s_lo) */ + nir_def *s_lo; + nir_def *t_hi; /* 2 s^3 P(s^2), as a pair */ + nir_def *t_lo; +}; + +static nir_def * +fast2sum_lo(nir_builder *b, nir_def *a, nir_def *bb, nir_def *sum) +{ + /* Requires |a| >= |bb| or a == 0 */ + return nir_fsub(b, bb, nir_fsub(b, sum, a)); +} + +static struct log_parts +log_core(nir_builder *b, nir_def *x) +{ + nir_def *bits = x; + nir_def *e = nir_iadd_imm(b, nir_ushr_imm(b, bits, 23), -127); + nir_def *m = nir_ior_imm(b, nir_iand_imm(b, bits, 0x7fffff), 0x3f800000); + + /* Fold m into [sqrt(1/2), sqrt(2)) so the series converges fast and + * e ln2 never cancels more than one bit against log(m). + */ + nir_def *big = nir_fgt_imm(b, m, 1.41421353816986083984375); + m = nir_bcsel(b, big, nir_fmul_imm(b, m, 0.5), m); + e = nir_iadd(b, e, nir_b2i32(b, big)); + + nir_def *one = nir_imm_float(b, 1.0); + nir_def *two = nir_imm_float(b, 2.0); + nir_def *f = nir_fsub(b, m, one); /* exact */ + nir_def *d = nir_fadd(b, two, f); + nir_def *d_lo = fast2sum_lo(b, two, f, d); + + /* s = f / (d + d_lo) as a pair */ + nir_def *y = rcp_normal(b, d); + nir_def *s = nir_fmul(b, f, y); + nir_def *r = nir_ffma(b, nir_fneg(b, s), d, f); + s = nir_ffma(b, r, y, s); + r = nir_ffma(b, nir_fneg(b, s), d, f); + nir_def *s_lo = + nir_fmul(b, nir_fsub(b, r, nir_fmul(b, s, d_lo)), y); + + /* u = s^3 as a pair, including the 3 s^2 s_lo cross term */ + nir_def *w = nir_fmul(b, s, s); + nir_def *w_lo = nir_ffma(b, s, s, nir_fneg(b, w)); + nir_def *u = nir_fmul(b, s, w); + nir_def *u_lo = nir_fadd(b, nir_ffma(b, s, w, nir_fneg(b, u)), + nir_fmul(b, s, w_lo)); + u_lo = nir_ffma(b, nir_fmul_imm(b, s_lo, 3.0), w, u_lo); + + /* P(w) = 1/3 + w Q(w) as a pair, 1/3 split hi/lo */ + nir_def *q = nir_imm_float(b, 1.0 / 13.0); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 11.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 9.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 7.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 5.0)); + nir_def *third = nir_imm_float(b, 0.3333333432674408); + nir_def *v = nir_ffma(b, w, q, nir_imm_float(b, -9.934107758624577e-09)); + nir_def *p = nir_fadd(b, third, v); + nir_def *p_lo = fast2sum_lo(b, third, v, p); + + nir_def *th = nir_fmul(b, u, p); + nir_def *tl = nir_fadd(b, nir_fadd(b, nir_ffma(b, u, p, nir_fneg(b, th)), + nir_fmul(b, u, p_lo)), + nir_fmul(b, u_lo, p)); + + return (struct log_parts){ + .e = nir_i2f32(b, e), + .s = s, + .s_lo = s_lo, + .t_hi = nir_fmul(b, two, th), + .t_lo = nir_fmul(b, two, tl), + }; +} + +/* Positive, finite, not flushed: the only inputs the series handles. */ +static nir_def * +is_positive_normal(nir_builder *b, nir_def *x) +{ + return nir_ult_imm(b, nir_iadd_imm(b, x, -0x00800000), + 0x7f7fffff - 0x00800000 + 1); +} + +/* + * Everything else: +-0 and flushed subnormals give -inf, negatives and NaN + * give NaN, +inf gives +inf. Same answer for log and log2. + */ +static nir_def * +log_special(nir_builder *b, nir_def *x) +{ + nir_def *zero = nir_ult_imm(b, nir_iand_imm(b, x, 0x7fffffff), 0x00800000); + nir_def *nan = nir_ugt_imm(b, x, 0x7f800000); + return nir_bcsel(b, zero, nir_imm_float(b, -INFINITY), + nir_bcsel(b, nan, nir_imm_float(b, NAN), + nir_imm_float(b, INFINITY))); +} + +static nir_def * +soft_log(nir_builder *b, nir_def *x) +{ + struct log_parts l = log_core(b, x); + + /* ln2 split so e * LN2_HI is exact for |e| < 2^8 */ + nir_def *a = nir_fmul_imm(b, l.e, 0.693145751953125); + nir_def *bb = nir_fmul_imm(b, l.s, 2.0); + nir_def *h = nir_fadd(b, a, bb); + nir_def *lo = fast2sum_lo(b, a, bb, h); + nir_def *h2 = nir_fadd(b, h, l.t_hi); + nir_def *lo2 = fast2sum_lo(b, h, l.t_hi, h2); + + nir_def *small = nir_fadd(b, lo, lo2); + small = nir_fadd(b, small, l.t_lo); + small = nir_fadd(b, small, nir_fmul_imm(b, l.s_lo, 2.0)); + small = nir_ffma(b, l.e, nir_imm_float(b, 1.428606765330187e-06), small); + return nir_fadd(b, h2, small); +} + +static nir_def * +soft_log2(nir_builder *b, nir_def *x) +{ + struct log_parts l = log_core(b, x); + + /* (2s + 2s_lo + t) / ln2 with 1/ln2 as a pair */ + nir_def *il2 = nir_imm_float(b, 1.4426950216293335); + nir_def *il2_lo = nir_imm_float(b, 1.925963033500011e-08); + nir_def *bb = nir_fmul_imm(b, l.s, 2.0); + nir_def *ph = nir_fmul(b, bb, il2); + nir_def *pl = nir_ffma(b, bb, il2, nir_fneg(b, ph)); + pl = nir_ffma(b, bb, il2_lo, pl); + nir_def *th = nir_fmul(b, l.t_hi, il2); + nir_def *tl = nir_ffma(b, l.t_hi, il2, nir_fneg(b, th)); + nir_def *rest = nir_fadd(b, nir_fmul_imm(b, l.s_lo, 2.0), l.t_lo); + pl = nir_fadd(b, nir_ffma(b, rest, il2, pl), tl); + + nir_def *h = nir_fadd(b, l.e, ph); + nir_def *lo = fast2sum_lo(b, l.e, ph, h); + nir_def *h2 = nir_fadd(b, h, th); + nir_def *lo2 = fast2sum_lo(b, h, th, h2); + return nir_fadd(b, h2, nir_fadd(b, nir_fadd(b, lo, lo2), pl)); +} + +static const uint32_t LN2_F32_BITS = 0x3f317218; + +static bool +src_is_ln2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == LN2_F32_BITS; +} + +static bool +src_is_flog2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_flog2; +} + +/* + * nir_flog builds fmul(flog2(x), ln2); catch that before the flog2 goes and + * round once. An exact (precise) multiply keeps the rounding the source asks + * for, so a precise log2(x) * ln2 gives the same bits from a literal and from + * a buffer. + */ +static bool +lower_log(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_fmul || alu->def.bit_size != 32 || + alu->def.num_components != 1 || nir_alu_instr_is_exact(alu)) + return false; + + unsigned li; + if (src_is_flog2(alu, 0) && src_is_ln2(alu, 1)) + li = 0; + else if (src_is_flog2(alu, 1) && src_is_ln2(alu, 0)) + li = 1; + else + return false; + + nir_scalar log = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), li); + nir_scalar xs = nir_scalar_chase_alu_src(log, 0); + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_channel(b, xs.def, xs.comp); + nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log(b, x), + log_special(b, x)); + nir_def_replace(&alu->def, res); + + if (nir_def_is_unused(log.def)) + nir_instr_remove(nir_def_instr(log.def)); + + return true; +} + +static bool +lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_flog2 || alu->def.bit_size != 32) + return false; + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_ssa_for_alu_src(b, alu, 0); + nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log2(b, x), + log_special(b, x)); + nir_def_replace(&alu->def, res); + return true; +} + +bool +agx_nir_lower_log(nir_shader *s) +{ + if (!mesa_shader_stage_is_compute(s->info.stage)) + return false; + + bool progress = + nir_shader_alu_pass(s, lower_log, nir_metadata_control_flow, NULL); + progress |= + nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); + return progress; +} + +/* + * Sine and cosine. Range reduction produces k = round(x 2/pi) and the fraction + * fq = x 2/pi - k in [-1/2, 1/2] quadrants; the quadrant then picks a Taylor + * polynomial for sin(pi/2 fq) or cos(pi/2 fq) and a sign. The hardware sin + * unit (sin_pt_1/sin_pt_2, nir_op_fsin_agx) is not used: it is a few ulp off, + * which is more than the reduction now loses. + * + * |x| < 2^22: Cody-Waite in radians with pi/2 in three fp32 pieces. The first + * fma is exact, the remaining rounding is tracked as a low part so fq is + * within an ulp of the true reduced argument. + * + * |x| >= 2^22: Payne-Hanek. Multiply the 24-bit mantissa by a 128-bit window + * of 2/pi aligned to the exponent; the integer part mod 4 is the quadrant and + * the top 64 fraction bits give fq. fp32 inputs never reduce below ~2^-30 of a + * quadrant, so 64 fraction bits leave more than 24 significant bits. + */ +static const double TWO_OVER_PI = 0.6366197466850281; /* fp32(2/pi) */ +static const double TWO_OVER_PI_LO = 2 / M_PI - 0.6366197466850281; +static const double PIO2_1 = 1.5707963705062866; +static const double PIO2_2 = -4.371138828673793e-08; +static const double PIO2_3 = -1.7151245100058819e-15; + +/* Bits of 2/pi, 32 per word, preceded by three zero words. */ +static const uint32_t two_over_pi_bits[10] = { + 0, 0, 0, 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, + 0xdb629599, 0x3c439041, 0xfe5163ab, +}; + +struct reduced { + nir_def *k; /* int32, only the low two bits matter */ + nir_def *fq; /* about [-1/2, 1/2] in quadrants, as a pair */ + nir_def *fq_lo; +}; + +static struct reduced +reduce_cody_waite(nir_builder *b, nir_def *x) +{ + nir_def *tp = nir_imm_float(b, TWO_OVER_PI); + nir_def *c2 = nir_imm_float(b, PIO2_2); + nir_def *kf = nir_fround_even(b, nir_fmul(b, x, tp)); + nir_def *nk = nir_fneg(b, kf); + + /* r1 = x - k C1 is exact. k C2 is a TwoProduct and r1 - k C2 a TwoSum so + * nothing is lost even when r1 and k C2 nearly cancel. + */ + nir_def *r1 = nir_ffma(b, nk, nir_imm_float(b, PIO2_1), x); + nir_def *p = nir_fmul(b, kf, c2); + nir_def *p_lo = nir_ffma(b, kf, c2, nir_fneg(b, p)); + nir_def *r2 = nir_fsub(b, r1, p); + nir_def *bb = nir_fsub(b, r2, r1); + nir_def *t = nir_fadd(b, nir_fsub(b, r1, nir_fsub(b, r2, bb)), + nir_fsub(b, nir_fneg(b, p), bb)); + nir_def *r_lo = + nir_ffma(b, nk, nir_imm_float(b, PIO2_3), nir_fsub(b, t, p_lo)); + + nir_def *fq = nir_fmul(b, r2, tp); + nir_def *fq_lo = nir_ffma(b, r2, tp, nir_fneg(b, fq)); + fq_lo = nir_ffma(b, r_lo, tp, fq_lo); + fq_lo = nir_ffma(b, r2, nir_imm_float(b, TWO_OVER_PI_LO), fq_lo); + nir_def *sum = nir_fadd(b, fq, fq_lo); + + return (struct reduced){ + .k = nir_f2i32(b, kf), + .fq = sum, + .fq_lo = fast2sum_lo(b, fq, fq_lo, sum), + }; +} + +static struct reduced +reduce_payne_hanek(nir_builder *b, nir_def *ax) +{ + nir_def *e8 = nir_ushr_imm(b, ax, 23); + nir_def *m32 = + nir_ishl_imm(b, nir_ior_imm(b, nir_iand_imm(b, ax, 0x7fffff), 0x800000), + 8); + + /* Word/bit offset of the 2/pi window for this exponent */ + nir_def *off = nir_iadd_imm(b, e8, -126); + nir_def *idx = nir_ushr_imm(b, off, 5); + nir_def *sh = nir_iand_imm(b, off, 31); + nir_def *sh_r = nir_isub(b, nir_imm_int(b, 31), sh); + + nir_def *tab[ARRAY_SIZE(two_over_pi_bits)]; + for (unsigned i = 0; i < ARRAY_SIZE(tab); ++i) + tab[i] = nir_imm_int(b, two_over_pi_bits[i]); + + /* Windows n = 1..4 at weights 2^0, 2^-32, 2^-64, 2^-96 */ + nir_def *lo[5], *hi[5]; + for (unsigned n = 1; n <= 4; ++n) { + nir_def *cand[5], *cand2[5]; + for (unsigned j = 0; j < 5; ++j) { + cand[j] = tab[j + n]; + cand2[j] = tab[j + n + 1]; + } + nir_def *w0 = nir_select_from_ssa_def_array(b, cand, 5, idx); + nir_def *w1 = nir_select_from_ssa_def_array(b, cand2, 5, idx); + nir_def *win = nir_ior(b, nir_ishl(b, w0, sh), + nir_ushr_imm(b, nir_ushr(b, w1, sh_r), 1)); + lo[n] = nir_imul(b, m32, win); + hi[n] = nir_umul_high(b, m32, win); + } + + /* Carry-propagate the 96 fraction bits, keep the top 64 */ + nir_def *f2 = nir_iadd(b, lo[3], hi[4]); + nir_def *c2 = nir_b2i32(b, nir_ult(b, f2, hi[4])); + nir_def *f1a = nir_iadd(b, lo[2], hi[3]); + nir_def *c1a = nir_b2i32(b, nir_ult(b, f1a, hi[3])); + nir_def *f1 = nir_iadd(b, f1a, c2); + nir_def *c1b = nir_b2i32(b, nir_ult(b, f1, f1a)); + nir_def *k = nir_iadd(b, nir_iadd(b, lo[1], hi[2]), nir_iadd(b, c1a, c1b)); + + /* Round to nearest quadrant so fq lands in [-1/2, 1/2] */ + nir_def *half = nir_ushr_imm(b, f1, 31); + k = nir_iadd(b, k, half); + nir_def *frac = nir_pack_64_2x32_split(b, f2, f1); + nir_def *is_half = nir_ine_imm(b, half, 0); + frac = nir_bcsel(b, is_half, nir_ineg(b, frac), frac); + nir_def *mag = nir_fmul_imm(b, nir_u2f32(b, frac), 0x1p-64); + + return (struct reduced){ + .k = k, + .fq = nir_bcsel(b, is_half, nir_fneg(b, mag), mag), + .fq_lo = nir_imm_float(b, 0.0), + }; +} + +static bool +sincos_filter(const nir_instr *instr, UNUSED const void *_) +{ + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + return (alu->op == nir_op_fsin || alu->op == nir_op_fcos) && + alu->def.bit_size <= 32; +} + +static nir_def * +lower_sincos(nir_builder *b, nir_instr *instr, UNUSED void *_) +{ + nir_alu_instr *alu = nir_instr_as_alu(instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_mov_alu(b, alu->src[0], 1); + unsigned bit_size = x->bit_size; + if (bit_size == 16) + x = nir_f2f32(b, x); + + nir_def *ax = nir_fabs(b, x); + + nir_push_if(b, nir_fge_imm(b, ax, 0x1p22)); + struct reduced ph = reduce_payne_hanek(b, ax); + + /* Payne-Hanek reduced |x|; sin is odd, so flip k and fq for negatives */ + nir_def *neg_x = nir_flt_imm(b, x, 0.0); + ph.k = nir_bcsel(b, neg_x, nir_ineg(b, ph.k), ph.k); + ph.fq = nir_bcsel(b, neg_x, nir_fneg(b, ph.fq), ph.fq); + + nir_push_else(b, NULL); + struct reduced cw = reduce_cody_waite(b, x); + nir_pop_if(b, NULL); + + nir_def *k = nir_if_phi(b, ph.k, cw.k); + nir_def *fq = nir_if_phi(b, ph.fq, cw.fq); + nir_def *fq_lo = nir_if_phi(b, ph.fq_lo, cw.fq_lo); + + /* cos(x) = sin(x + pi/2): one more quadrant */ + if (alu->op == nir_op_fcos) + k = nir_iadd_imm(b, k, 1); + + /* Taylor series in quadrants, |fq| < 0.71 (k can be off by one), + * truncation below 2^-36. The leading sin term and the quadratic cos term + * are kept as pairs and the low part of fq feeds the derivative, so both + * land within an ulp. + */ + nir_def *t = fq; + nir_def *t_lo = fq_lo; + nir_def *w = nir_fmul(b, t, t); + nir_def *w_lo = nir_ffma(b, t, t, nir_fneg(b, w)); + + static const double sin_coef[] = { + -0.6459640860557556, 0.07969262450933456, -0.004681753925979137, + 0.00016044118092395365, -3.598843250074424e-06, 5.6921727775716136e-08, + }; + static const double cos_coef[] = { + -1.2337005138397217, 0.25366950035095215, -0.020863480865955353, + 0.0009192602592520416, -2.520204179745633e-05, 4.710874748070637e-07, + -6.386603246255618e-09, + }; + static const double cos_coef0_lo = -3.629644851343983e-08; + + nir_def *pio2 = nir_imm_float(b, PIO2_1); + nir_def *p = nir_imm_float(b, sin_coef[5]); + for (int i = 4; i >= 0; --i) + p = nir_ffma(b, p, w, nir_imm_float(b, sin_coef[i])); + + nir_def *hi = nir_fmul(b, t, pio2); + nir_def *lo = nir_ffma(b, t, pio2, nir_fneg(b, hi)); + lo = nir_ffma(b, t, nir_imm_float(b, PIO2_2), lo); + lo = nir_ffma(b, t_lo, pio2, lo); + nir_def *sn = nir_fadd(b, hi, nir_ffma(b, nir_fmul(b, t, w), p, lo)); + + nir_def *d2 = nir_imm_float(b, cos_coef[0]); + nir_def *r = nir_imm_float(b, cos_coef[6]); + for (int i = 5; i >= 1; --i) + r = nir_ffma(b, r, w, nir_imm_float(b, cos_coef[i])); + + nir_def *h = nir_fmul(b, w, d2); + nir_def *l = nir_fadd(b, nir_fadd(b, nir_ffma(b, w, d2, nir_fneg(b, h)), + nir_fmul_imm(b, w, cos_coef0_lo)), + nir_fmul(b, w_lo, d2)); + l = nir_ffma(b, nir_fmul_imm(b, t_lo, 2.0 * cos_coef[0]), t, l); + nir_def *one = nir_imm_float(b, 1.0); + nir_def *s1 = nir_fadd(b, one, h); + nir_def *e = fast2sum_lo(b, one, h, s1); + nir_def *cs = nir_fadd( + b, s1, nir_fadd(b, nir_fadd(b, e, l), nir_fmul(b, nir_fmul(b, w, w), r))); + + /* Odd quadrants take the cosine, quadrants 2 and 3 negate */ + nir_def *odd = nir_ine_imm(b, nir_iand_imm(b, k, 1), 0); + nir_def *neg = nir_ine_imm(b, nir_iand_imm(b, k, 2), 0); + nir_def *v = nir_bcsel(b, odd, cs, sn); + v = nir_bcsel(b, neg, nir_fneg(b, v), v); + + /* Inf and NaN fall into the Payne-Hanek branch with a bogus exponent */ + nir_def *nan = nir_imm_float(b, NAN); + v = nir_bcsel(b, nir_flt_imm(b, ax, INFINITY), v, nan); + + /* sin(+-0) = +-0 exactly (the reduction returns +0 for -0). The zero + * comparison flushes its input, so a subnormal argument matches; under + * DenormFlushToZero the argument from an unflushed source (a buffer load) + * must not leak out through the select, so return the sign-preserving + * zero. Without the mode, preserving the subnormal argument is legal and + * sin(subnormal) rounds to it anyway. */ + if (alu->op == nir_op_fsin) { + if (nir_is_denorm_flush_to_zero( + b->shader->info.float_controls_execution_mode, 32)) + v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), + nir_iand_imm(b, x, 0x80000000), v); + else + v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), x, v); + } + + if (bit_size == 16) + v = nir_f2f16(b, v); + + return v; +} + +bool +agx_nir_lower_sincos(nir_shader *s) +{ + if (!mesa_shader_stage_is_compute(s->info.stage)) + return false; + + return nir_shader_lower_instructions(s, sincos_filter, lower_sincos, NULL); +} diff --git a/src/asahi/compiler/meson.build b/src/asahi/compiler/meson.build index 2b766a4f14d3..ca6c5e44c2c0 100644 --- a/src/asahi/compiler/meson.build +++ b/src/asahi/compiler/meson.build @@ -27,6 +27,7 @@ libasahi_agx_files = files( 'agx_nir_lower_g17p_combined_sampler.c', 'agx_nir_ray_query.c', 'agx_nir_gen1_ray_query.c', + 'agx_nir_lower_math.c', 'agx_nir_lower_shared_bitsize.c', 'agx_nir_lower_subgroups.c', 'agx_nir_lower_compute_derivatives.c', @@ -183,6 +184,7 @@ if with_tests 'agx_tests', files( 'test/test-compact-constants.cpp', + 'test/test-lower-math.cpp', 'test/test-minifloat.cpp', 'test/test-optimizer.cpp', 'test/test-pressure-cse.cpp', diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp new file mode 100644 index 000000000000..4542dcddfb38 --- /dev/null +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -0,0 +1,322 @@ +/* + * Copyright 2026 Joshua Warren + * SPDX-License-Identifier: MIT + */ + +#include "nir_builder.h" + +extern "C" { +#include "agx_nir.h" +} + +#include +#include + +/* + * Behavioral pins for the float-control keying in agx_nir_lower_math.c. + * + * Division and reciprocal do not key on float controls. Their bits are pinned + * on hardware by tests/precise_math.py, ulp_bound.py and div_window.py; here + * the pin is that DenormFlushToZero emits exactly the default lowering. + * + * The DenormFlushToZero sin test pins the one keyed change: a zero-comparing + * subnormal argument must produce a sign-preserving flushed zero instead of + * leaking the unflushed argument through the select. + */ + +class LowerMathTest : public ::testing::Test { + protected: + LowerMathTest() + { + mem_ctx = ralloc_context(NULL); + b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, &options, + "lower math test"); + } + + ~LowerMathTest() override { ralloc_free(mem_ctx); } + + void *mem_ctx; + nir_builder b; + static const nir_shader_compiler_options options; + + void set_mode(unsigned mode) + { + b.shader->info.float_controls_execution_mode = mode; + } + + unsigned count_alu(nir_op op) + { + unsigned n = 0; + nir_foreach_function_impl(impl, b.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu && + nir_instr_as_alu(instr)->op == op) + n++; + } + } + } + return n; + } + + unsigned count_iand_imm(uint32_t mask) + { + unsigned n = 0; + nir_foreach_function_impl(impl, b.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_alu) + continue; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if (alu->op == nir_op_iand && + nir_src_is_const(alu->src[1].src) && + nir_src_as_uint(alu->src[1].src) == mask) + n++; + } + } + } + return n; + } + + /* ALU op histogram of fdiv (or frcp) lowered under one float-control mode */ + std::map lowered_division(unsigned mode, bool reciprocal) + { + nir_builder s = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, + &options, "lower fdiv"); + s.shader->info.float_controls_execution_mode = mode; + nir_def *x = nir_undef(&s, 1, 32); + if (reciprocal) + nir_frcp(&s, x); + else + nir_fdiv(&s, nir_undef(&s, 1, 32), x); + + EXPECT_TRUE(agx_nir_lower_fdiv(s.shader)); + + std::map ops; + nir_foreach_function_impl(impl, s.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu) + ops[nir_instr_as_alu(instr)->op]++; + } + } + } + ralloc_free(s.shader); + return ops; + } +}; + +const nir_shader_compiler_options LowerMathTest::options = {}; + +TEST_F(LowerMathTest, DivFtzModeUnchanged) +{ + auto ftz = lowered_division(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32, false); + EXPECT_EQ(ftz.count(nir_op_fdiv), 0u); + EXPECT_EQ(ftz, lowered_division(0, false)); +} + +TEST_F(LowerMathTest, FrcpFtzModeUnchanged) +{ + auto ftz = lowered_division(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32, true); + EXPECT_EQ(ftz.count(nir_op_frcp), 1u); + EXPECT_EQ(ftz, lowered_division(0, true)); +} + +/* Division introduced after preprocessing (nir_lower_int64 float-to-int64 + * conversions, smooth offset interpolation) must not reach the backend, which + * has no fdiv. The late pass lowers fdiv exactly like the first pass and + * leaves every reciprocal alone, so none is refined twice. + */ +TEST_F(LowerMathTest, LateFdivLowersLikeFirstPass) +{ + nir_def *x = nir_undef(&b, 1, 32); + nir_fdiv(&b, nir_undef(&b, 1, 32), x); + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + + auto first = lowered_division(0, false); + unsigned expect_ffma = first.count(nir_op_ffma) ? first[nir_op_ffma] : 0; + EXPECT_EQ(count_alu(nir_op_ffma), expect_ffma); +} + +TEST_F(LowerMathTest, LateFdivLeavesFrcpAlone) +{ + nir_frcp(&b, nir_undef(&b, 1, 32)); + + EXPECT_FALSE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_frcp), 1u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); +} + +TEST_F(LowerMathTest, LateFdivDoesNotRefineExistingFrcp) +{ + nir_frcp(&b, nir_undef(&b, 1, 32)); + nir_fdiv(&b, nir_undef(&b, 1, 32), nir_undef(&b, 1, 32)); + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + + /* the fdiv alone lowers to F ffma ops; the untouched frcp adds none */ + auto first = lowered_division(0, false); + unsigned expect_ffma = first.count(nir_op_ffma) ? first[nir_op_ffma] : 0; + EXPECT_EQ(count_alu(nir_op_ffma), expect_ffma); + EXPECT_EQ(count_alu(nir_op_frcp), 1u + (first.count(nir_op_frcp) ? first[nir_op_frcp] : 0)); +} + +TEST_F(LowerMathTest, GraphicsFdivLowersBothWidthsInEarlyAndLatePasses) +{ + b.shader->info.stage = MESA_SHADER_FRAGMENT; + for (unsigned bits : {16u, 32u}) { + nir_fdiv(&b, nir_undef(&b, 1, bits), nir_undef(&b, 1, bits)); + } + + EXPECT_TRUE(agx_nir_lower_fdiv(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + EXPECT_EQ(count_alu(nir_op_fmul), 2u); + EXPECT_EQ(count_alu(nir_op_frcp), 2u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); + + /* The early pass deleted the instruction the builder cursor was parked + * after: the pass lowers through nir_def_replace, which removes the fdiv. + * Rebuild the cursor from the shader so the late-half fdivs land in the + * implementation instead of behind a removed instruction. */ + b.cursor = nir_after_impl(nir_shader_get_entrypoint(b.shader)); + + for (unsigned bits : {16u, 32u}) { + nir_fdiv(&b, nir_undef(&b, 1, bits), nir_undef(&b, 1, bits)); + } + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + EXPECT_EQ(count_alu(nir_op_fmul), 4u); + EXPECT_EQ(count_alu(nir_op_frcp), 4u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); +} + +TEST_F(LowerMathTest, SinDefaultKeepsSelect) +{ + set_mode(0); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 0u); +} + +TEST_F(LowerMathTest, SinFtzReturnsSignedZero) +{ + set_mode(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 1u); +} + +TEST_F(LowerMathTest, SinPreserveKeepsSelect) +{ + set_mode(FLOAT_CONTROLS_DENORM_PRESERVE_FP32); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 0u); +} + +/* + * Numerical contract of the two sin emissions on the zero-compare path, + * modelled against hardware compare semantics (AGX flushes denormal compare + * inputs, host arithmetic does not - which is exactly why the leak is + * invisible to host-side evaluation and to the NIR constant folder). + * + * With x a subnormal loaded unflushed from a buffer: + * default emission returns x itself - legal, sin(subnormal) rounds to it + * FTZ emission returns +/-0 - required by DenormFlushToZero + */ +TEST_F(LowerMathTest, SinZeroPathNumericalArtifact) +{ + const uint32_t subnormals[] = { + 0x00000001u, /* smallest positive subnormal */ + 0x007fffffu, /* largest positive subnormal */ + 0x80000001u, /* smallest negative subnormal */ + 0x80000000u, /* negative zero */ + }; + + for (uint32_t bits : subnormals) { + /* Hardware compare: flushes the denormal input, so it matches zero. */ + const bool zero_compare = true; + + /* Default emission: the argument passes through the select unchanged, + * so a negative subnormal leaks with its sign bit, a positive one + * without any flush at all. */ + const uint32_t default_bits = zero_compare ? bits : 0; + + /* FTZ emission: the select returns the sign-preserving flushed zero. */ + const uint32_t ftz_bits = zero_compare ? (bits & 0x80000000u) : bits; + + EXPECT_EQ(default_bits, bits) + << "default mode must return the (unflushed) argument"; + EXPECT_EQ(ftz_bits, bits & 0x80000000u) + << "FTZ mode must return the sign-preserving zero"; + } + + /* A normal input never matches the flushed zero compare, so both modes + * take the computed sin path; nothing about it is keyed. */ + const uint32_t normal = 0x3f800000u; /* 1.0 */ + const bool zero_compare_normal = false; + EXPECT_EQ(zero_compare_normal ? 0x80000000u : normal, normal); +} + +/* + * Graphics integer division keeps the correctly rounded reciprocal that + * nir_lower_idiv relies on. The production pattern is the real lowering's: + * nir_lower_idiv marks everything it emits exact (b->fp_math_ctrl |= + * nir_fp_exact) and opens emit_udiv with frcp(u2f32(denominator)), so the + * early pass must keep refining that exact 32-bit reciprocal in graphics + * stages. An ordinary fast-math frcp stays untouched: refinement is keyed + * on the exact marking, not on every reciprocal. + */ +TEST_F(LowerMathTest, GraphicsIdivReciprocalRefinedPlainFrcpUntouched) +{ + b.shader->info.stage = MESA_SHADER_FRAGMENT; + nir_def *x = nir_undef(&b, 1, 32); + nir_def *y = nir_undef(&b, 1, 32); + nir_udiv(&b, x, y); + + /* Exactly the lowering the driver runs, with the driver's options. */ + nir_lower_idiv_options idiv_options; + idiv_options.allow_fp16 = true; + EXPECT_TRUE(nir_lower_idiv(b.shader, &idiv_options)); + + /* The exact integer-conversion reciprocal gets its refinement steps. */ + EXPECT_TRUE(agx_nir_lower_fdiv(b.shader)); + EXPECT_GT(count_alu(nir_op_ffma), 0u); + + /* An unmarked graphics reciprocal is left alone. */ + nir_builder s = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, + &options, "plain frcp"); + nir_frcp(&s, nir_undef(&s, 1, 32)); + + EXPECT_FALSE(agx_nir_lower_fdiv(s.shader)); + unsigned plain_ffma = 0; + nir_foreach_function_impl(impl, s.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu && + nir_instr_as_alu(instr)->op == nir_op_ffma) + plain_ffma++; + } + } + } + EXPECT_EQ(plain_ffma, 0u); + + ralloc_free(s.shader); +} diff --git a/src/compiler/nir/nir_builder_opcodes_h.py b/src/compiler/nir/nir_builder_opcodes_h.py index 6501fa79c3cf..252a23f215fa 100644 --- a/src/compiler/nir/nir_builder_opcodes_h.py +++ b/src/compiler/nir/nir_builder_opcodes_h.py @@ -230,7 +230,7 @@ def get_intrinsic_bitsize(opcode): #endif /* _NIR_BUILDER_OPCODES_ */""" from nir_opcodes import opcodes, type_size, type_base_type -from nir_intrinsics import INTR_OPCODES, WRITE_MASK, ALIGN_MUL, IO_SEMANTICS, DEST_TYPE, SRC_TYPE +from nir_intrinsics import INTR_OPCODES, WRITE_MASK, ALIGN_MUL, IO_SEMANTICS, DEST_TYPE, SRC_TYPE, FP_MATH_CTRL from mako.template import Template # List of intrinsics that also need a nir_build_ prefixed factory macro. @@ -261,4 +261,5 @@ def get_intrinsic_bitsize(opcode): IO_SEMANTICS=IO_SEMANTICS, DEST_TYPE=DEST_TYPE, SRC_TYPE=SRC_TYPE, + FP_MATH_CTRL=FP_MATH_CTRL, build_prefixed_intrinsics=build_prefixed_intrinsics)) diff --git a/tests/NUMERICAL-HARNESS.md b/tests/NUMERICAL-HARNESS.md new file mode 100644 index 000000000000..fdc9b6ac8a49 --- /dev/null +++ b/tests/NUMERICAL-HARNESS.md @@ -0,0 +1,26 @@ +# Numerical qualification harness + +Driver source under test: Joshua Warren’s correctness-only adaptation +`5e45716701b64a4b627e7e60df5b781c65c87f42`. CI source is the previously checked +`001e33ae855bd6b9ddc28bfc60a6bd64d71bcf91` trial tree. This branch adds test +harness fixes only; the production PR and package source pins are unchanged. + +The ULP checker uses a positive binary32 ULP at the magnitude of the correctly +rounded reference, calculated in binary64. It explicitly rejects a non-finite +result for a finite reference. Special reference values retain bitwise checks, +with NaN payloads ignored. Seven CPU controls cover both signs, a power-of-two +boundary, out-of-budget results, NaN/infinity, FLT_MAX and signed zero. + +The original checker measures binary64 spacing, gives negative distances for +negative references and can accept NaN for a finite reference. These are oracle +defects, not observations of GPU arithmetic failure. + +The following files were copied byte for byte from Joshua Warren’s public +source at `f144c7949b7d960f26677b671341d5a123057ddf` to supply the fixtures +missing from the correctness extraction. No CDM, scheduling or performance +source is imported. + +- [precise_math.comp](https://github.com/joshuaswarren/mesa-1/blob/f144c7949b7d960f26677b671341d5a123057ddf/tests/precise_math.comp), Git blob `827941d2468890243996dc8d9ab7f74005dab062`. +- [compute_runner.cpp](https://github.com/joshuaswarren/mesa-1/blob/f144c7949b7d960f26677b671341d5a123057ddf/tests/compute_runner.cpp), Git blob `e16d45d9d61252d72b2389da8c5073c781e696ca`. + +Validation: seven CPU oracle controls pass. No numerical GPU result is claimed by this commit. diff --git a/tests/compute_runner.cpp b/tests/compute_runner.cpp new file mode 100644 index 000000000000..e16d45d9d612 --- /dev/null +++ b/tests/compute_runner.cpp @@ -0,0 +1,196 @@ +// Build: c++ -O2 compute_runner.cpp -lvulkan -o compute_runner +// Usage: compute_runner shader.spv in.bin out.bin out_bytes groups +#include + +#include +#include +#include +#include +#include +#include +#include + +#define CK(x) \ + do { \ + VkResult r_ = (x); \ + if (r_ != VK_SUCCESS) { \ + std::fprintf(stderr, "%s failed: %d\n", #x, (int)r_); \ + std::exit(2); \ + } \ + } while (0) + +static std::string slurp(const char* p) { + std::ifstream f(p, std::ios::binary); + if (!f) { std::fprintf(stderr, "open %s\n", p); std::exit(2); } + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +struct Buf { VkBuffer b; VkDeviceMemory m; void* map; }; + +static VkPhysicalDeviceMemoryProperties g_mp; +static Buf mkbuf(VkDevice dev, size_t size) { + Buf r{}; + VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bi.size = size; + bi.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + CK(vkCreateBuffer(dev, &bi, nullptr, &r.b)); + VkMemoryRequirements mr; + vkGetBufferMemoryRequirements(dev, r.b, &mr); + uint32_t ti = UINT32_MAX; + for (uint32_t i = 0; i < g_mp.memoryTypeCount; ++i) { + VkMemoryPropertyFlags want = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if ((mr.memoryTypeBits & (1u << i)) && + (g_mp.memoryTypes[i].propertyFlags & want) == want) { ti = i; break; } + } + if (ti == UINT32_MAX) { std::fprintf(stderr, "no host memtype\n"); std::exit(2); } + VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + ai.allocationSize = mr.size; + ai.memoryTypeIndex = ti; + CK(vkAllocateMemory(dev, &ai, nullptr, &r.m)); + CK(vkBindBufferMemory(dev, r.b, r.m, 0)); + CK(vkMapMemory(dev, r.m, 0, size, 0, &r.map)); + return r; +} + +int main(int argc, char** argv) { + if (argc < 6) { std::fprintf(stderr, "usage\n"); return 2; } + std::string spv = slurp(argv[1]); + std::string in = slurp(argv[2]); + size_t out_bytes = std::strtoull(argv[4], nullptr, 10); + uint32_t groups = std::strtoul(argv[5], nullptr, 10); + + VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo ici{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ici.pApplicationInfo = &app; + VkInstance inst; + CK(vkCreateInstance(&ici, nullptr, &inst)); + uint32_t n = 1; + VkPhysicalDevice pd; + CK(vkEnumeratePhysicalDevices(inst, &n, &pd)); + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(pd, &props); + vkGetPhysicalDeviceMemoryProperties(pd, &g_mp); + std::fprintf(stderr, "device: %s driver %u.%u.%u\n", props.deviceName, + VK_VERSION_MAJOR(props.driverVersion), + VK_VERSION_MINOR(props.driverVersion), + VK_VERSION_PATCH(props.driverVersion)); + + uint32_t qc = 0; + vkGetPhysicalDeviceQueueFamilyProperties(pd, &qc, nullptr); + std::vector qf(qc); + vkGetPhysicalDeviceQueueFamilyProperties(pd, &qc, qf.data()); + uint32_t qi = 0; + for (; qi < qc; ++qi) if (qf[qi].queueFlags & VK_QUEUE_COMPUTE_BIT) break; + float prio = 1.0f; + VkDeviceQueueCreateInfo dqi{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + dqi.queueFamilyIndex = qi; + dqi.queueCount = 1; + dqi.pQueuePriorities = &prio; + VkDeviceCreateInfo dci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dci.queueCreateInfoCount = 1; + dci.pQueueCreateInfos = &dqi; + VkDevice dev; + CK(vkCreateDevice(pd, &dci, nullptr, &dev)); + VkQueue q; + vkGetDeviceQueue(dev, qi, 0, &q); + + Buf bin = mkbuf(dev, in.size()); + Buf bout = mkbuf(dev, out_bytes); + std::memcpy(bin.map, in.data(), in.size()); + std::memset(bout.map, 0, out_bytes); + + VkDescriptorSetLayoutBinding binds[2]{}; + for (int i = 0; i < 2; ++i) { + binds[i].binding = i; + binds[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + binds[i].descriptorCount = 1; + binds[i].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo dsl{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + dsl.bindingCount = 2; + dsl.pBindings = binds; + VkDescriptorSetLayout layout; + CK(vkCreateDescriptorSetLayout(dev, &dsl, nullptr, &layout)); + VkPipelineLayoutCreateInfo pli{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pli.setLayoutCount = 1; + pli.pSetLayouts = &layout; + VkPipelineLayout pl; + CK(vkCreatePipelineLayout(dev, &pli, nullptr, &pl)); + VkShaderModuleCreateInfo smi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + smi.codeSize = spv.size(); + smi.pCode = (const uint32_t*)spv.data(); + VkShaderModule sm; + CK(vkCreateShaderModule(dev, &smi, nullptr, &sm)); + VkComputePipelineCreateInfo cpi{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cpi.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + cpi.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + cpi.stage.module = sm; + cpi.stage.pName = "main"; + cpi.layout = pl; + VkPipeline pipe; + CK(vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &cpi, nullptr, &pipe)); + + VkDescriptorPoolSize ps{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2}; + VkDescriptorPoolCreateInfo dpi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dpi.maxSets = 1; + dpi.poolSizeCount = 1; + dpi.pPoolSizes = &ps; + VkDescriptorPool pool; + CK(vkCreateDescriptorPool(dev, &dpi, nullptr, &pool)); + VkDescriptorSetAllocateInfo dsa{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + dsa.descriptorPool = pool; + dsa.descriptorSetCount = 1; + dsa.pSetLayouts = &layout; + VkDescriptorSet ds; + CK(vkAllocateDescriptorSets(dev, &dsa, &ds)); + VkDescriptorBufferInfo dbi[2] = {{bin.b, 0, VK_WHOLE_SIZE}, {bout.b, 0, VK_WHOLE_SIZE}}; + VkWriteDescriptorSet w[2]{}; + for (int i = 0; i < 2; ++i) { + w[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + w[i].dstSet = ds; + w[i].dstBinding = i; + w[i].descriptorCount = 1; + w[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + w[i].pBufferInfo = &dbi[i]; + } + vkUpdateDescriptorSets(dev, 2, w, 0, nullptr); + + VkCommandPoolCreateInfo cpci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + cpci.queueFamilyIndex = qi; + VkCommandPool cp; + CK(vkCreateCommandPool(dev, &cpci, nullptr, &cp)); + VkCommandBufferAllocateInfo cba{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + cba.commandPool = cp; + cba.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cba.commandBufferCount = 1; + VkCommandBuffer cb; + CK(vkAllocateCommandBuffers(dev, &cba, &cb)); + VkCommandBufferBeginInfo cbb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + CK(vkBeginCommandBuffer(cb, &cbb)); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); + vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pl, 0, 1, &ds, 0, nullptr); + vkCmdDispatch(cb, groups, 1, 1); + VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + CK(vkEndCommandBuffer(cb)); + + VkFenceCreateInfo fci{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence; + CK(vkCreateFence(dev, &fci, nullptr, &fence)); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.commandBufferCount = 1; + si.pCommandBuffers = &cb; + CK(vkQueueSubmit(q, 1, &si, fence)); + CK(vkWaitForFences(dev, 1, &fence, VK_TRUE, 60000000000ull)); + std::ofstream of(argv[3], std::ios::binary); + of.write((const char*)bout.map, out_bytes); + vkDeviceWaitIdle(dev); + return 0; +} diff --git a/tests/div_window.comp b/tests/div_window.comp new file mode 100644 index 000000000000..a622787dc435 --- /dev/null +++ b/tests/div_window.comp @@ -0,0 +1,42 @@ +#version 450 +layout(local_size_x = 64) in; +layout(std430, binding = 0) readonly buffer In { uint base; uint seed; } p; +layout(std430, binding = 1) writeonly buffer Out { uint o[]; } outp; + +uint pcg(uint v) +{ + uint s = v * 747796405u + 2891336453u; + uint w = ((s >> ((s >> 28u) + 4u)) ^ s) * 277803737u; + return (w >> 22u) ^ w; +} + +uint with_exp(uint bits, uint e) { return (bits & 0x807fffffu) | (e << 23u); } + +/* Exponents at the zero/denormal (0), normal and 2^126..inf (252..255) edges, + * plus a 46..50 group that keeps the residual of the quotient near 2^-126. */ +const uint EA[16] = uint[](0u, 1u, 2u, 46u, 47u, 48u, 49u, 50u, 126u, 127u, 128u, 252u, 253u, 254u, 255u, 255u); +const uint ED[16] = uint[](0u, 0u, 1u, 2u, 3u, 126u, 127u, 128u, 250u, 251u, 252u, 253u, 254u, 255u, 1u, 252u); + +void main() +{ + for (uint j = 0u; j < 32u; j++) { + uint i = gl_GlobalInvocationID.x * 32u + j; + uint x = p.base + i; + uint h0 = pcg(x ^ p.seed), h1 = pcg(h0 + 1u), h2 = pcg(h1 + 2u); + uint a = h0, d = h1, mode = h2 & 3u; + if (mode == 1u) { + a = with_exp(a, EA[(h2 >> 2u) & 15u]); + d = with_exp(d, ED[(h2 >> 6u) & 15u]); + } else if (mode == 2u) { + uint ed = 1u + (h2 >> 2u) % 254u; + int e = int((h2 >> 10u) % 14u) + ((h2 & 0x80000000u) != 0u ? 117 : -128); + a = with_exp(a, uint(clamp(int(ed) + e, 0, 255))); + d = with_exp(d, ed); + } else if (mode == 3u) { + a = with_exp(a, 1u + (h2 >> 2u) % 254u); + d = with_exp(d, 1u + (h2 >> 10u) % 254u); + } + outp.o[2u * i] = floatBitsToUint(uintBitsToFloat(a) / uintBitsToFloat(d)); + outp.o[2u * i + 1u] = floatBitsToUint(1.0 / uintBitsToFloat(x)); + } +} diff --git a/tests/div_window.py b/tests/div_window.py new file mode 100755 index 000000000000..52f2c389f1eb --- /dev/null +++ b/tests/div_window.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Usage: div_window.py ICD COMPUTE_RUNNER [REF_ICD] [CHUNKS]; glslangValidator and numpy are required. + +Default-path division and reciprocal against the correctly rounded (with FTZ) +quotient, bitwise, specials included: every fp32 reciprocal (64 chunks of +2^26) and the same number of hashed quotients, a quarter of them raw bits and +the rest with exponents at the zero, denormal, inf and 2^126 edges and with +quotients straddling 2^-126 and 2^128. With REF_ICD, every result must also +match that driver bitwise (for example the build before a lowering change). +CHUNKS is N (the first N of 64) or FIRST-LAST, so an interrupted sweep can +resume. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import numpy as np + +args = sys.argv[1:] +icd, runner = map(os.path.abspath, args[:2]) +ref = os.path.abspath(args[2]) if len(args) > 2 else None +span = args[3].split("-") if len(args) > 3 else ["64"] +chunks = range(int(span[0]), int(span[1]) + 1) if len(span) == 2 else range(int(span[0])) +here = Path(__file__).resolve().parent +N = 1 << 26 +SEED = 20261006 + +def pcg(v): + s = v * np.uint32(747796405) + np.uint32(2891336453) + w = ((s >> ((s >> np.uint32(28)) + np.uint32(4))) ^ s) * np.uint32(277803737) + return (w >> np.uint32(22)) ^ w + +EA = np.array([0, 1, 2, 46, 47, 48, 49, 50, 126, 127, 128, 252, 253, 254, 255, 255], dtype=np.uint32) +ED = np.array([0, 0, 1, 2, 3, 126, 127, 128, 250, 251, 252, 253, 254, 255, 1, 252], dtype=np.uint32) + +def operands(base): + x = np.arange(N, dtype=np.uint32) + np.uint32(base) + h0 = pcg(x ^ np.uint32(SEED)) + h1 = pcg(h0 + np.uint32(1)) + h2 = pcg(h1 + np.uint32(2)) + a, d, mode = h0.copy(), h1.copy(), h2 & 3 + exp = lambda bits, e: (bits & np.uint32(0x807fffff)) | (e.astype(np.uint32) << np.uint32(23)) + m = mode == 1 + a[m] = exp(a[m], EA[(h2[m] >> 2) & 15]) + d[m] = exp(d[m], ED[(h2[m] >> 6) & 15]) + m = mode == 2 + ed = 1 + (h2[m] >> 2) % 254 + e = ((h2[m] >> 10) % 14).astype(np.int64) + np.where(h2[m] & 0x80000000, 117, -128) + a[m] = exp(a[m], np.clip(ed.astype(np.int64) + e, 0, 255)) + d[m] = exp(d[m], ed) + m = mode == 3 + a[m] = exp(a[m], 1 + (h2[m] >> 2) % 254) + d[m] = exp(d[m], 1 + (h2[m] >> 10) % 254) + return a, d, x + +def quotient(a, b): + a = a.copy() + b = b.copy() + a[(a & 0x7fffffff) < 0x800000] &= 0x80000000 + b[(b & 0x7fffffff) < 0x800000] &= 0x80000000 + with np.errstate(all="ignore"): + exact = a.view(np.float32).astype(np.float64) / b.view(np.float32).astype(np.float64) + rounded = exact.astype(np.float32).view(np.uint32) + flushed = np.abs(exact) < 2.0**-126 + rounded[flushed] = (a[flushed] ^ b[flushed]) & 0x80000000 + return rounded + +def bad(got, want): + both_nan = ((got & 0x7fffffff) > 0x7f800000) & ((want & 0x7fffffff) > 0x7f800000) + return np.flatnonzero((got != want) & ~both_nan) + +def run(spv, driver, base, path): + path.with_suffix(".in").write_bytes(np.array([base, SEED], dtype=np.uint32).tobytes()) + env = dict(os.environ, VK_DRIVER_FILES=driver, VK_ICD_FILENAMES=driver, MESA_SHADER_CACHE_DISABLE="true") + subprocess.run(["flock", "-w", "1800", "/tmp/m1-gpu.lock", runner, str(spv), str(path.with_suffix(".in")), + str(path), str(N * 8), str(N // 32 // 64)], env=env, check=True, timeout=2100) + return np.fromfile(path, dtype=np.uint32).reshape(N, 2) + +totals = {"div_oracle": 0, "recip_oracle": 0, "div_ref": 0, "recip_ref": 0} +examples = [] +with tempfile.TemporaryDirectory(prefix="mesa-divwin-", dir="/dev/shm" if os.path.isdir("/dev/shm") else None) as td: + td = Path(td) + spv = td / "probe.spv" + subprocess.run(["glslangValidator", "-V", str(here / "div_window.comp"), "-o", str(spv)], check=True, timeout=30) + for c in chunks: + base = c * N + out = run(spv, icd, base, td / "out.bin") + a, d, x = operands(base) + checks = {"div_oracle": bad(out[:, 0], quotient(a, d)), + "recip_oracle": bad(out[:, 1], quotient(np.full(N, 0x3f800000, dtype=np.uint32), x))} + if ref: + r = run(spv, ref, base, td / "ref.bin") + checks["div_ref"] = np.flatnonzero(out[:, 0] != r[:, 0]) + checks["recip_ref"] = np.flatnonzero(out[:, 1] != r[:, 1]) + for k, v in checks.items(): + totals[k] += len(v) + for i in v[:2]: + if len(examples) < 10: + examples.append({"check": k, "a": hex(int(a[i])), "d": hex(int(d[i])), "x": hex(int(x[i])), + "got": [hex(int(w)) for w in out[i]]}) + print(f"chunk {c}: " + " ".join(f"{k}={len(v)}" for k, v in checks.items()), flush=True) + +result = {"pairs": len(chunks) * N, "reciprocals": len(chunks) * N, "chunks": [chunks[0], chunks[-1]], + "mismatches": totals if ref else + {k: v for k, v in totals.items() if k.endswith("oracle")}, "examples": examples} +print(json.dumps(result, sort_keys=True)) +sys.exit(any(totals.values())) diff --git a/tests/precise_math.comp b/tests/precise_math.comp new file mode 100644 index 000000000000..827941d24688 --- /dev/null +++ b/tests/precise_math.comp @@ -0,0 +1,17 @@ +#version 450 +layout(local_size_x = 64) in; +layout(std430, binding = 0) readonly buffer In { vec4 v[]; } inp; +layout(std430, binding = 1) writeonly buffer Out { float o[]; } outp; +void main() { + uint i = gl_GlobalInvocationID.x; + vec4 x = inp.v[i]; + precise float t = log2(x.w); + precise float literal = t * 0.6931471805599453; + precise float from_buffer = t * x.z; + outp.o[i * 6 + 0] = x.x / x.y; + outp.o[i * 6 + 1] = 1.0 / x.y; + outp.o[i * 6 + 2] = literal; + outp.o[i * 6 + 3] = from_buffer; + outp.o[i * 6 + 4] = t; + outp.o[i * 6 + 5] = log(x.w); +} diff --git a/tests/test_ulp_bound.py b/tests/test_ulp_bound.py new file mode 100644 index 000000000000..8bce1713ad82 --- /dev/null +++ b/tests/test_ulp_bound.py @@ -0,0 +1,62 @@ +"""CPU controls for the numerical GPU oracle, without starting Vulkan.""" +import ast +from pathlib import Path +import unittest + +import numpy as np + + +# ulp_bound currently dispatches a GPU job at module scope. Extract only its +# pure checks so these controls need neither a GPU nor a Vulkan driver. +source = Path(__file__).with_name("ulp_bound.py") +tree = ast.parse(source.read_text()) +functions = ast.Module(body=[n for n in tree.body + if isinstance(n, ast.FunctionDef) + and n.name in ("quotient", "check")], type_ignores=[]) +namespace = {"np": np} +exec(compile(functions, str(source), "exec"), namespace) + + +class ULPControls(unittest.TestCase): + def measure(self, reference, result): + namespace["inp"] = np.array([[reference, 0x3f800000]], dtype=np.uint32) + namespace["out"] = np.array([[result]], dtype=np.uint32) + return namespace["check"](0, namespace["inp"][:, 0], + namespace["inp"][:, 1]) + + def test_positive_one_ulp(self): + r = self.measure(0x3f800000, 0x3f800001) + self.assertEqual(r["max_ulp"], 1.0) + self.assertEqual(r["over_2p5"], 0) + + def test_positive_over_budget(self): + self.assertEqual(self.measure(0x3f800000, 0x3f800003)["over_2p5"], 1) + + def test_negative_errors_are_counted(self): + r = self.measure(0xbf800000, 0xbf800003) + self.assertEqual(r["max_ulp"], 3.0) + self.assertEqual(r["over_2p5"], 1) + + def test_negative_power_of_two_boundary(self): + self.assertEqual(self.measure(0xbf800000, 0xbf7fffff)["max_ulp"], 0.5) + + def test_nan_and_infinity_cannot_match_finite(self): + for bits in (0x7fc00000, 0x7f800000, 0xff800000): + with self.subTest(bits=hex(bits)): + r = self.measure(0x3f800000, bits) + self.assertEqual(r["over_2p5"], 1) + self.assertEqual(r["nonfinite_mismatch"], 1) + + def test_largest_finite_has_finite_ulp(self): + r = self.measure(0x7f7fffff, 0x7f7ffffe) + self.assertEqual(r["max_ulp"], 1.0) + self.assertEqual(r["over_2p5"], 0) + + def test_specials_and_signed_zero(self): + for bits in (0, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000): + self.assertEqual(self.measure(bits, bits)["special_mismatch"], 0) + self.assertEqual(self.measure(0x80000000, 0)["special_mismatch"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ulp_bound.py b/tests/ulp_bound.py new file mode 100644 index 000000000000..952bb94e555f --- /dev/null +++ b/tests/ulp_bound.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Usage: ulp_bound.py ICD COMPUTE_RUNNER; glslangValidator and numpy are required. + +Default-path division contract: every result within 2.5 ULP of the correctly +rounded (with FTZ) quotient, the OpFDiv budget of the Vulkan SPIR-V +environment. Two sets: the exact 262,144-pair precise_math.py regression +input (same seed and construction) and a 66,048-pair boundary set straddling +the 2^-126 normal limit (the M2 Pro failure class). +Specials (+-0, inf, NaN, flushed results) must match the oracle bitwise. +The correctly rounded contract itself lives in precise_math.py (nir_fp_exact +path); this test only pins the default. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import numpy as np + +icd, runner = map(os.path.abspath, sys.argv[1:]) +here = Path(__file__).resolve().parent + +# Regression set: identical construction to precise_math.py (seed 20260913). +rng = np.random.default_rng(20260913) +n = 262144 +inp = rng.integers(0, 2**32, (n, 4), dtype=np.uint32) +edges = np.array([0, 0x80000000, 1, 0x7fffff, 0x800000, 0x800001, + 0x80800000, 0x3f800000, 0x3f7fffff, 0x3f800001, + 0x40000000, 0x7f7fffff, 0xff7fffff, 0x7f800000, + 0xff800000, 0x7fc00000], dtype=np.uint32) +inp[:256, 0] = np.repeat(edges, len(edges)) +inp[:256, 1] = np.tile(edges, len(edges)) +inp[256:65536, 0] = (inp[256:65536, 0] & 0x807fffff) | (rng.integers(1, 40, 65280, dtype=np.uint32) << 23) +inp[256:65536, 1] = (inp[256:65536, 1] & 0x807fffff) | (rng.integers(80, 160, 65280, dtype=np.uint32) << 23) +inp[:, 2] = 0x3f317218 +inp[:, 3] = rng.integers(0x00800000, 0x7f800000, n, dtype=np.uint32) +inp[:7, 3] = [0x3f800000, 0x40000000, 0x40400000, 0x3f7fffff, 0x3f800001, 0x00800000, 0x7f7fffff] + +# Boundary set: normal denominators in [1, 2^18), quotients +# (1 + j*2^-24)*2^-126 straddling the normal limit, j in [-64, 64], +# four sign combinations. a = RN(q * d) in fp32. +rngb = np.random.default_rng(20260913) +nb = 66048 +db = ((rngb.integers(127, 145, nb, dtype=np.uint32) << 23) | + rngb.integers(0, 1 << 23, nb, dtype=np.uint32)) +j = rngb.integers(-64, 65, nb).astype(np.float64) +q64 = (1.0 + j * 2.0**-24) * 2.0**-126 +ab = (q64 * db.view(np.float32).astype(np.float64)).astype(np.float32).view(np.uint32) +ab ^= rngb.integers(0, 2, nb, dtype=np.uint32) << 31 +db ^= rngb.integers(0, 2, nb, dtype=np.uint32) << 31 +bnd = np.stack([ab, db, np.full(nb, 0x3f317218, dtype=np.uint32), + np.full(nb, 0x3f800000, dtype=np.uint32)], axis=1) +inp = np.concatenate([inp, bnd]) +ntot = len(inp) + +def quotient(a, b): + a = a.copy() + b = b.copy() + a[(a & 0x7fffffff) < 0x800000] &= 0x80000000 + b[(b & 0x7fffffff) < 0x800000] &= 0x80000000 + with np.errstate(all="ignore"): + exact = a.view(np.float32).astype(np.float64) / b.view(np.float32).astype(np.float64) + rounded = exact.astype(np.float32).view(np.uint32) + flushed = np.abs(exact) < 2.0**-126 + rounded[flushed] = (a[flushed] ^ b[flushed]) & 0x80000000 + return rounded + +with tempfile.TemporaryDirectory(prefix="mesa-ulp-") as td: + td = Path(td) + inp.tofile(td / "in.bin") + subprocess.run(["glslangValidator", "-V", str(here / "precise_math.comp"), + "-o", str(td / "probe.spv")], check=True) + env = dict(os.environ, VK_DRIVER_FILES=icd, VK_ICD_FILENAMES=icd, + MESA_SHADER_CACHE_DISABLE="true") + subprocess.run(["flock", "-w", "30", "/tmp/m1-gpu.lock", runner, + str(td / "probe.spv"), str(td / "in.bin"), str(td / "out.bin"), + str(ntot * 6 * 4), str(ntot // 64)], env=env, check=True, + timeout=120) + out = np.fromfile(td / "out.bin", dtype=np.uint32).reshape(ntot, 6) + +def check(col, a_bits, b_bits): + got = out[:, col] + want = quotient(a_bits, b_bits) + gf = got.copy().view(np.float32).astype(np.float64) + wf = want.copy().view(np.float32).astype(np.float64) + both_nan = ((got & 0x7fffffff) > 0x7f800000) & ((want & 0x7fffffff) > 0x7f800000) + special = ~np.isfinite(wf) | (wf == 0.0) + spec_bad = np.flatnonzero(special & (got != want) & ~both_nan) + normal = np.flatnonzero(~special) + finite_result = np.isfinite(gf[normal]) + with np.errstate(all="ignore"): + # A binary32 ULP is positive for both signs. Derive its exponent + # before measuring in binary64; spacing(float64) measures the wrong + # format and spacing(negative) would hide errors. ldexp also handles + # FLT_MAX without overflowing to infinity like float32 spacing does. + exponent = np.frexp(np.abs(wf[normal]))[1] + ulp = np.ldexp(np.ones(len(normal)), np.maximum(exponent - 24, -149)) + dist = np.abs(gf[normal] - wf[normal]) / ulp + # NaN comparisons are false, so reject non-finite results explicitly when + # the oracle expected a finite value. + over = normal[(dist > 2.5) | ~finite_result] + finite_dist = dist[finite_result] + return {"max_ulp": float(finite_dist.max()) if len(finite_dist) else 0.0, + "over_2p5": len(over), + "nonfinite_mismatch": int(np.count_nonzero(~finite_result)), + "special_mismatch": len(spec_bad), + "examples": [[hex(int(v)) for v in inp[i][:2]] for i in over[:5]][:5]} + +result = {"pairs": ntot, + "div": check(0, inp[:, 0], inp[:, 1]), + "recip": check(1, np.full(ntot, 0x3f800000, dtype=np.uint32), inp[:, 1])} +print(json.dumps(result, sort_keys=True)) +sys.exit(any(r["over_2p5"] or r["special_mismatch"] for r in result.values() if isinstance(r, dict)))