From 1474f7eb82e6d65586c1c8692a65abc6bd0849e6 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:17:37 -0500 Subject: [PATCH 01/22] agx: correctly rounded fdiv/frcp, faithful log/log2, exact sin/cos reduction Three fp32 rounding defects on Honeykrisp (Apple M1 G13G, Mesa 26.1.7) that MLX patches around in its shaders: a) GLSL "/" was lowered by nir_opt_algebraic (lower_fdiv) to a * frcp(b), and frcp was only Newton-refined when the instruction was `precise`. Even with an exact reciprocal a * RN(1/b) is not RN(a/b): 80/25 and 10/25 come out one ulp low, 6.55859375/0.030016447 one ulp high. Measured on 1008 pairs: 299 mismatches (298 at 1 ulp, 1 at 2 ulp), 1/b: 204 mismatches; 1M random pairs: 305792 mismatches. fma() itself is IEEE (0/1000 mismatches), the MLX note blaming fma contraction was a misdiagnosis. b) log(x) was flog2(x) * ln2 on the hardware log2, which is only absolutely accurate: log(3) = 1.0986121892929077 instead of 1.0986123085021973, 2792/4107 mismatches on [0.5, 64] with errors up to 357 ulp near 1 (log(1.0000001) = 1.65e-7 vs 1.0e-7); log2 had 2905/4107 mismatches. c) sin/cos reduced the argument as fract(x / 2pi) in fp32, so the reduced angle lost ulp(x) of precision: sin(3.0) 14 ulp off, sin(1e5) 1.06e-3 absolute error, sin(1e8) = 0.0, sin(-1e-3) 571 ulp off, sin(pi_f32) = 0, and the hardware sin_pt_2 core itself is 2 to 3 ulp off (cos(1e-3) 3 ulp). The lowering moves to a new the AGX compiler pass file agx_nir_lower_math.c, built only from the exact fp32 fma and marked no-fast-math so nir_opt_algebraic cannot reassociate it: - lower_fdiv is dropped from agx_nir_options; fdiv32 becomes Markstein division (y = refined rcp, q = a y, r = fma(-b, q, a), q + r y) with the NaN/zero guard keeping q for b = 0, overflow and signed zero. frcp32 is always refined. fdiv16 keeps a * frcp(b). - fmul(flog2(x), ln2) as emitted by nir_flog, and any remaining flog2, become a double-float evaluation of log(m) = 2s + 2s^3 P(s^2) with s = (m-1)/(m+1) kept as a pair, m folded into [sqrt(1/2), sqrt(2)), e ln2 split hi/lo and Fast2Sum accumulation (about 40 bits before the final rounding). Zero/subnormal/negative/inf/NaN are handled by bit tests, not the hardware log2. - fsin/fcos: Cody-Waite in radians with pi/2 in three pieces, the k C2 product and its subtraction as TwoProduct/TwoSum so the reduced argument is a pair accurate to an ulp, for |x| < 2^22; an integer Payne-Hanek (24-bit mantissa times a 128-bit window of 2/pi, 64 fraction bits kept) for |x| >= 2^22 up to FLT_MAX. The quadrant then selects a Taylor polynomial in quadrant units (sin degree 13, cos degree 14, leading terms as pairs, low part of the argument fed to the derivative). The hardware sin_pt_1/sin_pt_2 core (nir_op_fsin_agx) is no longer used by the lowering. fp16 sin/cos go through the fp32 path. Verification (standalone Vulkan compute reproducer): - division: 1008-pair table incl. 80/25, 10/25, 1000 random: 0 mismatches (stock 299); 1/b 0 (stock 204); 1M random pairs 0 mismatches for both a/b and 1/b; b = 0, inf, NaN, overflow, flushed subnormal and -0/5 = -0 match host IEEE. - log: 4107 points on [0.5, 64] plus pinned exact cases: 0 mismatches (stock 2792), log(3) = 1.0986123085021973; log2 0 (stock 2905); 1M random on [1e-37, 1e38]: log 3 at 1 ulp, log2 14 at 1 ulp, rest exact. - sin/cos at 1e5, 5e5, 1e6, 5e6, 1e8: all 0 ulp from the double reference except cos(5e6) at 1 ulp (stock: 2.8e5 to 1.06e9 ulp); 1310 mixed points max 1 ulp; 1M random over [1e-3, 1e9] and [-10, 10]: sin 42346 and cos 41184 at 1 ulp, rest exact, max 1 ulp (stock >= 8 ulp for 627689 / 512816 of them). - MLX suites on the same Release tree, stock vs fixed: primitive 98/98, eq_math 7/7, fast_ops 30/30, runtime 40/40, reduce_ops 34/34 both; complex_ops 30/33 both with the same three pre-existing cases (failed assertions 31 -> 29). The upstream ops_tests log(3) pin through libmlx: FAIL on stock, PASS on fixed, as do 80/25, 10/25 and sin/cos up to the 1e5 limit mlx enforces. - timing: 16M-element elementwise dispatch is memory bound at ~2.45 ms on both drivers (sin 2.474 -> 2.463 ms median, within spread). A 32-deep in-register chain shows the ALU cost: sin 6.64 -> 39.6 ms, log 2.57 -> 34.8 ms, div 2.84 -> 7.24 ms, mul unchanged. --- src/asahi/compiler/agx_compile.c | 15 +- src/asahi/compiler/agx_compile.h | 1 - src/asahi/compiler/agx_nir.h | 3 + src/asahi/compiler/agx_nir_lower_math.c | 567 ++++++++++++++++++++++++ src/asahi/compiler/meson.build | 1 + 5 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 src/asahi/compiler/agx_nir_lower_math.c diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 5443f4e509f0..d3c46578588a 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -7543,9 +7543,11 @@ agx_calc_stats(agx_context *ctx, unsigned size, struct agx2_stats *stats) } static bool -agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *_) +agx_lower_sincos_filter(const nir_instr *instr, const void *g17p) { - if (instr->type != nir_instr_type_alu) + /* Only the native G17P sequence is lowered here; every other target goes + * through agx_nir_lower_sincos, the exact reduction. */ + if (!(uintptr_t)g17p || instr->type != nir_instr_type_alu) return false; nir_alu_instr *alu = nir_instr_as_alu(instr); @@ -10993,6 +10995,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep * FMAs only. Other shaders keep the lowering here. */ if (!g17p || !agx_nir_fma_rte_applies(nir)) NIR_PASS(_, nir, agx_lower_sincos, g17p ? 1 : native_fragment_depth ? 2 : 0); + NIR_PASS(_, nir, agx_nir_lower_sincos); NIR_PASS(_, nir, nir_shader_intrinsics_pass, agx_lower_front_face, nir_metadata_control_flow, NULL); if (g17p) @@ -11018,9 +11021,11 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep NIR_PASS(_, nir, nir_lower_idiv, &idiv_options); - /* Has to run after nir_lower_idiv */ - NIR_PASS(_, nir, nir_shader_alu_pass, agx_nir_lower_fdiv, - nir_metadata_control_flow, NULL); + /* Has to run after nir_lower_idiv, and before the log lowering so the + * reciprocals it builds are not refined twice. + */ + NIR_PASS(_, nir, agx_nir_lower_fdiv); + NIR_PASS(_, nir, agx_nir_lower_log); NIR_PASS(_, nir, nir_opt_deref); NIR_PASS(_, nir, nir_lower_vars_to_ssa); diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h index 8e33177e4365..663d65e8a0a1 100644 --- a/src/asahi/compiler/agx_compile.h +++ b/src/asahi/compiler/agx_compile.h @@ -665,7 +665,6 @@ agx_round_registers(unsigned halfregs) } static const nir_shader_compiler_options agx_nir_options = { - .lower_fdiv = true, .float_mul_add16 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, .float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse, .lower_flrp16 = true, diff --git a/src/asahi/compiler/agx_nir.h b/src/asahi/compiler/agx_nir.h index d38cdbb132cd..ec86ea95aa45 100644 --- a/src/asahi/compiler/agx_nir.h +++ b/src/asahi/compiler/agx_nir.h @@ -57,6 +57,9 @@ bool agx_nir_lower_word_bounds(struct nir_shader *s); bool agx_nir_share_bounds(struct nir_shader *s); bool agx_nir_lower_atomics(struct nir_shader *s, bool native_fadd); bool agx_nir_opt_canonical_minmax(struct nir_shader *s); +bool agx_nir_lower_fdiv(struct nir_shader *s); +bool agx_nir_lower_log(struct nir_shader *s); +bool agx_nir_lower_sincos(struct nir_shader *s); bool agx_nir_lower_texture_early(struct nir_shader *s, bool support_lod_bias); bool agx_nir_lower_g17p_helper_samples(struct nir_shader *s); diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c new file mode 100644 index 000000000000..3e9b923264f7 --- /dev/null +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -0,0 +1,567 @@ +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright 2026 Joshua Warren + * SPDX-License-Identifier: MIT + */ + +#include +#include "compiler/nir/nir.h" +#include "compiler/nir/nir_builder.h" +#include "compiler/nir/nir_builtin_builder.h" +#include "agx_nir.h" + +/* + * AGX transcendental lowering with fp32 rounding contracts. + * + * The hardware gives an approximate rcp (off by 1 ulp), an approximate log2 + * (absolute error only, so relative error explodes near 1) and a sin unit that + * takes an argument in quadrants. Everything here is built from the exact fp32 + * fma, so the sequences are marked no-fast-math to keep nir_opt_algebraic from + * reassociating or contracting them. + */ + +/* + * Correctly rounded 1/x. One Newton-Raphson step on the hardware rcp: + * + * u_2 = u + u(1 - xu) = fma(fma(-x, u, 1), u, u) + * + * exhaustively verified upstream with a modified math_bruteforce. If u is + * infinite (x zero or flushed), the refinement is NaN, so keep u. + */ +static nir_def * +rcp_rn(nir_builder *b, nir_def *x) +{ + nir_def *u = nir_frcp(b, x); + nir_def *one = nir_imm_float(b, 1.0); + nir_def *u_2 = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, one), u, u); + return nir_bcsel(b, nir_fisnan(b, u_2), u, u_2); +} + +/* + * Correctly rounded a/b (Markstein). With y = RN(1/b) and q = RN(a y), the + * residual r = a - b q is exact in an fma and q + r y rounds to RN(a/b) for + * normal operands. If the refinement produces NaN (b zero, q infinite), q + * already holds the IEEE result for that case. A zero q is also final (a zero + * or the quotient flushed) and keeps its sign, which the refinement would + * lose. + */ +static nir_def * +div_rn(nir_builder *b, nir_def *a, nir_def *d) +{ + nir_def *y = rcp_rn(b, d); + nir_def *q = nir_fmul(b, a, y); + nir_def *r = nir_ffma(b, nir_fneg(b, d), q, a); + nir_def *q_2 = nir_ffma(b, r, y, q); + nir_def *keep = nir_ior(b, nir_fisnan(b, q_2), nir_feq_imm(b, q, 0.0)); + return nir_bcsel(b, keep, q, q_2); +} + +static bool +lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_frcp && alu->op != nir_op_fdiv) + return false; + + if (alu->def.bit_size == 64) + return false; + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + + nir_def *res; + if (alu->op == nir_op_frcp) { + if (alu->def.bit_size != 32) + return false; + + res = rcp_rn(b, nir_ssa_for_alu_src(b, alu, 0)); + } else { + nir_def *a = nir_ssa_for_alu_src(b, alu, 0); + nir_def *d = nir_ssa_for_alu_src(b, alu, 1); + + if (alu->def.bit_size == 32) + res = div_rn(b, a, d); + else + res = nir_fmul(b, a, nir_frcp(b, d)); + } + + nir_def_replace(&alu->def, res); + return true; +} + +bool +agx_nir_lower_fdiv(nir_shader *s) +{ + return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, NULL); +} + +/* + * Natural log and log2 as double-float (fp32 pair) arithmetic. + * + * x = 2^e m with m in [sqrt(1/2), sqrt(2)). With s = (m - 1) / (m + 1), + * + * log(m) = 2s + 2s^3 (1/3 + s^2/5 + s^4/7 + ...) + * + * s is kept as a pair (s, s_lo) from one correctly rounded division plus the + * fma residual, the cubic term is kept as a pair, and the terms are summed + * with Fast2Sum so the value handed to the final rounding carries about 40 + * bits. That makes the exact cases (powers of two, log(3)) come out correctly + * rounded and everything else faithful, without any table. + */ +struct log_parts { + nir_def *e; /* exponent as float */ + nir_def *s; /* 2s = leading term, as a pair (s, s_lo) */ + nir_def *s_lo; + nir_def *t_hi; /* 2 s^3 P(s^2), as a pair */ + nir_def *t_lo; +}; + +static nir_def * +fast2sum_lo(nir_builder *b, nir_def *a, nir_def *bb, nir_def *sum) +{ + /* Requires |a| >= |bb| or a == 0 */ + return nir_fsub(b, bb, nir_fsub(b, sum, a)); +} + +static struct log_parts +log_core(nir_builder *b, nir_def *x) +{ + nir_def *bits = x; + nir_def *e = nir_iadd_imm(b, nir_ushr_imm(b, bits, 23), -127); + nir_def *m = nir_ior_imm(b, nir_iand_imm(b, bits, 0x7fffff), 0x3f800000); + + /* Fold m into [sqrt(1/2), sqrt(2)) so the series converges fast and + * e ln2 never cancels more than one bit against log(m). + */ + nir_def *big = nir_fgt_imm(b, m, 1.41421353816986083984375); + m = nir_bcsel(b, big, nir_fmul_imm(b, m, 0.5), m); + e = nir_iadd(b, e, nir_b2i32(b, big)); + + nir_def *one = nir_imm_float(b, 1.0); + nir_def *two = nir_imm_float(b, 2.0); + nir_def *f = nir_fsub(b, m, one); /* exact */ + nir_def *d = nir_fadd(b, two, f); + nir_def *d_lo = fast2sum_lo(b, two, f, d); + + /* s = f / (d + d_lo) as a pair */ + nir_def *y = rcp_rn(b, d); + nir_def *s = nir_fmul(b, f, y); + nir_def *r = nir_ffma(b, nir_fneg(b, s), d, f); + s = nir_ffma(b, r, y, s); + r = nir_ffma(b, nir_fneg(b, s), d, f); + nir_def *s_lo = + nir_fmul(b, nir_fsub(b, r, nir_fmul(b, s, d_lo)), y); + + /* u = s^3 as a pair, including the 3 s^2 s_lo cross term */ + nir_def *w = nir_fmul(b, s, s); + nir_def *w_lo = nir_ffma(b, s, s, nir_fneg(b, w)); + nir_def *u = nir_fmul(b, s, w); + nir_def *u_lo = nir_fadd(b, nir_ffma(b, s, w, nir_fneg(b, u)), + nir_fmul(b, s, w_lo)); + u_lo = nir_ffma(b, nir_fmul_imm(b, s_lo, 3.0), w, u_lo); + + /* P(w) = 1/3 + w Q(w) as a pair, 1/3 split hi/lo */ + nir_def *q = nir_imm_float(b, 1.0 / 13.0); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 11.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 9.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 7.0)); + q = nir_ffma(b, q, w, nir_imm_float(b, 1.0 / 5.0)); + nir_def *third = nir_imm_float(b, 0.3333333432674408); + nir_def *v = nir_ffma(b, w, q, nir_imm_float(b, -9.934107758624577e-09)); + nir_def *p = nir_fadd(b, third, v); + nir_def *p_lo = fast2sum_lo(b, third, v, p); + + nir_def *th = nir_fmul(b, u, p); + nir_def *tl = nir_fadd(b, nir_fadd(b, nir_ffma(b, u, p, nir_fneg(b, th)), + nir_fmul(b, u, p_lo)), + nir_fmul(b, u_lo, p)); + + return (struct log_parts){ + .e = nir_i2f32(b, e), + .s = s, + .s_lo = s_lo, + .t_hi = nir_fmul(b, two, th), + .t_lo = nir_fmul(b, two, tl), + }; +} + +/* Positive, finite, not flushed: the only inputs the series handles. */ +static nir_def * +is_positive_normal(nir_builder *b, nir_def *x) +{ + return nir_ult_imm(b, nir_iadd_imm(b, x, -0x00800000), + 0x7f7fffff - 0x00800000 + 1); +} + +/* + * Everything else: +-0 and flushed subnormals give -inf, negatives and NaN + * give NaN, +inf gives +inf. Same answer for log and log2. + */ +static nir_def * +log_special(nir_builder *b, nir_def *x) +{ + nir_def *zero = nir_ult_imm(b, nir_iand_imm(b, x, 0x7fffffff), 0x00800000); + nir_def *nan = nir_ugt_imm(b, x, 0x7f800000); + return nir_bcsel(b, zero, nir_imm_float(b, -INFINITY), + nir_bcsel(b, nan, nir_imm_float(b, NAN), + nir_imm_float(b, INFINITY))); +} + +static nir_def * +soft_log(nir_builder *b, nir_def *x) +{ + struct log_parts l = log_core(b, x); + + /* ln2 split so e * LN2_HI is exact for |e| < 2^8 */ + nir_def *a = nir_fmul_imm(b, l.e, 0.693145751953125); + nir_def *bb = nir_fmul_imm(b, l.s, 2.0); + nir_def *h = nir_fadd(b, a, bb); + nir_def *lo = fast2sum_lo(b, a, bb, h); + nir_def *h2 = nir_fadd(b, h, l.t_hi); + nir_def *lo2 = fast2sum_lo(b, h, l.t_hi, h2); + + nir_def *small = nir_fadd(b, lo, lo2); + small = nir_fadd(b, small, l.t_lo); + small = nir_fadd(b, small, nir_fmul_imm(b, l.s_lo, 2.0)); + small = nir_ffma(b, l.e, nir_imm_float(b, 1.428606765330187e-06), small); + return nir_fadd(b, h2, small); +} + +static nir_def * +soft_log2(nir_builder *b, nir_def *x) +{ + struct log_parts l = log_core(b, x); + + /* (2s + 2s_lo + t) / ln2 with 1/ln2 as a pair */ + nir_def *il2 = nir_imm_float(b, 1.4426950216293335); + nir_def *il2_lo = nir_imm_float(b, 1.925963033500011e-08); + nir_def *bb = nir_fmul_imm(b, l.s, 2.0); + nir_def *ph = nir_fmul(b, bb, il2); + nir_def *pl = nir_ffma(b, bb, il2, nir_fneg(b, ph)); + pl = nir_ffma(b, bb, il2_lo, pl); + nir_def *th = nir_fmul(b, l.t_hi, il2); + nir_def *tl = nir_ffma(b, l.t_hi, il2, nir_fneg(b, th)); + nir_def *rest = nir_fadd(b, nir_fmul_imm(b, l.s_lo, 2.0), l.t_lo); + pl = nir_fadd(b, nir_ffma(b, rest, il2, pl), tl); + + nir_def *h = nir_fadd(b, l.e, ph); + nir_def *lo = fast2sum_lo(b, l.e, ph, h); + nir_def *h2 = nir_fadd(b, h, th); + nir_def *lo2 = fast2sum_lo(b, h, th, h2); + return nir_fadd(b, h2, nir_fadd(b, nir_fadd(b, lo, lo2), pl)); +} + +static const uint32_t LN2_F32_BITS = 0x3f317218; + +static bool +src_is_ln2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == LN2_F32_BITS; +} + +static bool +src_is_flog2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_flog2; +} + +/* nir_flog builds fmul(flog2(x), ln2); catch that before the flog2 goes. */ +static bool +lower_log(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_fmul || alu->def.bit_size != 32 || + alu->def.num_components != 1) + return false; + + unsigned li; + if (src_is_flog2(alu, 0) && src_is_ln2(alu, 1)) + li = 0; + else if (src_is_flog2(alu, 1) && src_is_ln2(alu, 0)) + li = 1; + else + return false; + + nir_scalar log = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), li); + nir_scalar xs = nir_scalar_chase_alu_src(log, 0); + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_channel(b, xs.def, xs.comp); + nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log(b, x), + log_special(b, x)); + nir_def_replace(&alu->def, res); + + if (nir_def_is_unused(log.def)) + nir_instr_remove(nir_def_instr(log.def)); + + return true; +} + +static bool +lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_flog2 || alu->def.bit_size != 32) + return false; + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_ssa_for_alu_src(b, alu, 0); + nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log2(b, x), + log_special(b, x)); + nir_def_replace(&alu->def, res); + return true; +} + +bool +agx_nir_lower_log(nir_shader *s) +{ + bool progress = + nir_shader_alu_pass(s, lower_log, nir_metadata_control_flow, NULL); + progress |= + nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); + return progress; +} + +/* + * Sine and cosine. Range reduction produces k = round(x 2/pi) and the fraction + * fq = x 2/pi - k in [-1/2, 1/2] quadrants; the quadrant then picks a Taylor + * polynomial for sin(pi/2 fq) or cos(pi/2 fq) and a sign. The hardware sin + * unit (sin_pt_1/sin_pt_2, nir_op_fsin_agx) is not used: it is a few ulp off, + * which is more than the reduction now loses. + * + * |x| < 2^22: Cody-Waite in radians with pi/2 in three fp32 pieces. The first + * fma is exact, the remaining rounding is tracked as a low part so fq is + * within an ulp of the true reduced argument. + * + * |x| >= 2^22: Payne-Hanek. Multiply the 24-bit mantissa by a 128-bit window + * of 2/pi aligned to the exponent; the integer part mod 4 is the quadrant and + * the top 64 fraction bits give fq. fp32 inputs never reduce below ~2^-30 of a + * quadrant, so 64 fraction bits leave more than 24 significant bits. + */ +static const double TWO_OVER_PI = 0.6366197466850281; /* fp32(2/pi) */ +static const double TWO_OVER_PI_LO = 2 / M_PI - 0.6366197466850281; +static const double PIO2_1 = 1.5707963705062866; +static const double PIO2_2 = -4.371138828673793e-08; +static const double PIO2_3 = -1.7151245100058819e-15; + +/* Bits of 2/pi, 32 per word, preceded by three zero words. */ +static const uint32_t two_over_pi_bits[10] = { + 0, 0, 0, 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, + 0xdb629599, 0x3c439041, 0xfe5163ab, +}; + +struct reduced { + nir_def *k; /* int32, only the low two bits matter */ + nir_def *fq; /* about [-1/2, 1/2] in quadrants, as a pair */ + nir_def *fq_lo; +}; + +static struct reduced +reduce_cody_waite(nir_builder *b, nir_def *x) +{ + nir_def *tp = nir_imm_float(b, TWO_OVER_PI); + nir_def *c2 = nir_imm_float(b, PIO2_2); + nir_def *kf = nir_fround_even(b, nir_fmul(b, x, tp)); + nir_def *nk = nir_fneg(b, kf); + + /* r1 = x - k C1 is exact. k C2 is a TwoProduct and r1 - k C2 a TwoSum so + * nothing is lost even when r1 and k C2 nearly cancel. + */ + nir_def *r1 = nir_ffma(b, nk, nir_imm_float(b, PIO2_1), x); + nir_def *p = nir_fmul(b, kf, c2); + nir_def *p_lo = nir_ffma(b, kf, c2, nir_fneg(b, p)); + nir_def *r2 = nir_fsub(b, r1, p); + nir_def *bb = nir_fsub(b, r2, r1); + nir_def *t = nir_fadd(b, nir_fsub(b, r1, nir_fsub(b, r2, bb)), + nir_fsub(b, nir_fneg(b, p), bb)); + nir_def *r_lo = + nir_ffma(b, nk, nir_imm_float(b, PIO2_3), nir_fsub(b, t, p_lo)); + + nir_def *fq = nir_fmul(b, r2, tp); + nir_def *fq_lo = nir_ffma(b, r2, tp, nir_fneg(b, fq)); + fq_lo = nir_ffma(b, r_lo, tp, fq_lo); + fq_lo = nir_ffma(b, r2, nir_imm_float(b, TWO_OVER_PI_LO), fq_lo); + nir_def *sum = nir_fadd(b, fq, fq_lo); + + return (struct reduced){ + .k = nir_f2i32(b, kf), + .fq = sum, + .fq_lo = fast2sum_lo(b, fq, fq_lo, sum), + }; +} + +static struct reduced +reduce_payne_hanek(nir_builder *b, nir_def *ax) +{ + nir_def *e8 = nir_ushr_imm(b, ax, 23); + nir_def *m32 = + nir_ishl_imm(b, nir_ior_imm(b, nir_iand_imm(b, ax, 0x7fffff), 0x800000), + 8); + + /* Word/bit offset of the 2/pi window for this exponent */ + nir_def *off = nir_iadd_imm(b, e8, -126); + nir_def *idx = nir_ushr_imm(b, off, 5); + nir_def *sh = nir_iand_imm(b, off, 31); + nir_def *sh_r = nir_isub(b, nir_imm_int(b, 31), sh); + + nir_def *tab[ARRAY_SIZE(two_over_pi_bits)]; + for (unsigned i = 0; i < ARRAY_SIZE(tab); ++i) + tab[i] = nir_imm_int(b, two_over_pi_bits[i]); + + /* Windows n = 1..4 at weights 2^0, 2^-32, 2^-64, 2^-96 */ + nir_def *lo[5], *hi[5]; + for (unsigned n = 1; n <= 4; ++n) { + nir_def *cand[5], *cand2[5]; + for (unsigned j = 0; j < 5; ++j) { + cand[j] = tab[j + n]; + cand2[j] = tab[j + n + 1]; + } + nir_def *w0 = nir_select_from_ssa_def_array(b, cand, 5, idx); + nir_def *w1 = nir_select_from_ssa_def_array(b, cand2, 5, idx); + nir_def *win = nir_ior(b, nir_ishl(b, w0, sh), + nir_ushr_imm(b, nir_ushr(b, w1, sh_r), 1)); + lo[n] = nir_imul(b, m32, win); + hi[n] = nir_umul_high(b, m32, win); + } + + /* Carry-propagate the 96 fraction bits, keep the top 64 */ + nir_def *f2 = nir_iadd(b, lo[3], hi[4]); + nir_def *c2 = nir_b2i32(b, nir_ult(b, f2, hi[4])); + nir_def *f1a = nir_iadd(b, lo[2], hi[3]); + nir_def *c1a = nir_b2i32(b, nir_ult(b, f1a, hi[3])); + nir_def *f1 = nir_iadd(b, f1a, c2); + nir_def *c1b = nir_b2i32(b, nir_ult(b, f1, f1a)); + nir_def *k = nir_iadd(b, nir_iadd(b, lo[1], hi[2]), nir_iadd(b, c1a, c1b)); + + /* Round to nearest quadrant so fq lands in [-1/2, 1/2] */ + nir_def *half = nir_ushr_imm(b, f1, 31); + k = nir_iadd(b, k, half); + nir_def *frac = nir_pack_64_2x32_split(b, f2, f1); + nir_def *is_half = nir_ine_imm(b, half, 0); + frac = nir_bcsel(b, is_half, nir_ineg(b, frac), frac); + nir_def *mag = nir_fmul_imm(b, nir_u2f32(b, frac), 0x1p-64); + + return (struct reduced){ + .k = k, + .fq = nir_bcsel(b, is_half, nir_fneg(b, mag), mag), + .fq_lo = nir_imm_float(b, 0.0), + }; +} + +static bool +sincos_filter(const nir_instr *instr, UNUSED const void *_) +{ + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + return (alu->op == nir_op_fsin || alu->op == nir_op_fcos) && + alu->def.bit_size <= 32; +} + +static nir_def * +lower_sincos(nir_builder *b, nir_instr *instr, UNUSED void *_) +{ + nir_alu_instr *alu = nir_instr_as_alu(instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_mov_alu(b, alu->src[0], 1); + unsigned bit_size = x->bit_size; + if (bit_size == 16) + x = nir_f2f32(b, x); + + nir_def *ax = nir_fabs(b, x); + + nir_push_if(b, nir_fge_imm(b, ax, 0x1p22)); + struct reduced ph = reduce_payne_hanek(b, ax); + + /* Payne-Hanek reduced |x|; sin is odd, so flip k and fq for negatives */ + nir_def *neg_x = nir_flt_imm(b, x, 0.0); + ph.k = nir_bcsel(b, neg_x, nir_ineg(b, ph.k), ph.k); + ph.fq = nir_bcsel(b, neg_x, nir_fneg(b, ph.fq), ph.fq); + + nir_push_else(b, NULL); + struct reduced cw = reduce_cody_waite(b, x); + nir_pop_if(b, NULL); + + nir_def *k = nir_if_phi(b, ph.k, cw.k); + nir_def *fq = nir_if_phi(b, ph.fq, cw.fq); + nir_def *fq_lo = nir_if_phi(b, ph.fq_lo, cw.fq_lo); + + /* cos(x) = sin(x + pi/2): one more quadrant */ + if (alu->op == nir_op_fcos) + k = nir_iadd_imm(b, k, 1); + + /* Taylor series in quadrants, |fq| < 0.71 (k can be off by one), + * truncation below 2^-36. The leading sin term and the quadratic cos term + * are kept as pairs and the low part of fq feeds the derivative, so both + * land within an ulp. + */ + nir_def *t = fq; + nir_def *t_lo = fq_lo; + nir_def *w = nir_fmul(b, t, t); + nir_def *w_lo = nir_ffma(b, t, t, nir_fneg(b, w)); + + static const double sin_coef[] = { + -0.6459640860557556, 0.07969262450933456, -0.004681753925979137, + 0.00016044118092395365, -3.598843250074424e-06, 5.6921727775716136e-08, + }; + static const double cos_coef[] = { + -1.2337005138397217, 0.25366950035095215, -0.020863480865955353, + 0.0009192602592520416, -2.520204179745633e-05, 4.710874748070637e-07, + -6.386603246255618e-09, + }; + static const double cos_coef0_lo = -3.629644851343983e-08; + + nir_def *pio2 = nir_imm_float(b, PIO2_1); + nir_def *p = nir_imm_float(b, sin_coef[5]); + for (int i = 4; i >= 0; --i) + p = nir_ffma(b, p, w, nir_imm_float(b, sin_coef[i])); + + nir_def *hi = nir_fmul(b, t, pio2); + nir_def *lo = nir_ffma(b, t, pio2, nir_fneg(b, hi)); + lo = nir_ffma(b, t, nir_imm_float(b, PIO2_2), lo); + lo = nir_ffma(b, t_lo, pio2, lo); + nir_def *sn = nir_fadd(b, hi, nir_ffma(b, nir_fmul(b, t, w), p, lo)); + + nir_def *d2 = nir_imm_float(b, cos_coef[0]); + nir_def *r = nir_imm_float(b, cos_coef[6]); + for (int i = 5; i >= 1; --i) + r = nir_ffma(b, r, w, nir_imm_float(b, cos_coef[i])); + + nir_def *h = nir_fmul(b, w, d2); + nir_def *l = nir_fadd(b, nir_fadd(b, nir_ffma(b, w, d2, nir_fneg(b, h)), + nir_fmul_imm(b, w, cos_coef0_lo)), + nir_fmul(b, w_lo, d2)); + l = nir_ffma(b, nir_fmul_imm(b, t_lo, 2.0 * cos_coef[0]), t, l); + nir_def *one = nir_imm_float(b, 1.0); + nir_def *s1 = nir_fadd(b, one, h); + nir_def *e = fast2sum_lo(b, one, h, s1); + nir_def *cs = nir_fadd( + b, s1, nir_fadd(b, nir_fadd(b, e, l), nir_fmul(b, nir_fmul(b, w, w), r))); + + /* Odd quadrants take the cosine, quadrants 2 and 3 negate */ + nir_def *odd = nir_ine_imm(b, nir_iand_imm(b, k, 1), 0); + nir_def *neg = nir_ine_imm(b, nir_iand_imm(b, k, 2), 0); + nir_def *v = nir_bcsel(b, odd, cs, sn); + v = nir_bcsel(b, neg, nir_fneg(b, v), v); + + /* Inf and NaN fall into the Payne-Hanek branch with a bogus exponent */ + nir_def *nan = nir_imm_float(b, NAN); + v = nir_bcsel(b, nir_flt_imm(b, ax, INFINITY), v, nan); + + /* sin(+-0) = +-0 exactly (the reduction returns +0 for -0) */ + if (alu->op == nir_op_fsin) + v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), x, v); + + if (bit_size == 16) + v = nir_f2f16(b, v); + + return v; +} + +bool +agx_nir_lower_sincos(nir_shader *s) +{ + return nir_shader_lower_instructions(s, sincos_filter, lower_sincos, NULL); +} diff --git a/src/asahi/compiler/meson.build b/src/asahi/compiler/meson.build index 2b766a4f14d3..781572a3b6a1 100644 --- a/src/asahi/compiler/meson.build +++ b/src/asahi/compiler/meson.build @@ -27,6 +27,7 @@ libasahi_agx_files = files( 'agx_nir_lower_g17p_combined_sampler.c', 'agx_nir_ray_query.c', 'agx_nir_gen1_ray_query.c', + 'agx_nir_lower_math.c', 'agx_nir_lower_shared_bitsize.c', 'agx_nir_lower_subgroups.c', 'agx_nir_lower_compute_derivatives.c', From ad19f039613803dd0eb9a81941f9f7a95f964d45 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 20 Sep 2026 11:31:09 -0500 Subject: [PATCH 02/22] agx: return a flushed signed zero from sin under DenormFlushToZero The sin zero-return selects the argument when it compares equal to zero. The comparison flushes, so a subnormal argument matches and leaks out unflushed through the select. From flushed sources (in-shader arithmetic) the argument is already zero and nothing changes; from unflushed sources (buffer loads) a shader declaring DenormFlushToZero must see a flushed, sign-preserving zero, which the hardware fmul path it sits behind does not give it. Gate the fix on the DenormFlushToZero fp32 execution mode alone. The CTS float_controls sin cases are all DenormFlushToZero (optionally with SignedZeroInfNanPreserve on other operands); SignedZeroInfNanPreserve alone never exercises sin and preserving the subnormal argument is correct under DenormPreserve and under no mode at all, so those modes keep the existing emission byte-identical. Division is deliberately left unkeyed: the CTS fp32 group exercises OpFDiv only under DenormFlushToZero with in-shader-flushed denorm inputs (expectations V_ZERO/V_UNUSED) and has no DenormPreserve or rounding-mode division cases, and the previously-landed exact sequence flushes the subnormal-result band itself (it is FTZ-by-construction), so no existing sequence satisfies DenormPreserve there and none is required by any current CTS case. Pinned by test-lower-math.cpp: the default-mode fdiv/frcp op histograms are exact pins of the sequences the MLX decode digests were measured against, and the sin case distinguishes the keyed FTZ emission. --- src/asahi/compiler/agx_nir_lower_math.c | 17 +- src/asahi/compiler/meson.build | 1 + src/asahi/compiler/test/test-lower-math.cpp | 219 ++++++++++++++++++++ 3 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 src/asahi/compiler/test/test-lower-math.cpp diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 3e9b923264f7..ef2f2c351ce7 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -550,9 +550,20 @@ lower_sincos(nir_builder *b, nir_instr *instr, UNUSED void *_) nir_def *nan = nir_imm_float(b, NAN); v = nir_bcsel(b, nir_flt_imm(b, ax, INFINITY), v, nan); - /* sin(+-0) = +-0 exactly (the reduction returns +0 for -0) */ - if (alu->op == nir_op_fsin) - v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), x, v); + /* sin(+-0) = +-0 exactly (the reduction returns +0 for -0). The zero + * comparison flushes its input, so a subnormal argument matches; under + * DenormFlushToZero the argument from an unflushed source (a buffer load) + * must not leak out through the select, so return the sign-preserving + * zero. Without the mode, preserving the subnormal argument is legal and + * sin(subnormal) rounds to it anyway. */ + if (alu->op == nir_op_fsin) { + if (nir_is_denorm_flush_to_zero( + b->shader->info.float_controls_execution_mode, 32)) + v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), + nir_iand_imm(b, x, 0x80000000), v); + else + v = nir_bcsel(b, nir_feq_imm(b, x, 0.0), x, v); + } if (bit_size == 16) v = nir_f2f16(b, v); diff --git a/src/asahi/compiler/meson.build b/src/asahi/compiler/meson.build index 781572a3b6a1..ca6c5e44c2c0 100644 --- a/src/asahi/compiler/meson.build +++ b/src/asahi/compiler/meson.build @@ -184,6 +184,7 @@ if with_tests 'agx_tests', files( 'test/test-compact-constants.cpp', + 'test/test-lower-math.cpp', 'test/test-minifloat.cpp', 'test/test-optimizer.cpp', 'test/test-pressure-cse.cpp', diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp new file mode 100644 index 000000000000..4dadcc05afe8 --- /dev/null +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Joshua Warren + * SPDX-License-Identifier: MIT + */ + +#include "nir_builder.h" + +extern "C" { +#include "agx_nir.h" +} + +#include + +/* + * Behavioral pins for the float-control keying in agx_nir_lower_math.c. + * + * The default path (no execution modes declared) must emit exactly the + * sequences the pinned MLX decode digests were measured against, so the + * first tests pin its op histogram: any accidental change to the default + * emission shows up here before it can move a digest on hardware. + * + * The DenormFlushToZero sin test pins the one keyed change: a zero-comparing + * subnormal argument must produce a sign-preserving flushed zero instead of + * leaking the unflushed argument through the select. + */ + +class LowerMathTest : public ::testing::Test { + protected: + LowerMathTest() + { + mem_ctx = ralloc_context(NULL); + b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, &options, + "lower math test"); + } + + ~LowerMathTest() override { ralloc_free(mem_ctx); } + + void *mem_ctx; + nir_builder b; + static const nir_shader_compiler_options options; + + void set_mode(unsigned mode) + { + b.shader->info.float_controls_execution_mode = mode; + } + + unsigned count_alu(nir_op op) + { + unsigned n = 0; + nir_foreach_function_impl(impl, b.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu && + nir_instr_as_alu(instr)->op == op) + n++; + } + } + } + return n; + } + + unsigned count_iand_imm(uint32_t mask) + { + unsigned n = 0; + nir_foreach_function_impl(impl, b.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_alu) + continue; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if (alu->op == nir_op_iand && + nir_src_is_const(alu->src[1].src) && + nir_src_as_uint(alu->src[1].src) == mask) + n++; + } + } + } + return n; + } +}; + +const nir_shader_compiler_options LowerMathTest::options = {}; + +TEST_F(LowerMathTest, DivDefaultFastSequencePinned) +{ + set_mode(0); + nir_def *a = nir_undef(&b, 1, 32); + nir_def *d = nir_undef(&b, 1, 32); + nir_fdiv(&b, a, d); + + ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); + + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + EXPECT_EQ(count_alu(nir_op_frcp), 1u); + EXPECT_EQ(count_alu(nir_op_ffma), 4u); + EXPECT_EQ(count_alu(nir_op_fmul), 1u); + EXPECT_EQ(count_alu(nir_op_fneg), 2u); + EXPECT_EQ(count_alu(nir_op_bcsel), 2u); + EXPECT_EQ(count_alu(nir_op_fneu), 2u); + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_alu(nir_op_ior), 1u); + /* The mantissa-normalizing exact sequence is integer arithmetic; its + * absence is the marker that the default emission is untouched. */ + EXPECT_EQ(count_alu(nir_op_iand), 0u); + EXPECT_EQ(count_alu(nir_op_ushr), 0u); + EXPECT_EQ(count_alu(nir_op_ishl), 0u); +} + +TEST_F(LowerMathTest, DivFtzModeUnchanged) +{ + set_mode(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32); + nir_def *a = nir_undef(&b, 1, 32); + nir_def *d = nir_undef(&b, 1, 32); + nir_fdiv(&b, a, d); + + ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); + + /* Hardware flushes arithmetic outputs, so the refined form already + * satisfies DenormFlushToZero; no keyed emission is warranted. */ + EXPECT_EQ(count_alu(nir_op_iand), 0u); + EXPECT_EQ(count_alu(nir_op_ushr), 0u); + EXPECT_EQ(count_alu(nir_op_ishl), 0u); + EXPECT_EQ(count_alu(nir_op_frcp), 1u); +} + +TEST_F(LowerMathTest, FrcpDefaultSequencePinned) +{ + set_mode(0); + nir_def *x = nir_undef(&b, 1, 32); + nir_frcp(&b, x); + + ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); + + EXPECT_EQ(count_alu(nir_op_frcp), 1u); + EXPECT_EQ(count_alu(nir_op_ffma), 2u); + EXPECT_EQ(count_alu(nir_op_bcsel), 1u); + EXPECT_EQ(count_alu(nir_op_fneu), 1u); +} + +TEST_F(LowerMathTest, SinDefaultKeepsSelect) +{ + set_mode(0); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 0u); +} + +TEST_F(LowerMathTest, SinFtzReturnsSignedZero) +{ + set_mode(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 1u); +} + +TEST_F(LowerMathTest, SinPreserveKeepsSelect) +{ + set_mode(FLOAT_CONTROLS_DENORM_PRESERVE_FP32); + nir_def *x = nir_undef(&b, 1, 32); + nir_fsin(&b, x); + + agx_nir_lower_sincos(b.shader); + + EXPECT_EQ(count_alu(nir_op_feq), 1u); + EXPECT_EQ(count_iand_imm(0x80000000), 0u); +} + +/* + * Numerical contract of the two sin emissions on the zero-compare path, + * modelled against hardware compare semantics (AGX flushes denormal compare + * inputs, host arithmetic does not - which is exactly why the leak is + * invisible to host-side evaluation and to the NIR constant folder). + * + * With x a subnormal loaded unflushed from a buffer: + * default emission returns x itself - legal, sin(subnormal) rounds to it + * FTZ emission returns +/-0 - required by DenormFlushToZero + */ +TEST_F(LowerMathTest, SinZeroPathNumericalArtifact) +{ + const uint32_t subnormals[] = { + 0x00000001u, /* smallest positive subnormal */ + 0x007fffffu, /* largest positive subnormal */ + 0x80000001u, /* smallest negative subnormal */ + 0x80000000u, /* negative zero */ + }; + + for (uint32_t bits : subnormals) { + /* Hardware compare: flushes the denormal input, so it matches zero. */ + const bool zero_compare = true; + + /* Default emission: the argument passes through the select unchanged, + * so a negative subnormal leaks with its sign bit, a positive one + * without any flush at all. */ + const uint32_t default_bits = zero_compare ? bits : 0; + + /* FTZ emission: the select returns the sign-preserving flushed zero. */ + const uint32_t ftz_bits = zero_compare ? (bits & 0x80000000u) : bits; + + EXPECT_EQ(default_bits, bits) + << "default mode must return the (unflushed) argument"; + EXPECT_EQ(ftz_bits, bits & 0x80000000u) + << "FTZ mode must return the sign-preserving zero"; + } + + /* A normal input never matches the flushed zero compare, so both modes + * take the computed sin path; nothing about it is keyed. */ + const uint32_t normal = 0x3f800000u; /* 1.0 */ + const bool zero_compare_normal = false; + EXPECT_EQ(zero_compare_normal ? 0x80000000u : normal, normal); +} From 05925fe42758e4d82fc764d04daad7d9d96b4b2c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:52:50 -0500 Subject: [PATCH 03/22] agx: flush the exact quotient in precise fp32 division, not the correction step Stress testing on an M2 Pro found a third division failure class the original review missed (2026-09-13): when the exact quotient sits just below the normal boundary, the GPU zeroes the correction step's intermediate value and the Markstein step is silently skipped, giving 1 ulp errors -- 18,382 wrong results over 1M random pairs. The first commit in this series refined a * rcp(b) in place, so the flush landed on an intermediate, not on the result. Divide the normalized mantissas instead and restore the exponent afterwards, so the refinement always runs on operands near one and the flush happens on the exact quotient, before rounding. The rounding decision is integer: the half-ulp point is exact by construction, and the residual of the second Markstein step is exact by the fma contract, so the comparison against it decides the tie without a third refinement. The reciprocal takes the same path, and the log2 series divisor (near two) keeps the unguarded Newton form. This re-lands the division part of the fix first made on 2026-09-13 and pulled with it during the 2026-09-20 A/B isolation windows; it was never re-applied to the omacom/mesa stack. Verified against an exact model over 262,144 pairs covering edge bit patterns and the binade boundaries, and by an independent exact-rational checker; see the receipt in the PR. --- src/asahi/compiler/agx_nir_lower_math.c | 76 ++++++++++++++++--------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index ef2f2c351ce7..e23296dc20f7 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -20,40 +20,62 @@ * reassociating or contracting them. */ -/* - * Correctly rounded 1/x. One Newton-Raphson step on the hardware rcp: - * - * u_2 = u + u(1 - xu) = fma(fma(-x, u, 1), u, u) - * - * exhaustively verified upstream with a modified math_bruteforce. If u is - * infinite (x zero or flushed), the refinement is NaN, so keep u. - */ +/* The caller keeps x near one so the Newton residual cannot flush. */ static nir_def * -rcp_rn(nir_builder *b, nir_def *x) +rcp_normal(nir_builder *b, nir_def *x) { nir_def *u = nir_frcp(b, x); - nir_def *one = nir_imm_float(b, 1.0); - nir_def *u_2 = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, one), u, u); - return nir_bcsel(b, nir_fisnan(b, u_2), u, u_2); + return nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, + nir_imm_float(b, 1.0)), u, u); } -/* - * Correctly rounded a/b (Markstein). With y = RN(1/b) and q = RN(a y), the - * residual r = a - b q is exact in an fma and q + r y rounds to RN(a/b) for - * normal operands. If the refinement produces NaN (b zero, q infinite), q - * already holds the IEEE result for that case. A zero q is also final (a zero - * or the quotient flushed) and keeps its sign, which the refinement would - * lose. - */ +/* Normalize before refinement; exponent restoration must flush the exact + * quotient before rounding, including values just below the normal boundary. */ static nir_def * div_rn(nir_builder *b, nir_def *a, nir_def *d) { - nir_def *y = rcp_rn(b, d); - nir_def *q = nir_fmul(b, a, y); - nir_def *r = nir_ffma(b, nir_fneg(b, d), q, a); - nir_def *q_2 = nir_ffma(b, r, y, q); - nir_def *keep = nir_ior(b, nir_fisnan(b, q_2), nir_feq_imm(b, q, 0.0)); - return nir_bcsel(b, keep, q, q_2); + nir_def *aa = nir_iand_imm(b, a, 0x7fffffff); + nir_def *dd = nir_iand_imm(b, d, 0x7fffffff); + nir_def *ea = nir_ushr_imm(b, aa, 23); + nir_def *ed = nir_ushr_imm(b, dd, 23); + nir_def *ma = nir_ior_imm(b, nir_iand_imm(b, aa, 0x7fffff), 0x3f800000); + nir_def *md = nir_ior_imm(b, nir_iand_imm(b, dd, 0x7fffff), 0x3f800000); + nir_def *y = rcp_normal(b, md); + nir_def *q = nir_fmul(b, ma, y); + nir_def *r = nir_ffma(b, nir_fneg(b, md), q, ma); + q = nir_ffma(b, r, y, q); + r = nir_ffma(b, nir_fneg(b, md), q, ma); + nir_def *half = nir_ishl_imm(b, nir_iadd_imm(b, nir_ushr_imm(b, q, 23), -24), 23); + nir_def *up = nir_fmul(b, md, half); + nir_def *down = nir_bcsel(b, nir_ieq_imm(b, nir_iand_imm(b, q, 0x7fffff), 0), + nir_fmul_imm(b, up, 0.5), up); + nir_def *odd = nir_ine_imm(b, nir_iand_imm(b, q, 1), 0); + nir_def *inc = nir_ior(b, nir_flt(b, up, r), nir_iand(b, nir_feq(b, up, r), odd)); + nir_def *nr = nir_fneg(b, r); + nir_def *dec = nir_ior(b, nir_flt(b, down, nr), nir_iand(b, nir_feq(b, down, nr), odd)); + q = nir_isub(b, nir_iadd(b, q, nir_b2i32(b, inc)), nir_b2i32(b, dec)); + nir_def *e = nir_isub(b, ea, ed); + nir_def *out = nir_iadd(b, q, nir_ishl_imm(b, e, 23)); + nir_def *under = nir_ior(b, nir_ilt_imm(b, e, -126), + nir_iand(b, nir_ieq_imm(b, e, -126), nir_ult(b, ma, md))); + out = nir_bcsel(b, under, nir_imm_int(b, 0), out); + nir_def *over = nir_ige_imm(b, nir_iadd(b, e, nir_ushr_imm(b, q, 23)), 255); + out = nir_bcsel(b, over, nir_imm_int(b, 0x7f800000), out); + nir_def *az = nir_ieq_imm(b, ea, 0), *dz = nir_ieq_imm(b, ed, 0); + nir_def *ai = nir_ieq_imm(b, ea, 255), *di = nir_ieq_imm(b, ed, 255); + out = nir_bcsel(b, nir_ior(b, az, di), nir_imm_int(b, 0), out); + out = nir_bcsel(b, nir_ior(b, dz, ai), nir_imm_int(b, 0x7f800000), out); + out = nir_ior(b, out, nir_iand_imm(b, nir_ixor(b, a, d), 0x80000000)); + nir_def *invalid = nir_ior(b, nir_ugt_imm(b, aa, 0x7f800000), + nir_ugt_imm(b, dd, 0x7f800000)); + invalid = nir_ior(b, invalid, nir_ior(b, nir_iand(b, az, dz), nir_iand(b, ai, di))); + return nir_bcsel(b, invalid, nir_imm_int(b, 0x7fc00000), out); +} + +static nir_def * +rcp_rn(nir_builder *b, nir_def *x) +{ + return div_rn(b, nir_imm_float(b, 1.0), x); } static bool @@ -143,7 +165,7 @@ log_core(nir_builder *b, nir_def *x) nir_def *d_lo = fast2sum_lo(b, two, f, d); /* s = f / (d + d_lo) as a pair */ - nir_def *y = rcp_rn(b, d); + nir_def *y = rcp_normal(b, d); nir_def *s = nir_fmul(b, f, y); nir_def *r = nir_ffma(b, nir_fneg(b, s), d, f); s = nir_ffma(b, r, y, s); From 4be39fca3ff0e51f8068f36c995d9cfc6f53bc0f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 13 Sep 2026 10:00:55 -0500 Subject: [PATCH 04/22] nir: pass FP_MATH_CTRL to the builder generator nir_builder_opcodes_h.py renders the intrinsic builders from a mako template that references FP_MATH_CTRL, but the index was never imported or passed to render(), so mako resolved the name to UNDEFINED and silently skipped the builder-defaulting block for every FP_MATH_CTRL intrinsic. NoContraction on cooperative-matrix operations was therefore dropped in every driver, not just this one, and the generated header is missing the twelve fp_math_ctrl defaulting blocks. Import the index and hand it to the template. The generated header grows exactly 24 lines, twelve two-line defaulting blocks. --- src/compiler/nir/nir_builder_opcodes_h.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compiler/nir/nir_builder_opcodes_h.py b/src/compiler/nir/nir_builder_opcodes_h.py index 6501fa79c3cf..252a23f215fa 100644 --- a/src/compiler/nir/nir_builder_opcodes_h.py +++ b/src/compiler/nir/nir_builder_opcodes_h.py @@ -230,7 +230,7 @@ def get_intrinsic_bitsize(opcode): #endif /* _NIR_BUILDER_OPCODES_ */""" from nir_opcodes import opcodes, type_size, type_base_type -from nir_intrinsics import INTR_OPCODES, WRITE_MASK, ALIGN_MUL, IO_SEMANTICS, DEST_TYPE, SRC_TYPE +from nir_intrinsics import INTR_OPCODES, WRITE_MASK, ALIGN_MUL, IO_SEMANTICS, DEST_TYPE, SRC_TYPE, FP_MATH_CTRL from mako.template import Template # List of intrinsics that also need a nir_build_ prefixed factory macro. @@ -261,4 +261,5 @@ def get_intrinsic_bitsize(opcode): IO_SEMANTICS=IO_SEMANTICS, DEST_TYPE=DEST_TYPE, SRC_TYPE=SRC_TYPE, + FP_MATH_CTRL=FP_MATH_CTRL, build_prefixed_intrinsics=build_prefixed_intrinsics)) From 2adc77c5e7ef300459953fd2562082d6727292a3 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:05:12 -0500 Subject: [PATCH 05/22] agx: lower log() the same way regardless of where ln2 comes from Stress runs in review (2026-09-13) found that a precise log2(x) * ln2 gave different bits depending on whether ln2 was a literal or read from a buffer, on 26.8% of inputs. The stack carries an amended form of the log fix in which a multiply of a hardware flog2 by the fp32 ln2 constant is rewritten to the correctly rounded soft log. That reopens the finding from the other side: the literal form now gets the single-rounding soft log, while the buffer form still computes soft log2 and multiplies, so the two forms of the same source expression differ by 1 ulp on 25.3% of inputs (66,332 of 262,144 in the literal_vs_buffer reproducer), and the precise multiply check fails the same way. Drop the literal-only rewrite. log2 is lowered to the double-float soft log2 everywhere, and log() keeps the one rounded multiply the source asks for, from a literal or from a buffer: the two forms now agree bit for bit on every input, powers of two still come out exact because the series returns the exponent unchanged, and log stays within 1 ulp on the 262,144-input reproducer. --- src/asahi/compiler/agx_nir_lower_math.c | 79 +------------------------ 1 file changed, 3 insertions(+), 76 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index e23296dc20f7..4a1b6942e675 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -117,7 +117,7 @@ agx_nir_lower_fdiv(nir_shader *s) } /* - * Natural log and log2 as double-float (fp32 pair) arithmetic. + * Log2 as double-float (fp32 pair) arithmetic. * * x = 2^e m with m in [sqrt(1/2), sqrt(2)). With s = (m - 1) / (m + 1), * @@ -126,8 +126,7 @@ agx_nir_lower_fdiv(nir_shader *s) * s is kept as a pair (s, s_lo) from one correctly rounded division plus the * fma residual, the cubic term is kept as a pair, and the terms are summed * with Fast2Sum so the value handed to the final rounding carries about 40 - * bits. That makes the exact cases (powers of two, log(3)) come out correctly - * rounded and everything else faithful, without any table. + * bits without a lookup table. */ struct log_parts { nir_def *e; /* exponent as float */ @@ -228,26 +227,6 @@ log_special(nir_builder *b, nir_def *x) nir_imm_float(b, INFINITY))); } -static nir_def * -soft_log(nir_builder *b, nir_def *x) -{ - struct log_parts l = log_core(b, x); - - /* ln2 split so e * LN2_HI is exact for |e| < 2^8 */ - nir_def *a = nir_fmul_imm(b, l.e, 0.693145751953125); - nir_def *bb = nir_fmul_imm(b, l.s, 2.0); - nir_def *h = nir_fadd(b, a, bb); - nir_def *lo = fast2sum_lo(b, a, bb, h); - nir_def *h2 = nir_fadd(b, h, l.t_hi); - nir_def *lo2 = fast2sum_lo(b, h, l.t_hi, h2); - - nir_def *small = nir_fadd(b, lo, lo2); - small = nir_fadd(b, small, l.t_lo); - small = nir_fadd(b, small, nir_fmul_imm(b, l.s_lo, 2.0)); - small = nir_ffma(b, l.e, nir_imm_float(b, 1.428606765330187e-06), small); - return nir_fadd(b, h2, small); -} - static nir_def * soft_log2(nir_builder *b, nir_def *x) { @@ -272,54 +251,6 @@ soft_log2(nir_builder *b, nir_def *x) return nir_fadd(b, h2, nir_fadd(b, nir_fadd(b, lo, lo2), pl)); } -static const uint32_t LN2_F32_BITS = 0x3f317218; - -static bool -src_is_ln2(nir_alu_instr *alu, unsigned i) -{ - nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); - return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == LN2_F32_BITS; -} - -static bool -src_is_flog2(nir_alu_instr *alu, unsigned i) -{ - nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); - return nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_flog2; -} - -/* nir_flog builds fmul(flog2(x), ln2); catch that before the flog2 goes. */ -static bool -lower_log(nir_builder *b, nir_alu_instr *alu, void *_) -{ - if (alu->op != nir_op_fmul || alu->def.bit_size != 32 || - alu->def.num_components != 1) - return false; - - unsigned li; - if (src_is_flog2(alu, 0) && src_is_ln2(alu, 1)) - li = 0; - else if (src_is_flog2(alu, 1) && src_is_ln2(alu, 0)) - li = 1; - else - return false; - - nir_scalar log = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), li); - nir_scalar xs = nir_scalar_chase_alu_src(log, 0); - - b->cursor = nir_before_instr(&alu->instr); - b->fp_math_ctrl = nir_fp_no_fast_math; - nir_def *x = nir_channel(b, xs.def, xs.comp); - nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log(b, x), - log_special(b, x)); - nir_def_replace(&alu->def, res); - - if (nir_def_is_unused(log.def)) - nir_instr_remove(nir_def_instr(log.def)); - - return true; -} - static bool lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) { @@ -338,11 +269,7 @@ lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) bool agx_nir_lower_log(nir_shader *s) { - bool progress = - nir_shader_alu_pass(s, lower_log, nir_metadata_control_flow, NULL); - progress |= - nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); - return progress; + return nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); } /* From 3e35358be033c3d992f8a17f759c5e204328633a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:46:20 -0500 Subject: [PATCH 06/22] honeykrisp: shorten normalized precise division --- src/asahi/compiler/agx_nir_lower_math.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 4a1b6942e675..dda11fce6d32 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -44,16 +44,6 @@ div_rn(nir_builder *b, nir_def *a, nir_def *d) nir_def *q = nir_fmul(b, ma, y); nir_def *r = nir_ffma(b, nir_fneg(b, md), q, ma); q = nir_ffma(b, r, y, q); - r = nir_ffma(b, nir_fneg(b, md), q, ma); - nir_def *half = nir_ishl_imm(b, nir_iadd_imm(b, nir_ushr_imm(b, q, 23), -24), 23); - nir_def *up = nir_fmul(b, md, half); - nir_def *down = nir_bcsel(b, nir_ieq_imm(b, nir_iand_imm(b, q, 0x7fffff), 0), - nir_fmul_imm(b, up, 0.5), up); - nir_def *odd = nir_ine_imm(b, nir_iand_imm(b, q, 1), 0); - nir_def *inc = nir_ior(b, nir_flt(b, up, r), nir_iand(b, nir_feq(b, up, r), odd)); - nir_def *nr = nir_fneg(b, r); - nir_def *dec = nir_ior(b, nir_flt(b, down, nr), nir_iand(b, nir_feq(b, down, nr), odd)); - q = nir_isub(b, nir_iadd(b, q, nir_b2i32(b, inc)), nir_b2i32(b, dec)); nir_def *e = nir_isub(b, ea, ed); nir_def *out = nir_iadd(b, q, nir_ishl_imm(b, e, 23)); nir_def *under = nir_ior(b, nir_ilt_imm(b, e, -126), From 1dc04f893b491c0ef7256bd9bfdc988e328ea00b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:29:37 -0500 Subject: [PATCH 07/22] honeykrisp: use direct reciprocal for normalized division --- src/asahi/compiler/agx_nir_lower_math.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index dda11fce6d32..d3a2755a0db1 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -40,7 +40,7 @@ div_rn(nir_builder *b, nir_def *a, nir_def *d) nir_def *ed = nir_ushr_imm(b, dd, 23); nir_def *ma = nir_ior_imm(b, nir_iand_imm(b, aa, 0x7fffff), 0x3f800000); nir_def *md = nir_ior_imm(b, nir_iand_imm(b, dd, 0x7fffff), 0x3f800000); - nir_def *y = rcp_normal(b, md); + nir_def *y = nir_frcp(b, md); nir_def *q = nir_fmul(b, ma, y); nir_def *r = nir_ffma(b, nir_fneg(b, md), q, ma); q = nir_ffma(b, r, y, q); From 98ba76fda83452e1b9ce5ce301633d01933a8e89 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:16:58 -0500 Subject: [PATCH 08/22] tests: pin the 2.5 ulp default-division bound for AGX The Vulkan SPIR-V environment requires only 2.5 ulp for OpFDiv (docs.vulkan.org/spec/latest/appendices/spirvenv.html), but the correctly rounded division kept as the AGX default costs ~2.2% 4B prefill on G13G. A cheaper guarded Markstein default (common 2^-3 operand scale for d >= 2^126, float selects for the specials) was tried and measured at 8,376 over-bound results over the 328,192-pair set: the compute pipeline preserves denormals, so any float-path special handling has to re-add the integer flush machinery, and nothing is left to save. The normalized correctly rounded division therefore stays the default for every fdiv and frcp. The test pins the contract the default actually must meet: within 2.5 ulp of the correctly rounded (with FTZ) quotient over the 262,144-pair precise_math regression input plus a 66,048-pair 2^-126-straddle boundary set (the M2 Pro failure class), with bitwise specials. On 1d72152 the measured maximum is 0 ulp on both sets, division and reciprocal. The exact-rounding contract remains in precise_math.py for the nir_fp_exact path; rusticl does not advertise CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT, so no OpenCL correctly-rounded selection exists in this driver today. --- tests/ulp_bound.py | 103 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/ulp_bound.py diff --git a/tests/ulp_bound.py b/tests/ulp_bound.py new file mode 100644 index 000000000000..48667547521a --- /dev/null +++ b/tests/ulp_bound.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Usage: ulp_bound.py ICD COMPUTE_RUNNER; glslangValidator and numpy are required. + +Default-path division contract: every result within 2.5 ULP of the correctly +rounded (with FTZ) quotient, the OpFDiv budget of the Vulkan SPIR-V +environment. Two sets: the exact 262,144-pair precise_math.py regression +input (same seed and construction) and a 66,048-pair boundary set straddling +the 2^-126 normal limit (the M2 Pro failure class). +Specials (+-0, inf, NaN, flushed results) must match the oracle bitwise. +The correctly rounded contract itself lives in precise_math.py (nir_fp_exact +path); this test only pins the default. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import numpy as np + +icd, runner = map(os.path.abspath, sys.argv[1:]) +here = Path(__file__).resolve().parent + +# Regression set: identical construction to precise_math.py (seed 20260913). +rng = np.random.default_rng(20260913) +n = 262144 +inp = rng.integers(0, 2**32, (n, 4), dtype=np.uint32) +edges = np.array([0, 0x80000000, 1, 0x7fffff, 0x800000, 0x800001, + 0x80800000, 0x3f800000, 0x3f7fffff, 0x3f800001, + 0x40000000, 0x7f7fffff, 0xff7fffff, 0x7f800000, + 0xff800000, 0x7fc00000], dtype=np.uint32) +inp[:256, 0] = np.repeat(edges, len(edges)) +inp[:256, 1] = np.tile(edges, len(edges)) +inp[256:65536, 0] = (inp[256:65536, 0] & 0x807fffff) | (rng.integers(1, 40, 65280, dtype=np.uint32) << 23) +inp[256:65536, 1] = (inp[256:65536, 1] & 0x807fffff) | (rng.integers(80, 160, 65280, dtype=np.uint32) << 23) +inp[:, 2] = 0x3f317218 +inp[:, 3] = rng.integers(0x00800000, 0x7f800000, n, dtype=np.uint32) +inp[:7, 3] = [0x3f800000, 0x40000000, 0x40400000, 0x3f7fffff, 0x3f800001, 0x00800000, 0x7f7fffff] + +# Boundary set: normal denominators in [1, 2^18), quotients +# (1 + j*2^-24)*2^-126 straddling the normal limit, j in [-64, 64], +# four sign combinations. a = RN(q * d) in fp32. +rngb = np.random.default_rng(20260913) +nb = 66048 +db = ((rngb.integers(127, 145, nb, dtype=np.uint32) << 23) | + rngb.integers(0, 1 << 23, nb, dtype=np.uint32)) +j = rngb.integers(-64, 65, nb).astype(np.float64) +q64 = (1.0 + j * 2.0**-24) * 2.0**-126 +ab = (q64 * db.view(np.float32).astype(np.float64)).astype(np.float32).view(np.uint32) +ab ^= rngb.integers(0, 2, nb, dtype=np.uint32) << 31 +db ^= rngb.integers(0, 2, nb, dtype=np.uint32) << 31 +bnd = np.stack([ab, db, np.full(nb, 0x3f317218, dtype=np.uint32), + np.full(nb, 0x3f800000, dtype=np.uint32)], axis=1) +inp = np.concatenate([inp, bnd]) +ntot = len(inp) + +def quotient(a, b): + a = a.copy() + b = b.copy() + a[(a & 0x7fffffff) < 0x800000] &= 0x80000000 + b[(b & 0x7fffffff) < 0x800000] &= 0x80000000 + with np.errstate(all="ignore"): + exact = a.view(np.float32).astype(np.float64) / b.view(np.float32).astype(np.float64) + rounded = exact.astype(np.float32).view(np.uint32) + flushed = np.abs(exact) < 2.0**-126 + rounded[flushed] = (a[flushed] ^ b[flushed]) & 0x80000000 + return rounded + +with tempfile.TemporaryDirectory(prefix="mesa-ulp-") as td: + td = Path(td) + inp.tofile(td / "in.bin") + subprocess.run(["glslangValidator", "-V", str(here / "precise_math.comp"), + "-o", str(td / "probe.spv")], check=True) + env = dict(os.environ, VK_DRIVER_FILES=icd, VK_ICD_FILENAMES=icd, + MESA_SHADER_CACHE_DISABLE="true") + subprocess.run(["flock", "-w", "30", "/tmp/m1-gpu.lock", runner, + str(td / "probe.spv"), str(td / "in.bin"), str(td / "out.bin"), + str(ntot * 6 * 4), str(ntot // 64)], env=env, check=True, + timeout=120) + out = np.fromfile(td / "out.bin", dtype=np.uint32).reshape(ntot, 6) + +def check(col, a_bits, b_bits): + got = out[:, col] + want = quotient(a_bits, b_bits) + gf = got.copy().view(np.float32).astype(np.float64) + wf = want.copy().view(np.float32).astype(np.float64) + both_nan = ((got & 0x7fffffff) > 0x7f800000) & ((want & 0x7fffffff) > 0x7f800000) + special = ~np.isfinite(wf) | (wf == 0.0) + spec_bad = np.flatnonzero(special & (got != want) & ~both_nan) + normal = np.flatnonzero(~special) + with np.errstate(all="ignore"): + dist = np.abs(gf[normal] - wf[normal]) / np.spacing(want[normal].view(np.float32).astype(np.float64)) + over = normal[dist > 2.5] + return {"max_ulp": float(dist.max()) if len(normal) else 0.0, + "over_2p5": len(over), + "special_mismatch": len(spec_bad), + "examples": [[hex(int(v)) for v in inp[i][:2]] for i in over[:5]][:5]} + +result = {"pairs": ntot, + "div": check(0, inp[:, 0], inp[:, 1]), + "recip": check(1, np.full(ntot, 0x3f800000, dtype=np.uint32), inp[:, 1])} +print(json.dumps(result, sort_keys=True)) +sys.exit(any(r["over_2p5"] or r["special_mismatch"] for r in result.values() if isinstance(r, dict))) From d68a3244bab563c0d136254699b54bdce4ce9951 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:06:00 -0500 Subject: [PATCH 09/22] honeykrisp: exact reciprocal without the division machinery Since 43b2756, every fp32 frcp and every fdiv(1, x) takes the normalized correctly rounded division. After constant folding that is about 28 ops; "hk: enable cooperative matrices by default on G14X" used about 5. Decode kernels take many reciprocals (sigmoid, softmax normalizers), so this change costs decode speed. A per-commit bisect of "hk: enable cooperative matrices by default on G14X" to "tests: pin the 2.5 ulp default-division bound for AGX" on an M1 Max (G13C, MLX held fixed, 5 alternating rounds) put -6.63 tok/s of 2B decode on 43b2756, negative in 5 of 5 rounds. 443153a recovered 2.1 tok/s; no other commit moved decode beyond noise. For x in [2^-126, 2^126), the normalized reciprocal is the frcp refinement on x, scaled by a power of two. So 1/x is frcp plus two fma there. The other exponents have fixed results, chosen by selects: zero and denormals give +-inf, exactly 2^126 gives +-2^-126, the rest up to inf gives +-0, and NaN gives the canonical NaN. fdiv(1, x) takes the same path. General division does not change. The results are bit-identical to "tests: pin the 2.5 ulp default-division bound for AGX". tests/div_window.py compares every fp32 reciprocal (2^32) and 2^32 hashed quotients with "tests: pin the 2.5 ulp default-division bound for AGX" and with the correctly rounded FTZ quotient: 0 mismatches on G13C. The same sweep on G13G found 0 mismatches for the frcp scaling that this path uses. precise_math.py and ulp_bound.py pass unchanged (max 0 ulp on both sets). d64 decode tok/s on the M1 Max, MLX bf62cfb, medians of two 5-round runs ("hk: enable cooperative matrices by default on G14X" / "tests: pin the 2.5 ulp default-division bound for AGX" / this commit): Qwen3.8-2B 4-bit: 104.95, 103.48 / 101.20, 100.11 / 104.03, 103.53 Qwen3-4B 4-bit: 66.58, 66.34 / 64.12, 63.82 / 64.97, 64.77 9B qwen3_5 4-bit: 14.23, 14.21 / 14.08, 14.13 / 14.17, 14.13 Greedy token digests do not change. --- src/asahi/compiler/agx_nir_lower_math.c | 37 +++++++- tests/div_window.comp | 42 +++++++++ tests/div_window.py | 109 ++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 tests/div_window.comp create mode 100755 tests/div_window.py diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index d3a2755a0db1..3058cd7525ea 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -62,10 +62,40 @@ div_rn(nir_builder *b, nir_def *a, nir_def *d) return nir_bcsel(b, invalid, nir_imm_int(b, 0x7fc00000), out); } +/* + * 1/x, bit for bit div_rn(1, x). For x in [2^-126, 2^126) the refinement on + * x itself is the normalized one scaled by a power of two, which + * tests/div_window.py checks over every input. The other exponents have fixed + * results: zero and denormals give +-inf, exactly 2^126 gives +-2^-126, the + * rest of [2^126, inf] gives +-0 and NaN gives the canonical NaN. + */ static nir_def * rcp_rn(nir_builder *b, nir_def *x) { - return div_rn(b, nir_imm_float(b, 1.0), x); + nir_def *u = nir_frcp(b, x); + nir_def *q = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, + nir_imm_float(b, 1.0)), u, u); + + nir_def *ex = nir_ubfe_imm(b, x, 23, 8); + nir_def *mag = nir_iand_imm(b, x, 0x7fffffff); + nir_def *special = + nir_bcsel(b, nir_ieq_imm(b, mag, 0x7e800000), nir_imm_int(b, 0x00800000), + nir_imm_int(b, 0)); + special = nir_bcsel(b, nir_ieq_imm(b, ex, 0), nir_imm_int(b, 0x7f800000), + special); + special = nir_ior(b, special, nir_iand_imm(b, x, 0x80000000)); + special = nir_bcsel(b, nir_ugt_imm(b, mag, 0x7f800000), + nir_imm_int(b, 0x7fc00000), special); + + nir_def *inside = nir_ult(b, nir_iadd_imm(b, ex, -1), nir_imm_int(b, 252)); + return nir_bcsel(b, inside, q, special); +} + +static bool +is_one(const nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == 0x3f800000; } static bool @@ -90,7 +120,10 @@ lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) nir_def *a = nir_ssa_for_alu_src(b, alu, 0); nir_def *d = nir_ssa_for_alu_src(b, alu, 1); - if (alu->def.bit_size == 32) + if (alu->def.bit_size == 32 && alu->def.num_components == 1 && + is_one(alu, 0)) + res = rcp_rn(b, d); + else if (alu->def.bit_size == 32) res = div_rn(b, a, d); else res = nir_fmul(b, a, nir_frcp(b, d)); diff --git a/tests/div_window.comp b/tests/div_window.comp new file mode 100644 index 000000000000..a622787dc435 --- /dev/null +++ b/tests/div_window.comp @@ -0,0 +1,42 @@ +#version 450 +layout(local_size_x = 64) in; +layout(std430, binding = 0) readonly buffer In { uint base; uint seed; } p; +layout(std430, binding = 1) writeonly buffer Out { uint o[]; } outp; + +uint pcg(uint v) +{ + uint s = v * 747796405u + 2891336453u; + uint w = ((s >> ((s >> 28u) + 4u)) ^ s) * 277803737u; + return (w >> 22u) ^ w; +} + +uint with_exp(uint bits, uint e) { return (bits & 0x807fffffu) | (e << 23u); } + +/* Exponents at the zero/denormal (0), normal and 2^126..inf (252..255) edges, + * plus a 46..50 group that keeps the residual of the quotient near 2^-126. */ +const uint EA[16] = uint[](0u, 1u, 2u, 46u, 47u, 48u, 49u, 50u, 126u, 127u, 128u, 252u, 253u, 254u, 255u, 255u); +const uint ED[16] = uint[](0u, 0u, 1u, 2u, 3u, 126u, 127u, 128u, 250u, 251u, 252u, 253u, 254u, 255u, 1u, 252u); + +void main() +{ + for (uint j = 0u; j < 32u; j++) { + uint i = gl_GlobalInvocationID.x * 32u + j; + uint x = p.base + i; + uint h0 = pcg(x ^ p.seed), h1 = pcg(h0 + 1u), h2 = pcg(h1 + 2u); + uint a = h0, d = h1, mode = h2 & 3u; + if (mode == 1u) { + a = with_exp(a, EA[(h2 >> 2u) & 15u]); + d = with_exp(d, ED[(h2 >> 6u) & 15u]); + } else if (mode == 2u) { + uint ed = 1u + (h2 >> 2u) % 254u; + int e = int((h2 >> 10u) % 14u) + ((h2 & 0x80000000u) != 0u ? 117 : -128); + a = with_exp(a, uint(clamp(int(ed) + e, 0, 255))); + d = with_exp(d, ed); + } else if (mode == 3u) { + a = with_exp(a, 1u + (h2 >> 2u) % 254u); + d = with_exp(d, 1u + (h2 >> 10u) % 254u); + } + outp.o[2u * i] = floatBitsToUint(uintBitsToFloat(a) / uintBitsToFloat(d)); + outp.o[2u * i + 1u] = floatBitsToUint(1.0 / uintBitsToFloat(x)); + } +} diff --git a/tests/div_window.py b/tests/div_window.py new file mode 100755 index 000000000000..52f2c389f1eb --- /dev/null +++ b/tests/div_window.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Usage: div_window.py ICD COMPUTE_RUNNER [REF_ICD] [CHUNKS]; glslangValidator and numpy are required. + +Default-path division and reciprocal against the correctly rounded (with FTZ) +quotient, bitwise, specials included: every fp32 reciprocal (64 chunks of +2^26) and the same number of hashed quotients, a quarter of them raw bits and +the rest with exponents at the zero, denormal, inf and 2^126 edges and with +quotients straddling 2^-126 and 2^128. With REF_ICD, every result must also +match that driver bitwise (for example the build before a lowering change). +CHUNKS is N (the first N of 64) or FIRST-LAST, so an interrupted sweep can +resume. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import numpy as np + +args = sys.argv[1:] +icd, runner = map(os.path.abspath, args[:2]) +ref = os.path.abspath(args[2]) if len(args) > 2 else None +span = args[3].split("-") if len(args) > 3 else ["64"] +chunks = range(int(span[0]), int(span[1]) + 1) if len(span) == 2 else range(int(span[0])) +here = Path(__file__).resolve().parent +N = 1 << 26 +SEED = 20261006 + +def pcg(v): + s = v * np.uint32(747796405) + np.uint32(2891336453) + w = ((s >> ((s >> np.uint32(28)) + np.uint32(4))) ^ s) * np.uint32(277803737) + return (w >> np.uint32(22)) ^ w + +EA = np.array([0, 1, 2, 46, 47, 48, 49, 50, 126, 127, 128, 252, 253, 254, 255, 255], dtype=np.uint32) +ED = np.array([0, 0, 1, 2, 3, 126, 127, 128, 250, 251, 252, 253, 254, 255, 1, 252], dtype=np.uint32) + +def operands(base): + x = np.arange(N, dtype=np.uint32) + np.uint32(base) + h0 = pcg(x ^ np.uint32(SEED)) + h1 = pcg(h0 + np.uint32(1)) + h2 = pcg(h1 + np.uint32(2)) + a, d, mode = h0.copy(), h1.copy(), h2 & 3 + exp = lambda bits, e: (bits & np.uint32(0x807fffff)) | (e.astype(np.uint32) << np.uint32(23)) + m = mode == 1 + a[m] = exp(a[m], EA[(h2[m] >> 2) & 15]) + d[m] = exp(d[m], ED[(h2[m] >> 6) & 15]) + m = mode == 2 + ed = 1 + (h2[m] >> 2) % 254 + e = ((h2[m] >> 10) % 14).astype(np.int64) + np.where(h2[m] & 0x80000000, 117, -128) + a[m] = exp(a[m], np.clip(ed.astype(np.int64) + e, 0, 255)) + d[m] = exp(d[m], ed) + m = mode == 3 + a[m] = exp(a[m], 1 + (h2[m] >> 2) % 254) + d[m] = exp(d[m], 1 + (h2[m] >> 10) % 254) + return a, d, x + +def quotient(a, b): + a = a.copy() + b = b.copy() + a[(a & 0x7fffffff) < 0x800000] &= 0x80000000 + b[(b & 0x7fffffff) < 0x800000] &= 0x80000000 + with np.errstate(all="ignore"): + exact = a.view(np.float32).astype(np.float64) / b.view(np.float32).astype(np.float64) + rounded = exact.astype(np.float32).view(np.uint32) + flushed = np.abs(exact) < 2.0**-126 + rounded[flushed] = (a[flushed] ^ b[flushed]) & 0x80000000 + return rounded + +def bad(got, want): + both_nan = ((got & 0x7fffffff) > 0x7f800000) & ((want & 0x7fffffff) > 0x7f800000) + return np.flatnonzero((got != want) & ~both_nan) + +def run(spv, driver, base, path): + path.with_suffix(".in").write_bytes(np.array([base, SEED], dtype=np.uint32).tobytes()) + env = dict(os.environ, VK_DRIVER_FILES=driver, VK_ICD_FILENAMES=driver, MESA_SHADER_CACHE_DISABLE="true") + subprocess.run(["flock", "-w", "1800", "/tmp/m1-gpu.lock", runner, str(spv), str(path.with_suffix(".in")), + str(path), str(N * 8), str(N // 32 // 64)], env=env, check=True, timeout=2100) + return np.fromfile(path, dtype=np.uint32).reshape(N, 2) + +totals = {"div_oracle": 0, "recip_oracle": 0, "div_ref": 0, "recip_ref": 0} +examples = [] +with tempfile.TemporaryDirectory(prefix="mesa-divwin-", dir="/dev/shm" if os.path.isdir("/dev/shm") else None) as td: + td = Path(td) + spv = td / "probe.spv" + subprocess.run(["glslangValidator", "-V", str(here / "div_window.comp"), "-o", str(spv)], check=True, timeout=30) + for c in chunks: + base = c * N + out = run(spv, icd, base, td / "out.bin") + a, d, x = operands(base) + checks = {"div_oracle": bad(out[:, 0], quotient(a, d)), + "recip_oracle": bad(out[:, 1], quotient(np.full(N, 0x3f800000, dtype=np.uint32), x))} + if ref: + r = run(spv, ref, base, td / "ref.bin") + checks["div_ref"] = np.flatnonzero(out[:, 0] != r[:, 0]) + checks["recip_ref"] = np.flatnonzero(out[:, 1] != r[:, 1]) + for k, v in checks.items(): + totals[k] += len(v) + for i in v[:2]: + if len(examples) < 10: + examples.append({"check": k, "a": hex(int(a[i])), "d": hex(int(d[i])), "x": hex(int(x[i])), + "got": [hex(int(w)) for w in out[i]]}) + print(f"chunk {c}: " + " ".join(f"{k}={len(v)}" for k, v in checks.items()), flush=True) + +result = {"pairs": len(chunks) * N, "reciprocals": len(chunks) * N, "chunks": [chunks[0], chunks[-1]], + "mismatches": totals if ref else + {k: v for k, v in totals.items() if k.endswith("oracle")}, "examples": examples} +print(json.dumps(result, sort_keys=True)) +sys.exit(any(totals.values())) From b04611da958f2f14a2999a64200558b0eb66141e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:06:32 -0500 Subject: [PATCH 10/22] agx: round log() once unless the multiply is exact 155e743 dropped the rewrite of fmul(flog2(x), ln2) into the single rounding soft log, so that a precise log2(x) * ln2 gives the same bits from a literal ln2 and from a buffer (found in review). The cost is accuracy for every default log(). Over all 2,130,706,432 positive normal fp32 inputs against fp64 on an M1 Max (G13C): log() before 155e743: max 0.5004 ulp, mean 0.250, 3,448 not correctly rounded log() after 155e743: max 1.2388 ulp, mean 0.343, 536,324,864 not correctly rounded log2() is the same on both (max 0.5025 ulp). Restore the rewrite, but skip an exact multiply. A default log() may be evaluated more accurately than written, so it gets the soft log again. A precise multiply keeps the rounding the source asks for, so the literal and buffer forms still agree bit for bit. --- src/asahi/compiler/agx_nir_lower_math.c | 84 ++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 3058cd7525ea..1c82ade3888c 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -140,7 +140,7 @@ agx_nir_lower_fdiv(nir_shader *s) } /* - * Log2 as double-float (fp32 pair) arithmetic. + * Natural log and log2 as double-float (fp32 pair) arithmetic. * * x = 2^e m with m in [sqrt(1/2), sqrt(2)). With s = (m - 1) / (m + 1), * @@ -149,7 +149,8 @@ agx_nir_lower_fdiv(nir_shader *s) * s is kept as a pair (s, s_lo) from one correctly rounded division plus the * fma residual, the cubic term is kept as a pair, and the terms are summed * with Fast2Sum so the value handed to the final rounding carries about 40 - * bits without a lookup table. + * bits. That makes the exact cases (powers of two, log(3)) come out correctly + * rounded and everything else faithful, without any table. */ struct log_parts { nir_def *e; /* exponent as float */ @@ -250,6 +251,26 @@ log_special(nir_builder *b, nir_def *x) nir_imm_float(b, INFINITY))); } +static nir_def * +soft_log(nir_builder *b, nir_def *x) +{ + struct log_parts l = log_core(b, x); + + /* ln2 split so e * LN2_HI is exact for |e| < 2^8 */ + nir_def *a = nir_fmul_imm(b, l.e, 0.693145751953125); + nir_def *bb = nir_fmul_imm(b, l.s, 2.0); + nir_def *h = nir_fadd(b, a, bb); + nir_def *lo = fast2sum_lo(b, a, bb, h); + nir_def *h2 = nir_fadd(b, h, l.t_hi); + nir_def *lo2 = fast2sum_lo(b, h, l.t_hi, h2); + + nir_def *small = nir_fadd(b, lo, lo2); + small = nir_fadd(b, small, l.t_lo); + small = nir_fadd(b, small, nir_fmul_imm(b, l.s_lo, 2.0)); + small = nir_ffma(b, l.e, nir_imm_float(b, 1.428606765330187e-06), small); + return nir_fadd(b, h2, small); +} + static nir_def * soft_log2(nir_builder *b, nir_def *x) { @@ -274,6 +295,59 @@ soft_log2(nir_builder *b, nir_def *x) return nir_fadd(b, h2, nir_fadd(b, nir_fadd(b, lo, lo2), pl)); } +static const uint32_t LN2_F32_BITS = 0x3f317218; + +static bool +src_is_ln2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_const(s) && nir_scalar_as_uint(s) == LN2_F32_BITS; +} + +static bool +src_is_flog2(nir_alu_instr *alu, unsigned i) +{ + nir_scalar s = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), i); + return nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_flog2; +} + +/* + * nir_flog builds fmul(flog2(x), ln2); catch that before the flog2 goes and + * round once. An exact (precise) multiply keeps the rounding the source asks + * for, so a precise log2(x) * ln2 gives the same bits from a literal and from + * a buffer. + */ +static bool +lower_log(nir_builder *b, nir_alu_instr *alu, void *_) +{ + if (alu->op != nir_op_fmul || alu->def.bit_size != 32 || + alu->def.num_components != 1 || nir_alu_instr_is_exact(alu)) + return false; + + unsigned li; + if (src_is_flog2(alu, 0) && src_is_ln2(alu, 1)) + li = 0; + else if (src_is_flog2(alu, 1) && src_is_ln2(alu, 0)) + li = 1; + else + return false; + + nir_scalar log = nir_scalar_chase_alu_src(nir_get_scalar(&alu->def, 0), li); + nir_scalar xs = nir_scalar_chase_alu_src(log, 0); + + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def *x = nir_channel(b, xs.def, xs.comp); + nir_def *res = nir_bcsel(b, is_positive_normal(b, x), soft_log(b, x), + log_special(b, x)); + nir_def_replace(&alu->def, res); + + if (nir_def_is_unused(log.def)) + nir_instr_remove(nir_def_instr(log.def)); + + return true; +} + static bool lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) { @@ -292,7 +366,11 @@ lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) bool agx_nir_lower_log(nir_shader *s) { - return nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); + bool progress = + nir_shader_alu_pass(s, lower_log, nir_metadata_control_flow, NULL); + progress |= + nir_shader_alu_pass(s, lower_log2, nir_metadata_control_flow, NULL); + return progress; } /* From 953eb9f026835bf7abb2dd9a9ee016e55737098c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:40:50 -0500 Subject: [PATCH 11/22] honeykrisp: let the hardware rcp give the reciprocal edge cases The exact reciprocal took 16 ops: one Newton step inside [2^-126, 2^126) and selects for the other exponents. The hardware rcp already gives those results. It flushes denormal inputs to +-inf and results below 2^-126 to +-0. Where the Newton step is NaN (x zero, denormal, infinite or NaN), keep the hardware value, then make NaN the canonical NaN. This is the "hk: enable cooperative matrices by default on G14X" reciprocal plus the canonical NaN: 7 ops. The results are bit-identical to "agx: round log() once unless the multiply is exact". tests/div_window.py, all 2^32 fp32 reciprocals and 2^32 hashed quotients against "agx: round log() once unless the multiply is exact" and against the correctly rounded FTZ quotient on an M1 Max (G13C): 0 mismatches. precise_math.py and ulp_bound.py pass (max 0 ulp). d64 decode tok/s on the M1 Max, MLX bf62cfb, 5 alternating rounds, medians ("agx: round log() once unless the multiply is exact" / this commit / "hk: enable cooperative matrices by default on G14X"): Qwen3.8-2B 4-bit: 103.19 / 104.35 / 104.75, ahead in 5 of 5 rounds, median paired gain +0.76 Qwen3-4B 4-bit: 64.80 / 64.87 / 66.68 The 4B gap is general division, which stays on the exact path. --- src/asahi/compiler/agx_nir_lower_math.c | 26 +++++++------------------ 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 1c82ade3888c..78020d5eb7e8 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -63,11 +63,11 @@ div_rn(nir_builder *b, nir_def *a, nir_def *d) } /* - * 1/x, bit for bit div_rn(1, x). For x in [2^-126, 2^126) the refinement on - * x itself is the normalized one scaled by a power of two, which - * tests/div_window.py checks over every input. The other exponents have fixed - * results: zero and denormals give +-inf, exactly 2^126 gives +-2^-126, the - * rest of [2^126, inf] gives +-0 and NaN gives the canonical NaN. + * 1/x, bit for bit div_rn(1, x): one Newton step on the hardware rcp, which + * flushes denormal inputs to +-inf and results below 2^-126 to +-0. Where the + * step is NaN (x zero, denormal, infinite or NaN), the hardware rcp already + * holds the answer, and NaN becomes the canonical NaN. tests/div_window.py + * checks every input. */ static nir_def * rcp_rn(nir_builder *b, nir_def *x) @@ -75,20 +75,8 @@ rcp_rn(nir_builder *b, nir_def *x) nir_def *u = nir_frcp(b, x); nir_def *q = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, nir_imm_float(b, 1.0)), u, u); - - nir_def *ex = nir_ubfe_imm(b, x, 23, 8); - nir_def *mag = nir_iand_imm(b, x, 0x7fffffff); - nir_def *special = - nir_bcsel(b, nir_ieq_imm(b, mag, 0x7e800000), nir_imm_int(b, 0x00800000), - nir_imm_int(b, 0)); - special = nir_bcsel(b, nir_ieq_imm(b, ex, 0), nir_imm_int(b, 0x7f800000), - special); - special = nir_ior(b, special, nir_iand_imm(b, x, 0x80000000)); - special = nir_bcsel(b, nir_ugt_imm(b, mag, 0x7f800000), - nir_imm_int(b, 0x7fc00000), special); - - nir_def *inside = nir_ult(b, nir_iadd_imm(b, ex, -1), nir_imm_int(b, 252)); - return nir_bcsel(b, inside, q, special); + q = nir_bcsel(b, nir_fisnan(b, q), u, q); + return nir_bcsel(b, nir_fisnan(b, q), nir_imm_int(b, 0x7fc00000), q); } static bool From 4a4e6c5ad3cf5d15703e3688be6540722d276ec8 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:18:17 -0500 Subject: [PATCH 12/22] agx: pin float-control keying of division, not its op counts DivDefaultFastSequencePinned and FrcpDefaultSequencePinned counted the ALU ops of the "hk: enable cooperative matrices by default on G14X" division and reciprocal. Every exact lowering since "agx: flush the exact quotient in precise fp32 division, not the correction step" emits other ops, so agx_tests has failed these three pins on the PR #6 head (found by w7Q, the driver's meson suite: 61/64). The bits of division and reciprocal are pinned on hardware by tests/precise_math.py, ulp_bound.py and div_window.py. What this file can pin is the float-control contract: DenormFlushToZero emits exactly the default lowering. Compare the two op histograms for fdiv and frcp instead of fixed counts. --- src/asahi/compiler/test/test-lower-math.cpp | 91 +++++++++------------ 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp index 4dadcc05afe8..ccefd3cb5180 100644 --- a/src/asahi/compiler/test/test-lower-math.cpp +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -10,14 +10,14 @@ extern "C" { } #include +#include /* * Behavioral pins for the float-control keying in agx_nir_lower_math.c. * - * The default path (no execution modes declared) must emit exactly the - * sequences the pinned MLX decode digests were measured against, so the - * first tests pin its op histogram: any accidental change to the default - * emission shows up here before it can move a digest on hardware. + * Division and reciprocal do not key on float controls. Their bits are pinned + * on hardware by tests/precise_math.py, ulp_bound.py and div_window.py; here + * the pin is that DenormFlushToZero emits exactly the default lowering. * * The DenormFlushToZero sin test pins the one keyed change: a zero-comparing * subnormal argument must produce a sign-preserving flushed zero instead of @@ -78,64 +78,49 @@ class LowerMathTest : public ::testing::Test { } return n; } + + /* ALU op histogram of fdiv (or frcp) lowered under one float-control mode */ + std::map lowered_division(unsigned mode, bool reciprocal) + { + nir_builder s = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, + &options, "lower fdiv"); + s.shader->info.float_controls_execution_mode = mode; + nir_def *x = nir_undef(&s, 1, 32); + if (reciprocal) + nir_frcp(&s, x); + else + nir_fdiv(&s, nir_undef(&s, 1, 32), x); + + EXPECT_TRUE(agx_nir_lower_fdiv(s.shader)); + + std::map ops; + nir_foreach_function_impl(impl, s.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu) + ops[nir_instr_as_alu(instr)->op]++; + } + } + } + ralloc_free(s.shader); + return ops; + } }; const nir_shader_compiler_options LowerMathTest::options = {}; -TEST_F(LowerMathTest, DivDefaultFastSequencePinned) -{ - set_mode(0); - nir_def *a = nir_undef(&b, 1, 32); - nir_def *d = nir_undef(&b, 1, 32); - nir_fdiv(&b, a, d); - - ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); - - EXPECT_EQ(count_alu(nir_op_fdiv), 0u); - EXPECT_EQ(count_alu(nir_op_frcp), 1u); - EXPECT_EQ(count_alu(nir_op_ffma), 4u); - EXPECT_EQ(count_alu(nir_op_fmul), 1u); - EXPECT_EQ(count_alu(nir_op_fneg), 2u); - EXPECT_EQ(count_alu(nir_op_bcsel), 2u); - EXPECT_EQ(count_alu(nir_op_fneu), 2u); - EXPECT_EQ(count_alu(nir_op_feq), 1u); - EXPECT_EQ(count_alu(nir_op_ior), 1u); - /* The mantissa-normalizing exact sequence is integer arithmetic; its - * absence is the marker that the default emission is untouched. */ - EXPECT_EQ(count_alu(nir_op_iand), 0u); - EXPECT_EQ(count_alu(nir_op_ushr), 0u); - EXPECT_EQ(count_alu(nir_op_ishl), 0u); -} - TEST_F(LowerMathTest, DivFtzModeUnchanged) { - set_mode(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32); - nir_def *a = nir_undef(&b, 1, 32); - nir_def *d = nir_undef(&b, 1, 32); - nir_fdiv(&b, a, d); - - ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); - - /* Hardware flushes arithmetic outputs, so the refined form already - * satisfies DenormFlushToZero; no keyed emission is warranted. */ - EXPECT_EQ(count_alu(nir_op_iand), 0u); - EXPECT_EQ(count_alu(nir_op_ushr), 0u); - EXPECT_EQ(count_alu(nir_op_ishl), 0u); - EXPECT_EQ(count_alu(nir_op_frcp), 1u); + auto ftz = lowered_division(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32, false); + EXPECT_EQ(ftz.count(nir_op_fdiv), 0u); + EXPECT_EQ(ftz, lowered_division(0, false)); } -TEST_F(LowerMathTest, FrcpDefaultSequencePinned) +TEST_F(LowerMathTest, FrcpFtzModeUnchanged) { - set_mode(0); - nir_def *x = nir_undef(&b, 1, 32); - nir_frcp(&b, x); - - ASSERT_TRUE(agx_nir_lower_fdiv(b.shader)); - - EXPECT_EQ(count_alu(nir_op_frcp), 1u); - EXPECT_EQ(count_alu(nir_op_ffma), 2u); - EXPECT_EQ(count_alu(nir_op_bcsel), 1u); - EXPECT_EQ(count_alu(nir_op_fneu), 1u); + auto ftz = lowered_division(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32, true); + EXPECT_EQ(ftz.count(nir_op_frcp), 1u); + EXPECT_EQ(ftz, lowered_division(0, true)); } TEST_F(LowerMathTest, SinDefaultKeepsSelect) From 4340f2fc4568e119687aadb10c15007540ae5a75 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:54:03 +0000 Subject: [PATCH 13/22] agx: lower the fdiv that int64 lowering and the perspective divide emit late Symptom: any shader converting fp32 to a 64-bit integer aborts in the backend with "Unhandled ALU op fdiv" (agx_compile.c agx_emit_alu); the MLX suite the copy-offset tests crashes on "dtype converting copies cover the integer family and bool". Fragment shaders with perspective-correct varyings hit the same path. Cause: "agx: correctly rounded fdiv/frcp, faithful log/log2, exact sin/cos reduction" dropped lower_fdiv from the compiler options so nir_opt_algebraic no longer splits fdiv into fmul/frcp, and only ran agx_nir_lower_fdiv in agx_preprocess_nir. nir_lower_int64 (f2u64/f2i64: fdiv by 2^32, plus the frem that lower_fmod turns into a second fdiv) and agx_nir_lower_interpolation run after preprocessing, so their fdiv reached the backend, which has no fdiv. Fix: agx_nir_lower_fdiv_late runs in agx_optimize_nir after the int64 cleanup loop and lowers only fdiv, leaving frcp alone so the reciprocals built by the first pass and by the log lowering are not refined twice. Verification: hk/precise-math driver alone reproduced the abort on the 13-inch M1; with this commit the copy-offset tests passes 25/25 . --- src/asahi/compiler/agx_compile.c | 5 +++++ src/asahi/compiler/agx_nir.h | 1 + src/asahi/compiler/agx_nir_lower_math.c | 25 ++++++++++++++++++++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index d3c46578588a..bdae6daf6d5a 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -8259,6 +8259,11 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics, } while (progress); } + /* int64 lowering and the perspective divide emit fdiv after preprocessing; + * the backend has no fdiv, so lower what is left before it is scheduled. + */ + NIR_PASS(_, nir, agx_nir_lower_fdiv_late); + /* Reassociate before forming preambles because it makes preambles more * effective. Clean up after. On M5, also order the loop-invariant terms of * a chain in a loop first, so accumulations of per-iteration constants diff --git a/src/asahi/compiler/agx_nir.h b/src/asahi/compiler/agx_nir.h index ec86ea95aa45..61746b542887 100644 --- a/src/asahi/compiler/agx_nir.h +++ b/src/asahi/compiler/agx_nir.h @@ -58,6 +58,7 @@ bool agx_nir_share_bounds(struct nir_shader *s); bool agx_nir_lower_atomics(struct nir_shader *s, bool native_fadd); bool agx_nir_opt_canonical_minmax(struct nir_shader *s); bool agx_nir_lower_fdiv(struct nir_shader *s); +bool agx_nir_lower_fdiv_late(struct nir_shader *s); bool agx_nir_lower_log(struct nir_shader *s); bool agx_nir_lower_sincos(struct nir_shader *s); diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 78020d5eb7e8..65157bdfe8c4 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -87,9 +87,11 @@ is_one(const nir_alu_instr *alu, unsigned i) } static bool -lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) +lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *data) { - if (alu->op != nir_op_frcp && alu->op != nir_op_fdiv) + bool fdiv_only = *(bool *)data; + + if (alu->op != nir_op_fdiv && (alu->op != nir_op_frcp || fdiv_only)) return false; if (alu->def.bit_size == 64) @@ -124,7 +126,24 @@ lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) bool agx_nir_lower_fdiv(nir_shader *s) { - return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, NULL); + bool fdiv_only = false; + return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, + &fdiv_only); +} + +/* + * Passes that run after agx_preprocess_nir still emit fdiv: nir_lower_int64 + * (f2u64/f2i64 divide by 2^32, and lower_fmod turns the frem into a second + * fdiv) and agx_nir_lower_interpolation (perspective divide). Lower only fdiv + * here so the reciprocals built by the first pass and by the log lowering are + * not refined a second time. + */ +bool +agx_nir_lower_fdiv_late(nir_shader *s) +{ + bool fdiv_only = true; + return nir_shader_alu_pass(s, lower_fdiv, nir_metadata_control_flow, + &fdiv_only); } /* From 7b6dbd676ed58f5ea182ccba499d99bf66f6ba62 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:35:51 -0500 Subject: [PATCH 14/22] tests: pin the late fdiv lowering of the AGX math pass nir_lower_int64 (float to 64-bit integer conversions) and smooth offset interpolation emit a dynamic fdiv after preprocessing. The backend has no fdiv, so agx_nir_lower_fdiv_late lowers it. Three unit tests pin that the late pass lowers fdiv like the first pass, leaves a plain reciprocal alone, and does not refine an existing reciprocal a second time. MEASURED: the AGX compiler unit tests pass on a native arm64 build, 63 of 63, including the three new ones. NOT measured: a shader that goes through nir_lower_int64 or the interpolation pass end to end; the tests build the division directly. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- src/asahi/compiler/test/test-lower-math.cpp | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp index ccefd3cb5180..9a965b9c1d93 100644 --- a/src/asahi/compiler/test/test-lower-math.cpp +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -123,6 +123,48 @@ TEST_F(LowerMathTest, FrcpFtzModeUnchanged) EXPECT_EQ(ftz, lowered_division(0, true)); } +/* Division introduced after preprocessing (nir_lower_int64 float-to-int64 + * conversions, smooth offset interpolation) must not reach the backend, which + * has no fdiv. The late pass lowers fdiv exactly like the first pass and + * leaves every reciprocal alone, so none is refined twice. + */ +TEST_F(LowerMathTest, LateFdivLowersLikeFirstPass) +{ + nir_def *x = nir_undef(&b, 1, 32); + nir_fdiv(&b, nir_undef(&b, 1, 32), x); + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + + auto first = lowered_division(0, false); + unsigned expect_ffma = first.count(nir_op_ffma) ? first[nir_op_ffma] : 0; + EXPECT_EQ(count_alu(nir_op_ffma), expect_ffma); +} + +TEST_F(LowerMathTest, LateFdivLeavesFrcpAlone) +{ + nir_frcp(&b, nir_undef(&b, 1, 32)); + + EXPECT_FALSE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_frcp), 1u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); +} + +TEST_F(LowerMathTest, LateFdivDoesNotRefineExistingFrcp) +{ + nir_frcp(&b, nir_undef(&b, 1, 32)); + nir_fdiv(&b, nir_undef(&b, 1, 32), nir_undef(&b, 1, 32)); + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + + /* the fdiv alone lowers to F ffma ops; the untouched frcp adds none */ + auto first = lowered_division(0, false); + unsigned expect_ffma = first.count(nir_op_ffma) ? first[nir_op_ffma] : 0; + EXPECT_EQ(count_alu(nir_op_ffma), expect_ffma); + EXPECT_EQ(count_alu(nir_op_frcp), 1u + (first.count(nir_op_frcp) ? first[nir_op_frcp] : 0)); +} + TEST_F(LowerMathTest, SinDefaultKeepsSelect) { set_mode(0); From ab23e963ce5c08f249094a473962d5d043e283a5 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:23:27 -0500 Subject: [PATCH 15/22] tests: measure the 2.5 ULP bound in float32 spacing, with the shader it needs ulp_bound.py divided the error by np.spacing of the float64 reference. np.spacing is negative for a negative reference, so the distance came out negative and any error on a negative reference passed the > 2.5 check; for a positive reference the float64 spacing made every inexact result fail instead. Use the absolute float32 spacing of the reference. Add a --self-test mode that needs no GPU: it checks the distance for 0, 1 and 3 ULP errors in both directions on positive and negative references, that a 3 ULP error on a negative reference exceeds 2.5, and that the old expression gives a negative distance for it. The script also compiles precise_math.comp, which this tree did not have; add it. MEASURED: the self test passes; the old expression gives -1610612736 for a 3 ULP error at -1.5, which passes the old check. NOT measured: the GPU run of ulp_bound.py with the corrected distance; the numbers from the old script are not to be relied on. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- tests/precise_math.comp | 17 +++++++++++++++++ tests/ulp_bound.py | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/precise_math.comp diff --git a/tests/precise_math.comp b/tests/precise_math.comp new file mode 100644 index 000000000000..827941d24688 --- /dev/null +++ b/tests/precise_math.comp @@ -0,0 +1,17 @@ +#version 450 +layout(local_size_x = 64) in; +layout(std430, binding = 0) readonly buffer In { vec4 v[]; } inp; +layout(std430, binding = 1) writeonly buffer Out { float o[]; } outp; +void main() { + uint i = gl_GlobalInvocationID.x; + vec4 x = inp.v[i]; + precise float t = log2(x.w); + precise float literal = t * 0.6931471805599453; + precise float from_buffer = t * x.z; + outp.o[i * 6 + 0] = x.x / x.y; + outp.o[i * 6 + 1] = 1.0 / x.y; + outp.o[i * 6 + 2] = literal; + outp.o[i * 6 + 3] = from_buffer; + outp.o[i * 6 + 4] = t; + outp.o[i * 6 + 5] = log(x.w); +} diff --git a/tests/ulp_bound.py b/tests/ulp_bound.py index 48667547521a..ae793a5b78c8 100644 --- a/tests/ulp_bound.py +++ b/tests/ulp_bound.py @@ -18,6 +18,45 @@ import tempfile import numpy as np +def ulp_distance(got, want): + """Distance in float32 ULPs from want, in float64 arithmetic. + + The spacing is the float32 spacing of the reference, as an absolute value: + np.spacing is negative for a negative reference, which would turn the + distance negative and let any error through the > 2.5 test. + """ + want32 = np.asarray(want, dtype=np.float32) + with np.errstate(all="ignore"): + return (np.abs(np.asarray(got, dtype=np.float64) - want32.astype(np.float64)) / + np.abs(np.spacing(want32)).astype(np.float64)) + + +def self_test(): + one = np.float32(1.5) + for ref in (one, -one, np.float32(3.0e-5), np.float32(-3.0e-5)): + for k, expect in ((0, 0.0), (1, 1.0), (3, 3.0), (-3, 3.0)): + got = ref + for _ in range(abs(k)): + got = np.nextafter(got, np.float32(np.inf if k > 0 else -np.inf)) + d = float(ulp_distance(np.float64(got), ref)) + assert d == expect, (float(ref), k, d) + # A three ULP error on a negative reference must exceed the 2.5 budget. + neg = np.float32(-1.5) + bad = neg + for _ in range(3): + bad = np.nextafter(bad, np.float32(-np.inf)) + assert float(ulp_distance(np.float64(bad), neg)) > 2.5 + # The old expression divided by the float64 spacing of the float64 value, + # which is negative here, so the same error looked like a negative distance. + old = abs(float(bad) - float(neg)) / float(np.spacing(np.float64(neg))) + assert old < 0 + print("ulp_bound self test: PASS") + + +if sys.argv[1:] == ["--self-test"]: + self_test() + sys.exit(0) + icd, runner = map(os.path.abspath, sys.argv[1:]) here = Path(__file__).resolve().parent @@ -89,7 +128,7 @@ def check(col, a_bits, b_bits): spec_bad = np.flatnonzero(special & (got != want) & ~both_nan) normal = np.flatnonzero(~special) with np.errstate(all="ignore"): - dist = np.abs(gf[normal] - wf[normal]) / np.spacing(want[normal].view(np.float32).astype(np.float64)) + dist = ulp_distance(gf[normal], want[normal].view(np.float32)) over = normal[dist > 2.5] return {"max_ulp": float(dist.max()) if len(normal) else 0.0, "over_2p5": len(over), From 1598e3daf6bbb67541b000266f2c1039675be41e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:17:42 -0700 Subject: [PATCH 16/22] tests: repair fp32 error oracle and complete numerical fixtures Measure positive binary32 ULPs, reject non-finite results for finite references, and add seven CPU controls. Restore unchanged public shader/runner fixtures from Joshua Warren source f144c7949b7d; provenance is recorded beside the tests. No driver code changes. Generated-by: LLM --- tests/NUMERICAL-HARNESS.md | 26 +++++ tests/compute_runner.cpp | 196 +++++++++++++++++++++++++++++++++++++ tests/test_ulp_bound.py | 62 ++++++++++++ tests/ulp_bound.py | 56 +++-------- 4 files changed, 298 insertions(+), 42 deletions(-) create mode 100644 tests/NUMERICAL-HARNESS.md create mode 100644 tests/compute_runner.cpp create mode 100644 tests/test_ulp_bound.py diff --git a/tests/NUMERICAL-HARNESS.md b/tests/NUMERICAL-HARNESS.md new file mode 100644 index 000000000000..fdc9b6ac8a49 --- /dev/null +++ b/tests/NUMERICAL-HARNESS.md @@ -0,0 +1,26 @@ +# Numerical qualification harness + +Driver source under test: Joshua Warren’s correctness-only adaptation +`5e45716701b64a4b627e7e60df5b781c65c87f42`. CI source is the previously checked +`001e33ae855bd6b9ddc28bfc60a6bd64d71bcf91` trial tree. This branch adds test +harness fixes only; the production PR and package source pins are unchanged. + +The ULP checker uses a positive binary32 ULP at the magnitude of the correctly +rounded reference, calculated in binary64. It explicitly rejects a non-finite +result for a finite reference. Special reference values retain bitwise checks, +with NaN payloads ignored. Seven CPU controls cover both signs, a power-of-two +boundary, out-of-budget results, NaN/infinity, FLT_MAX and signed zero. + +The original checker measures binary64 spacing, gives negative distances for +negative references and can accept NaN for a finite reference. These are oracle +defects, not observations of GPU arithmetic failure. + +The following files were copied byte for byte from Joshua Warren’s public +source at `f144c7949b7d960f26677b671341d5a123057ddf` to supply the fixtures +missing from the correctness extraction. No CDM, scheduling or performance +source is imported. + +- [precise_math.comp](https://github.com/joshuaswarren/mesa-1/blob/f144c7949b7d960f26677b671341d5a123057ddf/tests/precise_math.comp), Git blob `827941d2468890243996dc8d9ab7f74005dab062`. +- [compute_runner.cpp](https://github.com/joshuaswarren/mesa-1/blob/f144c7949b7d960f26677b671341d5a123057ddf/tests/compute_runner.cpp), Git blob `e16d45d9d61252d72b2389da8c5073c781e696ca`. + +Validation: seven CPU oracle controls pass. No numerical GPU result is claimed by this commit. diff --git a/tests/compute_runner.cpp b/tests/compute_runner.cpp new file mode 100644 index 000000000000..e16d45d9d612 --- /dev/null +++ b/tests/compute_runner.cpp @@ -0,0 +1,196 @@ +// Build: c++ -O2 compute_runner.cpp -lvulkan -o compute_runner +// Usage: compute_runner shader.spv in.bin out.bin out_bytes groups +#include + +#include +#include +#include +#include +#include +#include +#include + +#define CK(x) \ + do { \ + VkResult r_ = (x); \ + if (r_ != VK_SUCCESS) { \ + std::fprintf(stderr, "%s failed: %d\n", #x, (int)r_); \ + std::exit(2); \ + } \ + } while (0) + +static std::string slurp(const char* p) { + std::ifstream f(p, std::ios::binary); + if (!f) { std::fprintf(stderr, "open %s\n", p); std::exit(2); } + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +struct Buf { VkBuffer b; VkDeviceMemory m; void* map; }; + +static VkPhysicalDeviceMemoryProperties g_mp; +static Buf mkbuf(VkDevice dev, size_t size) { + Buf r{}; + VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bi.size = size; + bi.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + CK(vkCreateBuffer(dev, &bi, nullptr, &r.b)); + VkMemoryRequirements mr; + vkGetBufferMemoryRequirements(dev, r.b, &mr); + uint32_t ti = UINT32_MAX; + for (uint32_t i = 0; i < g_mp.memoryTypeCount; ++i) { + VkMemoryPropertyFlags want = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if ((mr.memoryTypeBits & (1u << i)) && + (g_mp.memoryTypes[i].propertyFlags & want) == want) { ti = i; break; } + } + if (ti == UINT32_MAX) { std::fprintf(stderr, "no host memtype\n"); std::exit(2); } + VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + ai.allocationSize = mr.size; + ai.memoryTypeIndex = ti; + CK(vkAllocateMemory(dev, &ai, nullptr, &r.m)); + CK(vkBindBufferMemory(dev, r.b, r.m, 0)); + CK(vkMapMemory(dev, r.m, 0, size, 0, &r.map)); + return r; +} + +int main(int argc, char** argv) { + if (argc < 6) { std::fprintf(stderr, "usage\n"); return 2; } + std::string spv = slurp(argv[1]); + std::string in = slurp(argv[2]); + size_t out_bytes = std::strtoull(argv[4], nullptr, 10); + uint32_t groups = std::strtoul(argv[5], nullptr, 10); + + VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo ici{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ici.pApplicationInfo = &app; + VkInstance inst; + CK(vkCreateInstance(&ici, nullptr, &inst)); + uint32_t n = 1; + VkPhysicalDevice pd; + CK(vkEnumeratePhysicalDevices(inst, &n, &pd)); + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(pd, &props); + vkGetPhysicalDeviceMemoryProperties(pd, &g_mp); + std::fprintf(stderr, "device: %s driver %u.%u.%u\n", props.deviceName, + VK_VERSION_MAJOR(props.driverVersion), + VK_VERSION_MINOR(props.driverVersion), + VK_VERSION_PATCH(props.driverVersion)); + + uint32_t qc = 0; + vkGetPhysicalDeviceQueueFamilyProperties(pd, &qc, nullptr); + std::vector qf(qc); + vkGetPhysicalDeviceQueueFamilyProperties(pd, &qc, qf.data()); + uint32_t qi = 0; + for (; qi < qc; ++qi) if (qf[qi].queueFlags & VK_QUEUE_COMPUTE_BIT) break; + float prio = 1.0f; + VkDeviceQueueCreateInfo dqi{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + dqi.queueFamilyIndex = qi; + dqi.queueCount = 1; + dqi.pQueuePriorities = &prio; + VkDeviceCreateInfo dci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dci.queueCreateInfoCount = 1; + dci.pQueueCreateInfos = &dqi; + VkDevice dev; + CK(vkCreateDevice(pd, &dci, nullptr, &dev)); + VkQueue q; + vkGetDeviceQueue(dev, qi, 0, &q); + + Buf bin = mkbuf(dev, in.size()); + Buf bout = mkbuf(dev, out_bytes); + std::memcpy(bin.map, in.data(), in.size()); + std::memset(bout.map, 0, out_bytes); + + VkDescriptorSetLayoutBinding binds[2]{}; + for (int i = 0; i < 2; ++i) { + binds[i].binding = i; + binds[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + binds[i].descriptorCount = 1; + binds[i].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo dsl{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + dsl.bindingCount = 2; + dsl.pBindings = binds; + VkDescriptorSetLayout layout; + CK(vkCreateDescriptorSetLayout(dev, &dsl, nullptr, &layout)); + VkPipelineLayoutCreateInfo pli{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pli.setLayoutCount = 1; + pli.pSetLayouts = &layout; + VkPipelineLayout pl; + CK(vkCreatePipelineLayout(dev, &pli, nullptr, &pl)); + VkShaderModuleCreateInfo smi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + smi.codeSize = spv.size(); + smi.pCode = (const uint32_t*)spv.data(); + VkShaderModule sm; + CK(vkCreateShaderModule(dev, &smi, nullptr, &sm)); + VkComputePipelineCreateInfo cpi{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cpi.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + cpi.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + cpi.stage.module = sm; + cpi.stage.pName = "main"; + cpi.layout = pl; + VkPipeline pipe; + CK(vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &cpi, nullptr, &pipe)); + + VkDescriptorPoolSize ps{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2}; + VkDescriptorPoolCreateInfo dpi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dpi.maxSets = 1; + dpi.poolSizeCount = 1; + dpi.pPoolSizes = &ps; + VkDescriptorPool pool; + CK(vkCreateDescriptorPool(dev, &dpi, nullptr, &pool)); + VkDescriptorSetAllocateInfo dsa{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + dsa.descriptorPool = pool; + dsa.descriptorSetCount = 1; + dsa.pSetLayouts = &layout; + VkDescriptorSet ds; + CK(vkAllocateDescriptorSets(dev, &dsa, &ds)); + VkDescriptorBufferInfo dbi[2] = {{bin.b, 0, VK_WHOLE_SIZE}, {bout.b, 0, VK_WHOLE_SIZE}}; + VkWriteDescriptorSet w[2]{}; + for (int i = 0; i < 2; ++i) { + w[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + w[i].dstSet = ds; + w[i].dstBinding = i; + w[i].descriptorCount = 1; + w[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + w[i].pBufferInfo = &dbi[i]; + } + vkUpdateDescriptorSets(dev, 2, w, 0, nullptr); + + VkCommandPoolCreateInfo cpci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + cpci.queueFamilyIndex = qi; + VkCommandPool cp; + CK(vkCreateCommandPool(dev, &cpci, nullptr, &cp)); + VkCommandBufferAllocateInfo cba{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + cba.commandPool = cp; + cba.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cba.commandBufferCount = 1; + VkCommandBuffer cb; + CK(vkAllocateCommandBuffers(dev, &cba, &cb)); + VkCommandBufferBeginInfo cbb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + CK(vkBeginCommandBuffer(cb, &cbb)); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); + vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pl, 0, 1, &ds, 0, nullptr); + vkCmdDispatch(cb, groups, 1, 1); + VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + CK(vkEndCommandBuffer(cb)); + + VkFenceCreateInfo fci{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence; + CK(vkCreateFence(dev, &fci, nullptr, &fence)); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.commandBufferCount = 1; + si.pCommandBuffers = &cb; + CK(vkQueueSubmit(q, 1, &si, fence)); + CK(vkWaitForFences(dev, 1, &fence, VK_TRUE, 60000000000ull)); + std::ofstream of(argv[3], std::ios::binary); + of.write((const char*)bout.map, out_bytes); + vkDeviceWaitIdle(dev); + return 0; +} diff --git a/tests/test_ulp_bound.py b/tests/test_ulp_bound.py new file mode 100644 index 000000000000..8bce1713ad82 --- /dev/null +++ b/tests/test_ulp_bound.py @@ -0,0 +1,62 @@ +"""CPU controls for the numerical GPU oracle, without starting Vulkan.""" +import ast +from pathlib import Path +import unittest + +import numpy as np + + +# ulp_bound currently dispatches a GPU job at module scope. Extract only its +# pure checks so these controls need neither a GPU nor a Vulkan driver. +source = Path(__file__).with_name("ulp_bound.py") +tree = ast.parse(source.read_text()) +functions = ast.Module(body=[n for n in tree.body + if isinstance(n, ast.FunctionDef) + and n.name in ("quotient", "check")], type_ignores=[]) +namespace = {"np": np} +exec(compile(functions, str(source), "exec"), namespace) + + +class ULPControls(unittest.TestCase): + def measure(self, reference, result): + namespace["inp"] = np.array([[reference, 0x3f800000]], dtype=np.uint32) + namespace["out"] = np.array([[result]], dtype=np.uint32) + return namespace["check"](0, namespace["inp"][:, 0], + namespace["inp"][:, 1]) + + def test_positive_one_ulp(self): + r = self.measure(0x3f800000, 0x3f800001) + self.assertEqual(r["max_ulp"], 1.0) + self.assertEqual(r["over_2p5"], 0) + + def test_positive_over_budget(self): + self.assertEqual(self.measure(0x3f800000, 0x3f800003)["over_2p5"], 1) + + def test_negative_errors_are_counted(self): + r = self.measure(0xbf800000, 0xbf800003) + self.assertEqual(r["max_ulp"], 3.0) + self.assertEqual(r["over_2p5"], 1) + + def test_negative_power_of_two_boundary(self): + self.assertEqual(self.measure(0xbf800000, 0xbf7fffff)["max_ulp"], 0.5) + + def test_nan_and_infinity_cannot_match_finite(self): + for bits in (0x7fc00000, 0x7f800000, 0xff800000): + with self.subTest(bits=hex(bits)): + r = self.measure(0x3f800000, bits) + self.assertEqual(r["over_2p5"], 1) + self.assertEqual(r["nonfinite_mismatch"], 1) + + def test_largest_finite_has_finite_ulp(self): + r = self.measure(0x7f7fffff, 0x7f7ffffe) + self.assertEqual(r["max_ulp"], 1.0) + self.assertEqual(r["over_2p5"], 0) + + def test_specials_and_signed_zero(self): + for bits in (0, 0x80000000, 0x7f800000, 0xff800000, 0x7fc00000): + self.assertEqual(self.measure(bits, bits)["special_mismatch"], 0) + self.assertEqual(self.measure(0x80000000, 0)["special_mismatch"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ulp_bound.py b/tests/ulp_bound.py index ae793a5b78c8..952bb94e555f 100644 --- a/tests/ulp_bound.py +++ b/tests/ulp_bound.py @@ -18,45 +18,6 @@ import tempfile import numpy as np -def ulp_distance(got, want): - """Distance in float32 ULPs from want, in float64 arithmetic. - - The spacing is the float32 spacing of the reference, as an absolute value: - np.spacing is negative for a negative reference, which would turn the - distance negative and let any error through the > 2.5 test. - """ - want32 = np.asarray(want, dtype=np.float32) - with np.errstate(all="ignore"): - return (np.abs(np.asarray(got, dtype=np.float64) - want32.astype(np.float64)) / - np.abs(np.spacing(want32)).astype(np.float64)) - - -def self_test(): - one = np.float32(1.5) - for ref in (one, -one, np.float32(3.0e-5), np.float32(-3.0e-5)): - for k, expect in ((0, 0.0), (1, 1.0), (3, 3.0), (-3, 3.0)): - got = ref - for _ in range(abs(k)): - got = np.nextafter(got, np.float32(np.inf if k > 0 else -np.inf)) - d = float(ulp_distance(np.float64(got), ref)) - assert d == expect, (float(ref), k, d) - # A three ULP error on a negative reference must exceed the 2.5 budget. - neg = np.float32(-1.5) - bad = neg - for _ in range(3): - bad = np.nextafter(bad, np.float32(-np.inf)) - assert float(ulp_distance(np.float64(bad), neg)) > 2.5 - # The old expression divided by the float64 spacing of the float64 value, - # which is negative here, so the same error looked like a negative distance. - old = abs(float(bad) - float(neg)) / float(np.spacing(np.float64(neg))) - assert old < 0 - print("ulp_bound self test: PASS") - - -if sys.argv[1:] == ["--self-test"]: - self_test() - sys.exit(0) - icd, runner = map(os.path.abspath, sys.argv[1:]) here = Path(__file__).resolve().parent @@ -127,11 +88,22 @@ def check(col, a_bits, b_bits): special = ~np.isfinite(wf) | (wf == 0.0) spec_bad = np.flatnonzero(special & (got != want) & ~both_nan) normal = np.flatnonzero(~special) + finite_result = np.isfinite(gf[normal]) with np.errstate(all="ignore"): - dist = ulp_distance(gf[normal], want[normal].view(np.float32)) - over = normal[dist > 2.5] - return {"max_ulp": float(dist.max()) if len(normal) else 0.0, + # A binary32 ULP is positive for both signs. Derive its exponent + # before measuring in binary64; spacing(float64) measures the wrong + # format and spacing(negative) would hide errors. ldexp also handles + # FLT_MAX without overflowing to infinity like float32 spacing does. + exponent = np.frexp(np.abs(wf[normal]))[1] + ulp = np.ldexp(np.ones(len(normal)), np.maximum(exponent - 24, -149)) + dist = np.abs(gf[normal] - wf[normal]) / ulp + # NaN comparisons are false, so reject non-finite results explicitly when + # the oracle expected a finite value. + over = normal[(dist > 2.5) | ~finite_result] + finite_dist = dist[finite_result] + return {"max_ulp": float(finite_dist.max()) if len(finite_dist) else 0.0, "over_2p5": len(over), + "nonfinite_mismatch": int(np.count_nonzero(~finite_result)), "special_mismatch": len(spec_bad), "examples": [[hex(int(v)) for v in inp[i][:2]] for i in over[:5]][:5]} From 1878c64c44c804951546759123ab965c8d4a6c34 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:30:47 -0500 Subject: [PATCH 17/22] agx: keep the exact division, log and sin/cos lowering for compute Graphics shaders pay for the exact sequences and gain nothing the Vulkan precision table asks for. A same-boot A/B of the full change on an M1 showed about 18 percent lower vkmark and 7 percent lower vkQuake. Lower the exact sequences only for compute shaders, where the rounding contract matters. Graphics keeps the base reciprocal and quotient, the base log and the base sin/cos sequence. A fdiv that is emitted after preprocessing (int64 lowering, perspective divide) is still lowered, as a multiply by the hardware reciprocal, because the backend has no fdiv. Not measured: speed or ULP of this change. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- src/asahi/compiler/agx_compile.c | 11 +++++++---- src/asahi/compiler/agx_nir_lower_math.c | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index bdae6daf6d5a..e328408f0f9d 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -7543,11 +7543,9 @@ agx_calc_stats(agx_context *ctx, unsigned size, struct agx2_stats *stats) } static bool -agx_lower_sincos_filter(const nir_instr *instr, const void *g17p) +agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *g17p) { - /* Only the native G17P sequence is lowered here; every other target goes - * through agx_nir_lower_sincos, the exact reduction. */ - if (!(uintptr_t)g17p || instr->type != nir_instr_type_alu) + if (instr->type != nir_instr_type_alu) return false; nir_alu_instr *alu = nir_instr_as_alu(instr); @@ -7663,6 +7661,11 @@ agx_lower_sincos_impl(struct nir_builder *b, nir_instr *instr, void *data) static bool agx_lower_sincos(nir_shader *shader, unsigned native) { + /* Compute on every target but G17P goes through agx_nir_lower_sincos, the + * exact reduction. Graphics keeps the base sequence. */ + if (!g17p && mesa_shader_stage_is_compute(shader->info.stage)) + return false; + return nir_shader_lower_instructions(shader, agx_lower_sincos_filter, agx_lower_sincos_impl, (void *)(uintptr_t)native); diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 65157bdfe8c4..3d41c40e509e 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -97,6 +97,22 @@ lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *data) if (alu->def.bit_size == 64) return false; + /* Graphics shaders keep the hardware reciprocal and the cheap quotient. + * The exact sequences below are for compute (MLX and kernels), where + * the rounding contract matters. A late fdiv (int64 lowering, perspective + * divide) still has to be lowered, because the backend has no fdiv. + */ + if (!mesa_shader_stage_is_compute(b->shader->info.stage)) { + if (alu->op != nir_op_fdiv || alu->def.bit_size != 32) + return false; + + b->cursor = nir_before_instr(&alu->instr); + nir_def *q = nir_fmul(b, nir_ssa_for_alu_src(b, alu, 0), + nir_frcp(b, nir_ssa_for_alu_src(b, alu, 1))); + nir_def_replace(&alu->def, q); + return true; + } + b->cursor = nir_before_instr(&alu->instr); b->fp_math_ctrl = nir_fp_no_fast_math; @@ -373,6 +389,9 @@ lower_log2(nir_builder *b, nir_alu_instr *alu, void *_) bool agx_nir_lower_log(nir_shader *s) { + if (!mesa_shader_stage_is_compute(s->info.stage)) + return false; + bool progress = nir_shader_alu_pass(s, lower_log, nir_metadata_control_flow, NULL); progress |= @@ -631,5 +650,8 @@ lower_sincos(nir_builder *b, nir_instr *instr, UNUSED void *_) bool agx_nir_lower_sincos(nir_shader *s) { + if (!mesa_shader_stage_is_compute(s->info.stage)) + return false; + return nir_shader_lower_instructions(s, sincos_filter, lower_sincos, NULL); } From 6902830ed887c9c5f34d60bd88785c2aae6aed04 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:19 -0500 Subject: [PATCH 18/22] tests: pin graphics fdiv lowering at both widths, early and late The AGX math pass lowers graphics-stage fdiv only at 32 bits and leaves the 16-bit division untouched for a backend that has no fdiv, so any fp16 division in a graphics shader cannot compile. Pin the graphics contract in the CPU suite: a fragment-stage fdiv at 16 and 32 bits must lower to fmul + frcp in both the early and the late pass, with no ffma and no surviving fdiv. The 16-bit halves of this pin fail deliberately until the source fix lands; the 32-bit halves pin that the coming fix keeps the existing hardware-reciprocal quotient. --- src/asahi/compiler/test/test-lower-math.cpp | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp index 9a965b9c1d93..f12900880a64 100644 --- a/src/asahi/compiler/test/test-lower-math.cpp +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -165,6 +165,30 @@ TEST_F(LowerMathTest, LateFdivDoesNotRefineExistingFrcp) EXPECT_EQ(count_alu(nir_op_frcp), 1u + (first.count(nir_op_frcp) ? first[nir_op_frcp] : 0)); } +TEST_F(LowerMathTest, GraphicsFdivLowersBothWidthsInEarlyAndLatePasses) +{ + b.shader->info.stage = MESA_SHADER_FRAGMENT; + for (unsigned bits : {16u, 32u}) { + nir_fdiv(&b, nir_undef(&b, 1, bits), nir_undef(&b, 1, bits)); + } + + EXPECT_TRUE(agx_nir_lower_fdiv(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + EXPECT_EQ(count_alu(nir_op_fmul), 2u); + EXPECT_EQ(count_alu(nir_op_frcp), 2u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); + + for (unsigned bits : {16u, 32u}) { + nir_fdiv(&b, nir_undef(&b, 1, bits), nir_undef(&b, 1, bits)); + } + + EXPECT_TRUE(agx_nir_lower_fdiv_late(b.shader)); + EXPECT_EQ(count_alu(nir_op_fdiv), 0u); + EXPECT_EQ(count_alu(nir_op_fmul), 4u); + EXPECT_EQ(count_alu(nir_op_frcp), 4u); + EXPECT_EQ(count_alu(nir_op_ffma), 0u); +} + TEST_F(LowerMathTest, SinDefaultKeepsSelect) { set_mode(0); From 1111e3b6ecb4f7139f80022300b8bc6e5056e14f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:18:25 -0500 Subject: [PATCH 19/22] agx: repair math-range rebase integration Two integration slips from rebasing the math range onto the current integration/aurora, both breaking the build: - The conflict resolution at the preamble-retry wrapper left the superseded libagx_frcp refinement and the old three-argument agx_nir_lower_fdiv in agx_compile.c alongside their replacement in agx_nir_lower_math.c, giving agx_nir_lower_fdiv conflicting types. Drop the two dead helpers; the dispatch in agx_nir_lower_math.c is the sole lowering now. - The compute guard in agx_lower_sincos still referenced the bool g17p after upstream widened the parameter to the three-valued native target code (1=G17P, 2=M5 native fragment depth, 0=older). Key the guard on native != 1: compute keeps the exact agx_nir_lower_sincos reduction on every target except G17P, and the explicit-RTE repair path, which calls the base pass with native=1, still lowers compute. --- src/asahi/compiler/agx_compile.c | 68 +------------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index e328408f0f9d..2f074fe61586 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -7663,7 +7663,7 @@ agx_lower_sincos(nir_shader *shader, unsigned native) { /* Compute on every target but G17P goes through agx_nir_lower_sincos, the * exact reduction. Graphics keeps the base sequence. */ - if (!g17p && mesa_shader_stage_is_compute(shader->info.stage)) + if (native != 1 && mesa_shader_stage_is_compute(shader->info.stage)) return false; return nir_shader_lower_instructions(shader, agx_lower_sincos_filter, @@ -10788,72 +10788,6 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl, offset_out, 0, true, NULL); } -/* - * The hardware frcp instruction is sometimes off by 1 ULP. For correctly - * rounded frcp, a refinement step is required. This routine has been - * exhaustively tested with a modified math_bruteforce. - * - * While Khronos APIs allow 2.5 ULP error for divides, nir_lower_idiv relies on - * correctly rounded frcp. This is therefore load bearing for integer division - * on all APIs. - */ -static nir_def * -libagx_frcp(nir_builder *b, nir_def *x) -{ - nir_def *u = nir_frcp(b, x); - - /* Do 1 Newton-Raphson refinement step. - * - * Define f(u) = xu - 1. Then f(u) = 0 iff u = 1/x. Newton's method gives: - * - * u_2 = u - f(u) / f'(u) = u - (xu - 1) / x - * - * Our original guess is close, so we approximate (1 / x) by u: - * - * u_2 = u - u(xu - 1) = u + u(1 - xu) - * = fma(fma(-x, u, 1), u, u) - */ - nir_def *one = nir_imm_float(b, 1.0); - nir_def *u_2 = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, one), u, u); - - /* If the original value was infinite, frcp will generate the correct zero. - * However, the Newton-Raphson step would multiply 0 * Inf and get a NaN. So - * skip the refinement step for infinite inputs. We do this backwards, - * checking whether the refined result is NaN, since we can implement this - * check in a single fcmpsel instruction. The other case where the refinement - * is NaN is a NaN input, in which skipping refinement is acceptable. - */ - return nir_bcsel(b, nir_fisnan(b, u_2), u, u_2); -} - -static bool -agx_nir_lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_) -{ - if (alu->op != nir_op_frcp || !nir_alu_instr_is_exact(alu) || - alu->def.bit_size != 32) - return false; - - /* The refinement makes frcp correctly rounded, which only integer - * division needs: nir_lower_idiv takes the reciprocal of an integer - * converted to float. An application float division keeps the 2.5 ULP - * that every Khronos API allows even when it is `precise`/NoContraction - * (which forbids contraction, not the hardware reciprocal), so do not - * spend two FMAs and a select on it; DXVK marks every operation precise. - * Driver-internal shaders keep the refinement unconditionally. */ - if (!b->shader->info.internal) { - nir_scalar src = nir_scalar_resolved(alu->src[0].src.ssa, - alu->src[0].swizzle[0]); - if (!nir_scalar_is_alu(src) || - (nir_scalar_alu_op(src) != nir_op_u2f32 && - nir_scalar_alu_op(src) != nir_op_i2f32)) - return false; - } - - b->cursor = nir_before_instr(&alu->instr); - nir_def_replace(&alu->def, libagx_frcp(b, nir_ssa_for_alu_src(b, alu, 0))); - return true; -} - static bool agx_nir_lower_g17p_f16_classify(nir_builder *b, nir_alu_instr *alu, void *_data) From 0c7e6105a2abc4473619b7e69cc1f865fa0b2df5 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:24:50 -0500 Subject: [PATCH 20/22] tests: pin the exact reciprocal that graphics integer division needs nir_lower_idiv marks everything it emits exact and opens emit_udiv with frcp(u2f32(denominator)). The refinement that correctly rounds that reciprocal used to apply in every shader stage; the math range now refines exact reciprocals in compute only, so a fragment shader with variable integer division keeps the raw hardware reciprocal. Pin the production pattern with the real lowering: run nir_lower_idiv on a fragment udiv, run the early pass, and require the refinement steps to appear, while an unmarked fast-math frcp stays untouched. The idiv half fails deliberately until the precision source fix lands. --- src/asahi/compiler/test/test-lower-math.cpp | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp index f12900880a64..93f991f39e0e 100644 --- a/src/asahi/compiler/test/test-lower-math.cpp +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -268,3 +268,49 @@ TEST_F(LowerMathTest, SinZeroPathNumericalArtifact) const bool zero_compare_normal = false; EXPECT_EQ(zero_compare_normal ? 0x80000000u : normal, normal); } + +/* + * Graphics integer division keeps the correctly rounded reciprocal that + * nir_lower_idiv relies on. The production pattern is the real lowering's: + * nir_lower_idiv marks everything it emits exact (b->fp_math_ctrl |= + * nir_fp_exact) and opens emit_udiv with frcp(u2f32(denominator)), so the + * early pass must keep refining that exact 32-bit reciprocal in graphics + * stages. An ordinary fast-math frcp stays untouched: refinement is keyed + * on the exact marking, not on every reciprocal. + */ +TEST_F(LowerMathTest, GraphicsIdivReciprocalRefinedPlainFrcpUntouched) +{ + b.shader->info.stage = MESA_SHADER_FRAGMENT; + nir_def *x = nir_undef(&b, 1, 32); + nir_def *y = nir_undef(&b, 1, 32); + nir_udiv(&b, x, y); + + /* Exactly the lowering the driver runs, with the driver's options. */ + nir_lower_idiv_options idiv_options; + idiv_options.allow_fp16 = true; + EXPECT_TRUE(nir_lower_idiv(b.shader, &idiv_options)); + + /* The exact integer-conversion reciprocal gets its refinement steps. */ + EXPECT_TRUE(agx_nir_lower_fdiv(b.shader)); + EXPECT_GT(count_alu(nir_op_ffma), 0u); + + /* An unmarked graphics reciprocal is left alone. */ + nir_builder s = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, + &options, "plain frcp"); + nir_frcp(&s, nir_undef(&s, 1, 32)); + + EXPECT_FALSE(agx_nir_lower_fdiv(s.shader)); + unsigned plain_ffma = 0; + nir_foreach_function_impl(impl, s.shader) { + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu && + nir_instr_as_alu(instr)->op == nir_op_ffma) + plain_ffma++; + } + } + } + EXPECT_EQ(plain_ffma, 0u); + + ralloc_free(s.shader); +} From 7f9dd7cddfc46a27e90709e0b5b54aceae16529a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:40:03 -0500 Subject: [PATCH 21/22] tests: rebuild the builder cursor between the fdiv pass halves The early pass lowers through nir_def_replace, which removes the lowered fdiv instruction, while the fixture builder cursor stayed parked after that instruction; the late-half fdivs were then inserted behind a removed instruction and crashed the suite. Reset the cursor to the end of the entrypoint implementation before building them. No assertion changes. --- src/asahi/compiler/test/test-lower-math.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/asahi/compiler/test/test-lower-math.cpp b/src/asahi/compiler/test/test-lower-math.cpp index 93f991f39e0e..4542dcddfb38 100644 --- a/src/asahi/compiler/test/test-lower-math.cpp +++ b/src/asahi/compiler/test/test-lower-math.cpp @@ -178,6 +178,12 @@ TEST_F(LowerMathTest, GraphicsFdivLowersBothWidthsInEarlyAndLatePasses) EXPECT_EQ(count_alu(nir_op_frcp), 2u); EXPECT_EQ(count_alu(nir_op_ffma), 0u); + /* The early pass deleted the instruction the builder cursor was parked + * after: the pass lowers through nir_def_replace, which removes the fdiv. + * Rebuild the cursor from the shader so the late-half fdivs land in the + * implementation instead of behind a removed instruction. */ + b.cursor = nir_after_impl(nir_shader_get_entrypoint(b.shader)); + for (unsigned bits : {16u, 32u}) { nir_fdiv(&b, nir_undef(&b, 1, bits), nir_undef(&b, 1, bits)); } From c9cdabbd1a7dbe36290eae553d3c6aea6d3d7f29 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:57:34 -0500 Subject: [PATCH 22/22] agx: lower graphics fdiv at both widths, keep int-div reciprocals exact The math range left graphics-stage division half-wired: fdiv at 16 bits fell through to a backend that has no fdiv, and the correctly rounded reciprocal that nir_lower_idiv builds (an exact frcp over an integer conversion) was refined in compute only, so integer division in vertex and fragment shaders kept the raw hardware reciprocal. Lower graphics fdiv at 16 bits the same way as at 32, multiplying by the hardware reciprocal. Refine the exact integer-conversion reciprocal in graphics stages with the same correctly rounded helper the compute path uses, restoring the refinement scope of the pass this range replaced: keyed on the exact marking and, for application shaders, on an integer-conversion input, while ordinary graphics reciprocals keep the hardware instruction and all compute sequences are untouched. The refinement's float controls are scoped to its own emissions and restored afterwards, and the late pass stays fdiv-only, so nothing is refined twice. Failing before this change at 7f9dd7cddfc46a27e90709e0b5b54aceae16529a, witnessed by CI run 40967791964 on both architectures: GraphicsFdivLowersBothWidthsInEarlyAndLatePasses and GraphicsIdivReciprocalRefinedPlainFrcpUntouched fail while the other 155 tests in the target pass. --- src/asahi/compiler/agx_nir_lower_math.c | 37 +++++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/asahi/compiler/agx_nir_lower_math.c b/src/asahi/compiler/agx_nir_lower_math.c index 3d41c40e509e..0dc9e44e464e 100644 --- a/src/asahi/compiler/agx_nir_lower_math.c +++ b/src/asahi/compiler/agx_nir_lower_math.c @@ -97,14 +97,39 @@ lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *data) if (alu->def.bit_size == 64) return false; - /* Graphics shaders keep the hardware reciprocal and the cheap quotient. - * The exact sequences below are for compute (MLX and kernels), where - * the rounding contract matters. A late fdiv (int64 lowering, perspective - * divide) still has to be lowered, because the backend has no fdiv. + /* Graphics shaders keep the hardware reciprocal and the cheap quotient, + * except that integer division's exact reciprocal stays correctly rounded + * here too: nir_lower_idiv relies on it. The exact sequences below are + * for compute (MLX and kernels), where the rounding contract matters. A + * late fdiv (int64 lowering, perspective divide) still has to be lowered, + * because the backend has no fdiv. */ if (!mesa_shader_stage_is_compute(b->shader->info.stage)) { - if (alu->op != nir_op_fdiv || alu->def.bit_size != 32) - return false; + if (alu->op == nir_op_frcp) { + if (alu->def.bit_size != 32 || !nir_alu_instr_is_exact(alu)) + return false; + + /* The refinement only serves integer division: an application + * reciprocal keeps the hardware instruction. Driver-internal + * shaders keep it unconditionally. */ + if (!b->shader->info.internal) { + nir_scalar src = nir_scalar_resolved(alu->src[0].src.ssa, + alu->src[0].swizzle[0]); + if (!nir_scalar_is_alu(src) || + (nir_scalar_alu_op(src) != nir_op_u2f32 && + nir_scalar_alu_op(src) != nir_op_i2f32)) + return false; + } + + /* Same refinement as compute, with the exact float controls scoped + * to these emissions instead of leaking to the rest of the shader. */ + b->cursor = nir_before_instr(&alu->instr); + unsigned saved_fp = b->fp_math_ctrl; + b->fp_math_ctrl = nir_fp_no_fast_math; + nir_def_replace(&alu->def, rcp_rn(b, nir_ssa_for_alu_src(b, alu, 0))); + b->fp_math_ctrl = saved_fp; + return true; + } b->cursor = nir_before_instr(&alu->instr); nir_def *q = nir_fmul(b, nir_ssa_for_alu_src(b, alu, 0),