Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1474f7e
agx: correctly rounded fdiv/frcp, faithful log/log2, exact sin/cos re…
joshuaswarren Sep 8, 2026
ad19f03
agx: return a flushed signed zero from sin under DenormFlushToZero
joshuaswarren Sep 20, 2026
05925fe
agx: flush the exact quotient in precise fp32 division, not the corre…
joshuaswarren Oct 5, 2026
4be39fc
nir: pass FP_MATH_CTRL to the builder generator
joshuaswarren Sep 13, 2026
2adc77c
agx: lower log() the same way regardless of where ln2 comes from
joshuaswarren Oct 5, 2026
3e35358
honeykrisp: shorten normalized precise division
joshuaswarren Oct 6, 2026
1dc04f8
honeykrisp: use direct reciprocal for normalized division
joshuaswarren Oct 6, 2026
98ba76f
tests: pin the 2.5 ulp default-division bound for AGX
joshuaswarren Oct 6, 2026
d68a324
honeykrisp: exact reciprocal without the division machinery
joshuaswarren Oct 6, 2026
b04611d
agx: round log() once unless the multiply is exact
joshuaswarren Oct 6, 2026
953eb9f
honeykrisp: let the hardware rcp give the reciprocal edge cases
joshuaswarren Oct 7, 2026
4a4e6c5
agx: pin float-control keying of division, not its op counts
joshuaswarren Oct 7, 2026
4340f2f
agx: lower the fdiv that int64 lowering and the perspective divide em…
joshuaswarren Sep 8, 2026
7b6dbd6
tests: pin the late fdiv lowering of the AGX math pass
joshuaswarren Oct 25, 2026
ab23e96
tests: measure the 2.5 ULP bound in float32 spacing, with the shader …
joshuaswarren Oct 25, 2026
1598e3d
tests: repair fp32 error oracle and complete numerical fixtures
chriskearney Oct 25, 2026
1878c64
agx: keep the exact division, log and sin/cos lowering for compute
joshuaswarren Oct 25, 2026
6902830
tests: pin graphics fdiv lowering at both widths, early and late
joshuaswarren Oct 29, 2026
1111e3b
agx: repair math-range rebase integration
joshuaswarren Oct 29, 2026
0c7e610
tests: pin the exact reciprocal that graphics integer division needs
joshuaswarren Oct 29, 2026
7f9dd7c
tests: rebuild the builder cursor between the fdiv pass halves
joshuaswarren Oct 29, 2026
c9cdabb
agx: lower graphics fdiv at both widths, keep int-div reciprocals exact
joshuaswarren Oct 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 17 additions & 70 deletions src/asahi/compiler/agx_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -7543,7 +7543,7 @@ agx_calc_stats(agx_context *ctx, unsigned size, struct agx2_stats *stats)
}

static bool
agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *_)
agx_lower_sincos_filter(const nir_instr *instr, UNUSED const void *g17p)
{
if (instr->type != nir_instr_type_alu)
return false;
Expand Down Expand Up @@ -7661,6 +7661,11 @@ agx_lower_sincos_impl(struct nir_builder *b, nir_instr *instr, void *data)
static bool
agx_lower_sincos(nir_shader *shader, unsigned native)
{
/* Compute on every target but G17P goes through agx_nir_lower_sincos, the
* exact reduction. Graphics keeps the base sequence. */
if (native != 1 && mesa_shader_stage_is_compute(shader->info.stage))
return false;

return nir_shader_lower_instructions(shader, agx_lower_sincos_filter,
agx_lower_sincos_impl,
(void *)(uintptr_t)native);
Expand Down Expand Up @@ -8257,6 +8262,11 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
} while (progress);
}

/* int64 lowering and the perspective divide emit fdiv after preprocessing;
* the backend has no fdiv, so lower what is left before it is scheduled.
*/
NIR_PASS(_, nir, agx_nir_lower_fdiv_late);

/* Reassociate before forming preambles because it makes preambles more
* effective. Clean up after. On M5, also order the loop-invariant terms of
* a chain in a loop first, so accumulations of per-iteration constants
Expand Down Expand Up @@ -10778,72 +10788,6 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
offset_out, 0, true, NULL);
}

/*
* The hardware frcp instruction is sometimes off by 1 ULP. For correctly
* rounded frcp, a refinement step is required. This routine has been
* exhaustively tested with a modified math_bruteforce.
*
* While Khronos APIs allow 2.5 ULP error for divides, nir_lower_idiv relies on
* correctly rounded frcp. This is therefore load bearing for integer division
* on all APIs.
*/
static nir_def *
libagx_frcp(nir_builder *b, nir_def *x)
{
nir_def *u = nir_frcp(b, x);

/* Do 1 Newton-Raphson refinement step.
*
* Define f(u) = xu - 1. Then f(u) = 0 iff u = 1/x. Newton's method gives:
*
* u_2 = u - f(u) / f'(u) = u - (xu - 1) / x
*
* Our original guess is close, so we approximate (1 / x) by u:
*
* u_2 = u - u(xu - 1) = u + u(1 - xu)
* = fma(fma(-x, u, 1), u, u)
*/
nir_def *one = nir_imm_float(b, 1.0);
nir_def *u_2 = nir_ffma(b, nir_ffma(b, nir_fneg(b, x), u, one), u, u);

/* If the original value was infinite, frcp will generate the correct zero.
* However, the Newton-Raphson step would multiply 0 * Inf and get a NaN. So
* skip the refinement step for infinite inputs. We do this backwards,
* checking whether the refined result is NaN, since we can implement this
* check in a single fcmpsel instruction. The other case where the refinement
* is NaN is a NaN input, in which skipping refinement is acceptable.
*/
return nir_bcsel(b, nir_fisnan(b, u_2), u, u_2);
}

static bool
agx_nir_lower_fdiv(nir_builder *b, nir_alu_instr *alu, void *_)
{
if (alu->op != nir_op_frcp || !nir_alu_instr_is_exact(alu) ||
alu->def.bit_size != 32)
return false;

/* The refinement makes frcp correctly rounded, which only integer
* division needs: nir_lower_idiv takes the reciprocal of an integer
* converted to float. An application float division keeps the 2.5 ULP
* that every Khronos API allows even when it is `precise`/NoContraction
* (which forbids contraction, not the hardware reciprocal), so do not
* spend two FMAs and a select on it; DXVK marks every operation precise.
* Driver-internal shaders keep the refinement unconditionally. */
if (!b->shader->info.internal) {
nir_scalar src = nir_scalar_resolved(alu->src[0].src.ssa,
alu->src[0].swizzle[0]);
if (!nir_scalar_is_alu(src) ||
(nir_scalar_alu_op(src) != nir_op_u2f32 &&
nir_scalar_alu_op(src) != nir_op_i2f32))
return false;
}

b->cursor = nir_before_instr(&alu->instr);
nir_def_replace(&alu->def, libagx_frcp(b, nir_ssa_for_alu_src(b, alu, 0)));
return true;
}

static bool
agx_nir_lower_g17p_f16_classify(nir_builder *b, nir_alu_instr *alu,
void *_data)
Expand Down Expand Up @@ -10993,6 +10937,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep
* FMAs only. Other shaders keep the lowering here. */
if (!g17p || !agx_nir_fma_rte_applies(nir))
NIR_PASS(_, nir, agx_lower_sincos, g17p ? 1 : native_fragment_depth ? 2 : 0);
NIR_PASS(_, nir, agx_nir_lower_sincos);
NIR_PASS(_, nir, nir_shader_intrinsics_pass, agx_lower_front_face,
nir_metadata_control_flow, NULL);
if (g17p)
Expand All @@ -11018,9 +10963,11 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_dep

NIR_PASS(_, nir, nir_lower_idiv, &idiv_options);

/* Has to run after nir_lower_idiv */
NIR_PASS(_, nir, nir_shader_alu_pass, agx_nir_lower_fdiv,
nir_metadata_control_flow, NULL);
/* Has to run after nir_lower_idiv, and before the log lowering so the
* reciprocals it builds are not refined twice.
*/
NIR_PASS(_, nir, agx_nir_lower_fdiv);
NIR_PASS(_, nir, agx_nir_lower_log);

NIR_PASS(_, nir, nir_opt_deref);
NIR_PASS(_, nir, nir_lower_vars_to_ssa);
Expand Down
1 change: 0 additions & 1 deletion src/asahi/compiler/agx_compile.h
Original file line number Diff line number Diff line change
Expand Up @@ -665,7 +665,6 @@ agx_round_registers(unsigned halfregs)
}

static const nir_shader_compiler_options agx_nir_options = {
.lower_fdiv = true,
.float_mul_add16 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse,
.float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse,
.lower_flrp16 = true,
Expand Down
4 changes: 4 additions & 0 deletions src/asahi/compiler/agx_nir.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ bool agx_nir_lower_word_bounds(struct nir_shader *s);
bool agx_nir_share_bounds(struct nir_shader *s);
bool agx_nir_lower_atomics(struct nir_shader *s, bool native_fadd);
bool agx_nir_opt_canonical_minmax(struct nir_shader *s);
bool agx_nir_lower_fdiv(struct nir_shader *s);
bool agx_nir_lower_fdiv_late(struct nir_shader *s);
bool agx_nir_lower_log(struct nir_shader *s);
bool agx_nir_lower_sincos(struct nir_shader *s);

bool agx_nir_lower_texture_early(struct nir_shader *s, bool support_lod_bias);
bool agx_nir_lower_g17p_helper_samples(struct nir_shader *s);
Expand Down
Loading
Loading