From 2168ba208f79ca40aa2fc7948cc245d79fa76f12 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:00:08 +0000 Subject: [PATCH 1/3] nir/lower_blend: add an option to emit the destination term first Additive blending currently emits src * src_factor + dst * dst_factor. With dst_term_first the destination term comes first, so contraction fuses the destination product into the final multiply-add and the source product becomes a separate multiply that a backend can schedule independently of the destination load. Off by default; no driver changes behaviour. --- src/compiler/nir/nir_lower_blend.c | 7 ++++--- src/compiler/nir/nir_lower_blend.h | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/compiler/nir/nir_lower_blend.c b/src/compiler/nir/nir_lower_blend.c index a5099116af4c..7f6e5a001f21 100644 --- a/src/compiler/nir/nir_lower_blend.c +++ b/src/compiler/nir/nir_lower_blend.c @@ -51,11 +51,11 @@ static nir_def * nir_blend_func( nir_builder *b, enum pipe_blend_func func, - nir_def *src, nir_def *dst) + nir_def *src, nir_def *dst, bool dst_first) { switch (func) { case PIPE_BLEND_ADD: - return nir_fadd(b, src, dst); + return dst_first ? nir_fadd(b, dst, src) : nir_fadd(b, src, dst); case PIPE_BLEND_SUBTRACT: return nir_fsub(b, src, dst); case PIPE_BLEND_REVERSE_SUBTRACT: @@ -1006,7 +1006,8 @@ nir_color_blend(nir_builder *b, nir_def *src, nir_def *src1, nir_def *dst, chan.dst_factor, rt->format); } - channels[c] = nir_blend_func(b, chan.func, psrc, pdst); + channels[c] = nir_blend_func(b, chan.func, psrc, pdst, + rt->dst_term_first); } return nir_vec(b, channels, 4); diff --git a/src/compiler/nir/nir_lower_blend.h b/src/compiler/nir/nir_lower_blend.h index bc979a79efc9..f5b865337a5f 100644 --- a/src/compiler/nir/nir_lower_blend.h +++ b/src/compiler/nir/nir_lower_blend.h @@ -53,6 +53,12 @@ typedef struct { bool src_premultiplied; bool dst_premultiplied; enum pipe_blend_overlap_mode overlap; + + /* Emit the destination term first in additive blending, so contraction + * fuses dst * dst_factor into the final multiply-add and the source + * product stays a separate multiply that does not depend on the + * destination. */ + bool dst_term_first; } nir_lower_blend_rt; typedef struct { From 39fd16ed11671a8c583e3b2401461143882fcb1e Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:00:09 +0000 Subject: [PATCH 2/3] asahi: emit the blend destination term first on G16G On G16G the epilog's additive blend now fuses dst * dst_factor into the final multiply-add, so each channel needs one dependent operation after the tilebuffer load instead of a multiply followed by a multiply-add. The source product no longer depends on the destination and is scheduled in the load's shadow. The association of the last add changes, so results may differ by one UNORM step from the previous order. The choice is carried in the epilog key, so it applies only to G16G and separates cached shaders. ASAHI_G16_BLEND_DST_FIRST=0 restores the previous order. --- src/asahi/compiler/agx_compile.c | 6 ++++++ src/asahi/compiler/agx_debug.h | 3 +++ src/asahi/lib/agx_linker.h | 3 +++ src/asahi/lib/agx_nir_prolog_epilog.c | 1 + src/asahi/vulkan/hk_cmd_draw.c | 2 ++ 5 files changed, 15 insertions(+) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 5443f4e509f0..917a557cdf43 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -82,6 +82,12 @@ agx_get_compiler_debug(void) return debug_get_option_agx_compiler_debug(); } +bool +agx_g16_blend_dst_first_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true); +} + static __thread char agx_refusal[192]; const char * diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index a1cc739e60b4..440b9a440f03 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -34,6 +34,9 @@ enum agx_compiler_dbg { uint64_t agx_get_compiler_debug(void); +/* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ +bool agx_g16_blend_dst_first_enabled(void); + /* Why the last agx_compile_shader_nir() on this thread returned false * ("" when it succeeded), for the driver's "cannot encode" error. */ const char *agx_last_compile_refusal(void); diff --git a/src/asahi/lib/agx_linker.h b/src/asahi/lib/agx_linker.h index e2993ee182a0..8dfaa4cbbae5 100644 --- a/src/asahi/lib/agx_linker.h +++ b/src/asahi/lib/agx_linker.h @@ -370,6 +370,9 @@ struct agx_fs_epilog_key { * the constants after the main shader's uniform footprint instead. */ uint16_t blend_constant_base; + /* Emit the blend destination term first (nir_lower_blend_rt::dst_term_first). */ + bool blend_dst_first; + /* Render targets whose tile value must be stored back unchanged (colour * write disabled while raster coverage still makes the tile nonempty). * Independent of blending, so it also holds for pure-integer formats. */ diff --git a/src/asahi/lib/agx_nir_prolog_epilog.c b/src/asahi/lib/agx_nir_prolog_epilog.c index d1bfa069d340..5e26c2677325 100644 --- a/src/asahi/lib/agx_nir_prolog_epilog.c +++ b/src/asahi/lib/agx_nir_prolog_epilog.c @@ -807,6 +807,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_) .alpha.src_factor = blend.alpha_src_factor, .alpha.dst_factor = blend.alpha_dst_factor, + .dst_term_first = key->blend_dst_first, .alpha.func = blend.alpha_func, .colormask = key->blend.rt[i].colormask, diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 81c7c77c5d5e..0d084597ec2d 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -9982,6 +9982,8 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs, key->epilog.link.already_ran_zs |= nontrivial_force_early; key->epilog.native_imageblock = in->tib->native_imageblock; key->epilog.max_sample_size_B = in->tib->max_sample_size_B; + key->epilog.blend_dst_first = + dev->dev.chip == AGX_CHIP_G16G && agx_g16_blend_dst_first_enabled(); key->epilog.resident_mask = in->tib->resident_mask; key->epilog.max_bytes_per_tile = in->tib->max_bytes_per_tile; key->epilog.max_samples_per_tile = in->tib->max_samples_per_tile; From 02437426637f3fecec1bfa71eaaccd0c5e7154c1 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:55:21 +0000 Subject: [PATCH 3/3] asahi: issue uniform-only G16 fragment arithmetic before the pixel wait After scheduling, move fadd, fmul, ffma and mov_imm computed only from uniforms and immediates ahead of the first wait_pix of a G16G fragment block. With the blend destination term first, the source terms of a uniform-colour blend (clamps, 1 - alpha and the source products) are then computed before the ordered colour section, which holds only the tile load, unpack, one multiply-add per channel, pack and store. Texture, varying and tile-dependent work keeps its scheduled place. Operations, modifiers and operands are unchanged, so results are bit-identical. The choice is part of the Vulkan compiler flags and the Gallium disk-cache identity. ASAHI_G16_PIX_HOIST=0 restores the previous placement. --- src/asahi/compiler/agx_compile.c | 72 ++++++++++++++++++++++ src/asahi/compiler/agx_debug.h | 2 + src/asahi/vulkan/hk_shader.c | 3 +- src/gallium/drivers/asahi/agx_disk_cache.c | 2 + 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 917a557cdf43..60e1d52d08ad 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -82,6 +82,12 @@ agx_get_compiler_debug(void) return debug_get_option_agx_compiler_debug(); } +bool +agx_g16_pix_hoist_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_PIX_HOIST", true); +} + bool agx_g16_blend_dst_first_enabled(void) { @@ -8878,6 +8884,71 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx) * adjacent run of non-coherent loads with identical operation and base; SSA * keeps their consumers attached to the correct result and no serialization * is introduced. */ +/* G16G (ASAHI_G16_PIX_HOIST=0 opts out): after scheduling, move + * float arithmetic computed only from uniforms ahead of the first + * pixel-ordering wait of its block, so it executes outside the ordered colour + * transaction instead of between WAIT_PIX and the tile store. Only fadd, fmul, + * ffma and mov_imm move, and only when every source is a uniform, an + * immediate, or the result of another such instruction. Moved instructions + * keep their relative order and their exact operations, modifiers and + * operands, so results are bit-identical. */ +static bool +agx3_g16_hoist_pixel_independent(agx_context *ctx) +{ + if (ctx->key->dev.chip != AGX_CHIP_G16G || + ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble || + !agx_g16_pix_hoist_enabled()) + return false; + + /* Values computed only from uniforms and immediates by the movable ops, + * wherever they are in the block. Texture, varying and tile results never + * qualify, so per-pixel work keeps its scheduled place. */ + BITSET_WORD *uniform = BITSET_CALLOC(ctx->alloc); + if (!uniform) + return false; + + bool progress = false; + agx_foreach_block(ctx, block) { + agx_instr *wait = NULL; + agx_foreach_instr_in_block_safe(block, I) { + if (!wait && I->op == AGX_OPCODE_WAIT_PIX) { + wait = I; + continue; + } + + bool uniform_only = (I->op == AGX_OPCODE_FADD || + I->op == AGX_OPCODE_FMUL || + I->op == AGX_OPCODE_FFMA || + I->op == AGX_OPCODE_MOV_IMM) && + I->nr_dests == 1 && + I->dest[0].type == AGX_INDEX_NORMAL; + agx_foreach_src(I, s) { + const agx_index src = I->src[s]; + if (src.type == AGX_INDEX_NORMAL) { + if (!BITSET_TEST(uniform, src.value)) + uniform_only = false; + } else if (src.type != AGX_INDEX_UNIFORM && + src.type != AGX_INDEX_IMMEDIATE && + src.type != AGX_INDEX_NULL) { + uniform_only = false; + } + } + + if (!uniform_only) + continue; + BITSET_SET(uniform, I->dest[0].value); + if (wait) { + list_del(&I->link); + list_addtail(&I->link, &wait->link); + progress = true; + } + } + } + + free(uniform); + return progress; +} + static bool agx3_order_fragment_device_loads(agx_context *ctx) { @@ -10389,6 +10460,7 @@ agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl, } AGX_PASS(ctx, agx3_order_fragment_device_loads); + AGX_PASS(ctx, agx3_g16_hoist_pixel_independent); if (agx_should_dump(nir, AGX_DBG_SHADERS)) agx_print_shader(ctx, stdout); diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index 440b9a440f03..0184f4fec68a 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -34,6 +34,8 @@ enum agx_compiler_dbg { uint64_t agx_get_compiler_debug(void); +/* Qualified on G16G; ASAHI_G16_PIX_HOIST=0 restores the scheduler placement. */ +bool agx_g16_pix_hoist_enabled(void); /* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ bool agx_g16_blend_dst_first_enabled(void); diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c index 77d3dbdf13bc..b26c3beb50a1 100644 --- a/src/asahi/vulkan/hk_shader.c +++ b/src/asahi/vulkan/hk_shader.c @@ -187,7 +187,8 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev) { /* This could be optimized but it doesn't matter. */ return pdev->dev.debug | - hk_fs_structure_key_bits(); + hk_fs_structure_key_bits() | + ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33); } const nir_shader_compiler_options * diff --git a/src/gallium/drivers/asahi/agx_disk_cache.c b/src/gallium/drivers/asahi/agx_disk_cache.c index 660795aec160..0141fdc73e7c 100644 --- a/src/gallium/drivers/asahi/agx_disk_cache.c +++ b/src/gallium/drivers/asahi/agx_disk_cache.c @@ -237,6 +237,8 @@ agx_disk_cache_init(struct agx_screen *screen) _mesa_sha1_format(timestamp, id_sha1); uint64_t driver_flags = screen->dev.debug; + if (screen->dev.chip == AGX_CHIP_G16G && agx_g16_pix_hoist_enabled()) + driver_flags |= UINT64_C(1) << 61; screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags); #endif }