From 2168ba208f79ca40aa2fc7948cc245d79fa76f12 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:00:08 +0000 Subject: [PATCH 1/5] nir/lower_blend: add an option to emit the destination term first Additive blending currently emits src * src_factor + dst * dst_factor. With dst_term_first the destination term comes first, so contraction fuses the destination product into the final multiply-add and the source product becomes a separate multiply that a backend can schedule independently of the destination load. Off by default; no driver changes behaviour. --- src/compiler/nir/nir_lower_blend.c | 7 ++++--- src/compiler/nir/nir_lower_blend.h | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/compiler/nir/nir_lower_blend.c b/src/compiler/nir/nir_lower_blend.c index a5099116af4c..7f6e5a001f21 100644 --- a/src/compiler/nir/nir_lower_blend.c +++ b/src/compiler/nir/nir_lower_blend.c @@ -51,11 +51,11 @@ static nir_def * nir_blend_func( nir_builder *b, enum pipe_blend_func func, - nir_def *src, nir_def *dst) + nir_def *src, nir_def *dst, bool dst_first) { switch (func) { case PIPE_BLEND_ADD: - return nir_fadd(b, src, dst); + return dst_first ? nir_fadd(b, dst, src) : nir_fadd(b, src, dst); case PIPE_BLEND_SUBTRACT: return nir_fsub(b, src, dst); case PIPE_BLEND_REVERSE_SUBTRACT: @@ -1006,7 +1006,8 @@ nir_color_blend(nir_builder *b, nir_def *src, nir_def *src1, nir_def *dst, chan.dst_factor, rt->format); } - channels[c] = nir_blend_func(b, chan.func, psrc, pdst); + channels[c] = nir_blend_func(b, chan.func, psrc, pdst, + rt->dst_term_first); } return nir_vec(b, channels, 4); diff --git a/src/compiler/nir/nir_lower_blend.h b/src/compiler/nir/nir_lower_blend.h index bc979a79efc9..f5b865337a5f 100644 --- a/src/compiler/nir/nir_lower_blend.h +++ b/src/compiler/nir/nir_lower_blend.h @@ -53,6 +53,12 @@ typedef struct { bool src_premultiplied; bool dst_premultiplied; enum pipe_blend_overlap_mode overlap; + + /* Emit the destination term first in additive blending, so contraction + * fuses dst * dst_factor into the final multiply-add and the source + * product stays a separate multiply that does not depend on the + * destination. */ + bool dst_term_first; } nir_lower_blend_rt; typedef struct { From 39fd16ed11671a8c583e3b2401461143882fcb1e Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:00:09 +0000 Subject: [PATCH 2/5] asahi: emit the blend destination term first on G16G On G16G the epilog's additive blend now fuses dst * dst_factor into the final multiply-add, so each channel needs one dependent operation after the tilebuffer load instead of a multiply followed by a multiply-add. The source product no longer depends on the destination and is scheduled in the load's shadow. The association of the last add changes, so results may differ by one UNORM step from the previous order. The choice is carried in the epilog key, so it applies only to G16G and separates cached shaders. ASAHI_G16_BLEND_DST_FIRST=0 restores the previous order. --- src/asahi/compiler/agx_compile.c | 6 ++++++ src/asahi/compiler/agx_debug.h | 3 +++ src/asahi/lib/agx_linker.h | 3 +++ src/asahi/lib/agx_nir_prolog_epilog.c | 1 + src/asahi/vulkan/hk_cmd_draw.c | 2 ++ 5 files changed, 15 insertions(+) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 5443f4e509f0..917a557cdf43 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -82,6 +82,12 @@ agx_get_compiler_debug(void) return debug_get_option_agx_compiler_debug(); } +bool +agx_g16_blend_dst_first_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true); +} + static __thread char agx_refusal[192]; const char * diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index a1cc739e60b4..440b9a440f03 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -34,6 +34,9 @@ enum agx_compiler_dbg { uint64_t agx_get_compiler_debug(void); +/* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ +bool agx_g16_blend_dst_first_enabled(void); + /* Why the last agx_compile_shader_nir() on this thread returned false * ("" when it succeeded), for the driver's "cannot encode" error. */ const char *agx_last_compile_refusal(void); diff --git a/src/asahi/lib/agx_linker.h b/src/asahi/lib/agx_linker.h index e2993ee182a0..8dfaa4cbbae5 100644 --- a/src/asahi/lib/agx_linker.h +++ b/src/asahi/lib/agx_linker.h @@ -370,6 +370,9 @@ struct agx_fs_epilog_key { * the constants after the main shader's uniform footprint instead. */ uint16_t blend_constant_base; + /* Emit the blend destination term first (nir_lower_blend_rt::dst_term_first). */ + bool blend_dst_first; + /* Render targets whose tile value must be stored back unchanged (colour * write disabled while raster coverage still makes the tile nonempty). * Independent of blending, so it also holds for pure-integer formats. */ diff --git a/src/asahi/lib/agx_nir_prolog_epilog.c b/src/asahi/lib/agx_nir_prolog_epilog.c index d1bfa069d340..5e26c2677325 100644 --- a/src/asahi/lib/agx_nir_prolog_epilog.c +++ b/src/asahi/lib/agx_nir_prolog_epilog.c @@ -807,6 +807,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_) .alpha.src_factor = blend.alpha_src_factor, .alpha.dst_factor = blend.alpha_dst_factor, + .dst_term_first = key->blend_dst_first, .alpha.func = blend.alpha_func, .colormask = key->blend.rt[i].colormask, diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 81c7c77c5d5e..0d084597ec2d 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -9982,6 +9982,8 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs, key->epilog.link.already_ran_zs |= nontrivial_force_early; key->epilog.native_imageblock = in->tib->native_imageblock; key->epilog.max_sample_size_B = in->tib->max_sample_size_B; + key->epilog.blend_dst_first = + dev->dev.chip == AGX_CHIP_G16G && agx_g16_blend_dst_first_enabled(); key->epilog.resident_mask = in->tib->resident_mask; key->epilog.max_bytes_per_tile = in->tib->max_bytes_per_tile; key->epilog.max_samples_per_tile = in->tib->max_samples_per_tile; From 02437426637f3fecec1bfa71eaaccd0c5e7154c1 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 18:55:21 +0000 Subject: [PATCH 3/5] asahi: issue uniform-only G16 fragment arithmetic before the pixel wait After scheduling, move fadd, fmul, ffma and mov_imm computed only from uniforms and immediates ahead of the first wait_pix of a G16G fragment block. With the blend destination term first, the source terms of a uniform-colour blend (clamps, 1 - alpha and the source products) are then computed before the ordered colour section, which holds only the tile load, unpack, one multiply-add per channel, pack and store. Texture, varying and tile-dependent work keeps its scheduled place. Operations, modifiers and operands are unchanged, so results are bit-identical. The choice is part of the Vulkan compiler flags and the Gallium disk-cache identity. ASAHI_G16_PIX_HOIST=0 restores the previous placement. --- src/asahi/compiler/agx_compile.c | 72 ++++++++++++++++++++++ src/asahi/compiler/agx_debug.h | 2 + src/asahi/vulkan/hk_shader.c | 3 +- src/gallium/drivers/asahi/agx_disk_cache.c | 2 + 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 917a557cdf43..60e1d52d08ad 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -82,6 +82,12 @@ agx_get_compiler_debug(void) return debug_get_option_agx_compiler_debug(); } +bool +agx_g16_pix_hoist_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_PIX_HOIST", true); +} + bool agx_g16_blend_dst_first_enabled(void) { @@ -8878,6 +8884,71 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx) * adjacent run of non-coherent loads with identical operation and base; SSA * keeps their consumers attached to the correct result and no serialization * is introduced. */ +/* G16G (ASAHI_G16_PIX_HOIST=0 opts out): after scheduling, move + * float arithmetic computed only from uniforms ahead of the first + * pixel-ordering wait of its block, so it executes outside the ordered colour + * transaction instead of between WAIT_PIX and the tile store. Only fadd, fmul, + * ffma and mov_imm move, and only when every source is a uniform, an + * immediate, or the result of another such instruction. Moved instructions + * keep their relative order and their exact operations, modifiers and + * operands, so results are bit-identical. */ +static bool +agx3_g16_hoist_pixel_independent(agx_context *ctx) +{ + if (ctx->key->dev.chip != AGX_CHIP_G16G || + ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble || + !agx_g16_pix_hoist_enabled()) + return false; + + /* Values computed only from uniforms and immediates by the movable ops, + * wherever they are in the block. Texture, varying and tile results never + * qualify, so per-pixel work keeps its scheduled place. */ + BITSET_WORD *uniform = BITSET_CALLOC(ctx->alloc); + if (!uniform) + return false; + + bool progress = false; + agx_foreach_block(ctx, block) { + agx_instr *wait = NULL; + agx_foreach_instr_in_block_safe(block, I) { + if (!wait && I->op == AGX_OPCODE_WAIT_PIX) { + wait = I; + continue; + } + + bool uniform_only = (I->op == AGX_OPCODE_FADD || + I->op == AGX_OPCODE_FMUL || + I->op == AGX_OPCODE_FFMA || + I->op == AGX_OPCODE_MOV_IMM) && + I->nr_dests == 1 && + I->dest[0].type == AGX_INDEX_NORMAL; + agx_foreach_src(I, s) { + const agx_index src = I->src[s]; + if (src.type == AGX_INDEX_NORMAL) { + if (!BITSET_TEST(uniform, src.value)) + uniform_only = false; + } else if (src.type != AGX_INDEX_UNIFORM && + src.type != AGX_INDEX_IMMEDIATE && + src.type != AGX_INDEX_NULL) { + uniform_only = false; + } + } + + if (!uniform_only) + continue; + BITSET_SET(uniform, I->dest[0].value); + if (wait) { + list_del(&I->link); + list_addtail(&I->link, &wait->link); + progress = true; + } + } + } + + free(uniform); + return progress; +} + static bool agx3_order_fragment_device_loads(agx_context *ctx) { @@ -10389,6 +10460,7 @@ agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl, } AGX_PASS(ctx, agx3_order_fragment_device_loads); + AGX_PASS(ctx, agx3_g16_hoist_pixel_independent); if (agx_should_dump(nir, AGX_DBG_SHADERS)) agx_print_shader(ctx, stdout); diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index 440b9a440f03..0184f4fec68a 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -34,6 +34,8 @@ enum agx_compiler_dbg { uint64_t agx_get_compiler_debug(void); +/* Qualified on G16G; ASAHI_G16_PIX_HOIST=0 restores the scheduler placement. */ +bool agx_g16_pix_hoist_enabled(void); /* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ bool agx_g16_blend_dst_first_enabled(void); diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c index 77d3dbdf13bc..b26c3beb50a1 100644 --- a/src/asahi/vulkan/hk_shader.c +++ b/src/asahi/vulkan/hk_shader.c @@ -187,7 +187,8 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev) { /* This could be optimized but it doesn't matter. */ return pdev->dev.debug | - hk_fs_structure_key_bits(); + hk_fs_structure_key_bits() | + ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33); } const nir_shader_compiler_options * diff --git a/src/gallium/drivers/asahi/agx_disk_cache.c b/src/gallium/drivers/asahi/agx_disk_cache.c index 660795aec160..0141fdc73e7c 100644 --- a/src/gallium/drivers/asahi/agx_disk_cache.c +++ b/src/gallium/drivers/asahi/agx_disk_cache.c @@ -237,6 +237,8 @@ agx_disk_cache_init(struct agx_screen *screen) _mesa_sha1_format(timestamp, id_sha1); uint64_t driver_flags = screen->dev.debug; + if (screen->dev.chip == AGX_CHIP_G16G && agx_g16_pix_hoist_enabled()) + driver_flags |= UINT64_C(1) << 61; screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags); #endif } From cc1294e701c38b8fa393958904fab81048ccbe80 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 19:50:38 +0000 Subject: [PATCH 4/5] asahi: evaluate uniform-only G16G fragment arithmetic per draw G16G forms no native constant program, so a fragment program whose colour comes from push constants recomputes the same clamps, 1 - alpha and source products for every pixel. For an inline-blend program, take the exact fp32 operations (fsat, fmul, fadd, fneg, fabs) whose operands are push constants or immediates, evaluate them on the host when the draw's root is built, and load the results from uniform halves 0..11. Those halves hold only the spill table and the blend constants, which such a program does not use; the program's uniform layout and launch words are unchanged. Fused multiply-add stays on the GPU because G16G FFMA does not round like the host. Only UNORM targets qualify, where host and GPU denormal or NaN handling cannot change a stored byte. For a uniform-colour alpha blend the fragment program becomes the tile load, unpack, one multiply-add per channel with uniform operands, pack and store. Programs built this way are not written to the disk cache. ASAHI_G16_HOST_BLEND_SRC=0 restores per-pixel evaluation; the setting is part of the physical device's compiler flags. --- src/asahi/compiler/agx_compile.c | 6 + src/asahi/compiler/agx_debug.h | 3 + src/asahi/vulkan/hk_cmd_buffer.c | 15 +- src/asahi/vulkan/hk_cmd_buffer.h | 4 + src/asahi/vulkan/hk_cmd_draw.c | 286 ++++++++++++++++++++++++++++++- src/asahi/vulkan/hk_shader.c | 3 +- src/asahi/vulkan/hk_shader.h | 44 +++++ 7 files changed, 356 insertions(+), 5 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 60e1d52d08ad..74129aa2590d 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -94,6 +94,12 @@ agx_g16_blend_dst_first_enabled(void) return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true); } +bool +agx_g16_host_blend_src_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_HOST_BLEND_SRC", true); +} + static __thread char agx_refusal[192]; const char * diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index 0184f4fec68a..cb8d6ccff76f 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -38,6 +38,9 @@ uint64_t agx_get_compiler_debug(void); bool agx_g16_pix_hoist_enabled(void); /* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ bool agx_g16_blend_dst_first_enabled(void); +/* Qualified on G16G; ASAHI_G16_HOST_BLEND_SRC=0 restores per-pixel evaluation + * of uniform-only inline-blend fragment arithmetic. */ +bool agx_g16_host_blend_src_enabled(void); /* Why the last agx_compile_shader_nir() on this thread returned false * ("" when it succeeded), for the driver's "cannot encode" error. */ diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 49772d7f0d66..6e4a3abf79d3 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -2538,7 +2538,20 @@ hk_g17p_graphics_uniform_ranges( .size_halfs = 4, .address = pointer, }; } - if (hk_cmd_uses_blend_constant(cmd)) { + if (linked && linked->g16_host.n_out) { + /* A G16G inline-blend program with host-evaluated uniforms reads + * them in halves [0, 2 * n_out), which only the spill table and the + * blend constants otherwise use: it is built for neither (an inline + * program refuses epilogs reading blend constants). */ + if (native_spills || !root_ptr) + return hk_g17p_graphics_ranges_refused(range_count, count); + sources[count] = HK_G17P_RANGE_SOURCE_ROOT; + ranges[count++] = (struct agx3_ioto_uniform_range){ + .start_halfs = 0, + .size_halfs = 2 * linked->g16_host.n_out, + .address = root_ptr + hk_root_descriptor_offset(g16_host_unif), + }; + } else if (hk_cmd_uses_blend_constant(cmd)) { /* The non-monolithic blend epilog reads the four constants from * the base selected by its linked key. It is half4 for the standard * ABI and moves beyond the main shader footprint when HAL200 native diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 38941cb1eb5c..b83c96ad28cc 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -258,6 +258,10 @@ struct hk_root_descriptor_table { * address of the word libagx_m5_task_dispatch writes (nonzero = execute); * zero for every other draw. Appended after every agx3b field. */ uint64_t mesh_task_flag; + + /* G16G host-evaluated fragment uniforms (hk_linked_shader::g16_host), + * published to uniform halves [0, 2 * n_out). */ + uint32_t g16_host_unif[HK_G16_HOST_MAX_OUT]; }; /* The graphics arm is the larger of the two, so everything after the union -- diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 0d084597ec2d..d0fe5435cc05 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -7003,10 +7003,267 @@ hk_g17p_mono_disk_key(struct hk_device *dev, const struct hk_shader *shader, return true; } +static bool +hk_g16_host_eval_allowed(const struct hk_device *dev, + const struct agx_fs_epilog_key *ekey) +{ + if (dev->dev.chip != AGX_CHIP_G16G || + !agx_g16_host_blend_src_enabled() || ekey->native_spills) + return false; + /* Host and GPU differ at most in denormal and NaN handling, which no + * UNORM store can observe. */ + for (unsigned rt = 0; rt < ARRAY_SIZE(ekey->rt_formats); ++rt) { + const enum pipe_format f = ekey->rt_formats[rt]; + if (f != PIPE_FORMAT_NONE && + (!util_format_is_unorm(f) || util_format_is_srgb(f))) + return false; + } + return true; +} + +/* Root byte offset of the 32-bit push constant the main's root uniform DMA + * places at uniform half `half`. */ +static bool +hk_g16_host_root_offset(const struct hk_shader *shader, unsigned half, + uint32_t *offset_B) +{ + for (unsigned i = 0; i < shader->b.info.agx3_uniform_dma_count; ++i) { + const struct agx3_uniform_dma_span *s = &shader->b.info.agx3_uniform_dma[i]; + if (s->source != AGX3_UNIFORM_DMA_POINTER || + s->pointer != shader->info.agx3_root_uniform || half < s->dest || + half + 2 > s->dest + s->halfs) + continue; + const uint32_t o = s->offset_B + (half - s->dest) * 2; + if ((o & 3) || o < hk_root_descriptor_offset(push) || + o + 4 > hk_root_descriptor_offset(push) + HK_MAX_PUSH_SIZE) + return false; + *offset_B = o; + return true; + } + return false; +} + +static bool +hk_g16_host_classify(const struct hk_shader *shader, struct hash_table *temps, + nir_scalar s, struct hk_g16_host_src *out) +{ + if (s.def->bit_size != 32) + return false; + struct hash_entry *he = _mesa_hash_table_search(temps, s.def); + if (he) { + if (s.comp != 0) + return false; + *out = (struct hk_g16_host_src){HK_G16_HOST_TEMP, + (uint32_t)(uintptr_t)he->data - 1}; + return true; + } + if (nir_scalar_is_const(s)) { + *out = (struct hk_g16_host_src){HK_G16_HOST_IMM, + (uint32_t)nir_scalar_as_uint(s)}; + return true; + } + nir_instr *parent = nir_def_instr(s.def); + if (parent->type != nir_instr_type_intrinsic) + return false; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(parent); + uint32_t offset_B; + if (intr->intrinsic == nir_intrinsic_load_preamble) { + if (!hk_g16_host_root_offset(shader, + nir_intrinsic_base(intr) + 2 * s.comp, + &offset_B)) + return false; + } else if (intr->intrinsic == nir_intrinsic_load_global_constant) { + /* Push constants before the backend moves them to the root uniform + * DMA: a load through the root pointer at a constant offset. */ + nir_scalar a = nir_scalar_chase_movs(nir_get_scalar(intr->src[0].ssa, 0)); + uint64_t off = 0; + if (nir_scalar_is_alu(a) && nir_scalar_alu_op(a) == nir_op_iadd) { + nir_scalar x = nir_scalar_chase_alu_src(a, 0); + nir_scalar y = nir_scalar_chase_alu_src(a, 1); + if (nir_scalar_is_const(x)) { + nir_scalar t = x; + x = y; + y = t; + } + if (!nir_scalar_is_const(y)) + return false; + off = nir_scalar_as_uint(y); + a = x; + } + nir_instr *base = nir_def_instr(a.def); + if (a.def->bit_size != 64 || base->type != nir_instr_type_intrinsic || + nir_instr_as_intrinsic(base)->intrinsic != + nir_intrinsic_load_preamble || + nir_intrinsic_base(nir_instr_as_intrinsic(base)) != + shader->info.agx3_root_uniform) + return false; + off += 4 * s.comp; + if ((off & 3) || off < hk_root_descriptor_offset(push) || + off + 4 > hk_root_descriptor_offset(push) + HK_MAX_PUSH_SIZE) + return false; + offset_B = off; + } else { + return false; + } + *out = (struct hk_g16_host_src){HK_G16_HOST_ROOT, offset_B}; + return true; +} + +/* Take the uniform-only exact fp32 arithmetic of `nir` into `prog` and have + * the program load its results from uniform halves [0, 2 * prog->n_out). + * Returns false, leaving the program's instructions as they were, when there + * is nothing to take or the halves are not free. */ +static bool +hk_g16_host_lower(nir_shader *nir, const struct hk_shader *shader, + struct hk_g16_host_prog *prog) +{ + memset(prog, 0, sizeof(*prog)); + nir_function_impl *impl = nir_shader_get_entrypoint(nir); + const unsigned reserved = 2 * HK_G16_HOST_MAX_OUT; + + if (shader->info.agx3_indexed_sampler_valid || + shader->info.agx3_combined_sampler_valid || + shader->info.agx3_gen1_fragment_rq || + shader->b.info.push_count < reserved) + return false; + for (unsigned i = 0; i < shader->b.info.agx3_uniform_dma_count; ++i) { + if (shader->b.info.agx3_uniform_dma[i].dest < reserved) + return false; + } + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(instr)->intrinsic == + nir_intrinsic_load_preamble && + nir_intrinsic_base(nir_instr_as_intrinsic(instr)) < reserved) + return false; + } + } + + nir_divergence_analysis(nir); + struct hash_table *temps = _mesa_pointer_hash_table_create(NULL); + if (!temps) + return false; + nir_def *defs[HK_G16_HOST_MAX_OPS]; + + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_alu) + continue; + nir_alu_instr *alu = nir_instr_as_alu(instr); + int opcode; + switch (alu->op) { + case nir_op_fsat: opcode = HK_G16_HOST_FSAT; break; + case nir_op_fmul: opcode = HK_G16_HOST_FMUL; break; + case nir_op_fadd: opcode = HK_G16_HOST_FADD; break; + case nir_op_fneg: opcode = HK_G16_HOST_FNEG; break; + case nir_op_fabs: opcode = HK_G16_HOST_FABS; break; + case nir_op_mov: opcode = HK_G16_HOST_MOV; break; + default: continue; + } + if (alu->def.bit_size != 32 || alu->def.num_components != 1 || + alu->def.divergent || prog->n_ops >= HK_G16_HOST_MAX_OPS) + continue; + struct hk_g16_host_op op = {.opcode = opcode}; + bool ok = true; + for (unsigned i = 0; i < nir_op_infos[alu->op].num_inputs && ok; ++i) + ok = hk_g16_host_classify( + shader, temps, + nir_get_scalar(alu->src[i].src.ssa, alu->src[i].swizzle[0]), + &op.src[i]); + if (!ok) + continue; + defs[prog->n_ops] = &alu->def; + prog->ops[prog->n_ops] = op; + _mesa_hash_table_insert(temps, &alu->def, + (void *)(uintptr_t)(prog->n_ops + 1)); + prog->n_ops++; + } + } + + /* Results some program instruction still reads. */ + for (unsigned i = 0; i < prog->n_ops; ++i) { + bool frontier = false; + nir_foreach_use_including_if(use, defs[i]) { + if (nir_src_is_if(use)) { + frontier = true; + continue; + } + nir_instr *user = nir_src_use_instr(use); + if (user->type != nir_instr_type_alu || + !_mesa_hash_table_search(temps, &nir_instr_as_alu(user)->def)) + frontier = true; + } + if (!frontier) + continue; + if (prog->n_out >= HK_G16_HOST_MAX_OUT) { + _mesa_hash_table_destroy(temps, NULL); + memset(prog, 0, sizeof(*prog)); + return false; + } + prog->out[prog->n_out++] = i; + } + _mesa_hash_table_destroy(temps, NULL); + if (!prog->n_out) { + memset(prog, 0, sizeof(*prog)); + return false; + } + + nir_builder b = nir_builder_at(nir_before_impl(impl)); + for (unsigned k = 0; k < prog->n_out; ++k) { + nir_def *u = nir_load_preamble(&b, 1, 32, .base = 2 * k); + nir_def_rewrite_uses(defs[prog->out[k]], u); + } + nir_progress(true, impl, nir_metadata_none); + return true; +} + +/* Bit-exact host counterpart of the taken GPU operations: each result is + * rounded to fp32 on its own (volatile: no contraction across operations). */ +static void +hk_g16_host_eval(const struct hk_g16_host_prog *p, const uint8_t *root, + uint32_t *out) +{ + volatile float t[HK_G16_HOST_MAX_OPS]; + for (unsigned i = 0; i < p->n_ops; ++i) { + float s[2] = {0.0f, 0.0f}; + for (unsigned j = 0; j < 2; ++j) { + const struct hk_g16_host_src *src = &p->ops[i].src[j]; + uint32_t u = 0; + if (src->kind == HK_G16_HOST_TEMP) { + s[j] = t[src->value]; + continue; + } + if (src->kind == HK_G16_HOST_IMM) + u = src->value; + else + memcpy(&u, root + src->value, sizeof(u)); + memcpy(&s[j], &u, sizeof(u)); + } + float r; + switch (p->ops[i].opcode) { + case HK_G16_HOST_FSAT: + r = isnan(s[0]) ? 0.0f : s[0] < 0.0f ? 0.0f : s[0] > 1.0f ? 1.0f : s[0]; + break; + case HK_G16_HOST_FMUL: r = s[0] * s[1]; break; + case HK_G16_HOST_FADD: r = s[0] + s[1]; break; + case HK_G16_HOST_FNEG: r = -s[0]; break; + case HK_G16_HOST_FABS: r = fabsf(s[0]); break; + default: r = s[0]; break; + } + t[i] = r; + } + for (unsigned k = 0; k < p->n_out; ++k) { + const float f = t[p->out[k]]; + memcpy(&out[k], &f, sizeof(f)); + } +} + static struct agx_shader_part * hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, const struct agx_fs_epilog_key *ekey, - const BITSET_WORD *partitions) + const BITSET_WORD *partitions, + struct hk_g16_host_prog *host) { /* A refusal of these inputs is final: remember it with the programs. */ #define HK_MONO_REFUSE(why) \ @@ -7035,7 +7292,13 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, MAX2(ekey->nr_samples, 1) != main_samples) HK_MONO_REFUSE("epilog shape"); - disk = hk_g17p_mono_disk_key(dev, shader, ekey, partitions, disk_key); + /* Host-evaluated programs are not stored: their host half lives in the + * linked shader only. */ + memset(host, 0, sizeof(*host)); + const bool host_eval = !inline_samples && + hk_g16_host_eval_allowed(dev, ekey); + disk = !host_eval && + hk_g17p_mono_disk_key(dev, shader, ekey, partitions, disk_key); if (disk) { size_t size = 0; uint8_t *data = @@ -7184,6 +7447,9 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, nir->info.fs.uses_sample_shading = false; } + if (host_eval && hk_g16_host_lower(nir, shader, host)) + NIR_PASS(_, nir, nir_opt_dce); + struct agx_shader_key key = shader->g17p_mono_key; if (inline_samples) key.fs.inside_sample_loop = false; @@ -7328,8 +7594,9 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader, (agx_native_graphics(&dev->dev) && (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND))) && !prolog && key->nr_samples_shaded == 0 && shader->g17p_mono_nir) { + struct hk_g16_host_prog host; struct agx_shader_part *mono = hk_g17p_inline_epilog_part( - dev, shader, &key->epilog, g17p_tile_partitions); + dev, shader, &key->epilog, g17p_tile_partitions, &host); if (mono) { struct hk_fast_link_key *mono_key_clone = ralloc_memdup(shader->linked.ht, key, sizeof(*key)); @@ -7360,6 +7627,7 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader, key->epilog.blend_constant_base; mono_linked->g17p_emrt_heap_base = key->epilog.emrt_heap_base; mono_linked->g17p_prolog_uses_root = key->prolog.statistics; + mono_linked->g16_host = host; _mesa_hash_table_insert(shader->linked.ht, mono_key_clone, mono_linked); return mono_linked; @@ -10800,6 +11068,18 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs, } } + /* G16G host-evaluated fragment uniforms follow the push constants and the + * linked program; both reach here before the root upload. */ + if (linked_fs && linked_fs->g16_host.n_out) { + uint32_t v[HK_G16_HOST_MAX_OUT]; + const size_t size = linked_fs->g16_host.n_out * sizeof(v[0]); + hk_g16_host_eval(&linked_fs->g16_host, (const uint8_t *)&desc->root, v); + if (memcmp(desc->root.g16_host_unif, v, size)) { + memcpy(desc->root.g16_host_unif, v, size); + desc->root_dirty = true; + } + } + /* Root must be uploaded after the above, which touch the root. */ if (gfx->descriptors.root_dirty) { gfx->root = diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c index b26c3beb50a1..c4355dcdf265 100644 --- a/src/asahi/vulkan/hk_shader.c +++ b/src/asahi/vulkan/hk_shader.c @@ -188,7 +188,8 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev) /* This could be optimized but it doesn't matter. */ return pdev->dev.debug | hk_fs_structure_key_bits() | - ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33); + ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33) | + ((uint64_t)!!agx_g16_host_blend_src_enabled() << 34); } const nir_shader_compiler_options * diff --git a/src/asahi/vulkan/hk_shader.h b/src/asahi/vulkan/hk_shader.h index 9e8dd7ffbeb2..b8636ee6e4f0 100644 --- a/src/asahi/vulkan/hk_shader.h +++ b/src/asahi/vulkan/hk_shader.h @@ -557,6 +557,47 @@ hk_rast_mesh_variant(struct hk_api_shader *obj) AGX_USC_REGISTERS_LENGTH + AGX_USC_SHADER_LENGTH + AGX_USC_SHARED_LENGTH + \ AGX_USC_SAMPLER_LENGTH + (AGX_USC_UNIFORM_LENGTH * 9)) +/* G16G (ASAHI_G16_HOST_BLEND_SRC=0 opts out): uniform-only fp32 arithmetic of an + * inline-blend fragment program, evaluated on the host for each draw and + * delivered in uniform halves [0, 2 * n_out) from the root's g16_host_unif. + * G16G forms no native constant program (agx_chip_native_preamble()), so + * without this every pixel recomputes these values. Only exact IEEE fp32 + * operations are taken: fused multiply-add stays on the GPU + * (agx_chip_exact_fma32()). */ +#define HK_G16_HOST_MAX_OPS 24 +#define HK_G16_HOST_MAX_OUT 6 + +enum hk_g16_host_src_kind { + HK_G16_HOST_TEMP, + HK_G16_HOST_IMM, + HK_G16_HOST_ROOT, +}; + +enum hk_g16_host_opcode { + HK_G16_HOST_FSAT, + HK_G16_HOST_FMUL, + HK_G16_HOST_FADD, + HK_G16_HOST_FNEG, + HK_G16_HOST_FABS, + HK_G16_HOST_MOV, +}; + +struct hk_g16_host_src { + uint8_t kind; /* enum hk_g16_host_src_kind */ + uint32_t value; /* temp index, immediate bits or root byte offset */ +}; + +struct hk_g16_host_op { + uint8_t opcode; /* enum hk_g16_host_opcode */ + struct hk_g16_host_src src[2]; +}; + +struct hk_g16_host_prog { + uint8_t n_ops, n_out; + uint8_t out[HK_G16_HOST_MAX_OUT]; + struct hk_g16_host_op ops[HK_G16_HOST_MAX_OPS]; +}; + struct hk_linked_shader { struct agx_linked_shader b; @@ -593,6 +634,9 @@ struct hk_linked_shader { * the descriptor root even when the application fragment shader does not. */ bool g17p_prolog_uses_root; + /* See struct hk_g16_host_prog; n_out == 0 when unused. */ + struct hk_g16_host_prog g16_host; + /* An inline-blend fragment program that forms its own constant program * (hk_g17p_inline_epilog_part()) has its own uniform layout: this shader * describes it, and the draw takes the program's launch (uniform ranges, From e5a8fc210144b16a1754cad2894034f6fe83a6c6 Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Tue, 17 Nov 2026 21:33:42 +0000 Subject: [PATCH 5/5] asahi: compile border-free twins on G16G Native graphics drops the custom-border emulation from a shader and keeps the emulating NIR as a twin until the physical device creates a sampler that needs it. The emulation branch is recognised by matching the border test with the index the clamp-to-0 sample uses. On G16G the test reads clamp_0_sampler_index_or_negative and the sample reads the unbiased clamp-to-0 index of the same sampled-image descriptor, a pair only the G17P path accepted, so every G16G texture sample kept the emulation: a flag load and compare, a branch, a second sample for border samplers and the merge. Accept that pair on G16G, with the offset between the two fields taken from struct hk_sampled_image_descriptor_sw. The textured-blend fragment program of a common benchmark goes from 79 to 42 instructions; shaders switch to the emulating twin exactly as on G17P once a border sampler exists. A twin is chosen when the shader is compiled and replaced when a command buffer binds it or draws after a border sampler exists. With update-after- bind sampled images a border sampler could reach an already recorded border-free draw, so devices that enable descriptorBindingSampledImageUpdateAfterBind keep the emulation on G16G. ASAHI_G16_BORDER_TWIN=0 keeps the emulation in every shader; the setting is part of the physical device's compiler flags. --- src/asahi/compiler/agx_compile.c | 6 ++++ src/asahi/compiler/agx_debug.h | 3 ++ src/asahi/vulkan/hk_shader.c | 47 ++++++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 74129aa2590d..a91f67bb1de9 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -100,6 +100,12 @@ agx_g16_host_blend_src_enabled(void) return debug_get_bool_option("ASAHI_G16_HOST_BLEND_SRC", true); } +bool +agx_g16_border_twin_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_BORDER_TWIN", true); +} + static __thread char agx_refusal[192]; const char * diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index cb8d6ccff76f..ec93dbcbd2f2 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -41,6 +41,9 @@ bool agx_g16_blend_dst_first_enabled(void); /* Qualified on G16G; ASAHI_G16_HOST_BLEND_SRC=0 restores per-pixel evaluation * of uniform-only inline-blend fragment arithmetic. */ bool agx_g16_host_blend_src_enabled(void); +/* Qualified on G16G; ASAHI_G16_BORDER_TWIN=0 keeps the custom-border emulation in + * every G16G texture sample instead of compiling border-free twins. */ +bool agx_g16_border_twin_enabled(void); /* Why the last agx_compile_shader_nir() on this thread returned false * ("" when it succeeded), for the driver's "cannot encode" error. */ diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c index c4355dcdf265..2988d9f1a3db 100644 --- a/src/asahi/vulkan/hk_shader.c +++ b/src/asahi/vulkan/hk_shader.c @@ -189,7 +189,8 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev) return pdev->dev.debug | hk_fs_structure_key_bits() | ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33) | - ((uint64_t)!!agx_g16_host_blend_src_enabled() << 34); + ((uint64_t)!!agx_g16_host_blend_src_enabled() << 34) | + ((uint64_t)!!agx_g16_border_twin_enabled() << 35); } const nir_shader_compiler_options * @@ -2389,8 +2390,29 @@ agx_nir_lower_custom_border(nir_shader *nir) * clamp-to-0 sample tex_0, after descriptor lowering: taken iff the sampler's * clamp-to-0 index c is not negative, i.e. `if (c >= 0)`, with tex_0 sampling * c without its integer-border bit (hk_nir_lower_descriptors()). */ +/* G16G: whether the border test reads clamp_0_sampler_index_or_negative and + * the clamp-to-0 sample reads the unbiased clamp-to-0 index of the same + * sampled-image descriptor, as hk_nir_lower_descriptors() emits them (two + * loads from one descriptor set address). */ static bool -hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) +hk_g16_border_test_and_index(nir_scalar tested, nir_scalar c) +{ + const uint64_t delta = + offsetof(struct hk_sampled_image_descriptor_sw, + clamp_0_sampler_index_or_negative) - + offsetof(struct hk_sampled_image_descriptor_sw, + g17p.unbiased_clamp_0_sampler_index); + nir_scalar tb, cb; + uint64_t to, co; + if (!hk_nir_border_address(tested, &tb, &to) || + !hk_nir_border_address(c, &cb, &co) || to - co != delta) + return false; + return nir_scalar_equal(tb, cb); +} + +static bool +hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p, + bool g16_twin) { const int s = nir_tex_instr_src_index(tex_0, nir_tex_src_sampler_handle); if (s < 0) @@ -2418,6 +2440,10 @@ hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) return false; if (nir_scalar_equal(tested, c)) return true; + /* G16G: the branch and the clamp-to-0 sample read the same descriptor + * field (clamp_0_sampler_index_or_negative) through two loads. */ + if (g16_twin) + return hk_g16_border_test_and_index(tested, c); if (!g17p) return false; const struct hk_sampled_image_descriptor_layout native = @@ -2434,7 +2460,8 @@ hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) * emulation, unless every clamp-to-0 sample sits directly in the then-list * of its branch (hk_is_border_emulation_if()). */ static bool -hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p) +hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p, + bool g16_twin) { nir_foreach_function_impl(impl, nir) { nir_foreach_block(block, impl) { @@ -2453,7 +2480,8 @@ hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p) foreach_list_typed(nir_cf_node, node, node, &nif->then_list) in_then |= node == &block->cf_node; if (!in_then || - !hk_is_border_emulation_if(nif, nir_instr_as_tex(instr), g17p)) { + !hk_is_border_emulation_if(nif, nir_instr_as_tex(instr), g17p, + g16_twin)) { return false; } @@ -5755,7 +5783,16 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf struct hk_physical_device *pdev = hk_device_physical(dev); struct util_dynarray ifs; util_dynarray_init(&ifs, NULL); - if (hk_find_border_emulation(nir, &ifs, dev->dev.chip == AGX_CHIP_G17P) && + /* G16G: a twin is chosen when the shader is compiled and replaced when + * a command buffer binds it or draws after a border sampler exists. + * Update-after-bind sampled images could put such a sampler under an + * already recorded border-free draw, so those devices keep the + * emulation (the feature is part of the shader key). */ + const bool g16_twin = + dev->dev.chip == AGX_CHIP_G16G && agx_g16_border_twin_enabled() && + !(features & HK_FEAT_SAMPLED_IMAGE_UPDATE_AFTER_BIND); + if (hk_find_border_emulation(nir, &ifs, dev->dev.chip == AGX_CHIP_G17P, + g16_twin) && ifs.size) { if (twin_of || p_atomic_read(&pdev->border_emulation)) { border_emulating = true;