diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index 5443f4e509f0..a91f67bb1de9 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -82,6 +82,30 @@ agx_get_compiler_debug(void) return debug_get_option_agx_compiler_debug(); } +bool +agx_g16_pix_hoist_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_PIX_HOIST", true); +} + +bool +agx_g16_blend_dst_first_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true); +} + +bool +agx_g16_host_blend_src_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_HOST_BLEND_SRC", true); +} + +bool +agx_g16_border_twin_enabled(void) +{ + return debug_get_bool_option("ASAHI_G16_BORDER_TWIN", true); +} + static __thread char agx_refusal[192]; const char * @@ -8872,6 +8896,71 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx) * adjacent run of non-coherent loads with identical operation and base; SSA * keeps their consumers attached to the correct result and no serialization * is introduced. */ +/* G16G (ASAHI_G16_PIX_HOIST=0 opts out): after scheduling, move + * float arithmetic computed only from uniforms ahead of the first + * pixel-ordering wait of its block, so it executes outside the ordered colour + * transaction instead of between WAIT_PIX and the tile store. Only fadd, fmul, + * ffma and mov_imm move, and only when every source is a uniform, an + * immediate, or the result of another such instruction. Moved instructions + * keep their relative order and their exact operations, modifiers and + * operands, so results are bit-identical. */ +static bool +agx3_g16_hoist_pixel_independent(agx_context *ctx) +{ + if (ctx->key->dev.chip != AGX_CHIP_G16G || + ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble || + !agx_g16_pix_hoist_enabled()) + return false; + + /* Values computed only from uniforms and immediates by the movable ops, + * wherever they are in the block. Texture, varying and tile results never + * qualify, so per-pixel work keeps its scheduled place. */ + BITSET_WORD *uniform = BITSET_CALLOC(ctx->alloc); + if (!uniform) + return false; + + bool progress = false; + agx_foreach_block(ctx, block) { + agx_instr *wait = NULL; + agx_foreach_instr_in_block_safe(block, I) { + if (!wait && I->op == AGX_OPCODE_WAIT_PIX) { + wait = I; + continue; + } + + bool uniform_only = (I->op == AGX_OPCODE_FADD || + I->op == AGX_OPCODE_FMUL || + I->op == AGX_OPCODE_FFMA || + I->op == AGX_OPCODE_MOV_IMM) && + I->nr_dests == 1 && + I->dest[0].type == AGX_INDEX_NORMAL; + agx_foreach_src(I, s) { + const agx_index src = I->src[s]; + if (src.type == AGX_INDEX_NORMAL) { + if (!BITSET_TEST(uniform, src.value)) + uniform_only = false; + } else if (src.type != AGX_INDEX_UNIFORM && + src.type != AGX_INDEX_IMMEDIATE && + src.type != AGX_INDEX_NULL) { + uniform_only = false; + } + } + + if (!uniform_only) + continue; + BITSET_SET(uniform, I->dest[0].value); + if (wait) { + list_del(&I->link); + list_addtail(&I->link, &wait->link); + progress = true; + } + } + } + + free(uniform); + return progress; +} + static bool agx3_order_fragment_device_loads(agx_context *ctx) { @@ -10383,6 +10472,7 @@ agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl, } AGX_PASS(ctx, agx3_order_fragment_device_loads); + AGX_PASS(ctx, agx3_g16_hoist_pixel_independent); if (agx_should_dump(nir, AGX_DBG_SHADERS)) agx_print_shader(ctx, stdout); diff --git a/src/asahi/compiler/agx_debug.h b/src/asahi/compiler/agx_debug.h index a1cc739e60b4..ec93dbcbd2f2 100644 --- a/src/asahi/compiler/agx_debug.h +++ b/src/asahi/compiler/agx_debug.h @@ -34,6 +34,17 @@ enum agx_compiler_dbg { uint64_t agx_get_compiler_debug(void); +/* Qualified on G16G; ASAHI_G16_PIX_HOIST=0 restores the scheduler placement. */ +bool agx_g16_pix_hoist_enabled(void); +/* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */ +bool agx_g16_blend_dst_first_enabled(void); +/* Qualified on G16G; ASAHI_G16_HOST_BLEND_SRC=0 restores per-pixel evaluation + * of uniform-only inline-blend fragment arithmetic. */ +bool agx_g16_host_blend_src_enabled(void); +/* Qualified on G16G; ASAHI_G16_BORDER_TWIN=0 keeps the custom-border emulation in + * every G16G texture sample instead of compiling border-free twins. */ +bool agx_g16_border_twin_enabled(void); + /* Why the last agx_compile_shader_nir() on this thread returned false * ("" when it succeeded), for the driver's "cannot encode" error. */ const char *agx_last_compile_refusal(void); diff --git a/src/asahi/lib/agx_linker.h b/src/asahi/lib/agx_linker.h index e2993ee182a0..8dfaa4cbbae5 100644 --- a/src/asahi/lib/agx_linker.h +++ b/src/asahi/lib/agx_linker.h @@ -370,6 +370,9 @@ struct agx_fs_epilog_key { * the constants after the main shader's uniform footprint instead. */ uint16_t blend_constant_base; + /* Emit the blend destination term first (nir_lower_blend_rt::dst_term_first). */ + bool blend_dst_first; + /* Render targets whose tile value must be stored back unchanged (colour * write disabled while raster coverage still makes the tile nonempty). * Independent of blending, so it also holds for pure-integer formats. */ diff --git a/src/asahi/lib/agx_nir_prolog_epilog.c b/src/asahi/lib/agx_nir_prolog_epilog.c index d1bfa069d340..5e26c2677325 100644 --- a/src/asahi/lib/agx_nir_prolog_epilog.c +++ b/src/asahi/lib/agx_nir_prolog_epilog.c @@ -807,6 +807,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_) .alpha.src_factor = blend.alpha_src_factor, .alpha.dst_factor = blend.alpha_dst_factor, + .dst_term_first = key->blend_dst_first, .alpha.func = blend.alpha_func, .colormask = key->blend.rt[i].colormask, diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c index 49772d7f0d66..6e4a3abf79d3 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.c +++ b/src/asahi/vulkan/hk_cmd_buffer.c @@ -2538,7 +2538,20 @@ hk_g17p_graphics_uniform_ranges( .size_halfs = 4, .address = pointer, }; } - if (hk_cmd_uses_blend_constant(cmd)) { + if (linked && linked->g16_host.n_out) { + /* A G16G inline-blend program with host-evaluated uniforms reads + * them in halves [0, 2 * n_out), which only the spill table and the + * blend constants otherwise use: it is built for neither (an inline + * program refuses epilogs reading blend constants). */ + if (native_spills || !root_ptr) + return hk_g17p_graphics_ranges_refused(range_count, count); + sources[count] = HK_G17P_RANGE_SOURCE_ROOT; + ranges[count++] = (struct agx3_ioto_uniform_range){ + .start_halfs = 0, + .size_halfs = 2 * linked->g16_host.n_out, + .address = root_ptr + hk_root_descriptor_offset(g16_host_unif), + }; + } else if (hk_cmd_uses_blend_constant(cmd)) { /* The non-monolithic blend epilog reads the four constants from * the base selected by its linked key. It is half4 for the standard * ABI and moves beyond the main shader footprint when HAL200 native diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h index 38941cb1eb5c..b83c96ad28cc 100644 --- a/src/asahi/vulkan/hk_cmd_buffer.h +++ b/src/asahi/vulkan/hk_cmd_buffer.h @@ -258,6 +258,10 @@ struct hk_root_descriptor_table { * address of the word libagx_m5_task_dispatch writes (nonzero = execute); * zero for every other draw. Appended after every agx3b field. */ uint64_t mesh_task_flag; + + /* G16G host-evaluated fragment uniforms (hk_linked_shader::g16_host), + * published to uniform halves [0, 2 * n_out). */ + uint32_t g16_host_unif[HK_G16_HOST_MAX_OUT]; }; /* The graphics arm is the larger of the two, so everything after the union -- diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c index 81c7c77c5d5e..d0fe5435cc05 100644 --- a/src/asahi/vulkan/hk_cmd_draw.c +++ b/src/asahi/vulkan/hk_cmd_draw.c @@ -7003,10 +7003,267 @@ hk_g17p_mono_disk_key(struct hk_device *dev, const struct hk_shader *shader, return true; } +static bool +hk_g16_host_eval_allowed(const struct hk_device *dev, + const struct agx_fs_epilog_key *ekey) +{ + if (dev->dev.chip != AGX_CHIP_G16G || + !agx_g16_host_blend_src_enabled() || ekey->native_spills) + return false; + /* Host and GPU differ at most in denormal and NaN handling, which no + * UNORM store can observe. */ + for (unsigned rt = 0; rt < ARRAY_SIZE(ekey->rt_formats); ++rt) { + const enum pipe_format f = ekey->rt_formats[rt]; + if (f != PIPE_FORMAT_NONE && + (!util_format_is_unorm(f) || util_format_is_srgb(f))) + return false; + } + return true; +} + +/* Root byte offset of the 32-bit push constant the main's root uniform DMA + * places at uniform half `half`. */ +static bool +hk_g16_host_root_offset(const struct hk_shader *shader, unsigned half, + uint32_t *offset_B) +{ + for (unsigned i = 0; i < shader->b.info.agx3_uniform_dma_count; ++i) { + const struct agx3_uniform_dma_span *s = &shader->b.info.agx3_uniform_dma[i]; + if (s->source != AGX3_UNIFORM_DMA_POINTER || + s->pointer != shader->info.agx3_root_uniform || half < s->dest || + half + 2 > s->dest + s->halfs) + continue; + const uint32_t o = s->offset_B + (half - s->dest) * 2; + if ((o & 3) || o < hk_root_descriptor_offset(push) || + o + 4 > hk_root_descriptor_offset(push) + HK_MAX_PUSH_SIZE) + return false; + *offset_B = o; + return true; + } + return false; +} + +static bool +hk_g16_host_classify(const struct hk_shader *shader, struct hash_table *temps, + nir_scalar s, struct hk_g16_host_src *out) +{ + if (s.def->bit_size != 32) + return false; + struct hash_entry *he = _mesa_hash_table_search(temps, s.def); + if (he) { + if (s.comp != 0) + return false; + *out = (struct hk_g16_host_src){HK_G16_HOST_TEMP, + (uint32_t)(uintptr_t)he->data - 1}; + return true; + } + if (nir_scalar_is_const(s)) { + *out = (struct hk_g16_host_src){HK_G16_HOST_IMM, + (uint32_t)nir_scalar_as_uint(s)}; + return true; + } + nir_instr *parent = nir_def_instr(s.def); + if (parent->type != nir_instr_type_intrinsic) + return false; + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(parent); + uint32_t offset_B; + if (intr->intrinsic == nir_intrinsic_load_preamble) { + if (!hk_g16_host_root_offset(shader, + nir_intrinsic_base(intr) + 2 * s.comp, + &offset_B)) + return false; + } else if (intr->intrinsic == nir_intrinsic_load_global_constant) { + /* Push constants before the backend moves them to the root uniform + * DMA: a load through the root pointer at a constant offset. */ + nir_scalar a = nir_scalar_chase_movs(nir_get_scalar(intr->src[0].ssa, 0)); + uint64_t off = 0; + if (nir_scalar_is_alu(a) && nir_scalar_alu_op(a) == nir_op_iadd) { + nir_scalar x = nir_scalar_chase_alu_src(a, 0); + nir_scalar y = nir_scalar_chase_alu_src(a, 1); + if (nir_scalar_is_const(x)) { + nir_scalar t = x; + x = y; + y = t; + } + if (!nir_scalar_is_const(y)) + return false; + off = nir_scalar_as_uint(y); + a = x; + } + nir_instr *base = nir_def_instr(a.def); + if (a.def->bit_size != 64 || base->type != nir_instr_type_intrinsic || + nir_instr_as_intrinsic(base)->intrinsic != + nir_intrinsic_load_preamble || + nir_intrinsic_base(nir_instr_as_intrinsic(base)) != + shader->info.agx3_root_uniform) + return false; + off += 4 * s.comp; + if ((off & 3) || off < hk_root_descriptor_offset(push) || + off + 4 > hk_root_descriptor_offset(push) + HK_MAX_PUSH_SIZE) + return false; + offset_B = off; + } else { + return false; + } + *out = (struct hk_g16_host_src){HK_G16_HOST_ROOT, offset_B}; + return true; +} + +/* Take the uniform-only exact fp32 arithmetic of `nir` into `prog` and have + * the program load its results from uniform halves [0, 2 * prog->n_out). + * Returns false, leaving the program's instructions as they were, when there + * is nothing to take or the halves are not free. */ +static bool +hk_g16_host_lower(nir_shader *nir, const struct hk_shader *shader, + struct hk_g16_host_prog *prog) +{ + memset(prog, 0, sizeof(*prog)); + nir_function_impl *impl = nir_shader_get_entrypoint(nir); + const unsigned reserved = 2 * HK_G16_HOST_MAX_OUT; + + if (shader->info.agx3_indexed_sampler_valid || + shader->info.agx3_combined_sampler_valid || + shader->info.agx3_gen1_fragment_rq || + shader->b.info.push_count < reserved) + return false; + for (unsigned i = 0; i < shader->b.info.agx3_uniform_dma_count; ++i) { + if (shader->b.info.agx3_uniform_dma[i].dest < reserved) + return false; + } + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(instr)->intrinsic == + nir_intrinsic_load_preamble && + nir_intrinsic_base(nir_instr_as_intrinsic(instr)) < reserved) + return false; + } + } + + nir_divergence_analysis(nir); + struct hash_table *temps = _mesa_pointer_hash_table_create(NULL); + if (!temps) + return false; + nir_def *defs[HK_G16_HOST_MAX_OPS]; + + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_alu) + continue; + nir_alu_instr *alu = nir_instr_as_alu(instr); + int opcode; + switch (alu->op) { + case nir_op_fsat: opcode = HK_G16_HOST_FSAT; break; + case nir_op_fmul: opcode = HK_G16_HOST_FMUL; break; + case nir_op_fadd: opcode = HK_G16_HOST_FADD; break; + case nir_op_fneg: opcode = HK_G16_HOST_FNEG; break; + case nir_op_fabs: opcode = HK_G16_HOST_FABS; break; + case nir_op_mov: opcode = HK_G16_HOST_MOV; break; + default: continue; + } + if (alu->def.bit_size != 32 || alu->def.num_components != 1 || + alu->def.divergent || prog->n_ops >= HK_G16_HOST_MAX_OPS) + continue; + struct hk_g16_host_op op = {.opcode = opcode}; + bool ok = true; + for (unsigned i = 0; i < nir_op_infos[alu->op].num_inputs && ok; ++i) + ok = hk_g16_host_classify( + shader, temps, + nir_get_scalar(alu->src[i].src.ssa, alu->src[i].swizzle[0]), + &op.src[i]); + if (!ok) + continue; + defs[prog->n_ops] = &alu->def; + prog->ops[prog->n_ops] = op; + _mesa_hash_table_insert(temps, &alu->def, + (void *)(uintptr_t)(prog->n_ops + 1)); + prog->n_ops++; + } + } + + /* Results some program instruction still reads. */ + for (unsigned i = 0; i < prog->n_ops; ++i) { + bool frontier = false; + nir_foreach_use_including_if(use, defs[i]) { + if (nir_src_is_if(use)) { + frontier = true; + continue; + } + nir_instr *user = nir_src_use_instr(use); + if (user->type != nir_instr_type_alu || + !_mesa_hash_table_search(temps, &nir_instr_as_alu(user)->def)) + frontier = true; + } + if (!frontier) + continue; + if (prog->n_out >= HK_G16_HOST_MAX_OUT) { + _mesa_hash_table_destroy(temps, NULL); + memset(prog, 0, sizeof(*prog)); + return false; + } + prog->out[prog->n_out++] = i; + } + _mesa_hash_table_destroy(temps, NULL); + if (!prog->n_out) { + memset(prog, 0, sizeof(*prog)); + return false; + } + + nir_builder b = nir_builder_at(nir_before_impl(impl)); + for (unsigned k = 0; k < prog->n_out; ++k) { + nir_def *u = nir_load_preamble(&b, 1, 32, .base = 2 * k); + nir_def_rewrite_uses(defs[prog->out[k]], u); + } + nir_progress(true, impl, nir_metadata_none); + return true; +} + +/* Bit-exact host counterpart of the taken GPU operations: each result is + * rounded to fp32 on its own (volatile: no contraction across operations). */ +static void +hk_g16_host_eval(const struct hk_g16_host_prog *p, const uint8_t *root, + uint32_t *out) +{ + volatile float t[HK_G16_HOST_MAX_OPS]; + for (unsigned i = 0; i < p->n_ops; ++i) { + float s[2] = {0.0f, 0.0f}; + for (unsigned j = 0; j < 2; ++j) { + const struct hk_g16_host_src *src = &p->ops[i].src[j]; + uint32_t u = 0; + if (src->kind == HK_G16_HOST_TEMP) { + s[j] = t[src->value]; + continue; + } + if (src->kind == HK_G16_HOST_IMM) + u = src->value; + else + memcpy(&u, root + src->value, sizeof(u)); + memcpy(&s[j], &u, sizeof(u)); + } + float r; + switch (p->ops[i].opcode) { + case HK_G16_HOST_FSAT: + r = isnan(s[0]) ? 0.0f : s[0] < 0.0f ? 0.0f : s[0] > 1.0f ? 1.0f : s[0]; + break; + case HK_G16_HOST_FMUL: r = s[0] * s[1]; break; + case HK_G16_HOST_FADD: r = s[0] + s[1]; break; + case HK_G16_HOST_FNEG: r = -s[0]; break; + case HK_G16_HOST_FABS: r = fabsf(s[0]); break; + default: r = s[0]; break; + } + t[i] = r; + } + for (unsigned k = 0; k < p->n_out; ++k) { + const float f = t[p->out[k]]; + memcpy(&out[k], &f, sizeof(f)); + } +} + static struct agx_shader_part * hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, const struct agx_fs_epilog_key *ekey, - const BITSET_WORD *partitions) + const BITSET_WORD *partitions, + struct hk_g16_host_prog *host) { /* A refusal of these inputs is final: remember it with the programs. */ #define HK_MONO_REFUSE(why) \ @@ -7035,7 +7292,13 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, MAX2(ekey->nr_samples, 1) != main_samples) HK_MONO_REFUSE("epilog shape"); - disk = hk_g17p_mono_disk_key(dev, shader, ekey, partitions, disk_key); + /* Host-evaluated programs are not stored: their host half lives in the + * linked shader only. */ + memset(host, 0, sizeof(*host)); + const bool host_eval = !inline_samples && + hk_g16_host_eval_allowed(dev, ekey); + disk = !host_eval && + hk_g17p_mono_disk_key(dev, shader, ekey, partitions, disk_key); if (disk) { size_t size = 0; uint8_t *data = @@ -7184,6 +7447,9 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader, nir->info.fs.uses_sample_shading = false; } + if (host_eval && hk_g16_host_lower(nir, shader, host)) + NIR_PASS(_, nir, nir_opt_dce); + struct agx_shader_key key = shader->g17p_mono_key; if (inline_samples) key.fs.inside_sample_loop = false; @@ -7328,8 +7594,9 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader, (agx_native_graphics(&dev->dev) && (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND))) && !prolog && key->nr_samples_shaded == 0 && shader->g17p_mono_nir) { + struct hk_g16_host_prog host; struct agx_shader_part *mono = hk_g17p_inline_epilog_part( - dev, shader, &key->epilog, g17p_tile_partitions); + dev, shader, &key->epilog, g17p_tile_partitions, &host); if (mono) { struct hk_fast_link_key *mono_key_clone = ralloc_memdup(shader->linked.ht, key, sizeof(*key)); @@ -7360,6 +7627,7 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader, key->epilog.blend_constant_base; mono_linked->g17p_emrt_heap_base = key->epilog.emrt_heap_base; mono_linked->g17p_prolog_uses_root = key->prolog.statistics; + mono_linked->g16_host = host; _mesa_hash_table_insert(shader->linked.ht, mono_key_clone, mono_linked); return mono_linked; @@ -9982,6 +10250,8 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs, key->epilog.link.already_ran_zs |= nontrivial_force_early; key->epilog.native_imageblock = in->tib->native_imageblock; key->epilog.max_sample_size_B = in->tib->max_sample_size_B; + key->epilog.blend_dst_first = + dev->dev.chip == AGX_CHIP_G16G && agx_g16_blend_dst_first_enabled(); key->epilog.resident_mask = in->tib->resident_mask; key->epilog.max_bytes_per_tile = in->tib->max_bytes_per_tile; key->epilog.max_samples_per_tile = in->tib->max_samples_per_tile; @@ -10798,6 +11068,18 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs, } } + /* G16G host-evaluated fragment uniforms follow the push constants and the + * linked program; both reach here before the root upload. */ + if (linked_fs && linked_fs->g16_host.n_out) { + uint32_t v[HK_G16_HOST_MAX_OUT]; + const size_t size = linked_fs->g16_host.n_out * sizeof(v[0]); + hk_g16_host_eval(&linked_fs->g16_host, (const uint8_t *)&desc->root, v); + if (memcmp(desc->root.g16_host_unif, v, size)) { + memcpy(desc->root.g16_host_unif, v, size); + desc->root_dirty = true; + } + } + /* Root must be uploaded after the above, which touch the root. */ if (gfx->descriptors.root_dirty) { gfx->root = diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c index 77d3dbdf13bc..2988d9f1a3db 100644 --- a/src/asahi/vulkan/hk_shader.c +++ b/src/asahi/vulkan/hk_shader.c @@ -187,7 +187,10 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev) { /* This could be optimized but it doesn't matter. */ return pdev->dev.debug | - hk_fs_structure_key_bits(); + hk_fs_structure_key_bits() | + ((uint64_t)!!agx_g16_pix_hoist_enabled() << 33) | + ((uint64_t)!!agx_g16_host_blend_src_enabled() << 34) | + ((uint64_t)!!agx_g16_border_twin_enabled() << 35); } const nir_shader_compiler_options * @@ -2387,8 +2390,29 @@ agx_nir_lower_custom_border(nir_shader *nir) * clamp-to-0 sample tex_0, after descriptor lowering: taken iff the sampler's * clamp-to-0 index c is not negative, i.e. `if (c >= 0)`, with tex_0 sampling * c without its integer-border bit (hk_nir_lower_descriptors()). */ +/* G16G: whether the border test reads clamp_0_sampler_index_or_negative and + * the clamp-to-0 sample reads the unbiased clamp-to-0 index of the same + * sampled-image descriptor, as hk_nir_lower_descriptors() emits them (two + * loads from one descriptor set address). */ static bool -hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) +hk_g16_border_test_and_index(nir_scalar tested, nir_scalar c) +{ + const uint64_t delta = + offsetof(struct hk_sampled_image_descriptor_sw, + clamp_0_sampler_index_or_negative) - + offsetof(struct hk_sampled_image_descriptor_sw, + g17p.unbiased_clamp_0_sampler_index); + nir_scalar tb, cb; + uint64_t to, co; + if (!hk_nir_border_address(tested, &tb, &to) || + !hk_nir_border_address(c, &cb, &co) || to - co != delta) + return false; + return nir_scalar_equal(tb, cb); +} + +static bool +hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p, + bool g16_twin) { const int s = nir_tex_instr_src_index(tex_0, nir_tex_src_sampler_handle); if (s < 0) @@ -2416,6 +2440,10 @@ hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) return false; if (nir_scalar_equal(tested, c)) return true; + /* G16G: the branch and the clamp-to-0 sample read the same descriptor + * field (clamp_0_sampler_index_or_negative) through two loads. */ + if (g16_twin) + return hk_g16_border_test_and_index(tested, c); if (!g17p) return false; const struct hk_sampled_image_descriptor_layout native = @@ -2432,7 +2460,8 @@ hk_is_border_emulation_if(nir_if *nif, nir_tex_instr *tex_0, bool g17p) * emulation, unless every clamp-to-0 sample sits directly in the then-list * of its branch (hk_is_border_emulation_if()). */ static bool -hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p) +hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p, + bool g16_twin) { nir_foreach_function_impl(impl, nir) { nir_foreach_block(block, impl) { @@ -2451,7 +2480,8 @@ hk_find_border_emulation(nir_shader *nir, struct util_dynarray *ifs, bool g17p) foreach_list_typed(nir_cf_node, node, node, &nif->then_list) in_then |= node == &block->cf_node; if (!in_then || - !hk_is_border_emulation_if(nif, nir_instr_as_tex(instr), g17p)) { + !hk_is_border_emulation_if(nif, nir_instr_as_tex(instr), g17p, + g16_twin)) { return false; } @@ -5753,7 +5783,16 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf struct hk_physical_device *pdev = hk_device_physical(dev); struct util_dynarray ifs; util_dynarray_init(&ifs, NULL); - if (hk_find_border_emulation(nir, &ifs, dev->dev.chip == AGX_CHIP_G17P) && + /* G16G: a twin is chosen when the shader is compiled and replaced when + * a command buffer binds it or draws after a border sampler exists. + * Update-after-bind sampled images could put such a sampler under an + * already recorded border-free draw, so those devices keep the + * emulation (the feature is part of the shader key). */ + const bool g16_twin = + dev->dev.chip == AGX_CHIP_G16G && agx_g16_border_twin_enabled() && + !(features & HK_FEAT_SAMPLED_IMAGE_UPDATE_AFTER_BIND); + if (hk_find_border_emulation(nir, &ifs, dev->dev.chip == AGX_CHIP_G17P, + g16_twin) && ifs.size) { if (twin_of || p_atomic_read(&pdev->border_emulation)) { border_emulating = true; diff --git a/src/asahi/vulkan/hk_shader.h b/src/asahi/vulkan/hk_shader.h index 9e8dd7ffbeb2..b8636ee6e4f0 100644 --- a/src/asahi/vulkan/hk_shader.h +++ b/src/asahi/vulkan/hk_shader.h @@ -557,6 +557,47 @@ hk_rast_mesh_variant(struct hk_api_shader *obj) AGX_USC_REGISTERS_LENGTH + AGX_USC_SHADER_LENGTH + AGX_USC_SHARED_LENGTH + \ AGX_USC_SAMPLER_LENGTH + (AGX_USC_UNIFORM_LENGTH * 9)) +/* G16G (ASAHI_G16_HOST_BLEND_SRC=0 opts out): uniform-only fp32 arithmetic of an + * inline-blend fragment program, evaluated on the host for each draw and + * delivered in uniform halves [0, 2 * n_out) from the root's g16_host_unif. + * G16G forms no native constant program (agx_chip_native_preamble()), so + * without this every pixel recomputes these values. Only exact IEEE fp32 + * operations are taken: fused multiply-add stays on the GPU + * (agx_chip_exact_fma32()). */ +#define HK_G16_HOST_MAX_OPS 24 +#define HK_G16_HOST_MAX_OUT 6 + +enum hk_g16_host_src_kind { + HK_G16_HOST_TEMP, + HK_G16_HOST_IMM, + HK_G16_HOST_ROOT, +}; + +enum hk_g16_host_opcode { + HK_G16_HOST_FSAT, + HK_G16_HOST_FMUL, + HK_G16_HOST_FADD, + HK_G16_HOST_FNEG, + HK_G16_HOST_FABS, + HK_G16_HOST_MOV, +}; + +struct hk_g16_host_src { + uint8_t kind; /* enum hk_g16_host_src_kind */ + uint32_t value; /* temp index, immediate bits or root byte offset */ +}; + +struct hk_g16_host_op { + uint8_t opcode; /* enum hk_g16_host_opcode */ + struct hk_g16_host_src src[2]; +}; + +struct hk_g16_host_prog { + uint8_t n_ops, n_out; + uint8_t out[HK_G16_HOST_MAX_OUT]; + struct hk_g16_host_op ops[HK_G16_HOST_MAX_OPS]; +}; + struct hk_linked_shader { struct agx_linked_shader b; @@ -593,6 +634,9 @@ struct hk_linked_shader { * the descriptor root even when the application fragment shader does not. */ bool g17p_prolog_uses_root; + /* See struct hk_g16_host_prog; n_out == 0 when unused. */ + struct hk_g16_host_prog g16_host; + /* An inline-blend fragment program that forms its own constant program * (hk_g17p_inline_epilog_part()) has its own uniform layout: this shader * describes it, and the draw takes the program's launch (uniform ranges, diff --git a/src/compiler/nir/nir_lower_blend.c b/src/compiler/nir/nir_lower_blend.c index a5099116af4c..7f6e5a001f21 100644 --- a/src/compiler/nir/nir_lower_blend.c +++ b/src/compiler/nir/nir_lower_blend.c @@ -51,11 +51,11 @@ static nir_def * nir_blend_func( nir_builder *b, enum pipe_blend_func func, - nir_def *src, nir_def *dst) + nir_def *src, nir_def *dst, bool dst_first) { switch (func) { case PIPE_BLEND_ADD: - return nir_fadd(b, src, dst); + return dst_first ? nir_fadd(b, dst, src) : nir_fadd(b, src, dst); case PIPE_BLEND_SUBTRACT: return nir_fsub(b, src, dst); case PIPE_BLEND_REVERSE_SUBTRACT: @@ -1006,7 +1006,8 @@ nir_color_blend(nir_builder *b, nir_def *src, nir_def *src1, nir_def *dst, chan.dst_factor, rt->format); } - channels[c] = nir_blend_func(b, chan.func, psrc, pdst); + channels[c] = nir_blend_func(b, chan.func, psrc, pdst, + rt->dst_term_first); } return nir_vec(b, channels, 4); diff --git a/src/compiler/nir/nir_lower_blend.h b/src/compiler/nir/nir_lower_blend.h index bc979a79efc9..f5b865337a5f 100644 --- a/src/compiler/nir/nir_lower_blend.h +++ b/src/compiler/nir/nir_lower_blend.h @@ -53,6 +53,12 @@ typedef struct { bool src_premultiplied; bool dst_premultiplied; enum pipe_blend_overlap_mode overlap; + + /* Emit the destination term first in additive blending, so contraction + * fuses dst * dst_factor into the final multiply-add and the source + * product stays a separate multiply that does not depend on the + * destination. */ + bool dst_term_first; } nir_lower_blend_rt; typedef struct { diff --git a/src/gallium/drivers/asahi/agx_disk_cache.c b/src/gallium/drivers/asahi/agx_disk_cache.c index 660795aec160..0141fdc73e7c 100644 --- a/src/gallium/drivers/asahi/agx_disk_cache.c +++ b/src/gallium/drivers/asahi/agx_disk_cache.c @@ -237,6 +237,8 @@ agx_disk_cache_init(struct agx_screen *screen) _mesa_sha1_format(timestamp, id_sha1); uint64_t driver_flags = screen->dev.debug; + if (screen->dev.chip == AGX_CHIP_G16G && agx_g16_pix_hoist_enabled()) + driver_flags |= UINT64_C(1) << 61; screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags); #endif }