Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions src/asahi/compiler/agx_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,18 @@ agx_get_compiler_debug(void)
return debug_get_option_agx_compiler_debug();
}

bool
agx_g16_pix_hoist_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_PIX_HOIST", true);
}

bool
agx_g16_blend_dst_first_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true);
}

static __thread char agx_refusal[192];

const char *
Expand Down Expand Up @@ -8872,6 +8884,71 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx)
* adjacent run of non-coherent loads with identical operation and base; SSA
* keeps their consumers attached to the correct result and no serialization
* is introduced. */
/* G16G (ASAHI_G16_PIX_HOIST=0 opts out): after scheduling, move
* float arithmetic computed only from uniforms ahead of the first
* pixel-ordering wait of its block, so it executes outside the ordered colour
* transaction instead of between WAIT_PIX and the tile store. Only fadd, fmul,
* ffma and mov_imm move, and only when every source is a uniform, an
* immediate, or the result of another such instruction. Moved instructions
* keep their relative order and their exact operations, modifiers and
* operands, so results are bit-identical. */
static bool
agx3_g16_hoist_pixel_independent(agx_context *ctx)
Comment on lines +8895 to +8896

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Instruction moves lack tests

agx3_g16_hoist_pixel_independent changes instruction order after scheduling, but no test covers its dependency checks or chip gate. Add cases showing a chain of uniform-only operations moving together before WAIT_PIX, a tile-dependent operation staying put, and unchanged order on other chips or with the opt-out. Hardware readbacks do not protect these rules from later compiler changes.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

{
if (ctx->key->dev.chip != AGX_CHIP_G16G ||
ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
!agx_g16_pix_hoist_enabled())
return false;

/* Values computed only from uniforms and immediates by the movable ops,
* wherever they are in the block. Texture, varying and tile results never
* qualify, so per-pixel work keeps its scheduled place. */
BITSET_WORD *uniform = BITSET_CALLOC(ctx->alloc);
if (!uniform)
return false;

bool progress = false;
agx_foreach_block(ctx, block) {
agx_instr *wait = NULL;
agx_foreach_instr_in_block_safe(block, I) {
if (!wait && I->op == AGX_OPCODE_WAIT_PIX) {
wait = I;
continue;
}

bool uniform_only = (I->op == AGX_OPCODE_FADD ||
I->op == AGX_OPCODE_FMUL ||
I->op == AGX_OPCODE_FFMA ||
I->op == AGX_OPCODE_MOV_IMM) &&
I->nr_dests == 1 &&
I->dest[0].type == AGX_INDEX_NORMAL;
agx_foreach_src(I, s) {
const agx_index src = I->src[s];
if (src.type == AGX_INDEX_NORMAL) {
if (!BITSET_TEST(uniform, src.value))
uniform_only = false;
} else if (src.type != AGX_INDEX_UNIFORM &&
src.type != AGX_INDEX_IMMEDIATE &&
src.type != AGX_INDEX_NULL) {
uniform_only = false;
}
}

if (!uniform_only)
continue;
BITSET_SET(uniform, I->dest[0].value);
if (wait) {
list_del(&I->link);
list_addtail(&I->link, &wait->link);
progress = true;
}
}
}

free(uniform);
return progress;
}

static bool
agx3_order_fragment_device_loads(agx_context *ctx)
{
Expand Down Expand Up @@ -10383,6 +10460,7 @@ agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl,
}

AGX_PASS(ctx, agx3_order_fragment_device_loads);
AGX_PASS(ctx, agx3_g16_hoist_pixel_independent);

if (agx_should_dump(nir, AGX_DBG_SHADERS))
agx_print_shader(ctx, stdout);
Expand Down
5 changes: 5 additions & 0 deletions src/asahi/compiler/agx_debug.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ enum agx_compiler_dbg {

uint64_t agx_get_compiler_debug(void);

/* Qualified on G16G; ASAHI_G16_PIX_HOIST=0 restores the scheduler placement. */
bool agx_g16_pix_hoist_enabled(void);
/* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */
bool agx_g16_blend_dst_first_enabled(void);

/* Why the last agx_compile_shader_nir() on this thread returned false
* ("" when it succeeded), for the driver's "cannot encode" error. */
const char *agx_last_compile_refusal(void);
Expand Down
3 changes: 3 additions & 0 deletions src/asahi/lib/agx_linker.h
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,9 @@ struct agx_fs_epilog_key {
* the constants after the main shader's uniform footprint instead. */
uint16_t blend_constant_base;

/* Emit the blend destination term first (nir_lower_blend_rt::dst_term_first). */
bool blend_dst_first;

/* Render targets whose tile value must be stored back unchanged (colour
* write disabled while raster coverage still makes the tile nonempty).
* Independent of blending, so it also holds for pure-integer formats. */
Expand Down
1 change: 1 addition & 0 deletions src/asahi/lib/agx_nir_prolog_epilog.c
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)

.alpha.src_factor = blend.alpha_src_factor,
.alpha.dst_factor = blend.alpha_dst_factor,
.dst_term_first = key->blend_dst_first,
.alpha.func = blend.alpha_func,

.colormask = key->blend.rt[i].colormask,
Expand Down
2 changes: 2 additions & 0 deletions src/asahi/vulkan/hk_cmd_draw.c
Original file line number Diff line number Diff line change
Expand Up @@ -9982,6 +9982,8 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
key->epilog.link.already_ran_zs |= nontrivial_force_early;
key->epilog.native_imageblock = in->tib->native_imageblock;
key->epilog.max_sample_size_B = in->tib->max_sample_size_B;
key->epilog.blend_dst_first =
dev->dev.chip == AGX_CHIP_G16G && agx_g16_blend_dst_first_enabled();
key->epilog.resident_mask = in->tib->resident_mask;
key->epilog.max_bytes_per_tile = in->tib->max_bytes_per_tile;
key->epilog.max_samples_per_tile = in->tib->max_samples_per_tile;
Expand Down
3 changes: 2 additions & 1 deletion src/asahi/vulkan/hk_shader.c
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,8 @@ hk_physical_device_compiler_flags(const struct hk_physical_device *pdev)
{
/* This could be optimized but it doesn't matter. */
return pdev->dev.debug |
hk_fs_structure_key_bits();
hk_fs_structure_key_bits() |
((uint64_t)!!agx_g16_pix_hoist_enabled() << 33);
}

const nir_shader_compiler_options *
Expand Down
7 changes: 4 additions & 3 deletions src/compiler/nir/nir_lower_blend.c
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@ static nir_def *
nir_blend_func(
nir_builder *b,
enum pipe_blend_func func,
nir_def *src, nir_def *dst)
nir_def *src, nir_def *dst, bool dst_first)
{
switch (func) {
case PIPE_BLEND_ADD:
return nir_fadd(b, src, dst);
return dst_first ? nir_fadd(b, dst, src) : nir_fadd(b, src, dst);
case PIPE_BLEND_SUBTRACT:
return nir_fsub(b, src, dst);
case PIPE_BLEND_REVERSE_SUBTRACT:
Expand Down Expand Up @@ -1006,7 +1006,8 @@ nir_color_blend(nir_builder *b, nir_def *src, nir_def *src1, nir_def *dst,
chan.dst_factor, rt->format);
}

channels[c] = nir_blend_func(b, chan.func, psrc, pdst);
channels[c] = nir_blend_func(b, chan.func, psrc, pdst,
rt->dst_term_first);
}

return nir_vec(b, channels, 4);
Expand Down
6 changes: 6 additions & 0 deletions src/compiler/nir/nir_lower_blend.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ typedef struct {
bool src_premultiplied;
bool dst_premultiplied;
enum pipe_blend_overlap_mode overlap;

/* Emit the destination term first in additive blending, so contraction
* fuses dst * dst_factor into the final multiply-add and the source
* product stays a separate multiply that does not depend on the
* destination. */
bool dst_term_first;
} nir_lower_blend_rt;

typedef struct {
Expand Down
2 changes: 2 additions & 0 deletions src/gallium/drivers/asahi/agx_disk_cache.c
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,8 @@ agx_disk_cache_init(struct agx_screen *screen)
_mesa_sha1_format(timestamp, id_sha1);

uint64_t driver_flags = screen->dev.debug;
if (screen->dev.chip == AGX_CHIP_G16G && agx_g16_pix_hoist_enabled())
driver_flags |= UINT64_C(1) << 61;
screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
#endif
}