Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions src/asahi/compiler/agx_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,30 @@ agx_get_compiler_debug(void)
return debug_get_option_agx_compiler_debug();
}

bool
agx_g16_pix_hoist_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_PIX_HOIST", true);
}

bool
agx_g16_blend_dst_first_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_BLEND_DST_FIRST", true);
}

bool
agx_g16_host_blend_src_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_HOST_BLEND_SRC", true);
}

bool
agx_g16_border_twin_enabled(void)
{
return debug_get_bool_option("ASAHI_G16_BORDER_TWIN", true);
}

static __thread char agx_refusal[192];

const char *
Expand Down Expand Up @@ -8872,6 +8896,71 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx)
* adjacent run of non-coherent loads with identical operation and base; SSA
* keeps their consumers attached to the correct result and no serialization
* is introduced. */
/* G16G (ASAHI_G16_PIX_HOIST=0 opts out): after scheduling, move
* float arithmetic computed only from uniforms ahead of the first
* pixel-ordering wait of its block, so it executes outside the ordered colour
* transaction instead of between WAIT_PIX and the tile store. Only fadd, fmul,
* ffma and mov_imm move, and only when every source is a uniform, an
* immediate, or the result of another such instruction. Moved instructions
* keep their relative order and their exact operations, modifiers and
* operands, so results are bit-identical. */
static bool
agx3_g16_hoist_pixel_independent(agx_context *ctx)
{
if (ctx->key->dev.chip != AGX_CHIP_G16G ||
ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
!agx_g16_pix_hoist_enabled())
return false;

/* Values computed only from uniforms and immediates by the movable ops,
* wherever they are in the block. Texture, varying and tile results never
* qualify, so per-pixel work keeps its scheduled place. */
BITSET_WORD *uniform = BITSET_CALLOC(ctx->alloc);
if (!uniform)
return false;

bool progress = false;
agx_foreach_block(ctx, block) {
agx_instr *wait = NULL;
agx_foreach_instr_in_block_safe(block, I) {
if (!wait && I->op == AGX_OPCODE_WAIT_PIX) {
wait = I;
continue;
}

bool uniform_only = (I->op == AGX_OPCODE_FADD ||
I->op == AGX_OPCODE_FMUL ||
I->op == AGX_OPCODE_FFMA ||
I->op == AGX_OPCODE_MOV_IMM) &&
I->nr_dests == 1 &&
I->dest[0].type == AGX_INDEX_NORMAL;
agx_foreach_src(I, s) {
const agx_index src = I->src[s];
if (src.type == AGX_INDEX_NORMAL) {
if (!BITSET_TEST(uniform, src.value))
uniform_only = false;
} else if (src.type != AGX_INDEX_UNIFORM &&
src.type != AGX_INDEX_IMMEDIATE &&
src.type != AGX_INDEX_NULL) {
uniform_only = false;
}
}

if (!uniform_only)
continue;
BITSET_SET(uniform, I->dest[0].value);
if (wait) {
list_del(&I->link);
list_addtail(&I->link, &wait->link);
progress = true;
}
}
}

free(uniform);
return progress;
}

static bool
agx3_order_fragment_device_loads(agx_context *ctx)
{
Expand Down Expand Up @@ -10383,6 +10472,7 @@ agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl,
}

AGX_PASS(ctx, agx3_order_fragment_device_loads);
AGX_PASS(ctx, agx3_g16_hoist_pixel_independent);

if (agx_should_dump(nir, AGX_DBG_SHADERS))
agx_print_shader(ctx, stdout);
Expand Down
11 changes: 11 additions & 0 deletions src/asahi/compiler/agx_debug.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,17 @@ enum agx_compiler_dbg {

uint64_t agx_get_compiler_debug(void);

/* Qualified on G16G; ASAHI_G16_PIX_HOIST=0 restores the scheduler placement. */
bool agx_g16_pix_hoist_enabled(void);
/* Qualified on G16G; ASAHI_G16_BLEND_DST_FIRST=0 restores the source-first blend add. */
bool agx_g16_blend_dst_first_enabled(void);
/* Qualified on G16G; ASAHI_G16_HOST_BLEND_SRC=0 restores per-pixel evaluation
* of uniform-only inline-blend fragment arithmetic. */
bool agx_g16_host_blend_src_enabled(void);
/* Qualified on G16G; ASAHI_G16_BORDER_TWIN=0 keeps the custom-border emulation in
* every G16G texture sample instead of compiling border-free twins. */
bool agx_g16_border_twin_enabled(void);

/* Why the last agx_compile_shader_nir() on this thread returned false
* ("" when it succeeded), for the driver's "cannot encode" error. */
const char *agx_last_compile_refusal(void);
Expand Down
3 changes: 3 additions & 0 deletions src/asahi/lib/agx_linker.h
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,9 @@ struct agx_fs_epilog_key {
* the constants after the main shader's uniform footprint instead. */
uint16_t blend_constant_base;

/* Emit the blend destination term first (nir_lower_blend_rt::dst_term_first). */
bool blend_dst_first;

/* Render targets whose tile value must be stored back unchanged (colour
* write disabled while raster coverage still makes the tile nonempty).
* Independent of blending, so it also holds for pure-integer formats. */
Expand Down
1 change: 1 addition & 0 deletions src/asahi/lib/agx_nir_prolog_epilog.c
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)

.alpha.src_factor = blend.alpha_src_factor,
.alpha.dst_factor = blend.alpha_dst_factor,
.dst_term_first = key->blend_dst_first,
.alpha.func = blend.alpha_func,

.colormask = key->blend.rt[i].colormask,
Expand Down
15 changes: 14 additions & 1 deletion src/asahi/vulkan/hk_cmd_buffer.c
Original file line number Diff line number Diff line change
Expand Up @@ -2538,7 +2538,20 @@ hk_g17p_graphics_uniform_ranges(
.size_halfs = 4, .address = pointer,
};
}
if (hk_cmd_uses_blend_constant(cmd)) {
if (linked && linked->g16_host.n_out) {
/* A G16G inline-blend program with host-evaluated uniforms reads
* them in halves [0, 2 * n_out), which only the spill table and the
* blend constants otherwise use: it is built for neither (an inline
* program refuses epilogs reading blend constants). */
if (native_spills || !root_ptr)
return hk_g17p_graphics_ranges_refused(range_count, count);
sources[count] = HK_G17P_RANGE_SOURCE_ROOT;
ranges[count++] = (struct agx3_ioto_uniform_range){
.start_halfs = 0,
.size_halfs = 2 * linked->g16_host.n_out,
.address = root_ptr + hk_root_descriptor_offset(g16_host_unif),
};
} else if (hk_cmd_uses_blend_constant(cmd)) {
/* The non-monolithic blend epilog reads the four constants from
* the base selected by its linked key. It is half4 for the standard
* ABI and moves beyond the main shader footprint when HAL200 native
Expand Down
4 changes: 4 additions & 0 deletions src/asahi/vulkan/hk_cmd_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,10 @@ struct hk_root_descriptor_table {
* address of the word libagx_m5_task_dispatch writes (nonzero = execute);
* zero for every other draw. Appended after every agx3b field. */
uint64_t mesh_task_flag;

/* G16G host-evaluated fragment uniforms (hk_linked_shader::g16_host),
* published to uniform halves [0, 2 * n_out). */
uint32_t g16_host_unif[HK_G16_HOST_MAX_OUT];
};

/* The graphics arm is the larger of the two, so everything after the union --
Expand Down
Loading