diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index bcb0bcba00e..fa88fde874e 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -883,6 +883,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state) case nir_intrinsic_load_texture_scale: case nir_intrinsic_load_inline_data_intel: case nir_intrinsic_resource_intel: + case nir_intrinsic_load_smp_meta_pco: case nir_intrinsic_load_push_constant: { unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs; for (unsigned i = 0; i < num_srcs; i++) { diff --git a/src/imagination/g5p/g5p_texel_defs.h b/src/imagination/g5p/g5p_texel_defs.h index c68680b2b73..c9aac2f5d95 100644 --- a/src/imagination/g5p/g5p_texel_defs.h +++ b/src/imagination/g5p/g5p_texel_defs.h @@ -13,10 +13,16 @@ #define PVR_G5P_TBO_LIMIT 65536u #define PVR_G5P_TBO_MAGIC 0x54424f31u /* software metadata, never a GPU field */ #define PVR_G5P_TBO_NIR_FOLDED (1u << 31) -/* Texel buffers have not run on the device: not advertised */ +/* Uniform texel buffers in fragment and vertex programs. This path (a + * linear 4096-wide 2D state over the buffer, the NIR fold of the index, the + * range in software metadata) gave exact pixels on the A9 for 30 formats, + * 1 to 65536 elements, offsets 64 and 128, the size query and out-of-range + * reads, in fragment and vertex programs. The fetch instruction is the + * general texture encoder's. Storage texel buffers and compute fetches stay + * refused by name. */ static inline bool pvr_g5p_tbo_probe(void) { - return false; + return true; } #endif diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index ef9c7c6c372..9431b87526d 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -1720,15 +1720,19 @@ unsigned g5p_usc_position_xy(uint8_t *out, unsigned dst_half, unsigned words, un return finish(out, raw, source_half ? 6 : 4); } -unsigned g5p_usc_position_zw(uint8_t *out, unsigned dst_half, bool z, bool both, int long_form) +/* The screen-linear position iterator: one coefficient set (set 0, or set 1 + * with set1), or sets 0 and 1 together (both). The last byte is the set << 1, + * as in the varying iterator: Apple puts W at set 0 and Z at set 1, or Z at + * set 0 when no W is iterated (Apple's compiles f_raster_pos_z/w). */ +unsigned g5p_usc_position_zw(uint8_t *out, unsigned dst_half, bool set1, bool both, int long_form) { - CHECK(!(both && z), "invalid position iterator"); + CHECK(!(both && set1), "invalid position iterator"); CHECK(!(dst_half & 1), "position iterator needs a dword-aligned destination"); CHECK(dst_half < 256, "position destination must be u8"); - bool minimum = dst_half || z || both; + bool minimum = dst_half || set1 || both; bool wide = long_form < 0 ? minimum : long_form; CHECK(!(minimum && !wide), "short position iterator omits nonzero fields"); - uint8_t raw[4] = {8, (both ? 0x10 : 8) | wide, dst_half, z ? 2 : 0}; + uint8_t raw[4] = {8, (both ? 0x10 : 8) | wide, dst_half, set1 ? 2 : 0}; return finish(out, raw, wide ? 4 : 2); } diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index 8c05bb9e149..628698d7d81 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -350,7 +350,7 @@ unsigned g5p_usc_cond_select(uint8_t *out, unsigned dst_half, struct g5p_src con struct g5p_src f, unsigned bank, unsigned source_extension, unsigned header); /* Raster position, tile access and native-f16 blend arithmetic. */ unsigned g5p_usc_position_xy(uint8_t *out, unsigned dst_half, unsigned words, unsigned source_half); -unsigned g5p_usc_position_zw(uint8_t *out, unsigned dst_half, bool z, bool both, int long_form); +unsigned g5p_usc_position_zw(uint8_t *out, unsigned dst_half, bool set1, bool both, int long_form); unsigned g5p_usc_tile_read(uint8_t *out, unsigned dst_half, unsigned words); unsigned g5p_usc_half_binary(uint8_t *out, bool mul, unsigned dst_half, struct g5p_src a, struct g5p_src b, struct g5p_ctl k, int source_control); diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 8a0d74fca9c..7436e270c17 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -62,6 +62,10 @@ * buffer starts at CODE_INITIAL and doubles. */ #define CODE_INITIAL 16384 #define NUM_DWORDS 32 +/* Fragment registers above r31 (homes): r32..r127, the destination field's reach */ +#define G5P_MAX_HOMES 96 +/* home_reuse: values parked above r31 over a program's length (each attempt is one more compile) */ +#define G5P_MAX_VICTIMS 1024 /* Vertex input dwords: the PDS fetches up to this many into bank 1. Those above r31 are read through a move * (g5p_usc_vs_or_wide(), Apple's vertex form for attributes above r31), as the fragment homes are. */ #define VS_INPUT_DWORDS 64 @@ -82,6 +86,7 @@ struct val { int last; /* last reading instruction (-1: unread) */ unsigned uses; /* consumer references */ bool body; /* defined by an instruction of the body: may get a home */ + bool decl; /* a NIR register (out of SSA): may get a home in the home_reuse retry */ }; struct ref { @@ -300,6 +305,14 @@ struct gctx { * codegen_reset(). */ int8_t *home; unsigned nhomes; + /* The last fragment retry (home_reuse): a value marked in home[] gets the + * first register above r31 that is free when it is defined, and frees it + * at its last read, as r0..r31 are allocated. Kept by codegen_reset(); + * hbusy and hmax are per codegen pass. */ + bool home_reuse; + uint64_t hbusy[2]; /* bit i: r(32 + i) holds a live value */ + unsigned hmax; /* registers above r31 used: r32..r(31 + hmax) */ + unsigned need; /* dwords the last register-pressure refusal asked for */ }; static _Noreturn void PRINTFLIKE(2, 3) refuse(struct gctx *c, const char *fmt, ...) @@ -465,6 +478,7 @@ static unsigned alloc_run(struct gctx *c, unsigned n, const char *why) tex_flush(c); return alloc_run(c, n, why); } + c->need = n; refuse(c, "register pressure: %s needs %u dword(s), %u of %u in use", why, n, util_bitcount64((c->busy | c->reserved | c->tex_coord_pending | c->tex_pending) & BITFIELD64_MASK(NUM_DWORDS)), NUM_DWORDS); } @@ -549,9 +563,19 @@ static void free_scratch(struct gctx *c) static void release(struct gctx *c, int idx) { - for (unsigned i = 0; i < c->nvals; i++) - if (c->vals[i].held && c->vals[i].last <= idx) - drop(c, &c->vals[i]); + for (unsigned i = 0; i < c->nvals; i++) { + struct val *v = &c->vals[i]; + if (v->held && v->last <= idx) { + drop(c, v); + } else if (c->home_reuse && v->state == VAL_HOME && v->last <= idx) { + /* past its last read (a loop's end for a value read in a loop it is defined outside of, mark()): the + * register above r31 takes a later value. The value has no register any more: a read now would be + * refused by name (reg_of()), never served from a register another value took over. */ + assert(!c->vs && v->reg >= 32 && v->reg < 32 + G5P_MAX_HOMES); + c->hbusy[(v->reg - 32) / 64] &= ~BITFIELD64_BIT((v->reg - 32) % 64); + v->state = VAL_NONE; + } + } free_scratch(c); } @@ -1168,22 +1192,14 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * ok = !c->vs; break; case nir_intrinsic_load_frag_coord: + /* x/y: the raster position; z: the TSP's unclamped Z (plan_varyings()); + * w: the RHW. The backend binds components, which only ALU reads use. */ ok = !c->vs; - nir_foreach_use(use, &in->def) { - nir_instr *ui = nir_src_use_instr(use); - if (ui->type == nir_instr_type_alu) { - nir_alu_instr *ua = nir_instr_as_alu(ui); - for (unsigned k = 0; k < nir_op_infos[ua->op].num_inputs; k++) - if (&ua->src[k].src == use) - for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(ua, k); j++) - if (ua->src[k].swizzle[j] == 2) - ok = false; - } else { + nir_foreach_use(use, &in->def) + if (nir_src_use_instr(use)->type != nir_instr_type_alu) ok = false; - } - } if (!ok && !c->vs) { - note(u, "load_frag_coord .z (Z coefficient layout is not qualified)"); + note(u, "load_frag_coord read by a non-ALU instruction"); continue; } break; @@ -1194,6 +1210,10 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * case nir_intrinsic_load_ubo: ok = true; break; + case nir_intrinsic_load_smp_meta_pco: + /* the LOD-bias variant's sampler bias (fragment only) */ + ok = !c->vs && in->def.bit_size == 32 && nir_src_is_const(in->src[0]); + break; case nir_intrinsic_get_ssbo_size: ok = in->def.bit_size == 32 && in->def.num_components == 1; break; @@ -1550,14 +1570,16 @@ static void plan_varyings(struct gctx *c) uint8_t interp[PCO_G5P_VARYING_SLOTS] = {0}, mask[PCO_G5P_VARYING_SLOTS] = {0}; bool centroid[PCO_G5P_VARYING_SLOTS] = {0}; uint32_t read = 0; - bool position_w = false; + bool position_w = false, position_z = false; memset(c->vary_idx, -1, sizeof(c->vary_idx)); foreach_instr_all(ins, c) { if (ins->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *in = nir_instr_as_intrinsic(ins); - if (in->intrinsic == nir_intrinsic_load_frag_coord) + if (in->intrinsic == nir_intrinsic_load_frag_coord) { position_w |= nir_def_components_read(&in->def) & BITFIELD_BIT(3); + position_z |= nir_def_components_read(&in->def) & BITFIELD_BIT(2); + } if (in->intrinsic != nir_intrinsic_load_interpolated_input && in->intrinsic != nir_intrinsic_load_input) continue; unsigned loc = nir_intrinsic_io_semantics(in).location; @@ -1590,17 +1612,22 @@ static void plan_varyings(struct gctx *c) * only after this plan. Reading V(...)->uses here left W-only programs * without a coefficient DOUTI despite emitting the native W iterator. */ bool w = position_w || smooth; + /* Apple's A9 coefficient layout (Apple's compiles f_raster_pos_z/w): + * W at set 0 when present, then position Z when the program reads it, then the read + * locations, 4 sets each. Z without W is set 0 (f_z_only, f_z_f, f_z_n). */ + const unsigned base = (w ? 1 : 0) + (position_z ? 1 : 0); /* the last location's last component must fit the u7 iterator slot */ - if ((w ? 1 : 0) + 4 * util_bitcount(read) > 128) - refuse(c, "%u varying locations with W exceed the u7 iterator slot", util_bitcount(read)); + if (base + 4 * util_bitcount(read) > 128) + refuse(c, "%u varying locations with W/Z exceed the u7 iterator slot", util_bitcount(read)); unsigned i = 0; u_foreach_bit(n, read) { - c->p->vary[i] = (struct pco_g5p_varying){.location = n, .set = (w ? 1 : 0) + 4 * i, .interp = interp[n], + c->p->vary[i] = (struct pco_g5p_varying){.location = n, .set = base + 4 * i, .interp = interp[n], .centroid = centroid[n], .mask = mask[n]}; c->vary_idx[n] = i++; } c->p->num_varyings = i; c->p->w_set = w; + c->p->z_set = position_z; } static const struct pco_g5p_varying *vary_of(struct gctx *c, nir_intrinsic_instr *in) @@ -2059,7 +2086,8 @@ static void tex_bindings(struct gctx *c, nir_tex_instr *t, struct tex_bind *b) sv->data.binding, element, selement}; b->needs_sampler = t->op != nir_texop_txf; b->needs_fetch_sampler = t->op == nir_texop_txf; - b->image_flags = (t->backend_flags & G5P_TEX_LAYER0_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_AS_2D : + b->image_flags = (t->backend_flags & PCO_G5P_TEX_ONE_LAYER_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_ONE_LAYER : + (t->backend_flags & G5P_TEX_LAYER0_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_AS_2D : t->is_array ? PCO_G5P_UPLOAD_IMAGE_ARRAY : 0; } @@ -2084,6 +2112,35 @@ static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) return &c->tex[c->ntex++]; } +/* A sample that uses its sampler's LOD (pco_g5p_program::lod_samplers): the draw selects the LOD-bias + * variant when such a sampler's mipLodBias is not zero, and refuses by name where no program adds it. */ +static void record_lod_sampler(struct gctx *c, nir_tex_instr *t) +{ + if (t->op != nir_texop_tex && t->op != nir_texop_txb && t->op != nir_texop_txl && t->op != nir_texop_txd) + return; + if (t->sampler_dim == GLSL_SAMPLER_DIM_BUF) + return; + struct tex_bind b; + tex_bindings(c, t, &b); + const bool biased = (t->backend_flags & PCO_G5P_TEX_SAMPLER_BIAS) != 0; + struct pco_g5p_program *p = c->p; + for (unsigned i = 0; i < p->num_lod_samplers; i++) { + struct pco_g5p_lod_sampler *l = &p->lod_samplers[i]; + if (l->set == b.sset && l->binding == b.sbinding && l->element == b.selement) { + /* one unbiased sample of the sampler leaves it unbiased */ + l->biased &= biased; + return; + } + } + if (p->num_lod_samplers == PCO_G5P_MAX_LOD_SAMPLERS || b.sset > UINT8_MAX || b.sbinding > UINT8_MAX || + b.selement > UINT16_MAX) { + p->lod_samplers_overflow = true; + return; + } + p->lod_samplers[p->num_lod_samplers++] = + (struct pco_g5p_lod_sampler){ (uint8_t)b.sset, (uint8_t)b.sbinding, (uint16_t)b.selement, biased }; +} + /* textypes: the size block of a texture binding at a constant level. */ static unsigned size_level(nir_tex_instr *t) { @@ -2126,10 +2183,12 @@ static void plan_common(struct gctx *c) foreach_instr_all(ins, c) { if (ins->type == nir_instr_type_tex) { nir_tex_instr *t = nir_instr_as_tex(ins); - if (size_query(t)) + if (size_query(t)) { size_for(c, t); - else + } else { tex_for(c, t); + record_lod_sampler(c, t); + } continue; } if (ins->type != nir_instr_type_intrinsic) @@ -2144,7 +2203,16 @@ static void plan_common(struct gctx *c) if (xfb_address(in->src[1].ssa, &b, &off, &k)) c->p->xfb_buffers |= 1u << b; } - if (in->intrinsic == nir_intrinsic_load_push_constant) { + if (in->intrinsic == nir_intrinsic_load_smp_meta_pco) { + /* the LOD-bias variant's per-binding bias: a sampler metadata dword, preloaded as a uniform */ + if (!nir_src_is_const(in->src[0])) + refuse(c, "load_smp_meta_pco: a non-constant array element"); + struct range *r = range_for(c, PCO_G5P_UPLOAD_SAMPLER_META, nir_intrinsic_desc_set(in), + nir_intrinsic_binding(in), nir_src_as_uint(in->src[0])); + const unsigned off = 4 * nir_intrinsic_component(in); + r->lo = MIN2(r->lo, off); + r->hi = MAX2(r->hi, off + 4 * in->def.num_components); + } else if (in->intrinsic == nir_intrinsic_load_push_constant) { unsigned off = load_offset(c, in, 0, nir_intrinsic_base(in)); struct range *r = range_for(c, PCO_G5P_UPLOAD_PUSH, 0, 0, 0); r->lo = MIN2(r->lo, off); @@ -2379,7 +2447,18 @@ static void plan_common(struct gctx *c) * for r32..r69 (control-flow.md ยง2.3). Apple allocates r32 and up for * every form; this keeps every other encoder on r0..r31. */ #define G5P_OP_OR 0xce -#define G5P_MAX_HOMES 96 + +/* home_reuse: the first register above r31 no live value holds */ +static unsigned alloc_home(struct gctx *c) +{ + for (unsigned i = 0; i < G5P_MAX_HOMES; i++) + if (!(c->hbusy[i / 64] & BITFIELD64_BIT(i % 64))) { + c->hbusy[i / 64] |= BITFIELD64_BIT(i % 64); + c->hmax = MAX2(c->hmax, i + 1); + return 32 + i; + } + refuse(c, "register pressure: all %u registers above r31 hold live values", G5P_MAX_HOMES); +} static void park(struct gctx *c, nir_instr *ins) { @@ -2394,6 +2473,8 @@ static void park(struct gctx *c, nir_instr *ins) int h = c->home ? c->home[d->index * 4 + k] : -1; if (h < 0) continue; + if (c->home_reuse) + h = alloc_home(c); if (c->tex_pending & BITFIELD64_BIT(v->reg)) tex_flush(c); uint8_t raw[G5P_USC_MAX]; @@ -2412,7 +2493,8 @@ static int pick_home(struct gctx *c) int best = -1; for (unsigned i = 0; i < c->nvals; i++) { struct val *v = &c->vals[i]; - if (!v->held || v->state != VAL_REG || !v->body || v->last <= c->now || c->home[i] >= 0 || + if (!v->held || v->state != VAL_REG || !(v->body || (c->home_reuse && v->decl)) || v->last <= c->now || + c->home[i] >= 0 || (c->reserved & BITFIELD64_BIT(v->reg))) continue; if (best < 0 || v->last > c->vals[best].last) @@ -3919,7 +4001,7 @@ static void vs_bitwise(struct gctx *c, nir_alu_instr *a) * bank 1, header 0x31 (m_imin_rr, m_isub_rr); a literal in A with header 0x11 * (m_imin_lit, m_imax_neg, m_isub_lit, m_isub_big; ineg is 0 - x, m_ineg); * a common-store operand in A (m_imin_u; m_isub_u is u - x, m_isub_u2 x - u - * by routing 0x70). The fragment form (bank 0, header 1) in a vertex program + * by routing 0x70), with either register in bank 0. The fragment form (bank 0, header 1) in a vertex program * reads other registers (the vertex shift, device run 1). x - k is an iadd of * -k (Apple's v_iadd_lit form). */ static void vs_minsub(struct gctx *c, nir_alu_instr *a) @@ -3969,7 +4051,11 @@ static void vs_minsub(struct gctx *c, nir_alu_instr *a) unsigned xr = reg_of(c, x, name); unsigned g = vgrp(c, x); unsigned dst = vs_dst(c, &x, 1); - vs_bank1_only(c, sub ? "vertex shared isub" : "vertex shared min/max", dst, xr, -1, -1); + /* Either register may be in bank 0: the instruction's bank is the + * destination's and header bit 0x10 marks B in bank 1, the same + * fields as the register forms. Apple's compiler emits this form with + * every destination and B bank pair (eight values clamped by eight + * uniforms under attribute pressure), byte-equal to these encoders. */ unsigned n = sub ? g5p_usc_alu_shared(raw, G5P_BIN_ISUB, H(c, dst), sd, H(c, xr), vbank(c, dst), g, vhdr(c, 0x11, -1, xr, -1), 0, 0, false, i ? 0x70 : -1, 1) @@ -4661,6 +4747,17 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) } break; } + case nir_intrinsic_load_smp_meta_pco: { + /* a preloaded sampler metadata dword (plan_common()) */ + struct range *r = range_for(c, PCO_G5P_UPLOAD_SAMPLER_META, nir_intrinsic_desc_set(in), + nir_intrinsic_binding(in), nir_src_as_uint(in->src[0])); + for (unsigned k = 0; k < in->def.num_components; k++) { + struct val *v = V(c, &in->def, k); + v->state = VAL_SHARED; + v->imm = r->common + (4 * nir_intrinsic_component(in) - r->lo) / 4 + k; + } + break; + } case nir_intrinsic_get_ssbo_size: { unsigned set, binding, element; if (!block_binding_element(in->src[0].ssa, &set, &binding, &element)) @@ -4703,6 +4800,16 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) case nir_intrinsic_load_reg: { /* a copy: nir_trivialize_registers adds movs, which are register maps here */ nir_def *decl = in->src[0].ssa; struct val *dv = V(c, decl, 0); + if (dv->state == VAL_HOME && !c->vs) { + /* home_reuse: the register's home above r31, G9P's move back (park()) */ + if (!V(c, &in->def, 0)->uses) + break; + unsigned dst = alloc_reg(c, -1); + put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, dst, dv->reg, 0, dv->reg, 0), + "register read from a home above r31"); + define(c, &in->def, dst); + break; + } if (!dv->held) refuse(c, "load_reg: NIR register without a register"); if (!V(c, &in->def, 0)->uses) @@ -4719,6 +4826,17 @@ static void emit_intrinsic(struct gctx *c, nir_intrinsic_instr *in) } case nir_intrinsic_store_reg: { struct val *dv = V(c, in->src[1].ssa, 0); + if (dv->state == VAL_HOME && !c->vs) { + /* home_reuse: the value into the register's home above r31 (park()'s move). The read is counted + * (vgrp()) so no earlier read of the value takes the last-read group; this one keeps group 0. */ + struct ref r = resolve_value(c, in->src[0].ssa, 0); + unsigned src = reg_of(c, r, "store_reg"); + (void)vgrp(c, r); + put(c, raw, g5p_usc_alu2_wide(raw, G5P_OP_OR, dv->reg, src, 0, src, 0), + "register write to a home above r31"); + free_scratch(c); + break; + } if (!dv->held) refuse(c, "store_reg: NIR register without a register"); struct ref r = resolve_value(c, in->src[0].ssa, 0); @@ -5449,15 +5567,20 @@ static void hoist_iterators(struct gctx *c) fc = nir_instr_as_intrinsic(ins); bool fc_used = fc && (V(c, &fc->def, 0)->uses || V(c, &fc->def, 1)->uses); bool fc_w = fc && V(c, &fc->def, 3)->uses; - if (fc_w) { - /* f_raster_pos_w: native position W reads coefficient set zero, - * followed by WDF. plan_varyings reserves W even for a shader with - * no smooth user input. Z has a different layout and stays refused. - * The token has no counter-1 form, so issue it first. */ + bool fc_z = fc && V(c, &fc->def, 2)->uses; + /* Apple's compiles f_raster_pos_w / f_raster_pos_z: the + * native position iterator reads one screen-linear coefficient set: W at + * set 0, Z at the set after it (set 0 without W; plan_varyings()). The + * token has no counter-1 form, so these are issued first. */ + for (unsigned k = 3; k >= 2; k--) { + if (!(k == 3 ? fc_w : fc_z)) + continue; + assert(k == 3 ? c->p->w_set : c->p->z_set); + const bool set1 = k == 2 && c->p->w_set; drc_take(c, &d); unsigned r = alloc_reg(c, -1); - put(c, raw, g5p_usc_position_zw(raw, 2 * r, false, false, -1), "raster position W"); - struct val *v = V(c, &fc->def, 3); + put(c, raw, g5p_usc_position_zw(raw, 2 * r, set1, false, -1), k == 3 ? "raster position W" : "raster position Z"); + struct val *v = V(c, &fc->def, k); v->state = VAL_REG; v->reg = r; v->held = true; @@ -5470,7 +5593,7 @@ static void hoist_iterators(struct gctx *c) reads += varying_reads(c, nir_instr_as_intrinsic(ins)); /* The raster position read has no witnessed counter-1 form: it goes * first when the batch spills onto counter 1, else last (round 18). */ - bool fc_first = fc_used && reads + 1 + fc_w > 8; + bool fc_first = fc_used && reads + 1 + fc_w + fc_z > 8; if (fc_first) { drc_take(c, &d); c->fragcoord = alloc_run(c, 2, "gl_FragCoord"); @@ -7130,6 +7253,7 @@ static bool speculable(nir_instr *i) case nir_intrinsic_store_reg: case nir_intrinsic_load_push_constant: case nir_intrinsic_load_ubo: + case nir_intrinsic_load_smp_meta_pco: case nir_intrinsic_load_frag_coord: return true; default: @@ -7313,6 +7437,15 @@ static void decl_alloc(struct gctx *c, int ip) if (c->decl_held[i] || c->decl_first[i] > ip) continue; c->decl_held[i] = true; + v->decl = true; + if (c->home_reuse && c->home[c->decls[i]->index * 4] >= 0) { + /* home_reuse: a loop-carried register lives above r31 from its first + * access to its last (released by release()); each load_reg moves it + * into a register, each store_reg moves the value there. */ + v->state = VAL_HOME; + v->reg = alloc_home(c); + continue; + } v->state = VAL_REG; const bool out = c->alloc_out; c->alloc_out = c->vs && c->vs_bank0 && c->decl_out[i]; @@ -7679,6 +7812,8 @@ static bool codegen(struct gctx *c) c->vals[i].last = -1; for (unsigned i = 0; i < c->impl->ssa_alloc; i++) c->rh_half[i] = -1; + c->hbusy[0] = c->hbusy[1] = 0; + c->hmax = 0; if (c->ndecls) memset(c->decl_held, 0, c->ndecls * sizeof(*c->decl_held)); c->stack_used = false; @@ -7758,8 +7893,8 @@ static bool codegen(struct gctx *c) c->max_dword = MAX2(c->max_dword, c->st + 1); out->temps = c->vs && c->vs_bank0 ? MAX2(8, align(c->max_dword, 4)) : MAX3(8, align(c->max_dword, 4), align(c->max_bank0, 4)); - if (c->nhomes) { - out->temps = MAX2(out->temps, align(32 + c->nhomes, 4)); + if (c->home_reuse ? c->hmax : c->nhomes) { + out->temps = MAX2(out->temps, align(32 + (c->home_reuse ? c->hmax : c->nhomes), 4)); out->evidence |= PCO_G5P_EV_HIGH_REGS; } out->written_temps = c->vs ? (c->vs_bank0 ? MAX2(8, align(c->max_written, 4)) : @@ -8250,7 +8385,7 @@ bool pco_g5p_compile_general_key(nir_shader *nir, bool half_output, const uint8_ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_t *attrib_components, const struct pco_g5p_fs_key *key, const struct pco_g5p_vs_link *link, - struct pco_g5p_program *out, bool preserve_cf, bool evict) + struct pco_g5p_program *out, bool preserve_cf, bool evict, bool last) { static const struct pco_g5p_fs_key default_key = {.write_mask = 0xf}; memset(out, 0, sizeof(*out)); @@ -8349,6 +8484,15 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ out->evidence |= PCO_G5P_EV_CF; if (preserve_cf) NIR_PASS(_, nir, nir_lower_all_phis_to_scalar); + /* The last fragment attempt: a phi whose value is another phi's (an + * if/else chain's result passed out level by level) reads it through + * the scalarized vec4 and its movs, so out-of-SSA gives every level its + * own register, all held across the whole chain. Copy propagation makes + * the outer phi read the inner one, and they become one register web. */ + if (last && c->cf && !c->vs) { + NIR_PASS(_, nir, nir_opt_copy_prop); + NIR_PASS(_, nir, nir_opt_dce); + } cf_out_of_ssa(nir); /* Vertex control flow (Apple's v_loop, v_brk, v_ifelse, v_nested): the * loop state, the compares and the mask stack in bank 0, the values of @@ -8467,6 +8611,41 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ ok = two_pass(c); goto homes_retry; } + /* Last fragment retry, only for a program every step above refused: the + * homes as an allocator. G9P's homes give each parked value its own + * register above r31 for the whole program, so a program that parks more + * than 96 values over its length runs out (Dolphin's output resampling: + * four 16-tap bicubic paths, each 16 vec4 samples live at once). Here a + * register above r31 is taken when its value is defined and free again + * after the value's last read (release()), as Apple's A9 programs use + * r32..r69 as ordinary temporaries. Each attempt + * parks as many more values as the failing allocation asked dwords for. + * The moves are the same G9P OR moves; only the register numbers are + * reused, and the temps count the highest register used. */ + if (!ok && last && !c->vs && strstr(out->refusal, "register pressure")) { + memset(c->home, -1, c->impl->ssa_alloc * 4); + c->nhomes = 0; + c->home_reuse = c->remat = c->lazy_iter = true; + c->need = 1; + codegen_reset(c); + ok = two_pass(c); + while (!ok && strstr(out->refusal, "register pressure") && !strstr(out->refusal, "above r31") && + c->nhomes < G5P_MAX_VICTIMS) { + unsigned picked = 0; + for (; picked < MAX2(c->need, 1); picked++) { + const int victim = pick_home(c); + if (victim < 0) + break; + c->home[victim] = 1; /* home_reuse: a flag; park() picks the register */ + c->nhomes++; + } + if (!picked) + break; + c->need = 1; + codegen_reset(c); + ok = two_pass(c); + } + } for (unsigned split = 0; !ok && c->vs && !c->cf && split < 16 && strstr(out->refusal, "register pressure") && !impl_has_side_effects(c->impl); split++) { @@ -8519,18 +8698,30 @@ bool pco_g5p_compile_general_link(nir_shader *nir, bool half_output, const uint8 * from its own order, before the scheduling retries reordered it, with the * attributes moved out of bank 1 where a bank-1-only form needs room * (evict_vertex_bank1()). Programs that compile today never get there. */ - nir_shader *original = (nir->info.stage == MESA_SHADER_FRAGMENT && cf_left(nir)) || - nir->info.stage == MESA_SHADER_VERTEX - ? nir_shader_clone(NULL, nir) : NULL; - bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false, false); + /* A fragment program every attempt below refused for pressure is compiled + * a last time (compile_general_link()'s last): its own control flow with + * the if/else chains' phi webs coalesced, and the registers above r31 + * reused after each value's last read (home_reuse). Without control flow + * left the first attempt is the last. Programs that compile today never + * get there, so their bytes are unchanged. */ + const bool fs = nir->info.stage == MESA_SHADER_FRAGMENT; + const bool fs_cf = fs && cf_left(nir); + nir_shader *original = fs_cf || nir->info.stage == MESA_SHADER_VERTEX ? nir_shader_clone(NULL, nir) : NULL; + bool ok = compile_general_link(nir, half_output, attrib_components, key, link, out, false, false, fs && !fs_cf); + nir_shader *final = NULL; + if (!ok && fs_cf && strstr(out->refusal, "register pressure")) + final = nir_shader_clone(NULL, original); if (!ok && original && strstr(out->refusal, "register pressure")) { char first[sizeof(out->refusal)]; memcpy(first, out->refusal, sizeof(first)); const bool vs = nir->info.stage == MESA_SHADER_VERTEX; - ok = compile_general_link(original, half_output, attrib_components, key, link, out, !vs, vs); + ok = compile_general_link(original, half_output, attrib_components, key, link, out, !vs, vs, false); if (!ok && vs) /* the ladder's own reason stays the one reported */ memcpy(out->refusal, first, sizeof(first)); } + if (!ok && final && strstr(out->refusal, "register pressure")) + ok = compile_general_link(final, half_output, attrib_components, key, link, out, true, false, true); + ralloc_free(final); ralloc_free(original); return ok; } diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 284ab420ab6..867c30dfc81 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -152,7 +152,15 @@ enum pco_g5p_upload_kind { PCO_G5P_UPLOAD_XFB, /* per-draw driver block, PCO_G5P_XFB_* dwords */ PCO_G5P_UPLOAD_BLEND_CONSTANTS, /* 4 dwords: the dynamic blend constants (f32) */ PCO_G5P_UPLOAD_FETCH_SAMPLER, /* two generated samplerless-read policy words */ + /* sampler metadata dwords [offset / 4, offset / 4 + dwords) of the sampler at (set, binding, element): + * pvr_sampler_descriptor::meta (load_smp_meta_pco; the LOD-bias variant reads the sampler's mipLodBias, + * PVR_G5P_SAMPLER_META_LOD_BIAS) */ + PCO_G5P_UPLOAD_SAMPLER_META, }; +/* nir_tex_instr.backend_flags, set by the driver's LOD-bias lowering (pvr_g5p_nir.c lod_bias_lower()): the + * sample adds its sampler's mipLodBias (pco_g5p_program::lod_samplers[].biased) */ +#define PCO_G5P_TEX_SAMPLER_BIAS (1u << 28) +#define PCO_G5P_MAX_LOD_SAMPLERS 16 /* PCO_G5P_UPLOAD_IMAGE's offset (the state is always descriptor +0): how the * program samples it. AS_2D: an array sample at the constant layer 0, drawn * as a 2D sample, so the driver uploads the view's base layer as a 2D state @@ -164,6 +172,17 @@ enum pco_g5p_upload_kind { #define PCO_G5P_SSBO_SIZE_OFFSET 0xfffcu #define PCO_G5P_UPLOAD_IMAGE_AS_2D 1u #define PCO_G5P_UPLOAD_IMAGE_ARRAY 2u +/* ONE_LAYER: an array sample at any layer of a one-layer view, drawn as a 2D + * sample. Vulkan's layer is clamp(RNE(z), 0, layers - 1), so with one layer + * every z reads the base layer. Only the driver's one-layer program variant + * has such samples (pvr_g5p_nir.c one_layer_as_2d()), and the draw selects + * that variant only when every array view it samples has one layer. The + * driver uploads a 2D state of that layer, and refuses a multi-layer view by + * name (pvr_g5p_tex_state_for()). */ +#define PCO_G5P_UPLOAD_IMAGE_ONE_LAYER 4u +/* nir_tex_instr.backend_flags, set by the driver's one-layer lowering: this + * former array sample reads PCO_G5P_UPLOAD_IMAGE_ONE_LAYER state */ +#define PCO_G5P_TEX_ONE_LAYER_AS_2D (1u << 29) /* Varyings (docs/gpu/experiments/varyings.md). Apple's A9 compiler takes at @@ -178,6 +197,8 @@ enum pco_g5p_upload_kind { * included). Before: 16 vec4, 64 components. */ #define PCO_G5P_MAX_FS_VARYINGS 28 #define PCO_G5P_VARYING_SLOTS 32 /* VARYING_SLOT_VAR0..VAR31 */ +/* pco_g5p_program::z_set: fragment position Z; host tests check for it */ +#define PCO_G5P_HAVE_Z_SET 1 enum pco_g5p_interp { PCO_G5P_INTERP_UNREAD = 0, /* VS link: the fragment program does not read it */ @@ -250,6 +271,10 @@ struct pco_g5p_program { unsigned num_varyings; struct pco_g5p_varying vary[PCO_G5P_MAX_FS_VARYINGS]; bool w_set; /* coefficient set 0 is W (some varying is perspective-correct) */ + /* gl_FragCoord.z is read: the next coefficient set (1 after W, else 0) is the + * TSP's unclamped Z, which TA_OUTPUT_SEL tsp_unclamped_z_pres puts right after + * the RHW (AGXMetalA9 FragmentProgramVariant 0x207d28be0, varying item type 6) */ + bool z_set; /* VS: first output dword of each VARn (0 = not written) and its * interpolation; outputs are grouped linear, flat, npc (STATE_VARYING0). */ uint8_t vout_dword[PCO_G5P_VARYING_SLOTS]; @@ -271,6 +296,18 @@ struct pco_g5p_program { * (set by pvr_g5p_nir.c after compilation; polygon mode POINT refuses it) */ bool psize_dropped; uint8_t clip_count; /* VS: user clip distances, the last output dwords (TA user planes 0..n-1) */ + /* The samplers whose LOD a sample uses (tex, txb, txl, txd; not txf, size queries or gathers): a sampler's + * mipLodBias applies to them (Vulkan's LOD operation: lambda' = lambda_base + clamp(sampler bias + shader + * bias)). `biased`: the program adds the bound sampler's bias itself (the LOD-bias variant, + * PCO_G5P_TEX_SAMPLER_BIAS); the draw selects programs from these (pvr_arch_g5p_gfx.c). More than + * PCO_G5P_MAX_LOD_SAMPLERS sets lod_samplers_overflow. */ + unsigned num_lod_samplers; + bool lod_samplers_overflow; + struct pco_g5p_lod_sampler { + uint8_t set, binding; + uint16_t element; + bool biased; + } lod_samplers[PCO_G5P_MAX_LOD_SAMPLERS]; char refusal[512]; }; diff --git a/src/imagination/g9p/g9p_a9_tex.h b/src/imagination/g9p/g9p_a9_tex.h index 0069a620322..9eecee45851 100644 --- a/src/imagination/g9p/g9p_a9_tex.h +++ b/src/imagination/g9p/g9p_a9_tex.h @@ -147,6 +147,12 @@ a9_txs(struct g9p_ctx *ctx, nir_tex_instr *tex) a9_tex_refuse(ctx, "texture size query of a texture without a constant binding"); return false; } + /* a buffer view's size is its element count (pvr_g5p_tbo_upload()), which + * the compute uploads do not carry; texel-buffer fetches are refused too */ + if (tex->sampler_dim == GLSL_SAMPLER_DIM_BUF) { + a9_tex_refuse(ctx, "texel-buffer size query in a compute shader"); + return false; + } if (tex->op == nir_texop_txs && li >= 0) { if (!nir_src_is_const(tex->src[li].src)) { a9_tex_refuse(ctx, "textureDimensions at a non-constant level in a compute shader"); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index d96550fc5a6..5213860ead6 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -456,6 +456,14 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, mesa_logw("G5P refuses: compute set %u binding %u: a sampler without A9 words",u->set,u->binding); return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a sampler without A9 words"); } + /* the A9 sampler word has no LOD bias: graphics adds it in the LOD-bias fragment variant + * (pvr_g5p_nir.c lod_bias_lower()); compute samples at a shared constant level and has no such + * program, so a biased sampler is refused rather than sampled at the unbiased level */ + if(d[2*ROGUE_NUM_TEXSTATE_SAMPLER_WORDS+PVR_G5P_SAMPLER_META_LOD_BIAS]) { + mesa_logw("G5P refuses: compute set %u binding %u: a sampler LOD bias of %g (no LOD-bias compute program)", + u->set,u->binding,uif(d[2*ROGUE_NUM_TEXSTATE_SAMPLER_WORDS+PVR_G5P_SAMPLER_META_LOD_BIAS])); + return vk_errorf(cmd_buffer,VK_ERROR_OUT_OF_HOST_MEMORY,"G5P compute: a sampler LOD bias"); + } memcpy(out,d,4*2); break; case G9P_A9_UP_SIZE: diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index 68d7fc3b21b..cdbd14e4833 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -26,6 +26,7 @@ #include "vk_enum_to_str.h" #include "vk_format.h" #include "vk_graphics_state.h" +#include "pvr_g5p_depth_clip.h" #include "vk_log.h" #include "vk_shader_module.h" @@ -326,11 +327,16 @@ launch_program(uint32_t *dest) * The coefficient (varying) program, as Apple's PDSEncoderGen1:: * EncodeFragmentVaryingsPDSProgram builds it: one DOUTI per item, item i's * source in c(2i), the data first and padded to 16 bytes (the code starts - * at VARYINGBASE + 16 * PDS_VARYINGSIZE). Items: W at coefficient set 0 - * when some varying is perspective-correct, then every varying the fragment - * program reads (dest: its first set; offset: its vertex output dword in - * the TA vertex, which drops x, y, z: dword d is TA dword d - 3, in 16-bit - * units). Returns the data dwords; *code_dwords the code's. + * at VARYINGBASE + 16 * PDS_VARYINGSIZE). Items: position Z when the + * fragment program reads it (coefficient set 1 after W, else 0), W at set 0 + * when some varying is perspective-correct, then every varying it reads + * (dest: its first set; offset: its vertex output dword in the TSP vertex, + * in 16-bit units). The TSP vertex is the RHW, then the unclamped Z when + * TA_OUTPUT_SEL tsp_unclamped_z_pres is set, then the varyings: vertex + * output dword d (x, y, z dropped, w kept as the RHW) is TSP dword d - 3, + * one dword later with Z. Apple's Z item (FragmentProgramVariant 0x207d28be0, + * varying type 6): f32/f16 offset 2, Gouraud, not perspective, and every + * other item's offsets + 2. Returns the data dwords; *code_dwords the code's. */ /* pvr_physical_device.c reports what the fragment compiler admits */ static_assert(4 * PCO_G5P_MAX_FS_VARYINGS == 112, "G5P varying components"); @@ -341,13 +347,23 @@ varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program * { pdsc_init(); pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-varying"); - struct pdsc_douti_src0 item[1 + PCO_G5P_MAX_FS_VARYINGS]; + struct pdsc_douti_src0 item[2 + PCO_G5P_MAX_FS_VARYINGS]; unsigned items = 0; + /* Apple's item order: position Z, then W, then the varyings (the A9 + * compiler's reflection lists the type-6 item first: f_z_s, f_xyz_s); + * the sets stay W 0, Z 1 */ + if (fs->z_set) + item[items++] = (struct pdsc_douti_src0){ + .dest = fs->w_set ? 1 : 0, + .shademodel = PDSC_DOUTI_SHADEMODEL_GOURAUD, + .f32_offset = 2, + .f16_offset = 2, + }; if (fs->w_set) item[items++] = (struct pdsc_douti_src0){ .shademodel = PDSC_DOUTI_SHADEMODEL_GOURAUD }; for (unsigned i = 0; i < fs->num_varyings; i++) { const struct pco_g5p_varying *v = &fs->vary[i]; - const unsigned off = 2 * (vs->vout_dword[v->location] - 3); + const unsigned off = 2 * (vs->vout_dword[v->location] - 3) + (fs->z_set ? 2 : 0); const bool flat = v->interp == PCO_G5P_INTERP_FLAT; item[items++] = (struct pdsc_douti_src0){ .dest = v->set, @@ -395,6 +411,21 @@ g5p_native_bresenham_line(const struct vk_graphics_pipeline_state *s) s->rs->line.mode == VK_LINE_RASTERIZATION_MODE_BRESENHAM; } +/* PPP_CTRL clip_mode values pvr_g5p_depth_clip.h names */ +static_assert(ROGUE_TA_CLIP_MODE_FRONT_REAR == PVR_G5P_CLIP_MODE_CLIP, "clip mode 1"); +static_assert(ROGUE_TA_CLIP_MODE_FRONT_REAR_DEPTH == PVR_G5P_CLIP_MODE_CLAMP, "clip mode 2"); + +/* The attachments g5p_check_state() judges: from the render pass object or the rendering state at creation, + * kept for a variant built at a later draw (the render pass may be gone by then) */ +struct g5p_rp_summary { + unsigned colors; + VkFormat cf; + VkFormat cfs[PVR_G5P_MRT_MAX_COLORS]; + bool ds; +}; +static thread_local const struct g5p_rp_summary *g5p_rp_override; +static thread_local struct g5p_rp_summary *g5p_rp_record; + /* What initial triangle path draws: refused by name otherwise (the ICD's validate_state() * with initial triangle path's limits) */ static VkResult @@ -407,7 +438,13 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c VkFormat cf; VkFormat cfs[PVR_G5P_MRT_MAX_COLORS] = { VK_FORMAT_UNDEFINED }; bool ds; - if (create_info->renderPass) { + if (g5p_rp_override) { + /* a variant built after creation (the LOD-bias variant): the attachments the pipeline was made for */ + colors = g5p_rp_override->colors; + cf = g5p_rp_override->cf; + memcpy(cfs, g5p_rp_override->cfs, sizeof(cfs)); + ds = g5p_rp_override->ds; + } else if (create_info->renderPass) { VK_FROM_HANDLE(pvr_render_pass, pass, create_info->renderPass); const struct pvr_render_subpass *sub = &pass->subpasses[create_info->subpass]; colors = 0; @@ -431,12 +468,25 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c } else { return G5P_REFUSE(device, "no render pass state"); } + if (g5p_rp_record) { + *g5p_rp_record = (struct g5p_rp_summary){ .colors = colors, .cf = cf, .ds = ds }; + memcpy(g5p_rp_record->cfs, cfs, sizeof(cfs)); + } if (s->shader_stages != (VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT) || s->ts) return G5P_REFUSE(device, "shader stages %#x (a vertex and a fragment shader)", s->shader_stages); - if (s->rs && (s->rs->depth_clamp_enable || - (s->rs->polygon_mode != VK_POLYGON_MODE_FILL && !pvr_g5p_polygon_mode_enabled()))) - return G5P_REFUSE(device, "depth clamp or polygon mode %d", s->rs->polygon_mode); + if (s->rs && s->rs->polygon_mode != VK_POLYGON_MODE_FILL && !pvr_g5p_polygon_mode_enabled()) + return G5P_REFUSE(device, "polygon mode %d", s->rs->polygon_mode); + /* depth clip and clamp: the draw judges them with its viewport and bias + * (pvr_g5p_depth_clip.h); a static state no viewport admits fails here */ + if (s->rs && !BITSET_TEST(s->dynamic, MESA_VK_DYNAMIC_RS_DEPTH_CLAMP_ENABLE) && + !BITSET_TEST(s->dynamic, MESA_VK_DYNAMIC_RS_DEPTH_CLIP_ENABLE)) { + unsigned mode; + const char *why = pvr_g5p_depth_clip_mode(vk_rasterization_state_depth_clip_enable(s->rs), + s->rs->depth_clamp_enable, false, 0.0f, 1.0f, false, &mode); + if (why) + return G5P_REFUSE(device, "%s", why); + } /* line and point fill: the qualified scope only */ if (s->rs && s->rs->polygon_mode != VK_POLYGON_MODE_FILL) { if (s->rs->polygon_mode != VK_POLYGON_MODE_LINE && s->rs->polygon_mode != VK_POLYGON_MODE_POINT) @@ -547,6 +597,60 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c return VK_SUCCESS; } +/* A pipeline node (the pipeline, or one of its variants) */ +static struct pvr_g5p_pipeline * +g5p_node_alloc(struct pvr_device *device) +{ + struct pvr_g5p_pipeline *g = + vk_zalloc(&device->vk.alloc, sizeof(*g), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (g) + simple_mtx_init(&g->lod_bias_lock, mtx_plain); + return g; +} + +/* The sampler words and metadata of a bound sampler or combined image/sampler descriptor (a standalone + * sampler at +0, a combined descriptor's sampler after its image, at +32), or NULL for another type */ +static const uint32_t * +g5p_bound_sampler(const struct pvr_descriptor_set_layout_binding *lb, const uint32_t *desc) +{ + static_assert(offsetof(struct pvr_combined_image_sampler_descriptor, sampler) == 32, "combined sampler"); + static_assert(offsetof(struct pvr_sampler_descriptor, meta) == 8 * ROGUE_NUM_TEXSTATE_SAMPLER_WORDS, + "sampler metadata"); + if (lb->type == VK_DESCRIPTOR_TYPE_SAMPLER) + return desc; + if (lb->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) + return desc + offsetof(struct pvr_combined_image_sampler_descriptor, sampler) / 4; + return NULL; +} + +/* + * What a pipeline keeps to build its LOD-bias variant at a later draw (pvr_g5p_nir.c lod_bias_lower()): its + * two stages' SPIR-V and entry points, the graphics state and the attachments it was made for. Kept only by + * a pipeline whose fragment program samples with a sampler's LOD; its xfb and one-layer nodes share it. A + * pipeline created from the driver's own NIR (internal pipelines, which use the driver's samplers) keeps + * none: a biased draw of it refuses by name. + */ +struct pvr_g5p_lod_bias_src { + VkPipelineShaderStageCreateInfo stage[2]; + VkShaderModuleCreateInfo module[2]; + struct vk_graphics_pipeline_state state; + void *state_mem; + struct g5p_rp_summary rp; +}; + +static void +g5p_lod_bias_src_free(struct pvr_device *device, struct pvr_g5p_lod_bias_src *src) +{ + if (!src) + return; + for (unsigned i = 0; i < 2; i++) { + vk_free(&device->vk.alloc, (void *)src->module[i].pCode); + vk_free(&device->vk.alloc, (void *)src->stage[i].pName); + } + vk_free(&device->vk.alloc, src->state_mem); + vk_free(&device->vk.alloc, src); +} + static void g5p_pipeline_free(struct pvr_device *device, struct pvr_g5p_pipeline *g) { @@ -569,6 +673,11 @@ g5p_pipeline_free(struct pvr_device *device, struct pvr_g5p_pipeline *g) } g5p_pipeline_free(device, g->xfb); g5p_pipeline_free(device, g->punch); + g5p_pipeline_free(device, g->one_layer); + g5p_pipeline_free(device, g->lod_bias); + if (g->lod_bias_src_owner) + g5p_lod_bias_src_free(device, g->lod_bias_src); + simple_mtx_destroy(&g->lod_bias_lock); vk_free(&device->vk.alloc, g); } @@ -701,6 +810,10 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo case PCO_G5P_UPLOAD_SAMPLER: case PCO_G5P_UPLOAD_IMAGE_SIZE: case PCO_G5P_UPLOAD_BLEND_CONSTANTS: + case PCO_G5P_UPLOAD_SAMPLER_META: + if (u->kind == PCO_G5P_UPLOAD_SAMPLER_META && u->offset + 4 * u->dwords > 4 * PCO_SAMPLER_META_COUNT) + return G5P_REFUSE(device, "fragment upload %u: sampler metadata [%u, %u) bytes", i, u->offset, + u->offset + 4 * u->dwords); if (u->dwords > 8) return G5P_REFUSE(device, "fragment upload %u: %u dwords of descriptor words", i, u->dwords); break; @@ -724,6 +837,30 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo } g->fs_num_uploads = fs->num_uploads; g->fs_common_dwords = fs->common_dwords; + /* the array-sampled views the draw classifies for the one-layer variant (g5p_one_layer_select()) */ + g->num_array_tex = 0; + for (unsigned st = 0; st < 2; st++) { + const struct pco_g5p_program *p = st ? vs : fs; + for (unsigned i = 0; i < p->num_uploads; i++) { + const struct pco_g5p_upload *u = &p->uploads[i]; + if (u->kind != PCO_G5P_UPLOAD_IMAGE || !(u->offset & PCO_G5P_UPLOAD_IMAGE_ARRAY)) + continue; + if (g->num_array_tex < ARRAY_SIZE(g->array_tex)) + g->array_tex[g->num_array_tex] = (__typeof__(g->array_tex[0])){ u->set, u->binding, u->element }; + g->num_array_tex++; + } + } + /* the samplers whose LOD the programs use, for the LOD-bias variant (g5p_lod_bias_select()) */ + static_assert(PVR_G5P_MAX_LOD_SAMPLERS == PCO_G5P_MAX_LOD_SAMPLERS, "LOD sampler list"); + for (unsigned st = 0; st < 2; st++) { + const struct pco_g5p_program *p = st ? vs : fs; + g->num_lod_samplers[st] = p->num_lod_samplers; + g->lod_samplers_overflow[st] = p->lod_samplers_overflow; + for (unsigned i = 0; i < p->num_lod_samplers; i++) + g->lod_samplers[st][i] = (struct pvr_g5p_lod_sampler){ p->lod_samplers[i].set, p->lod_samplers[i].binding, + p->lod_samplers[i].element, + p->lod_samplers[i].biased }; + } /* PDS has 128 constant dwords, four per transfer. Coalesce copied * descriptor intervals when the original upload list exceeds twelve. * GPU-written UBOs and push snapshots retain their own source addresses. */ @@ -732,7 +869,8 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo const struct pco_g5p_upload *u = &fs->uploads[i]; bool copy = fs->num_uploads > 12 && (u->kind == PCO_G5P_UPLOAD_IMAGE || u->kind == PCO_G5P_UPLOAD_SAMPLER || - u->kind == PCO_G5P_UPLOAD_IMAGE_SIZE || u->kind == PCO_G5P_UPLOAD_BLEND_CONSTANTS); + u->kind == PCO_G5P_UPLOAD_IMAGE_SIZE || u->kind == PCO_G5P_UPLOAD_BLEND_CONSTANTS || + u->kind == PCO_G5P_UPLOAD_SAMPLER_META); if (u->common + u->dwords > ARRAY_SIZE(cpu)) return G5P_REFUSE(device, "fragment upload outside common store"); if (copy) { @@ -786,6 +924,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo g->fs_side_effects = fs->side_effects; g->vs_side_effects = vs->side_effects; g->fs_depth_feedback = fs->depth_feedback; + g->fs_position_z = fs->z_set; if (g->fs_depth_feedback && g->samples != 1) return G5P_REFUSE(device, "multisample fragment depth export is not qualified"); g->fs_blend_mode = fs->blend_mode; @@ -863,7 +1002,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo /* PDS: the coefficient program (the device HALT without varyings) */ unsigned varying_data_dwords = 0; - if (fs->num_varyings || fs->w_set) { + if (fs->num_varyings || fs->w_set || fs->z_set) { unsigned code_dwords; varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords, 0); result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, &g->varying); @@ -933,10 +1072,11 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo s1.pds_uniformsize = g->fs_num_uploads ? DIV_ROUND_UP(4 * g->fs_num_dmas, ROGUE_TA_STATE_PDS_SIZEINFO1_PDS_UNIFORMSIZE_UNIT_SIZE) : 1; - if (fs->num_varyings || fs->w_set) { + if (fs->num_varyings || fs->w_set || fs->z_set) { s1.pds_varyingsize = DIV_ROUND_UP(varying_data_dwords, 4); - /* coefficient sets: W, then four per varying (4 dwords each) */ - s1.usc_varyingsize = DIV_ROUND_UP(4 * ((fs->w_set ? 1 : 0) + 4 * fs->num_varyings), 16); + /* coefficient sets: W, Z, then four per varying (4 dwords each) */ + s1.usc_varyingsize = + DIV_ROUND_UP(4 * ((fs->w_set ? 1 : 0) + (fs->z_set ? 1 : 0) + 4 * fs->num_varyings), 16); } } /* A discarding program runs punch-through with triangle merging off @@ -948,13 +1088,14 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo ROGUE_TA_STATE_PDS_SIZEINFO2_USC_SHAREDSIZE_UNIT_SIZE); } pvr_csb_pack (&w[5], TA_STATE_PDS_VARYINGBASE, vb) { - vb.addr = PVR_DEV_ADDR(fs->num_varyings || fs->w_set ? g->varying.data_offset : dp->halt.code_offset); + vb.addr = PVR_DEV_ADDR(fs->num_varyings || fs->w_set || fs->z_set ? g->varying.data_offset + : dp->halt.code_offset); } result = pvr_gpu_upload(device, device->heaps.general_heap, w, 24, 16, &g->static_frag); if (result != VK_SUCCESS) return result; /* the same pointers naming the flat programs of vertices 1 and 2 */ - for (unsigned pv = 1; g->fs_flat && (fs->num_varyings || fs->w_set) && pv <= 2; pv++) { + for (unsigned pv = 1; g->fs_flat && (fs->num_varyings || fs->w_set || fs->z_set) && pv <= 2; pv++) { uint32_t *v = g->static_frag_pv_words[pv - 1]; memcpy(v, w, sizeof(g->static_frag_words)); pvr_csb_pack (&v[5], TA_STATE_PDS_VARYINGBASE, vb) { @@ -983,7 +1124,7 @@ g5p_pipeline_build_punch(struct pvr_device *device, return VK_SUCCESS; /* General-blend punch variants use Apple's tile-read-before-kill order. */ struct pvr_g5p_pipeline *p = - vk_zalloc(&device->vk.alloc, sizeof(*p), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + g5p_node_alloc(device); if (!p) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); p->punch_variant = true; @@ -1006,6 +1147,318 @@ g5p_pipeline_build_punch(struct pvr_device *device, return result; } +static VkResult g5p_pipeline_check_draw_variants(struct pvr_device *device, + const struct vk_graphics_pipeline_state *state, + const struct pvr_g5p_pipeline *g); + +/* The one-layer variant (pvr_g5p.h): built only for a pipeline with array samples, as a full linked + * pipeline (its fragment program may read fewer varying components, so its vertex program is its own), + * with its own punch child. The base programs are unchanged. A refused variant leaves the pipeline as it + * was: a draw that needs it refuses by name. */ +static VkResult +g5p_pipeline_build_one_layer(struct pvr_device *device, + const VkGraphicsPipelineCreateInfo *create_info, + const struct vk_graphics_pipeline_state *state, + struct pvr_g5p_pipeline *g) +{ + if (!g->num_array_tex) + return VK_SUCCESS; + if (g->num_array_tex > ARRAY_SIZE(g->array_tex)) { + g->one_layer_why = "more than 16 array-sampled views"; + return VK_SUCCESS; + } + struct pvr_g5p_pipeline *p = + g5p_node_alloc(device); + if (!p) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + p->one_layer_variant = true; + p->xfb_variant = g->xfb_variant; + struct pco_g5p_program vs = { 0 }, fs = { 0 }; + const bool was = pvr_g5p_one_layer_compile; + pvr_g5p_one_layer_compile = true; + VkResult result = g5p_pipeline_build(device, create_info, state, p, &vs, &fs); + free(vs.code); + free(fs.code); + if (result == VK_SUCCESS) + result = g5p_pipeline_build_punch(device, create_info, state, p); + pvr_g5p_one_layer_compile = was; + if (result == VK_SUCCESS) + result = g5p_pipeline_check_draw_variants(device, state, p); + if (result == VK_SUCCESS) { + g->one_layer = p; + return VK_SUCCESS; + } + g5p_pipeline_free(device, p); + if (result == VK_ERROR_FEATURE_NOT_PRESENT) { + g->one_layer_why = "the variant's compile was refused (see the refusal above)"; + return VK_SUCCESS; + } + return result; +} + +/* A draw's program for its bound views: the one-layer variant when an array sample's view has no array + * state (a one-layer view of a linear image), else the pipeline itself. The variant's uploads refuse a + * multi-layer view among the others by name; a view without texture state is the uploads' to refuse. */ +static VkResult +g5p_one_layer_select(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline **gp) +{ + const struct pvr_g5p_pipeline *g = *gp; + if (!g->num_array_tex || g->num_array_tex > ARRAY_SIZE(g->array_tex)) + return VK_SUCCESS; + const struct pvr_descriptor_state *ds = &cmd_buffer->state.gfx_desc_state; + for (unsigned i = 0; i < g->num_array_tex; i++) { + const unsigned set_index = g->array_tex[i].set, binding = g->array_tex[i].binding; + const struct pvr_descriptor_set *set = set_index < PVR_MAX_DESCRIPTOR_SETS ? ds->sets[set_index] : NULL; + if (!set || !set->layout || binding >= set->layout->binding_count) + continue; + const struct pvr_descriptor_set_layout_binding *lb = &set->layout->bindings[binding]; + if (lb->type != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && lb->type != VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE) + continue; + if (g->array_tex[i].element >= MAX2(lb->descriptor_count, 1)) + continue; + const uint32_t *desc = (const uint32_t *)((const uint8_t *)set->mapping + lb->offset + + (size_t)g->array_tex[i].element * lb->stride); + if (!pvr_g5p_tex_state_valid(desc) || (desc[0] & 7) == PVR_G5P_TEX_LAYOUT_2D_ARRAY) + continue; + if (!g->one_layer) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: an array sample at a non-constant layer of a view " + "without array state, and no one-layer variant: %s", set_index, binding, + g->one_layer_why ? g->one_layer_why : "none built"); + *gp = g->one_layer; + return VK_SUCCESS; + } + return VK_SUCCESS; +} + +/* Keep what the LOD-bias variant is built from (struct pvr_g5p_lod_bias_src). *why names a pipeline that + * keeps none (its stages are the driver's NIR); OOM fails creation. */ +static VkResult +g5p_lod_bias_capture(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *create_info, + const struct vk_graphics_pipeline_state *state, const struct g5p_rp_summary *rp, + struct pvr_g5p_lod_bias_src **out, const char **why) +{ + *out = NULL; + *why = NULL; + const VkPipelineShaderStageCreateInfo *st[2] = { NULL, NULL }; + for (uint32_t i = 0; i < create_info->stageCount; i++) { + const unsigned k = create_info->pStages[i].stage == VK_SHADER_STAGE_FRAGMENT_BIT ? 1 : 0; + if (!st[k]) + st[k] = &create_info->pStages[i]; + } + const uint32_t *code[2] = { NULL, NULL }; + size_t size[2] = { 0, 0 }; + for (unsigned k = 0; k < 2; k++) { + if (!st[k]) { + *why = "a stage is missing"; + return VK_SUCCESS; + } + VK_FROM_HANDLE(vk_shader_module, module, st[k]->module); + const VkShaderModuleCreateInfo *inline_ci = vk_find_struct_const(st[k]->pNext, SHADER_MODULE_CREATE_INFO); + if (module && !module->nir) { + code[k] = (const uint32_t *)module->data; + size[k] = module->size; + } else if (!module && inline_ci && !vk_find_struct_const(st[k]->pNext, PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA)) { + code[k] = inline_ci->pCode; + size[k] = inline_ci->codeSize; + } else { + *why = "the pipeline's stages are the driver's NIR (no SPIR-V kept)"; + return VK_SUCCESS; + } + } + struct pvr_g5p_lod_bias_src *src = + vk_zalloc(&device->vk.alloc, sizeof(*src), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (!src) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + for (unsigned k = 0; k < 2; k++) { + const char *entry = st[k]->pName ? st[k]->pName : "main"; + uint32_t *words = vk_alloc(&device->vk.alloc, size[k], 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + char *name = vk_alloc(&device->vk.alloc, strlen(entry) + 1, 1, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (words) + memcpy(words, code[k], size[k]); + if (name) + memcpy(name, entry, strlen(entry) + 1); + src->module[k] = (VkShaderModuleCreateInfo){ .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .codeSize = size[k], .pCode = words }; + src->stage[k] = (VkPipelineShaderStageCreateInfo){ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = &src->module[k], .stage = st[k]->stage, + .pName = name, + /* specialization constants are refused at creation */ + .pSpecializationInfo = NULL }; + if (!words || !name) { + g5p_lod_bias_src_free(device, src); + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + } + } + VkResult result = vk_graphics_pipeline_state_copy(&device->vk, &src->state, state, &device->vk.alloc, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, &src->state_mem); + if (result != VK_SUCCESS) { + g5p_lod_bias_src_free(device, src); + return result; + } + src->rp = *rp; + *out = src; + return VK_SUCCESS; +} + +/* Hand a pipeline's retained stages (or why it has none) to its nodes the draw may bias: the pipeline, its xfb + * node and their one-layer nodes (punch children are selected after the LOD-bias variant) */ +static void +g5p_lod_bias_share(struct pvr_g5p_pipeline *g, struct pvr_g5p_lod_bias_src *src, const char *why, bool owner) +{ + if (!g) + return; + g->lod_bias_src = src; + g->lod_bias_src_owner = owner; + g->lod_bias_why = src ? NULL : why; + g5p_lod_bias_share(g->one_layer, src, why, false); +} + +static VkResult g5p_pipeline_check_draw_variants(struct pvr_device *device, + const struct vk_graphics_pipeline_state *state, + const struct pvr_g5p_pipeline *g); + +/* Build node n's LOD-bias variant (at the first draw that needs it, under n->lod_bias_lock): the same + * pipeline compiled from the retained stages with pvr_g5p_lod_bias_compile, n's xfb and one-layer forms, + * and its own punch child. A refused compile leaves n->lod_bias_why; the draw refuses by name. */ +static VkResult +g5p_lod_bias_build(struct pvr_device *device, struct pvr_g5p_pipeline *n) +{ + struct pvr_g5p_lod_bias_src *src = n->lod_bias_src; + const VkGraphicsPipelineCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .stageCount = 2, .pStages = src->stage }; + struct pvr_g5p_pipeline *p = g5p_node_alloc(device); + if (!p) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + p->lod_bias_variant = true; + p->xfb_variant = n->xfb_variant; + p->one_layer_variant = n->one_layer_variant; + const bool was_one_layer = pvr_g5p_one_layer_compile, was_lod_bias = pvr_g5p_lod_bias_compile; + const struct g5p_rp_summary *was_rp = g5p_rp_override; + pvr_g5p_one_layer_compile = n->one_layer_variant; + pvr_g5p_lod_bias_compile = true; + g5p_rp_override = &src->rp; + struct pco_g5p_program vs = { 0 }, fs = { 0 }; + VkResult result = g5p_pipeline_build(device, &ci, &src->state, p, &vs, &fs); + free(vs.code); + free(fs.code); + if (result == VK_SUCCESS) + result = g5p_pipeline_build_punch(device, &ci, &src->state, p); + pvr_g5p_one_layer_compile = was_one_layer; + pvr_g5p_lod_bias_compile = was_lod_bias; + g5p_rp_override = was_rp; + if (result == VK_SUCCESS) + result = g5p_pipeline_check_draw_variants(device, &src->state, p); + /* the same pipeline: what the draw already judged from n must hold for the variant */ + if (result == VK_SUCCESS && + (p->samples != n->samples || p->color_targets != n->color_targets || p->fs_write_mask != n->fs_write_mask || + p->fs_blend_mode != n->fs_blend_mode || p->vs_outputs != n->vs_outputs || + p->fs_position_z != n->fs_position_z || + p->fs_discard != n->fs_discard || p->xfb_layout.buffers != n->xfb_layout.buffers)) { + g5p_pipeline_free(device, p); + n->lod_bias_why = "the variant's state differs from the pipeline's"; + return VK_SUCCESS; + } + if (result == VK_SUCCESS) { + n->lod_bias = p; + return VK_SUCCESS; + } + g5p_pipeline_free(device, p); + if (result == VK_ERROR_FEATURE_NOT_PRESENT) { + n->lod_bias_why = "the variant's compile was refused (see the refusal above)"; + return VK_SUCCESS; + } + return result; +} + +/* The mipLodBias of the sampler bound at a LOD sampler's (set, binding, element); 0 when the binding holds + * no sampler (the uploads refuse that by name) */ +static float +g5p_bound_lod_bias(const struct pvr_descriptor_state *ds, const struct pvr_g5p_lod_sampler *l) +{ + const struct pvr_descriptor_set *set = l->set < PVR_MAX_DESCRIPTOR_SETS ? ds->sets[l->set] : NULL; + if (!set || !set->layout || l->binding >= set->layout->binding_count) + return 0.0f; + const struct pvr_descriptor_set_layout_binding *lb = &set->layout->bindings[l->binding]; + if (l->element >= MAX2(lb->descriptor_count, 1)) + return 0.0f; + const uint32_t *smp = g5p_bound_sampler(lb, (const uint32_t *)((const uint8_t *)set->mapping + lb->offset + + (size_t)l->element * lb->stride)); + return smp ? uif(smp[2 * ROGUE_NUM_TEXSTATE_SAMPLER_WORDS + PVR_G5P_SAMPLER_META_LOD_BIAS]) : 0.0f; +} + +/* A draw's program for its bound samplers' LOD biases: the LOD-bias variant (pvr_g5p_nir.c lod_bias_lower()) + * when a sampler that a sample of the fragment program reads the LOD of has a non-zero mipLodBias, else the + * pipeline itself. The A9 sampler word has no bias, so a program that does not add it would draw the + * unbiased LOD: a vertex-stage one (no vertex variant), or a missing variant, refuses by name. */ +static VkResult +g5p_lod_bias_select(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline **gp) +{ + const struct pvr_g5p_pipeline *g = *gp; + if (!g->num_lod_samplers[0] && !g->num_lod_samplers[1] && !g->lod_samplers_overflow[0] && + !g->lod_samplers_overflow[1]) + return VK_SUCCESS; + const struct pvr_descriptor_state *ds = &cmd_buffer->state.gfx_desc_state; + const struct pvr_g5p_lod_sampler *need = NULL; + float need_bias = 0.0f; + for (unsigned st = 0; st < 2; st++) { + if (g->lod_samplers_overflow[st]) + return G5P_REFUSE(cmd_buffer, "%s samples at more than %u samplers, or at a binding past 255: their LOD " + "biases are not checked", st ? "vertex" : "fragment", PVR_G5P_MAX_LOD_SAMPLERS); + for (unsigned i = 0; i < g->num_lod_samplers[st]; i++) { + const struct pvr_g5p_lod_sampler *l = &g->lod_samplers[st][i]; + const float bias = g5p_bound_lod_bias(ds, l); + if (bias == 0.0f || l->biased) + continue; + if (st) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: a sampler LOD bias of %g under a vertex-stage sample " + "(no LOD-bias vertex program)", l->set, l->binding, bias); + if (!need) { + need = l; + need_bias = bias; + } + } + } + if (!need) + return VK_SUCCESS; + if (g->lod_bias_variant) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: the LOD-bias variant leaves a sample of it unbiased", + need->set, need->binding); + struct pvr_g5p_pipeline *m = (struct pvr_g5p_pipeline *)g; /* the variant is built at its first use */ + VkResult result = VK_SUCCESS; + simple_mtx_lock(&m->lod_bias_lock); + if (!m->lod_bias && !m->lod_bias_why) { + if (m->lod_bias_src) + result = g5p_lod_bias_build(cmd_buffer->device, m); + else + m->lod_bias_why = "no retained stages"; + } + const struct pvr_g5p_pipeline *v = m->lod_bias; + const char *why = m->lod_bias_why; + simple_mtx_unlock(&m->lod_bias_lock); + if (result != VK_SUCCESS) + return vk_command_buffer_set_error(&cmd_buffer->vk, result); + if (!v) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: a sampler LOD bias of %g and no LOD-bias variant: %s", + need->set, need->binding, need_bias, why ? why : "none built"); + /* every biased sampler is one the variant biases */ + for (unsigned i = 0; i < g->num_lod_samplers[0]; i++) { + const struct pvr_g5p_lod_sampler *l = &g->lod_samplers[0][i]; + if (g5p_bound_lod_bias(ds, l) == 0.0f) + continue; + bool biased = false; + for (unsigned j = 0; j < v->num_lod_samplers[0]; j++) { + const struct pvr_g5p_lod_sampler *w = &v->lod_samplers[0][j]; + if (w->set == l->set && w->binding == l->binding && w->element == l->element) + biased = w->biased; + } + if (!biased) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: the LOD-bias variant leaves a sample of it unbiased", + l->set, l->binding); + } + *gp = v; + return VK_SUCCESS; +} + /* Program variants are checked at creation under the selected refusal * policy. The draw selects its punch variant from dynamic depth/stencil * state (g5p_draw()); this takes every state the pipeline allows: a dynamic enable may be on, a @@ -1060,11 +1513,14 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, { struct pco_g5p_program vs = { 0 }, fs = { 0 }; struct pvr_g5p_pipeline *g = - vk_zalloc(&device->vk.alloc, sizeof(*g), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + g5p_node_alloc(device); if (!g) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + struct g5p_rp_summary rp = { 0 }; + g5p_rp_record = &rp; VkResult result = g5p_pipeline_build(device, create_info, vk_state, g, &vs, &fs); + g5p_rp_record = NULL; /* the program bytes are the backend's (malloc'd); uploaded or refused */ free(vs.code); free(fs.code); @@ -1083,10 +1539,18 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, result = g5p_pipeline_build_punch(device, create_info, vk_state, g); if (result == VK_SUCCESS) result = g5p_pipeline_check_draw_variants(device, vk_state, g); + if (result == VK_SUCCESS) + result = g5p_pipeline_build_one_layer(device, create_info, vk_state, g); + /* a fragment program that samples with a sampler's LOD: keep its stages for the LOD-bias variant */ + struct pvr_g5p_lod_bias_src *lod_bias_src = NULL; + const char *lod_bias_why = NULL; + if (result == VK_SUCCESS && (g->num_lod_samplers[0] || g->lod_samplers_overflow[0])) + result = g5p_lod_bias_capture(device, create_info, vk_state, &rp, &lod_bias_src, &lod_bias_why); if (result != VK_SUCCESS) { g5p_pipeline_free(device, g); return pvr_refusal_to_api(result); } + g5p_lod_bias_share(g, lod_bias_src, lod_bias_why, true); pipeline->g5p = g; /* transform feedback: the same pipeline with the capturing vertex @@ -1098,7 +1562,7 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, if (vs_info && pvr_g5p_stage_has_xfb(vs_info)) { struct pco_g5p_program xvs = { 0 }, xfs = { 0 }; struct pvr_g5p_pipeline *x = - vk_zalloc(&device->vk.alloc, sizeof(*x), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + g5p_node_alloc(device); if (!x) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); x->xfb_variant = true; @@ -1109,8 +1573,11 @@ PVR_PER_ARCH(g5p_graphics_pipeline_compile)(struct pvr_device *device, result = g5p_pipeline_build_punch(device, create_info, vk_state, x); if (result == VK_SUCCESS && x->xfb_layout.buffers) result = g5p_pipeline_check_draw_variants(device, vk_state, x); + if (result == VK_SUCCESS && x->xfb_layout.buffers) + result = g5p_pipeline_build_one_layer(device, create_info, vk_state, x); if (result == VK_SUCCESS && x->xfb_layout.buffers) { g->xfb = x; + g5p_lod_bias_share(x, lod_bias_src, lod_bias_why, false); } else { g5p_pipeline_free(device, x); if (result != VK_SUCCESS) { @@ -1607,6 +2074,16 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, return G5P_REFUSE(cmd_buffer, "set %u binding %u: an image size without A9 texture state", u->set, u->binding); break; + case PCO_G5P_UPLOAD_SAMPLER_META: { + /* the sampler's metadata dwords (the LOD-bias variant's bias): a standalone sampler's at +16, a + * combined descriptor's sampler at +32 */ + const uint32_t *smp = g5p_bound_sampler(lb, desc); + if (!smp || !pvr_g5p_sampler_valid(smp)) + return G5P_REFUSE(cmd_buffer, "set %u binding %u: sampler metadata of a binding without an A9 sampler", + u->set, u->binding); + memcpy(out, smp + 2 * ROGUE_NUM_TEXSTATE_SAMPLER_WORDS + u->offset / 4, 4 * u->dwords); + break; + } default: return G5P_REFUSE(cmd_buffer, "fragment upload kind %u", u->kind); } @@ -1807,6 +2284,18 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g } if (!draw->count || !draw->instances) return VK_SUCCESS; + /* a one-layer view of a linear image under an array sample: the one-layer variant (pvr_g5p.h) */ + { + VkResult lr = g5p_one_layer_select(cmd_buffer, &g); + if (lr != VK_SUCCESS) + return lr; + } + /* a sampler with a non-zero mipLodBias under a fragment sample: the LOD-bias variant (pvr_g5p.h) */ + { + VkResult br = g5p_lod_bias_select(cmd_buffer, &g); + if (br != VK_SUCCESS) + return br; + } if (!g5p_topology(dyn->ia.primitive_topology, &hw_topology, &split, &lines)) return G5P_REFUSE(cmd_buffer, "topology %d", dyn->ia.primitive_topology); VkResult result; @@ -1948,6 +2437,17 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g (dyn->rs.depth_bias.constant_factor != 0.0f || dyn->rs.depth_bias.slope_factor != 0.0f); if (g->fs_depth_feedback && dbias) return G5P_REFUSE(cmd_buffer, "fragment depth export with depth bias is not qualified"); + unsigned clip_mode = PVR_G5P_CLIP_MODE_CLIP; + if (!discard) { + const VkViewport *vp0 = &dyn->vp.viewports[0]; + const char *why = pvr_g5p_depth_clip_mode(vk_rasterization_state_depth_clip_enable(&dyn->rs), + dyn->rs.depth_clamp_enable, true, vp0->minDepth, vp0->maxDepth, + dbias, &clip_mode); + if (!why && clip_mode != PVR_G5P_CLIP_MODE_CLIP && g->fs_depth_feedback) + why = "fragment depth export with depth clipping disabled is not qualified"; + if (why) + return G5P_REFUSE(cmd_buffer, "%s", why); + } uint32_t dbias_index = 0; /* Real single-token feedback defers depth/stencil writes until the kill * result is known (g5p-punch-ds@8150a8195df); preserve all DS test/ops. */ @@ -2060,6 +2560,11 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[1], TA_OUTPUT_SEL, os) { os.vtxsize = g->vs_outputs; os.rhw_pres = true; + /* the fragment program reads gl_FragCoord.z: the TSP vertex carries the + * unclamped Z after the RHW (AGXMetalA9: FragmentProgramVariant + * 0x207d28be0 sets 0x210000 for a type-6 varying, else 0x10000, and + * setRenderPipelineStateCommon 0x207d65d74 merges it under 0x210000) */ + os.tsp_unclamped_z_pres = g->fs_position_z; /* points: the size follows the position (g3b-points) */ os.psprite_size_pres = g->vs_point_size; /* user planes 0..n-1 from the declared clip distances, the vertex's @@ -2200,7 +2705,8 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g } w[1] = G5P_WCLAMP; pvr_csb_pack (&w[2], TA_STATE_PPP_CTRL, c) { - c.clip_mode = ROGUE_TA_CLIP_MODE_FRONT_REAR; + /* 1 clips at near/far, 2 keeps that geometry and clamps its depth (pvr_g5p_depth_clip.h) */ + c.clip_mode = (enum ROGUE_TA_CLIP_MODE)clip_mode; c.flatshade_vtx = ROGUE_TA_FLATSHADE_VTX_VERTEX_0; c.wclampen = true; if (dyn->rs.cull_mode == VK_CULL_MODE_BACK_BIT || dyn->rs.cull_mode == VK_CULL_MODE_FRONT_BIT) { diff --git a/src/imagination/vulkan/pvr_arch_g5p_tex.c b/src/imagination/vulkan/pvr_arch_g5p_tex.c index edb1debaab9..858a23a6d5d 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_tex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_tex.c @@ -27,6 +27,8 @@ #include "pvr_g5p_formats.h" #include "pvr_g5p_tex_encode.h" #include "pvr_image.h" +#include "pvr_g5p.h" +#include "util/u_math.h" /* A view format's texture key and sRGB bit: the A9 format table's * (pvr_g5p_formats.h, lane f) */ @@ -245,6 +247,10 @@ PVR_PER_ARCH(g5p_pack_sampler)(const VkSamplerCreateInfo *info, struct pvr_sampl desc->words[0] = general; desc->meta[PCO_SAMPLER_META_RSVD0] = (uint32_t)single; desc->meta[PCO_SAMPLER_META_RSVD1] = (uint32_t)(single >> 32); + /* the LOD bias the shader adds (the LOD-bias variant), clamped to maxSamplerLodBias as Vulkan's LOD + * operation does; -0.0 is no bias */ + const float bias = CLAMP(info->mipLodBias, -PVR_G5P_MAX_SAMPLER_LOD_BIAS, PVR_G5P_MAX_SAMPLER_LOD_BIAS); + desc->meta[PVR_G5P_SAMPLER_META_LOD_BIAS] = bias == 0.0f ? 0u : fui(bias); return NULL; } diff --git a/src/imagination/vulkan/pvr_formats.c b/src/imagination/vulkan/pvr_formats.c index 9c92664d401..613833baead 100644 --- a/src/imagination/vulkan/pvr_formats.c +++ b/src/imagination/vulkan/pvr_formats.c @@ -500,16 +500,15 @@ pvr_get_buffer_format_features2(struct pvr_physical_device *pdevice, } if (vk_format_is_scaled(vk_format)) - /* + return flags; + + /* * G9P: no texel buffers yet. Mesa's PBO path samples one when it is offered, and - * its fallback works; ES 3.0 does not have them. + * its fallback works; ES 3.0 does not have them. (This returned for every + * architecture once the scaled-format return above lost its statement.) */ if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g9p)) - flags &= ~(VK_FORMAT_FEATURE_2_UNIFORM_TEXEL_BUFFER_BIT | - VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_BIT | - VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_ATOMIC_BIT); - - return flags; + return flags; flags |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT; @@ -564,12 +563,16 @@ pvr_get_buffer_format_features2(struct pvr_physical_device *pdevice, break; } - /* no texel buffers on G5P: the backend refuses buffer textures - * (tex_shape_refusal()), so neither uniform nor storage texel buffers - * (nor the storage bits derived from them below) are claimed */ + /* G5P: Rogue's texel-buffer bits are not the A9's. Its uniform texel + * buffers are the qualified formats of pvr_g5p_tbo_features(); storage + * texel buffers (and the storage bits derived from them below) are not + * claimed */ if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p)) flags &= ~(VK_FORMAT_FEATURE_2_UNIFORM_TEXEL_BUFFER_BIT | VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_BIT | - VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_ATOMIC_BIT); + VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_ATOMIC_BIT | + /* image features (Rogue lists them for buffers too): the A9's buffer features are its + * vertex formats and the texel buffers below */ + VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT); PVR_G5P_HOOK(texel_features, pdevice, vk_format, flags) if (flags & VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_BIT) { flags |= VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT | diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index 3f1b72011e9..d0b6151cd6d 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -47,6 +47,14 @@ static inline VkResult pvr_refusal_to_api(VkResult result) #include "pvr_common.h" #include "pvr_limits.h" +/* A sampler's mipLodBias, clamped to +-maxSamplerLodBias, as an f32 in its descriptor's metadata + * (pvr_sampler_descriptor::meta): the A9 sampler word has no bias field, so the LOD-bias program variant adds + * it in the shader (pvr_g5p_nir.c lod_bias_lower()). */ +#define PVR_G5P_SAMPLER_META_LOD_BIAS PCO_SAMPLER_META_RSVD2 +#define PVR_G5P_MAX_SAMPLER_LOD_BIAS 16.0f +#define PVR_G5P_MAX_LOD_SAMPLERS 16 /* PCO_G5P_MAX_LOD_SAMPLERS */ +struct pvr_g5p_lod_bias_src; + /* User clip distances are written to the A9 TA user planes. */ static inline bool pvr_g5p_clip_distance_probe(void) @@ -273,6 +281,8 @@ struct pvr_g5p_pipeline { /* the fragment program discards (punch-through, no triangle merging); * it reads a flat varying */ bool fs_discard, fs_flat; + /* it reads gl_FragCoord.z (TA_OUTPUT_SEL tsp_unclamped_z_pres) */ + bool fs_position_z; unsigned fs_discard_tokens; bool fs_discard_select; /* Full linked variant: its uploads/varyings may differ after retaining a kill. @@ -323,6 +333,31 @@ struct pvr_g5p_pipeline { struct pvr_g5p_pipeline *xfb; bool xfb_variant; struct g5p_xfb_layout xfb_layout; + /* The one-layer variant (pvr_g5p_nir.c one_layer_as_2d()): the same pipeline with its array samples drawn + * as 2D samples of a one-layer view, a full linked variant with its own punch child. Draws use it when an + * array sample's view has no array state (a one-layer view of a linear image). array_tex: the + * (set, binding, element) of every array-sampled image (PCO_G5P_UPLOAD_IMAGE_ARRAY), vertex and + * fragment, that the draw classifies; NULL one_layer: none, or its compile was refused (one_layer_why). */ + struct pvr_g5p_pipeline *one_layer; + bool one_layer_variant; + const char *one_layer_why; + unsigned num_array_tex; + struct { uint8_t set, binding; uint16_t element; } array_tex[16]; + /* The LOD-bias variant (pvr_g5p_nir.c lod_bias_lower()): the samplers whose LOD the programs use + * (pco_g5p_program::lod_samplers, [0] fragment, [1] vertex); a draw with a non-zero mipLodBias on one the + * program does not bias selects the variant, built at that first draw from the retained stages + * (lod_bias_src, owned by the pipeline node, NULL for a node without one) and kept. The variant has its + * own punch child. lod_bias_variant: this node is one. */ + unsigned num_lod_samplers[2]; + bool lod_samplers_overflow[2]; + struct pvr_g5p_lod_sampler { uint8_t set, binding; uint16_t element; bool biased; } + lod_samplers[2][PVR_G5P_MAX_LOD_SAMPLERS]; + struct pvr_g5p_lod_bias_src *lod_bias_src; + bool lod_bias_src_owner; /* the top-level node frees it; its xfb and one-layer nodes borrow it */ + bool lod_bias_variant; + simple_mtx_t lod_bias_lock; /* the draw-time build of lod_bias */ + struct pvr_g5p_pipeline *lod_bias; + const char *lod_bias_why; /* no variant: why (no retained stages, or its compile was refused) */ }; struct pvr_buffer; diff --git a/src/imagination/vulkan/pvr_g5p_depth_clip.h b/src/imagination/vulkan/pvr_g5p_depth_clip.h new file mode 100644 index 00000000000..f3c491bf82e --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_depth_clip.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT + * + * Apple's A9 GPU (G5P): the PPP clip mode of a draw's depth clip and clamp + * state, or the reason the A9 path cannot draw it. Standalone (no Mesa + * headers), so a host test can include it as it is. + * + * AGXMetalA9 -[AGXA9FamilyRenderContext setDepthClipMode:] writes + * TA_STATE_PPP_CTRL bits 10:11 as 0x400 << mode: Metal's clip mode writes 1 + * (ROGUE_TA_CLIP_MODE_FRONT_REAR), its clamp mode 2 (FRONT_REAR_DEPTH), + * which keeps geometry beyond the near and far planes and clamps its depth + * to [0, 1]. On the A9: Vulkan's explicit + * complementary clip/clamp pairs 12/12 exact; GL depth clamp exact except + * custom depth ranges, which the A9 clamps to [0, 1], not to the range. + * So mode 2 is exactly: + * - depth clip off, clamp off: Vulkan clamps z_f to [0, 1] with + * depthClampZeroOne enabled (Zink enables it), and leaves depth outside + * the viewport range undefined without it; + * - depth clip off, clamp on: Vulkan's clamp to [min, max] of the + * viewport's depth range, when that range is [0, 1] (either order). Zink + * sends this pair for GL_DEPTH_CLAMP. + * Depth bias with mode 2, and clamp with clipping on, have no hardware run. + */ +#ifndef PVR_G5P_DEPTH_CLIP_H +#define PVR_G5P_DEPTH_CLIP_H + +#include +#include + +#define PVR_G5P_CLIP_MODE_CLIP 1u /* ROGUE_TA_CLIP_MODE_FRONT_REAR */ +#define PVR_G5P_CLIP_MODE_CLAMP 2u /* ROGUE_TA_CLIP_MODE_FRONT_REAR_DEPTH */ + +/* clip: the effective depth clip enable (vk_rasterization_state_depth_clip_enable()); + * has_range false: the pipeline's static check, before the draw knows its + * viewport; dbias: a depth bias with a non-zero factor applies. */ +static inline const char * +pvr_g5p_depth_clip_mode(bool clip, bool clamp, bool has_range, float min_depth, float max_depth, bool dbias, + unsigned *mode) +{ + *mode = PVR_G5P_CLIP_MODE_CLIP; + if (clip && !clamp) + return NULL; + if (clip) + return "depth clamp with depth clipping enabled is not qualified"; + if (dbias) + return "depth bias with depth clipping disabled is not qualified"; + if (clamp && has_range) { + const float lo = min_depth < max_depth ? min_depth : max_depth; + const float hi = min_depth < max_depth ? max_depth : min_depth; + if (lo != 0.0f || hi != 1.0f) + return "depth clamp to a viewport depth range other than [0, 1] (the A9 clamps to [0, 1])"; + } + *mode = PVR_G5P_CLIP_MODE_CLAMP; + return NULL; +} + +#endif diff --git a/src/imagination/vulkan/pvr_g5p_nir.c b/src/imagination/vulkan/pvr_g5p_nir.c index 4bc078ac35f..a614a776ba6 100644 --- a/src/imagination/vulkan/pvr_g5p_nir.c +++ b/src/imagination/vulkan/pvr_g5p_nir.c @@ -29,6 +29,7 @@ #include "pco/pco.h" #include "pco_g5p_general.h" #include "pvr_g5p.h" +#include "pvr_g5p_shader_dump.h" #include "pvr_g5p_vertex.h" /* nir->options must outlive the shader (the general backend runs NIR passes @@ -228,6 +229,308 @@ ubo_block_bytes(nir_shader *nir, nir_def *d, unsigned *bytes) return false; } +/* A constant term of a UBO offset as the backend evaluates it + * (pco_g5p_general.c eval_offset): ubo_offset_const() or a sum of them. */ +static bool +ubo_offset_eval(nir_scalar s, uint32_t *v) +{ + s = nir_scalar_chase_movs(s); + if (ubo_offset_const(s, v)) + return true; + uint32_t x, y; + if (nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_iadd && + ubo_offset_eval(nir_scalar_chase_alu_src(s, 0), &x) && ubo_offset_eval(nir_scalar_chase_alu_src(s, 1), &y)) { + *v = x + y; + return true; + } + return false; +} + +/* + * A register-indexed UBO load the backend's direct form does not take: + * offset = (x << s) + C with s from ishl or a multiply by a power of two + * (pco_g5p_general.c dyn_ubo_offset), read as a scalar at s != 1 or as a + * vec4 at s == 4, s <= 4 and C a multiple of the load. Anything else with a + * non-constant offset (a sum of two dynamic terms, a non-power-of-two + * stride, a larger or smaller shift) is general and goes through + * general_dyn_ubo(). A shift by a non-constant amount is neither. + */ +static bool +dyn_ubo_general(nir_intrinsic_instr *in) +{ + if (in->intrinsic != nir_intrinsic_load_ubo || in->def.bit_size != 32) + return false; + uint32_t c = 0, v; + nir_scalar s = nir_scalar_chase_movs(nir_get_scalar(in->src[1].ssa, 0)); + if (ubo_offset_eval(s, &v)) + return false; /* a constant offset: preloaded */ + for (unsigned guard = 0; guard < 8 && nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_iadd; guard++) { + nir_scalar a = nir_scalar_chase_alu_src(s, 0), b = nir_scalar_chase_alu_src(s, 1); + if (ubo_offset_eval(a, &v)) + s = nir_scalar_chase_movs(b); + else if (ubo_offset_eval(b, &v)) + s = nir_scalar_chase_movs(a); + else + return true; /* two dynamic terms */ + c += v; + } + unsigned shift = 0; + if (nir_scalar_is_alu(s) && (nir_scalar_alu_op(s) == nir_op_ishl || nir_scalar_alu_op(s) == nir_op_imul)) { + if (!ubo_offset_eval(nir_scalar_chase_alu_src(s, 1), &v)) + return false; + if (nir_scalar_alu_op(s) == nir_op_imul) { + if (!util_is_power_of_two_nonzero(v)) + return true; /* a non-power-of-two stride */ + v = util_logbase2(v); + } + shift = v; + } + /* the backend sees the load after its trailing unread components are trimmed */ + const unsigned n = MAX2(util_last_bit(nir_def_components_read(&in->def)), 1); + const unsigned loaded = (n == 2 || n == 3) && shift == 4 && !(c % 16) ? 4 : n; + return !((loaded == 4 && shift == 4) || (n == 1 && shift != 1)) || shift > 4 || c % (4 * loaded); +} + +/* s / 2^k for an offset term built from constants, iadd, a multiply by a + * constant and a shift by a constant, when each part divides exactly (the + * element index Apple's compiler computes); NULL otherwise. */ +static nir_def * +dyn_ubo_div(nir_builder *b, nir_scalar s, unsigned k, unsigned depth) +{ + const uint32_t mask = (1u << k) - 1; + uint32_t v; + s = nir_scalar_chase_movs(s); + if (ubo_offset_eval(s, &v)) + return v & mask ? NULL : nir_imm_int(b, (int32_t)v >> k); + if (!nir_scalar_is_alu(s) || depth > 8) + return NULL; + nir_scalar x = nir_scalar_chase_alu_src(s, 0), y = nir_scalar_chase_alu_src(s, 1); + switch (nir_scalar_alu_op(s)) { + case nir_op_iadd: { + nir_def *p = dyn_ubo_div(b, x, k, depth + 1); + nir_def *q = p ? dyn_ubo_div(b, y, k, depth + 1) : NULL; + if (!q) + return NULL; + if (nir_scalar_is_const(nir_get_scalar(q, 0))) + return nir_iadd_imm(b, p, nir_scalar_as_uint(nir_get_scalar(q, 0))); + if (nir_scalar_is_const(nir_get_scalar(p, 0))) + return nir_iadd_imm(b, q, nir_scalar_as_uint(nir_get_scalar(p, 0))); + return nir_iadd(b, p, q); + } + case nir_op_imul: { + if (!nir_scalar_is_const(y)) { + nir_scalar t = x; + x = y; + y = t; + } + if (!nir_scalar_is_const(y)) + return NULL; + const int32_t m = (int32_t)nir_scalar_as_uint(y); + if (!(m & mask)) + return nir_imul_imm(b, nir_channel(b, x.def, x.comp), (uint32_t)(m >> k)); + nir_def *p = dyn_ubo_div(b, x, k, depth + 1); + return p ? nir_imul_imm(b, p, (uint32_t)m) : NULL; + } + case nir_op_ishl: { + if (!nir_scalar_is_const(y)) + return NULL; + const unsigned n = nir_scalar_as_uint(y) & 31; + if (n >= k) + return nir_ishl_imm(b, nir_channel(b, x.def, x.comp), n - k); + return dyn_ubo_div(b, x, k - n, depth + 1); /* (x << n) / 2^k = x / 2^(k - n) */ + } + default: + return NULL; + } +} + +/* dyn_ubo_div() would succeed (nothing is built) */ +static bool +dyn_ubo_divides(nir_scalar s, unsigned k, unsigned depth) +{ + const uint32_t mask = (1u << k) - 1; + uint32_t v; + s = nir_scalar_chase_movs(s); + if (ubo_offset_eval(s, &v)) + return !(v & mask); + if (!nir_scalar_is_alu(s) || depth > 8) + return false; + nir_scalar x = nir_scalar_chase_alu_src(s, 0), y = nir_scalar_chase_alu_src(s, 1); + switch (nir_scalar_alu_op(s)) { + case nir_op_iadd: + return dyn_ubo_divides(x, k, depth + 1) && dyn_ubo_divides(y, k, depth + 1); + case nir_op_imul: + if (!nir_scalar_is_const(y)) { + nir_scalar t = x; + x = y; + y = t; + } + return nir_scalar_is_const(y) && + (!(nir_scalar_as_uint(y) & mask) || dyn_ubo_divides(x, k, depth + 1)); + case nir_op_ishl: { + if (!nir_scalar_is_const(y)) + return false; + const unsigned n = nir_scalar_as_uint(y) & 31; + return n >= k || dyn_ubo_divides(x, k - n, depth + 1); + } + default: + return false; + } +} + +/* offset = d + c: the dynamic part and the sum of the constant terms */ +static nir_scalar +dyn_ubo_peel(nir_intrinsic_instr *in, uint32_t *c) +{ + nir_scalar s = nir_scalar_chase_movs(nir_get_scalar(in->src[1].ssa, 0)); + uint32_t v; + *c = 0; + for (unsigned guard = 0; guard < 8 && nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_iadd; guard++) { + nir_scalar a = nir_scalar_chase_alu_src(s, 0), b = nir_scalar_chase_alu_src(s, 1); + if (ubo_offset_eval(a, &v)) + s = nir_scalar_chase_movs(b); + else if (ubo_offset_eval(b, &v)) + s = nir_scalar_chase_movs(a); + else + break; + *c += v; + } + return s; +} + +/* + * Zink reads a GL uniform block as an array of uints, so a vec4 at a general + * offset arrives as four scalar loads at d + c, c + 4, c + 8, c + 12. When + * d + c is a multiple of 16 by construction (k*48 + 32), the four are one + * vec4 load, which general_dyn_ubo() reads with one 16-byte load at shift 4 + * (as Apple's compiler reads a float4 at p[k*3 + 1]). Only complete quads of general loads are merged, as + * vec4_dyn_ubo() does for the direct form. + */ +static bool +vec4_general_dyn_ubo(nir_shader *nir) +{ + bool progress = false; + nir_foreach_function_impl (impl, nir) { + nir_builder b = nir_builder_create(impl); + nir_foreach_block (block, impl) { + bool again; + do { + again = false; + nir_foreach_instr (instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *lo = nir_instr_as_intrinsic(instr), *q[4] = { lo }; + uint32_t c, e; + if (lo->def.num_components != 1 || !dyn_ubo_general(lo)) + continue; + nir_scalar d = dyn_ubo_peel(lo, &c); + if (c % 16 || !dyn_ubo_divides(d, 4, 0)) + continue; + unsigned found = 1; + for (nir_instr *i = nir_instr_next(instr); i && found < 4; i = nir_instr_next(i)) { + if (i->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *in = nir_instr_as_intrinsic(i); + if (in->def.num_components != 1 || !dyn_ubo_general(in) || in->src[0].ssa != lo->src[0].ssa) + continue; + nir_scalar y = dyn_ubo_peel(in, &e); + if (!nir_scalar_equal(d, y) || e < c + 4 || e > c + 12 || (e - c) % 4 || q[(e - c) / 4]) + continue; + q[(e - c) / 4] = in; + found++; + } + if (found < 4) + continue; + /* the range that covers all four (the bound when the block size is unknown) */ + uint64_t end = 0; + for (unsigned j = 0; j < 4; j++) + end = MAX2(end, nir_intrinsic_range(q[j]) == ~0u ? UINT64_MAX + : (uint64_t)nir_intrinsic_range_base(q[j]) + nir_intrinsic_range(q[j])); + b.cursor = nir_before_instr(instr); + nir_def *v = nir_load_ubo(&b, 4, 32, lo->src[0].ssa, lo->src[1].ssa, + .access = nir_intrinsic_access(lo), .align_mul = 16, .align_offset = 0, + .range_base = 0, .range = end >= UINT32_MAX ? ~0u : (uint32_t)end); + for (unsigned j = 0; j < 4; j++) { + nir_def_rewrite_uses(&q[j]->def, nir_channel(&b, v, j)); + if (j) + nir_instr_remove(&q[j]->instr); + } + nir_instr_remove(instr); + progress = again = true; + break; /* the block changed under the iterator */ + } + } while (again); + } + nir_progress(progress, impl, nir_metadata_control_flow); + } + return progress; +} + +/* + * The general register-indexed UBO load, as Apple's A9 compiler reads a + * constant buffer at a computed offset (Metal, constant float *p: + * p[k*3 + j + 2] is imul, iadd, iadd and addr.add_shift at shift 2; a + * constant float4 *p read at p[k*3 + 1] is the same at shift 4, one 16-byte + * load): the element index is computed with integer + * ALU, then the pointer pair = buffer + (index << s) and one 4- or 16-byte + * load. Here offset' = umin(offset >> s, last) << s, the backend's direct + * form with a clamped index (robust access; the pair add does not + * sign-extend), with offset >> s built term by term where it divides + * exactly (k*12 + j*4 + 8 at s = 2 is k*3 + j + 2) and a ushr otherwise. + * s is 4 for a vec4 whose alignment is 16, else 2 per 32-bit component; a + * vec4 below 16-byte alignment becomes four scalar loads. A load whose + * block size is unknown, or below 4-byte alignment, keeps its shape (and + * stays refused by name). + */ +static void +dyn_ubo_rewrite(nir_builder *b, nir_intrinsic_instr *in, unsigned shift, unsigned size) +{ + const unsigned bytes = shift == 4 ? 16 : 4; + b->cursor = nir_before_instr(&in->instr); + nir_scalar off = nir_get_scalar(in->src[1].ssa, 0); + nir_def *index = dyn_ubo_div(b, off, shift, 0); + if (!index) + index = nir_ushr_imm(b, nir_channel(b, off.def, off.comp), shift); + index = nir_umin(b, index, nir_imm_int(b, (size - bytes) >> shift)); + nir_src_rewrite(&in->src[1], nir_ishl_imm(b, index, shift)); +} + +static bool +general_dyn_ubo(nir_builder *b, nir_intrinsic_instr *in, void *data) +{ + nir_shader *nir = data; + if (!dyn_ubo_general(in) || nir_intrinsic_align(in) < 4) + return false; + const unsigned n = in->def.num_components; + const unsigned shift = n == 4 && (nir_intrinsic_align(in) >= 16 || + dyn_ubo_divides(nir_get_scalar(in->src[1].ssa, 0), 4, 0)) ? 4 : 2; + /* the block's size; else the bytes the load's deref chain can reach + * (nir_lower_explicit_io's range, e.g. Zink's ubos[n] with n > 1) */ + unsigned size; + if (!ubo_block_bytes(nir, in->src[0].ssa, &size)) { + if (nir_intrinsic_range(in) == ~0u || nir_intrinsic_range_base(in) + (uint64_t)nir_intrinsic_range(in) > UINT32_MAX) + return false; + size = nir_intrinsic_range_base(in) + nir_intrinsic_range(in); + } + if (size < (shift == 4 ? 16 : 4)) + return false; + if (shift == 4 || n == 1) { + dyn_ubo_rewrite(b, in, shift, size); + return true; + } + b->cursor = nir_before_instr(&in->instr); + nir_def *comp[NIR_MAX_VEC_COMPONENTS]; + for (unsigned k = 0; k < n; k++) { + comp[k] = nir_load_ubo(b, 1, 32, in->src[0].ssa, nir_iadd_imm(b, in->src[1].ssa, 4 * k), + .access = nir_intrinsic_access(in), .align_mul = 4, .align_offset = 0, + .range_base = 0, .range = ~0u); + dyn_ubo_rewrite(b, nir_def_as_intrinsic(comp[k]), 2, size); + b->cursor = nir_after_instr(nir_def_instr(comp[k])); + } + nir_def_replace(&in->def, nir_vec(b, comp, n)); + return true; +} + static bool clamp_dyn_ubo(nir_shader *nir) { @@ -239,7 +542,7 @@ clamp_dyn_ubo(nir_shader *nir) if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr); - if (in->intrinsic != nir_intrinsic_load_ubo || in->def.bit_size != 32) + if (in->intrinsic != nir_intrinsic_load_ubo || in->def.bit_size != 32 || dyn_ubo_general(in)) continue; /* offset = shift(x) + c, the shape the backend folds */ nir_scalar s = nir_scalar_chase_movs(nir_get_scalar(in->src[1].ssa, 0)); @@ -372,7 +675,9 @@ g5p_nir(const uint32_t *spv, size_t words, mesa_shader_stage stage, const char * static util_once_flag once = UTIL_ONCE_FLAG_INIT; util_call_once(&once, g5p_init_options); struct spirv_capabilities caps = { .Shader = true, .Float16 = true, .Int16 = true, .Int8 = true, - .Int64 = true, .Float64 = true, .TransformFeedback = true }; + .Int64 = true, .Float64 = true, .TransformFeedback = true, + /* user clip distances: the TA user planes (pvr_g5p_clip_distance_probe()) */ + .ClipDistance = true }; struct spirv_to_nir_options opts = { .environment = NIR_SPIRV_VULKAN, .capabilities = &caps, @@ -444,6 +749,13 @@ g5p_nir_prepare(nir_shader *nir) NIR_PASS(progress, nir, vec4_dyn_ubo); if (stage == MESA_SHADER_FRAGMENT && pco_g5p_fs_dyn_ubo()) NIR_PASS(progress, nir, clamp_dyn_ubo); + /* after the clamp, which leaves these loads alone: their index is clamped here */ + NIR_PASS(_, nir, vec4_general_dyn_ubo); + if (nir_shader_intrinsics_pass(nir, general_dyn_ubo, nir_metadata_control_flow, nir)) { + NIR_PASS(_, nir, nir_opt_constant_folding); + NIR_PASS(_, nir, nir_opt_cse); + progress = true; + } if (progress) { NIR_PASS(_, nir, nir_opt_copy_prop); NIR_PASS(_, nir, nir_opt_dce); @@ -665,6 +977,152 @@ meta_blit_2d(nir_shader *nir) nir_shader_instructions_pass(nir, meta_tex_2d, nir_metadata_control_flow, NULL); } +/* + * The one-layer program variant (pvr_g5p_pipeline::one_layer): every 2D + * array sample the backend would draw as an array sample (all but the + * constant layer-0 ones, layer0_as_2d() in pco_g5p_general.c) becomes a 2D + * sample of the view's base layer. With one layer that is exact: Vulkan's + * layer is clamp(RNE(z), 0, layers - 1) for filtered samples, and a texel + * fetch at any other layer is out of bounds (undefined: robustImageAccess is + * not exposed). The A9 has no array state for a linear image (AGXMetalA9 + * ImageStateEncoderGen2::ImageStateFields: a linear 2D array encodes no + * texture type), so a one-layer view of a linear image (every single-level + * 2D colour image on G5P) needs this form when a program samples it at a + * layer it computes (Dolphin's EFB copy, restore and palette programs). Apple + * never meets the case: Metal types its textures at creation and makes every + * array twiddled. The draw selects this variant (g5p_cmd_draw()) only when an + * array sample's view has no array state, and its uploads refuse a + * multi-layer view by name (PCO_G5P_UPLOAD_IMAGE_ONE_LAYER). + */ +thread_local bool pvr_g5p_one_layer_compile; + +static bool +one_layer_as_2d(nir_builder *b, nir_instr *instr, void *data) +{ + if (instr->type != nir_instr_type_tex) + return false; + nir_tex_instr *t = nir_instr_as_tex(instr); + if (!t->is_array || t->sampler_dim != GLSL_SAMPLER_DIM_2D || t->coord_components != 3 || + (t->op != nir_texop_tex && t->op != nir_texop_txb && t->op != nir_texop_txl && + t->op != nir_texop_txd && t->op != nir_texop_txf && t->op != nir_texop_tg4)) + return false; + const int ci = nir_tex_instr_src_index(t, nir_tex_src_coord); + if (ci < 0) + return false; + /* layer0_as_2d()'s own samples keep their form (and their bytes) */ + nir_scalar z = nir_scalar_chase_movs(nir_get_scalar(t->src[ci].src.ssa, 2)); + if (nir_scalar_is_const(z) && + (t->op == nir_texop_txf ? nir_scalar_as_int(z) == 0 : nir_scalar_as_float(z) <= 0.5)) + return false; + b->cursor = nir_before_instr(instr); + nir_src_rewrite(&t->src[ci].src, nir_trim_vector(b, t->src[ci].src.ssa, 2)); + t->coord_components = 2; + t->is_array = false; + t->backend_flags |= PCO_G5P_TEX_ONE_LAYER_AS_2D; + return true; +} + +static void +one_layer_lower(nir_shader *nir) +{ + if (nir_shader_instructions_pass(nir, one_layer_as_2d, nir_metadata_control_flow, NULL)) { + nir_opt_dce(nir); + nir_index_ssa_defs(nir_shader_get_entrypoint(nir)); + } +} + +/* + * The LOD-bias program variant (pvr_g5p_pipeline::lod_bias): every fragment sample that uses its + * sampler's LOD adds the bound sampler's mipLodBias, read per binding from the sampler descriptor + * (load_smp_meta_pco of PVR_G5P_SAMPLER_META_LOD_BIAS, a preloaded uniform: PCO_G5P_UPLOAD_SAMPLER_META). + * The A9 sampler word has no bias field: AGXMetalA9 builds it from MTLSamplerDescriptor's filters, + * address modes, compare, anisotropy and LOD clamps only (-[AGXA9FamilyDevice + * newSamplerStateWithDescriptor:] 0x207ccd0bc reads no lodBias; AGX::SamplerStateEncoderGen2<>:: + * SamplerStateFields 0x207d80720 packs min/max LOD x 64), and Metal's LOD bias is the shader's sample + * argument (bias(b)), the biased sample form. Vulkan's LOD operation: + * lambda' = lambda_base + clamp(sampler bias + shader bias, -maxSamplerLodBias, maxSamplerLodBias), + * lambda_base the implicit LOD, or the explicit one (txl), then clamped to [minLod, maxLod] (the sampler + * word's clamps, applied after the bias as Metal's are). So: + * - tex: the biased sample (txb) by the sampler's bias (clamped when the sampler was made); + * - txb: by clamp(shader bias + sampler bias); + * - txl: at lod + sampler bias; + * - txd: gradients scaled by exp2(sampler bias): lambda_base = log2(rho), rho linear in the + * gradients, so log2(rho * 2^b) = lambda_base + b (anisotropic: the ratio is unchanged). + * txf, size queries and gathers do not use the sampler's LOD. The draw selects this variant only when a + * bound sampler of such a sample has a non-zero bias; base programs are unchanged. + */ +thread_local bool pvr_g5p_lod_bias_compile; + +static bool +lod_bias_tex(nir_builder *b, nir_instr *instr, void *data) +{ + if (instr->type != nir_instr_type_tex) + return false; + nir_tex_instr *t = nir_instr_as_tex(instr); + if (t->op != nir_texop_tex && t->op != nir_texop_txb && t->op != nir_texop_txl && t->op != nir_texop_txd) + return false; + if (t->sampler_dim == GLSL_SAMPLER_DIM_BUF || (t->backend_flags & PCO_G5P_TEX_SAMPLER_BIAS)) + return false; + nir_deref_instr *d = nir_get_tex_deref(t, nir_tex_src_sampler_deref); + if (!d) + d = nir_get_tex_deref(t, nir_tex_src_texture_deref); + unsigned element = 0; + if (d && d->deref_type == nir_deref_type_array) { + nir_deref_instr *parent = nir_deref_instr_parent(d); + /* the backend refuses these by name (tex_deref()) */ + if (!parent || parent->deref_type != nir_deref_type_var || !nir_src_is_const(d->arr.index)) + return false; + element = nir_src_as_uint(d->arr.index); + } else if (!d || d->deref_type != nir_deref_type_var) { + return false; + } + const nir_variable *var = nir_deref_instr_get_variable(d); + if (!var) + return false; + b->cursor = nir_before_instr(instr); + nir_def *bias = nir_load_smp_meta_pco(b, 1, nir_imm_int(b, element), .desc_set = var->data.descriptor_set, + .binding = var->data.binding, + .component = PVR_G5P_SAMPLER_META_LOD_BIAS); + switch (t->op) { + case nir_texop_tex: + nir_tex_instr_add_src(t, nir_tex_src_bias, bias); + t->op = nir_texop_txb; + break; + case nir_texop_txb: { + const int i = nir_tex_instr_src_index(t, nir_tex_src_bias); + nir_def *sum = nir_fadd(b, t->src[i].src.ssa, bias); + sum = nir_fmin(b, nir_fmax(b, sum, nir_imm_float(b, -PVR_G5P_MAX_SAMPLER_LOD_BIAS)), + nir_imm_float(b, PVR_G5P_MAX_SAMPLER_LOD_BIAS)); + nir_src_rewrite(&t->src[i].src, sum); + break; + } + case nir_texop_txl: { + const int i = nir_tex_instr_src_index(t, nir_tex_src_lod); + nir_src_rewrite(&t->src[i].src, nir_fadd(b, t->src[i].src.ssa, bias)); + break; + } + case nir_texop_txd: { + nir_def *scale = nir_fexp2(b, bias); + const int x = nir_tex_instr_src_index(t, nir_tex_src_ddx), y = nir_tex_instr_src_index(t, nir_tex_src_ddy); + nir_src_rewrite(&t->src[x].src, nir_fmul(b, t->src[x].src.ssa, scale)); + nir_src_rewrite(&t->src[y].src, nir_fmul(b, t->src[y].src.ssa, scale)); + break; + } + default: + UNREACHABLE("filtered above"); + } + t->backend_flags |= PCO_G5P_TEX_SAMPLER_BIAS; + return true; +} + +static void +lod_bias_lower(nir_shader *nir) +{ + if (nir->info.stage == MESA_SHADER_FRAGMENT && + nir_shader_instructions_pass(nir, lod_bias_tex, nir_metadata_control_flow, NULL)) + nir_index_ssa_defs(nir_shader_get_entrypoint(nir)); +} + /* The graphics lowerings of a prepared stage (the ICD's graphics_nir()) */ static nir_shader * graphics_lower(nir_shader *nir, unsigned color_targets, bool *sample_mask, bool point_size, bool *psize_dropped) @@ -673,6 +1131,8 @@ graphics_lower(nir_shader *nir, unsigned color_targets, bool *sample_mask, bool if (nir) meta_blit_2d(nir); + if (nir && pvr_g5p_one_layer_compile) + one_layer_lower(nir); if (nir && nir_shader_intrinsics_pass(nir, lower_layer_0, nir_metadata_control_flow, NULL)) { nir->info.outputs_written &= ~BITFIELD64_BIT(VARYING_SLOT_LAYER); BITSET_CLEAR(nir->info.system_values_read, SYSTEM_VALUE_LAYER_ID); @@ -680,6 +1140,8 @@ graphics_lower(nir_shader *nir, unsigned color_targets, bool *sample_mask, bool nir_opt_dce(nir); nir_index_ssa_defs(nir_shader_get_entrypoint(nir)); } + if (nir && pvr_g5p_lod_bias_compile) + lod_bias_lower(nir); if (nir && stage == MESA_SHADER_VERTEX && !point_size && !xfb_captures(nir, VARYING_SLOT_PSIZ) && nir_shader_intrinsics_pass(nir, drop_point_size, nir_metadata_control_flow, NULL)) { /* polygon mode POINT needs to know (pco_g5p_program::psize_dropped) */ diff --git a/src/imagination/vulkan/pvr_g5p_shader_dump.h b/src/imagination/vulkan/pvr_g5p_shader_dump.h index 3d3ddab56c0..e719924b0e5 100644 --- a/src/imagination/vulkan/pvr_g5p_shader_dump.h +++ b/src/imagination/vulkan/pvr_g5p_shader_dump.h @@ -60,4 +60,12 @@ const VkPipelineShaderStageCreateInfo *pvr_g5p_stub_stage(VkShaderStageFlagBits * raised: a refused one fails as before instead of being admitted without programs */ extern thread_local unsigned pvr_g5p_internal_pipeline; +/* Set while the driver compiles a pipeline's one-layer variant: its stages' array samples are drawn as 2D + * samples of a one-layer view (pvr_g5p_nir.c one_layer_as_2d()) */ +extern thread_local bool pvr_g5p_one_layer_compile; + +/* Set while the driver compiles a pipeline's LOD-bias variant: its fragment samples add their sampler's + * mipLodBias (pvr_g5p_nir.c lod_bias_lower()) */ +extern thread_local bool pvr_g5p_lod_bias_compile; + #endif diff --git a/src/imagination/vulkan/pvr_g5p_tex_encode.h b/src/imagination/vulkan/pvr_g5p_tex_encode.h index bc875761bec..3ae7948a467 100644 --- a/src/imagination/vulkan/pvr_g5p_tex_encode.h +++ b/src/imagination/vulkan/pvr_g5p_tex_encode.h @@ -243,8 +243,8 @@ pvr_g5p_sampler_encode(const VkSamplerCreateInfo *ci, bool extension, bool mip_n return "compare op"; if (ci->unnormalizedCoordinates) return "unnormalized coordinates"; - if (ci->mipLodBias != 0.0f) - return "LOD bias (no field in the sampler word)"; + /* mipLodBias: no field in the sampler word (AGXMetalA9's SamplerStateFields has none). The descriptor + * carries it (PVR_G5P_SAMPLER_META_LOD_BIAS) and the LOD-bias program variant adds it in the shader. */ if (extension) return "YCbCr conversion, reduction mode or custom border colour"; addr[0] = pvr_g5p_smp_address(ci->addressModeU, ci->borderColor); @@ -336,11 +336,31 @@ pvr_g5p_tex_size(const uint32_t state[4], unsigned level, uint32_t out[4]) * - ARRAY: the state must be an array's (a single-layer array view of a * linear image has only its 2D state: sampled at a non-constant layer). */ +/* + * - ONE_LAYER (an array sample at any layer of a one-layer view, drawn as 2D: + * the one-layer program variant): as AS_2D, but only for one layer. A + * 2D_ARRAY state of more layers would read its base layer where the + * application asked for another, so it is refused by name. + */ +static inline bool +pvr_g5p_tex_state_one_layer(const uint32_t dw[4]) +{ + const unsigned layout = dw[0] & 7; + const uint64_t q1 = dw[2] | (uint64_t)dw[3] << 32; + return layout == PVR_G5P_TEX_LAYOUT_2D || (layout == PVR_G5P_TEX_LAYOUT_2D_ARRAY && + !((q1 >> PVR_G5P_TEX_DEPTH_SHIFT) & 0x3fff)); +} + static inline const char * pvr_g5p_tex_state_for(uint32_t dw[8], unsigned flags) { const unsigned layout = dw[0] & 7; - if (flags & 1u /* PCO_G5P_UPLOAD_IMAGE_AS_2D */) { + if ((flags & 4u /* PCO_G5P_UPLOAD_IMAGE_ONE_LAYER */) && !pvr_g5p_tex_state_one_layer(dw)) + return layout == PVR_G5P_TEX_LAYOUT_2D_ARRAY + ? "a multi-layer array view where the one-layer program variant samples (a program samples a " + "one-layer view of a linear image and a multi-layer array view at non-constant layers)" + : "a one-layer array sample of a 3D or cube view"; + if (flags & 5u /* PCO_G5P_UPLOAD_IMAGE_AS_2D, _ONE_LAYER */) { if (layout == PVR_G5P_TEX_LAYOUT_2D_ARRAY) { uint64_t q0 = dw[0] | (uint64_t)dw[1] << 32, q1 = dw[2] | (uint64_t)dw[3] << 32; q0 = (q0 & ~UINT64_C(7)) | PVR_G5P_TEX_LAYOUT_2D;