Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/asahi/compiler/agx_nir_lower_cmat.c
Original file line number Diff line number Diff line change
Expand Up @@ -237,12 +237,15 @@ lower_matrix_instr(nir_builder *b, nir_instr *instr, struct hash_table *types)
enum glsl_base_type input_b = matrix_desc(intr->src[2]).element_type;
enum glsl_base_type accum = matrix_desc(intr->src[3]).element_type;
enum glsl_base_type result = matrix_desc(intr->src[0]).element_type;
/* Cooperative-matrix accumulators are always widened to f32 for
* the AGX path. */
assert((input == GLSL_TYPE_FLOAT16 || input == GLSL_TYPE_BFLOAT16 ||
(input == GLSL_TYPE_FLOAT && matrix_desc(intr->src[1]).rows == 8)) &&
(input_b == input || (matrix_desc(intr->src[1]).rows == 16 &&
((input == GLSL_TYPE_FLOAT16 && input_b == GLSL_TYPE_BFLOAT16) ||
(input == GLSL_TYPE_BFLOAT16 && input_b == GLSL_TYPE_FLOAT16)))) &&
(accum == GLSL_TYPE_FLOAT || accum == GLSL_TYPE_BFLOAT16) && result == accum);
(accum == GLSL_TYPE_FLOAT || accum == GLSL_TYPE_FLOAT16 ||
accum == GLSL_TYPE_BFLOAT16) && result == accum);
nir_def *a = load_matrix(b, intr->src[1]);
nir_def *bb = load_matrix(b, intr->src[2]);
nir_def *c = convert_matrix_value(b, load_matrix(b, intr->src[3]),
Expand Down
1 change: 1 addition & 0 deletions src/asahi/compiler/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ if with_tests
'test/test-compact-constants.cpp',
'test/test-minifloat.cpp',
'test/test-optimizer.cpp',
'test/test-cmat-accumulator.cpp',
'test/test-pressure-cse.cpp',
'test/test-rte-fma.cpp',
'test/test-border-index.cpp',
Expand Down
104 changes: 104 additions & 0 deletions src/asahi/compiler/test/test-cmat-accumulator.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* SPDX-License-Identifier: MIT
*/

#include "agx_compile.h"
extern "C" {
#include "agx_nir.h"
}
#include "compiler/glsl_types.h"
#include "nir.h"
#include "nir_builder.h"
#include <gtest/gtest.h>

static const glsl_type *
cmat_type(enum glsl_base_type element, enum glsl_cmat_use use)
{
struct glsl_cmat_description desc = {
.rows = 16,
.cols = 16,
.element_type = (uint16_t)element,
.scope = SCOPE_SUBGROUP,
.use = (uint8_t)use,
};
return glsl_cmat_type(&desc);
}

static nir_deref_instr *
cmat_var(nir_builder *b, enum glsl_base_type element, enum glsl_cmat_use use,
const char *name)
{
return nir_build_deref_var(
b, nir_local_variable_create(b->impl, cmat_type(element, use), name));
}

/* A 16x16x16 multiply-add of half inputs with the given accumulator type, as
* the flash attention shaders of llama.cpp write it for the PV product.
*/
static unsigned
lowered_muladd_count(enum glsl_base_type accum)
{
nir_builder b = nir_builder_init_simple_shader(
MESA_SHADER_COMPUTE, &agx_nir_options, "cmat");
b.shader->info.cs.has_cooperative_matrix = true;

nir_deref_instr *a = cmat_var(&b, GLSL_TYPE_FLOAT16, GLSL_CMAT_USE_A, "a");
nir_deref_instr *bb = cmat_var(&b, GLSL_TYPE_FLOAT16, GLSL_CMAT_USE_B, "b");
nir_deref_instr *c = cmat_var(&b, accum, GLSL_CMAT_USE_ACCUMULATOR, "c");
nir_deref_instr *d = cmat_var(&b, accum, GLSL_CMAT_USE_ACCUMULATOR, "d");

nir_cmat_construct(&b, &a->def, nir_imm_float16(&b, 1.0f));
nir_cmat_construct(&b, &bb->def, nir_imm_float16(&b, 1.0f));
nir_cmat_construct(&b, &c->def,
accum == GLSL_TYPE_FLOAT16 ? nir_imm_float16(&b, 0.0f)
: nir_imm_float(&b, 0.0f));
nir_cmat_muladd(&b, &d->def, &a->def, &bb->def, &c->def);

agx_nir_lower_cmat(b.shader);
nir_validate_shader(b.shader, "after agx_nir_lower_cmat");
nir_divergence_analysis(b.shader);

unsigned left = 0, agx = 0;
bool f16_store = false;
nir_foreach_function_impl(impl, b.shader) {
nir_foreach_block(block, impl) {
nir_foreach_instr(instr, block) {
if (instr->type == nir_instr_type_intrinsic) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
left += intr->intrinsic == nir_intrinsic_cmat_muladd;
/* The hardware accumulates in 32 bits whatever the matrix type,
* and every lane holds its own part of the result.
*/
agx += intr->intrinsic == nir_intrinsic_cmat_muladd_agx &&
intr->src[2].ssa->bit_size == 32 && intr->def.divergent;
} else if (instr->type == nir_instr_type_alu) {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (alu->op == nir_op_f2f16)
f16_store = true;
}
}
}
}
ralloc_free(b.shader);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: glsl_type_singleton_init_or_ref() at line 41 has no matching glsl_type_singleton_decref(). Please add one after ralloc_free(b.shader), as the other asahi tests do.

— Marcelo's agent

if (left || !agx)
return 0;
if (accum == GLSL_TYPE_FLOAT16 && !f16_store)
return 0;
return agx;
}

class CmatLower : public ::testing::Test {
protected:
CmatLower() { glsl_type_singleton_init_or_ref(); }
~CmatLower() { glsl_type_singleton_decref(); }
};

TEST_F(CmatLower, HalfInputsFloatAccumulator)
{
EXPECT_EQ(lowered_muladd_count(GLSL_TYPE_FLOAT), 1u);
}

TEST_F(CmatLower, HalfInputsHalfAccumulator)
{
EXPECT_EQ(lowered_muladd_count(GLSL_TYPE_FLOAT16), 1u);
}
1 change: 1 addition & 0 deletions src/compiler/nir/nir_divergence_analysis.c
Original file line number Diff line number Diff line change
Expand Up @@ -1094,6 +1094,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state)
case nir_intrinsic_load_ray_triangle_vertex_positions:
case nir_intrinsic_cmat_extract:
case nir_intrinsic_cmat_muladd_amd:
case nir_intrinsic_cmat_muladd_agx:
case nir_intrinsic_cmat_muladd_pan:
case nir_intrinsic_dpas_intel:
case nir_intrinsic_convert_cmat_intel:
Expand Down
Loading